From 148cf48d2b68354881066e2587e2673c91d2619a Mon Sep 17 00:00:00 2001 From: hrxi Date: Sat, 28 Dec 2019 03:11:51 +0100 Subject: [PATCH 01/56] Add libnice and listen for direct connections in Jingle SOCKS5 (#608) Add libnice as a plugin. If it is present, use libnice to enumerate local IP addresses and listen on them to support direct connections for Jingle SOCKS5. Tested with Conversations and Gajim. Created the nice.vapi file using ``` vapigen --library nice --pkg gio-2.0 --metadatadir metadata /usr/share/gir-1.0/Nice-0.1.gir ``` --- CMakeLists.txt | 2 +- cmake/FindNice.cmake | 13 + cmake/PkgConfigWithFallback.cmake | 2 +- libdino/src/service/connection_manager.vala | 2 + libdino/src/service/stream_interactor.vala | 5 +- plugins/CMakeLists.txt | 4 + plugins/ice/CMakeLists.txt | 30 ++ plugins/ice/src/plugin.vala | 30 ++ plugins/ice/src/register_plugin.vala | 3 + plugins/ice/vapi/metadata/Nice-0.1.metadata | 4 + plugins/ice/vapi/nice.vapi | 385 ++++++++++++++++++ .../module/xep/0065_socks5_bytestreams.vala | 17 +- .../xep/0260_jingle_socks5_bytestreams.vala | 217 +++++++++- 13 files changed, 698 insertions(+), 16 deletions(-) create mode 100644 cmake/FindNice.cmake create mode 100644 plugins/ice/CMakeLists.txt create mode 100644 plugins/ice/src/plugin.vala create mode 100644 plugins/ice/src/register_plugin.vala create mode 100644 plugins/ice/vapi/metadata/Nice-0.1.metadata create mode 100644 plugins/ice/vapi/nice.vapi diff --git a/CMakeLists.txt b/CMakeLists.txt index b738b585..2af0719b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ else () endif () # Prepare Plugins -set(DEFAULT_PLUGINS omemo;openpgp;http-files) +set(DEFAULT_PLUGINS omemo;openpgp;http-files;ice) foreach (plugin ${DEFAULT_PLUGINS}) if ("$CACHE{DINO_PLUGIN_ENABLED_${plugin}}" STREQUAL "") if (NOT DEFINED DINO_PLUGIN_ENABLED_${plugin}}) diff --git a/cmake/FindNice.cmake b/cmake/FindNice.cmake new file mode 100644 index 00000000..d40fc8c7 --- /dev/null +++ b/cmake/FindNice.cmake @@ -0,0 +1,13 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(Nice + PKG_CONFIG_NAME nice + LIB_NAMES nice + INCLUDE_NAMES nice.h + INCLUDE_DIR_SUFFIXES nice nice/include + DEPENDS GIO +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Nice + REQUIRED_VARS Nice_LIBRARY + VERSION_VAR Nice_VERSION) diff --git a/cmake/PkgConfigWithFallback.cmake b/cmake/PkgConfigWithFallback.cmake index ea14fa23..9124bb35 100644 --- a/cmake/PkgConfigWithFallback.cmake +++ b/cmake/PkgConfigWithFallback.cmake @@ -10,7 +10,7 @@ function(find_pkg_config_with_fallback name) endif(PKG_CONFIG_FOUND) if (${name}_PKG_CONFIG_FOUND) - # Found via pkg-config, using it's result values + # Found via pkg-config, using its result values set(${name}_FOUND ${${name}_PKG_CONFIG_FOUND}) # Try to find real file name of libraries diff --git a/libdino/src/service/connection_manager.vala b/libdino/src/service/connection_manager.vala index e0f4e19c..454bcc2c 100644 --- a/libdino/src/service/connection_manager.vala +++ b/libdino/src/service/connection_manager.vala @@ -8,6 +8,7 @@ namespace Dino { public class ConnectionManager : Object { public signal void stream_opened(Account account, XmppStream stream); + public signal void stream_attached_modules(Account account, XmppStream stream); public signal void connection_state_changed(Account account, ConnectionState state); public signal void connection_error(Account account, ConnectionError error); @@ -225,6 +226,7 @@ public class ConnectionManager : Object { connections[account].established = new DateTime.now_utc(); stream.attached_modules.connect((stream) => { + stream_attached_modules(account, stream); change_connection_state(account, ConnectionState.CONNECTED); }); stream.get_module(Sasl.Module.IDENTITY).received_auth_failure.connect((stream, node) => { diff --git a/libdino/src/service/stream_interactor.vala b/libdino/src/service/stream_interactor.vala index e60a43d6..192460d4 100644 --- a/libdino/src/service/stream_interactor.vala +++ b/libdino/src/service/stream_interactor.vala @@ -11,7 +11,7 @@ public class StreamInteractor : Object { public signal void account_removed(Account account); public signal void stream_resumed(Account account, XmppStream stream); public signal void stream_negotiated(Account account, XmppStream stream); - public signal void attached_modules(Account account, XmppStream stream); + public signal void stream_attached_modules(Account account, XmppStream stream); public ModuleManager module_manager; public ConnectionManager connection_manager; @@ -22,6 +22,9 @@ public class StreamInteractor : Object { connection_manager = new ConnectionManager(module_manager); connection_manager.stream_opened.connect(on_stream_opened); + connection_manager.stream_attached_modules.connect((account, stream) => { + stream_attached_modules(account, stream); + }); } public void connect_account(Account account) { diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 6cccec3b..48593c7a 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -2,6 +2,10 @@ if(DINO_PLUGIN_ENABLED_http-files) add_subdirectory(http-files) endif(DINO_PLUGIN_ENABLED_http-files) +if(DINO_PLUGIN_ENABLED_ice) + add_subdirectory(ice) +endif(DINO_PLUGIN_ENABLED_ice) + if(DINO_PLUGIN_ENABLED_openpgp) add_subdirectory(gpgme-vala) add_subdirectory(openpgp) diff --git a/plugins/ice/CMakeLists.txt b/plugins/ice/CMakeLists.txt new file mode 100644 index 00000000..76dba28f --- /dev/null +++ b/plugins/ice/CMakeLists.txt @@ -0,0 +1,30 @@ +find_packages(ICE_PACKAGES REQUIRED + Gee + GLib + GModule + GObject + GTK3 + Nice +) + +vala_precompile(ICE_VALA_C +SOURCES + src/plugin.vala + src/register_plugin.vala +CUSTOM_VAPIS + ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi + ${CMAKE_BINARY_DIR}/exports/dino.vapi + ${CMAKE_BINARY_DIR}/exports/qlite.vapi +PACKAGES + ${ICE_PACKAGES} +OPTIONS + --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi +) + +add_definitions(${VALA_CFLAGS}) +add_library(ice SHARED ${ICE_VALA_C}) +target_link_libraries(ice libdino ${ICE_PACKAGES}) +set_target_properties(ice PROPERTIES PREFIX "") +set_target_properties(ice PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) + +install(TARGETS ice ${PLUGIN_INSTALL}) diff --git a/plugins/ice/src/plugin.vala b/plugins/ice/src/plugin.vala new file mode 100644 index 00000000..f1c41a27 --- /dev/null +++ b/plugins/ice/src/plugin.vala @@ -0,0 +1,30 @@ +using Gee; +using Nice; +using Xmpp; + +namespace Dino.Plugins.Ice { + +public class Plugin : RootInterface, Object { + public Dino.Application app; + + public void registered(Dino.Application app) { + this.app = app; + app.stream_interactor.stream_attached_modules.connect((account, stream) => { + stream.get_module(Xmpp.Xep.Socks5Bytestreams.Module.IDENTITY).set_local_ip_address_handler(get_local_ip_addresses); + }); + } + + private Gee.List get_local_ip_addresses() { + Gee.List result = new ArrayList(); + foreach (string ip_address in Nice.interfaces_get_local_ips(false)) { + result.add(ip_address); + } + return result; + } + + public void shutdown() { + // Nothing to do + } +} + +} diff --git a/plugins/ice/src/register_plugin.vala b/plugins/ice/src/register_plugin.vala new file mode 100644 index 00000000..b2ed56c1 --- /dev/null +++ b/plugins/ice/src/register_plugin.vala @@ -0,0 +1,3 @@ +public Type register_plugin(Module module) { + return typeof (Dino.Plugins.Ice.Plugin); +} diff --git a/plugins/ice/vapi/metadata/Nice-0.1.metadata b/plugins/ice/vapi/metadata/Nice-0.1.metadata new file mode 100644 index 00000000..437da816 --- /dev/null +++ b/plugins/ice/vapi/metadata/Nice-0.1.metadata @@ -0,0 +1,4 @@ +Nice cheader_filename="nice.h" +Agent.new_reliable#constructor name="create_reliable" +PseudoTcpCallbacks#record skip +PseudoTcpSocket#class skip diff --git a/plugins/ice/vapi/nice.vapi b/plugins/ice/vapi/nice.vapi new file mode 100644 index 00000000..aa45cf08 --- /dev/null +++ b/plugins/ice/vapi/nice.vapi @@ -0,0 +1,385 @@ +/* nice.vapi generated by vapigen, do not modify. */ + +[CCode (cprefix = "Nice", gir_namespace = "Nice", gir_version = "0.1", lower_case_cprefix = "nice_")] +namespace Nice { + [CCode (cheader_filename = "nice.h", type_id = "nice_agent_get_type ()")] + public class Agent : GLib.Object { + [CCode (has_construct_function = false)] + public Agent (GLib.MainContext ctx, Nice.Compatibility compat); + public bool add_local_address (Nice.Address addr); + public uint add_stream (uint n_components); + [Version (since = "0.1.16")] + public async void close_async (); + [CCode (cname = "nice_agent_new_reliable", has_construct_function = false)] + [Version (since = "0.0.11")] + public Agent.create_reliable (GLib.MainContext ctx, Nice.Compatibility compat); + [Version (since = "0.1.6")] + public bool forget_relays (uint stream_id, uint component_id); + [CCode (has_construct_function = false)] + [Version (since = "0.1.15")] + public Agent.full (GLib.MainContext ctx, Nice.Compatibility compat, Nice.AgentOption flags); + public bool gather_candidates (uint stream_id); + [Version (since = "0.1.4")] + public string generate_local_candidate_sdp (Nice.Candidate candidate); + [Version (since = "0.1.4")] + public string generate_local_sdp (); + [Version (since = "0.1.4")] + public string generate_local_stream_sdp (uint stream_id, bool include_non_ice); + [Version (since = "0.1.8")] + public Nice.ComponentState get_component_state (uint stream_id, uint component_id); + public Nice.Candidate get_default_local_candidate (uint stream_id, uint component_id); + [Version (since = "0.1.5")] + public GLib.IOStream get_io_stream (uint stream_id, uint component_id); + public GLib.SList get_local_candidates (uint stream_id, uint component_id); + public bool get_local_credentials (uint stream_id, out string ufrag, out string pwd); + public GLib.SList get_remote_candidates (uint stream_id, uint component_id); + public bool get_selected_pair (uint stream_id, uint component_id, Nice.Candidate local, Nice.Candidate remote); + [Version (since = "0.1.5")] + public GLib.Socket? get_selected_socket (uint stream_id, uint component_id); + [Version (since = "0.1.4")] + public unowned string get_stream_name (uint stream_id); + [Version (since = "0.1.4")] + public Nice.Candidate parse_remote_candidate_sdp (uint stream_id, string sdp); + [Version (since = "0.1.4")] + public int parse_remote_sdp (string sdp); + [Version (since = "0.1.4")] + public GLib.SList parse_remote_stream_sdp (uint stream_id, string sdp, string ufrag, string pwd); + [Version (since = "0.1.16")] + public bool peer_candidate_gathering_done (uint stream_id); + [Version (since = "0.1.5")] + public ssize_t recv (uint stream_id, uint component_id, [CCode (array_length_cname = "buf_len", array_length_pos = 3.5, array_length_type = "gsize")] out unowned uint8[] buf, GLib.Cancellable? cancellable = null) throws GLib.Error; + [Version (since = "0.1.5")] + public int recv_messages (uint stream_id, uint component_id, [CCode (array_length_cname = "n_messages", array_length_pos = 3.5, array_length_type = "guint")] out unowned Nice.InputMessage[] messages, GLib.Cancellable? cancellable = null) throws GLib.Error; + [Version (since = "0.1.5")] + public int recv_messages_nonblocking (uint stream_id, uint component_id, [CCode (array_length_cname = "n_messages", array_length_pos = 3.5, array_length_type = "guint")] out unowned Nice.InputMessage[] messages, GLib.Cancellable? cancellable = null) throws GLib.Error; + [Version (since = "0.1.5")] + public ssize_t recv_nonblocking (uint stream_id, uint component_id, [CCode (array_length_cname = "buf_len", array_length_pos = 3.5, array_length_type = "gsize")] out unowned uint8[] buf, GLib.Cancellable? cancellable = null) throws GLib.Error; + public void remove_stream (uint stream_id); + public bool restart (); + [Version (since = "0.1.6")] + public bool restart_stream (uint stream_id); + public int send (uint stream_id, uint component_id, uint len, string buf); + [Version (since = "0.1.5")] + public int send_messages_nonblocking (uint stream_id, uint component_id, [CCode (array_length_cname = "n_messages", array_length_pos = 3.5, array_length_type = "guint")] Nice.OutputMessage[] messages, GLib.Cancellable? cancellable = null) throws GLib.Error; + public bool set_local_credentials (uint stream_id, string ufrag, string pwd); + public void set_port_range (uint stream_id, uint component_id, uint min_port, uint max_port); + public bool set_relay_info (uint stream_id, uint component_id, string server_ip, uint server_port, string username, string password, Nice.RelayType type); + public int set_remote_candidates (uint stream_id, uint component_id, GLib.SList candidates); + public bool set_remote_credentials (uint stream_id, string ufrag, string pwd); + public bool set_selected_pair (uint stream_id, uint component_id, string lfoundation, string rfoundation); + public bool set_selected_remote_candidate (uint stream_id, uint component_id, Nice.Candidate candidate); + [Version (since = "0.0.10")] + public void set_software (string software); + [Version (since = "0.1.4")] + public bool set_stream_name (uint stream_id, string name); + [Version (since = "0.0.9")] + public void set_stream_tos (uint stream_id, int tos); + [NoAccessorMethod] + [Version (since = "0.1.8")] + public bool bytestream_tcp { get; } + [NoAccessorMethod] + public uint compatibility { get; construct; } + [NoAccessorMethod] + public bool controlling_mode { get; set; } + [NoAccessorMethod] + [Version (since = "0.1.14")] + public bool force_relay { get; set; } + [NoAccessorMethod] + public bool full_mode { get; construct; } + [NoAccessorMethod] + [Version (since = "0.1.8")] + public bool ice_tcp { get; set; } + [NoAccessorMethod] + [Version (since = "0.1.16")] + public bool ice_trickle { get; set; } + [NoAccessorMethod] + [Version (since = "0.1.8")] + public bool ice_udp { get; set; } + [NoAccessorMethod] + [Version (since = "0.1.8")] + public bool keepalive_conncheck { get; set; } + [NoAccessorMethod] + public void* main_context { get; construct; } + [NoAccessorMethod] + public uint max_connectivity_checks { get; set; } + [NoAccessorMethod] + [Version (since = "0.0.4")] + public string proxy_ip { owned get; set; } + [NoAccessorMethod] + [Version (since = "0.0.4")] + public string proxy_password { owned get; set; } + [NoAccessorMethod] + [Version (since = "0.0.4")] + public uint proxy_port { get; set; } + [NoAccessorMethod] + [Version (since = "0.0.4")] + public uint proxy_type { get; set; } + [NoAccessorMethod] + [Version (since = "0.0.4")] + public string proxy_username { owned get; set; } + [NoAccessorMethod] + [Version (since = "0.0.11")] + public bool reliable { get; construct; } + [NoAccessorMethod] + [Version (since = "0.1.15")] + public uint stun_initial_timeout { get; set construct; } + [NoAccessorMethod] + [Version (since = "0.1.15")] + public uint stun_max_retransmissions { get; set construct; } + [NoAccessorMethod] + public uint stun_pacing_timer { get; set construct; } + [NoAccessorMethod] + [Version (since = "0.1.15")] + public uint stun_reliable_timeout { get; set construct; } + [NoAccessorMethod] + public string stun_server { owned get; set; } + [NoAccessorMethod] + public uint stun_server_port { get; set; } + [NoAccessorMethod] + public bool support_renomination { get; set; } + [NoAccessorMethod] + [Version (since = "0.0.7")] + public bool upnp { get; set construct; } + [NoAccessorMethod] + [Version (since = "0.0.7")] + public uint upnp_timeout { get; set construct; } + public signal void candidate_gathering_done (uint stream_id); + public signal void component_state_changed (uint stream_id, uint component_id, uint state); + public signal void initial_binding_request_received (uint stream_id); + [Version (deprecated = true, deprecated_since = "0.1.8")] + public signal void new_candidate (uint stream_id, uint component_id, string foundation); + [Version (since = "0.1.8")] + public signal void new_candidate_full (Nice.Candidate candidate); + [Version (deprecated = true, deprecated_since = "0.1.8")] + public signal void new_remote_candidate (uint stream_id, uint component_id, string foundation); + [Version (since = "0.1.8")] + public signal void new_remote_candidate_full (Nice.Candidate candidate); + [Version (deprecated = true, deprecated_since = "0.1.8")] + public signal void new_selected_pair (uint stream_id, uint component_id, string lfoundation, string rfoundation); + [Version (since = "0.1.8")] + public signal void new_selected_pair_full (uint stream_id, uint component_id, Nice.Candidate lcandidate, Nice.Candidate rcandidate); + [Version (since = "0.0.11")] + public signal void reliable_transport_writable (uint stream_id, uint component_id); + [Version (since = "0.1.5")] + public signal void streams_removed ([CCode (array_length = false, array_null_terminated = true)] uint[] stream_ids); + } + [CCode (cheader_filename = "nice.h", copy_function = "g_boxed_copy", free_function = "g_boxed_free", type_id = "nice_candidate_get_type ()")] + [Compact] + public class Candidate { + public Nice.Address addr; + public Nice.Address base_addr; + public uint component_id; + [CCode (array_length = false)] + public weak char foundation[33]; + public weak string password; + public uint32 priority; + public void* sockptr; + public uint stream_id; + public Nice.CandidateTransport transport; + public Nice.TurnServer turn; + public Nice.CandidateType type; + public weak string username; + [CCode (has_construct_function = false)] + public Candidate (Nice.CandidateType type); + public Nice.Candidate copy (); + [Version (since = "0.1.15")] + public bool equal_target (Nice.Candidate candidate2); + public void free (); + } + [CCode (cheader_filename = "nice.h", has_type_id = false)] + public struct Address { + [CCode (cname = "s.addr")] + public void* s_addr; + [CCode (cname = "s.ip4")] + public void* s_ip4; + [CCode (cname = "s.ip6")] + public void* s_ip6; + public void copy_to_sockaddr (void* sin); + public bool equal (Nice.Address b); + [Version (since = "0.1.8")] + public bool equal_no_port (Nice.Address b); + public void free (); + public uint get_port (); + public void init (); + public int ip_version (); + public bool is_private (); + public bool is_valid (); + public void set_from_sockaddr (void* sin); + public bool set_from_string (string str); + public void set_ipv4 (uint32 addr_ipv4); + public void set_ipv6 (uint8 addr_ipv6); + public void set_port (uint port); + public void to_string (string dst); + } + [CCode (cheader_filename = "nice.h", has_type_id = false)] + [Version (since = "0.1.5")] + public struct InputMessage { + [CCode (array_length_cname = "n_buffers")] + public weak GLib.InputVector[] buffers; + public int n_buffers; + public Nice.Address from; + public size_t length; + } + [CCode (cheader_filename = "nice.h", has_type_id = false)] + [Version (since = "0.1.5")] + public struct OutputMessage { + [CCode (array_length_cname = "n_buffers")] + public weak GLib.OutputVector[] buffers; + public int n_buffers; + } + [CCode (cheader_filename = "nice.h", cname = "TurnServer", has_type_id = false)] + public struct TurnServer { + public int ref_count; + public Nice.Address server; + public weak string username; + public weak string password; + public Nice.RelayType type; + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_AGENT_OPTION_", has_type_id = false)] + [Flags] + [Version (since = "0.1.15")] + public enum AgentOption { + REGULAR_NOMINATION, + RELIABLE, + LITE_MODE, + ICE_TRICKLE, + SUPPORT_RENOMINATION + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_CANDIDATE_TRANSPORT_", has_type_id = false)] + public enum CandidateTransport { + UDP, + TCP_ACTIVE, + TCP_PASSIVE, + TCP_SO + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_CANDIDATE_TYPE_", has_type_id = false)] + public enum CandidateType { + HOST, + SERVER_REFLEXIVE, + PEER_REFLEXIVE, + RELAYED + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_COMPATIBILITY_", has_type_id = false)] + public enum Compatibility { + RFC5245, + DRAFT19, + GOOGLE, + MSN, + WLM2009, + OC2007, + OC2007R2, + LAST + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_COMPONENT_STATE_", has_type_id = false)] + public enum ComponentState { + DISCONNECTED, + GATHERING, + CONNECTING, + CONNECTED, + READY, + FAILED, + LAST; + [Version (since = "0.1.6")] + public unowned string to_string (); + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_COMPONENT_TYPE_", has_type_id = false)] + public enum ComponentType { + RTP, + RTCP + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_NOMINATION_MODE_", has_type_id = false)] + [Version (since = "0.1.15")] + public enum NominationMode { + REGULAR, + AGGRESSIVE + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_PROXY_TYPE_", has_type_id = false)] + [Version (since = "0.0.4")] + public enum ProxyType { + NONE, + SOCKS5, + HTTP, + LAST + } + [CCode (cheader_filename = "nice.h", cname = "PseudoTcpDebugLevel", cprefix = "PSEUDO_TCP_DEBUG_", has_type_id = false)] + [Version (since = "0.0.11")] + public enum PseudoTcpDebugLevel { + NONE, + NORMAL, + VERBOSE + } + [CCode (cheader_filename = "nice.h", cname = "PseudoTcpShutdown", cprefix = "PSEUDO_TCP_SHUTDOWN_", has_type_id = false)] + [Version (since = "0.1.8")] + public enum PseudoTcpShutdown { + RD, + WR, + RDWR + } + [CCode (cheader_filename = "nice.h", cname = "PseudoTcpState", cprefix = "PSEUDO_TCP_", has_type_id = false)] + [Version (since = "0.0.11")] + public enum PseudoTcpState { + LISTEN, + SYN_SENT, + SYN_RECEIVED, + ESTABLISHED, + CLOSED, + FIN_WAIT_1, + FIN_WAIT_2, + CLOSING, + TIME_WAIT, + CLOSE_WAIT, + LAST_ACK + } + [CCode (cheader_filename = "nice.h", cname = "PseudoTcpWriteResult", cprefix = "WR_", has_type_id = false)] + [Version (since = "0.0.11")] + public enum PseudoTcpWriteResult { + SUCCESS, + TOO_LARGE, + FAIL + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_RELAY_TYPE_TURN_", has_type_id = false)] + public enum RelayType { + UDP, + TCP, + TLS + } + [CCode (cheader_filename = "nice.h", instance_pos = 5.9)] + public delegate void AgentRecvFunc (Nice.Agent agent, uint stream_id, uint component_id, uint len, string buf); + [CCode (cheader_filename = "nice.h", cname = "NICE_AGENT_MAX_REMOTE_CANDIDATES")] + public const int AGENT_MAX_REMOTE_CANDIDATES; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_DIRECTION_MS_PREF_ACTIVE")] + public const int CANDIDATE_DIRECTION_MS_PREF_ACTIVE; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_DIRECTION_MS_PREF_PASSIVE")] + public const int CANDIDATE_DIRECTION_MS_PREF_PASSIVE; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_MAX_FOUNDATION")] + public const int CANDIDATE_MAX_FOUNDATION; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TRANSPORT_MS_PREF_TCP")] + public const int CANDIDATE_TRANSPORT_MS_PREF_TCP; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TRANSPORT_MS_PREF_UDP")] + public const int CANDIDATE_TRANSPORT_MS_PREF_UDP; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_HOST")] + public const int CANDIDATE_TYPE_PREF_HOST; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_NAT_ASSISTED")] + public const int CANDIDATE_TYPE_PREF_NAT_ASSISTED; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_PEER_REFLEXIVE")] + public const int CANDIDATE_TYPE_PREF_PEER_REFLEXIVE; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_RELAYED")] + public const int CANDIDATE_TYPE_PREF_RELAYED; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_RELAYED_UDP")] + public const int CANDIDATE_TYPE_PREF_RELAYED_UDP; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_SERVER_REFLEXIVE")] + public const int CANDIDATE_TYPE_PREF_SERVER_REFLEXIVE; + [CCode (cheader_filename = "nice.h")] + public static void debug_disable (bool with_stun); + [CCode (cheader_filename = "nice.h")] + public static void debug_enable (bool with_stun); + [CCode (cheader_filename = "nice.h")] + public static string? interfaces_get_ip_for_interface (string interface_name); + [CCode (cheader_filename = "nice.h")] + public static GLib.List interfaces_get_local_interfaces (); + [CCode (cheader_filename = "nice.h")] + public static GLib.List interfaces_get_local_ips (bool include_loopback); + [CCode (cheader_filename = "nice.h", cname = "pseudo_tcp_set_debug_level")] + [Version (since = "0.0.11")] + public static void pseudo_tcp_set_debug_level (Nice.PseudoTcpDebugLevel level); +} diff --git a/xmpp-vala/src/module/xep/0065_socks5_bytestreams.vala b/xmpp-vala/src/module/xep/0065_socks5_bytestreams.vala index fdee2411..c184877c 100644 --- a/xmpp-vala/src/module/xep/0065_socks5_bytestreams.vala +++ b/xmpp-vala/src/module/xep/0065_socks5_bytestreams.vala @@ -18,21 +18,34 @@ public class Proxy : Object { } } +public delegate Gee.List GetLocalIpAddresses(); + public class Module : XmppStreamModule, Iq.Handler { public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "0065_socks5_bytestreams"); + private GetLocalIpAddresses? get_local_ip_addresses_impl = null; + public override void attach(XmppStream stream) { stream.add_flag(new Flag()); query_availability.begin(stream); } public override void detach(XmppStream stream) { } - public async void on_iq_set(XmppStream stream, Iq.Stanza iq) { } - public Gee.List get_proxies(XmppStream stream) { return stream.get_flag(Flag.IDENTITY).proxies; } + public void set_local_ip_address_handler(owned GetLocalIpAddresses get_local_ip_addresses) { + get_local_ip_addresses_impl = (owned)get_local_ip_addresses; + } + + public Gee.List get_local_ip_addresses() { + if (get_local_ip_addresses_impl == null) { + return Gee.List.empty(); + } + return get_local_ip_addresses_impl(); + } + private async void query_availability(XmppStream stream) { ServiceDiscovery.ItemsResult? items_result = yield stream.get_module(ServiceDiscovery.Module.IDENTITY).request_items(stream, stream.remote_name); if (items_result == null) return; diff --git a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala index ea7ef375..bb31cadc 100644 --- a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala +++ b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala @@ -5,6 +5,7 @@ using Xmpp.Xep; namespace Xmpp.Xep.JingleSocks5Bytestreams { private const string NS_URI = "urn:xmpp:jingle:transports:s5b:1"; +private const int NEGOTIATION_TIMEOUT = 3; public class Module : Jingle.Transport, XmppStreamModule { public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "0260_jingle_socks5_bytestreams"); @@ -33,7 +34,7 @@ public class Module : Jingle.Transport, XmppStreamModule { public int transport_priority() { return 1; } - private Gee.List get_local_candidates(XmppStream stream) { + private Gee.List get_proxies(XmppStream stream) { Gee.List result = new ArrayList(); int i = 1 << 15; foreach (Socks5Bytestreams.Proxy proxy in stream.get_module(Socks5Bytestreams.Module.IDENTITY).get_proxies(stream)) { @@ -42,18 +43,59 @@ public class Module : Jingle.Transport, XmppStreamModule { } return result; } + private Gee.List start_local_listeners(XmppStream stream, Jid local_full_jid, string dstaddr, out LocalListener? local_listener) { + Gee.List result = new ArrayList(); + SocketListener listener = new SocketListener(); + int i = 1 << 15; + foreach (string ip_address in stream.get_module(Socks5Bytestreams.Module.IDENTITY).get_local_ip_addresses()) { + InetSocketAddress addr = new InetSocketAddress.from_string(ip_address, 0); + SocketAddress effective_any; + string cid = random_uuid(); + try { + listener.add_address(addr, SocketType.STREAM, SocketProtocol.DEFAULT, new StringWrapper(cid), out effective_any); + } catch (Error e) { + continue; + } + InetSocketAddress effective = (InetSocketAddress)effective_any; + result.add(new Candidate.build(cid, ip_address, local_full_jid, (int)effective.port, i, CandidateType.DIRECT)); + i -= 1; + } + if (!result.is_empty) { + local_listener = new LocalListener(listener, dstaddr); + local_listener.start(); + } else { + local_listener = new LocalListener.empty(); + } + return result; + } + private void select_candidates(XmppStream stream, Jid local_full_jid, string dstaddr, Parameters result) { + result.local_candidates.add_all(get_proxies(stream)); + result.local_candidates.add_all(start_local_listeners(stream, local_full_jid, dstaddr, out result.listener)); + result.local_candidates.sort((c1, c2) => { + if (c1.priority < c2.priority) { return 1; } + if (c1.priority > c2.priority) { return -1; } + return 0; + }); + } public Jingle.TransportParameters create_transport_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid) { Parameters result = new Parameters.create(local_full_jid, peer_full_jid, random_uuid()); - result.local_candidates.add_all(get_local_candidates(stream)); + string dstaddr = calculate_dstaddr(result.sid, local_full_jid, peer_full_jid); + select_candidates(stream, local_full_jid, dstaddr, result); return result; } public Jingle.TransportParameters parse_transport_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { Parameters result = Parameters.parse(local_full_jid, peer_full_jid, transport); - result.local_candidates.add_all(get_local_candidates(stream)); + string dstaddr = calculate_dstaddr(result.sid, local_full_jid, peer_full_jid); + select_candidates(stream, local_full_jid, dstaddr, result); return result; } } +private string calculate_dstaddr(string sid, Jid first_jid, Jid second_jid) { + string hashed = sid + first_jid.to_string() + second_jid.to_string(); + return Checksum.compute_for_string(ChecksumType.SHA1, hashed); +} + public enum CandidateType { ASSISTED, DIRECT, @@ -156,6 +198,142 @@ bool bytes_equal(uint8[] a, uint8[] b) { return true; } +class StringWrapper : GLib.Object { + public string str { get; set; } + + public StringWrapper(string str) { + this.str = str; + } +} + +class LocalListener { + SocketListener? inner; + string dstaddr; + HashMap connections = new HashMap(); + + public LocalListener(SocketListener inner, string dstaddr) { + this.inner = inner; + this.dstaddr = dstaddr; + } + public LocalListener.empty() { + this.inner = null; + this.dstaddr = ""; + } + + public void start() { + if (inner == null) { + return; + } + run.begin(); + } + async void run() { + while (true) { + Object cid; + SocketConnection conn; + try { + conn = yield inner.accept_async(null, out cid); + } catch (Error e) { + break; + } + handle_conn.begin(((StringWrapper)cid).str, conn); + } + } + async void handle_conn(string cid, SocketConnection conn) { + conn.socket.timeout = NEGOTIATION_TIMEOUT; + size_t read; + size_t written; + uint8[] read_buffer = new uint8[1024]; + ByteArray write_buffer = new ByteArray(); + + try { + // 05 SOCKS version 5 + // ?? number of authentication methods + yield conn.input_stream.read_all_async(read_buffer[0:2], GLib.Priority.DEFAULT, null, out read); + if (read != 2) { + throw new IOError.PROXY_FAILED("wanted client hello message consisting of 2 bytes, only got %d bytes".printf((int)read)); + } + if (read_buffer[0] != 0x05 || read_buffer[1] == 0) { + throw new IOError.PROXY_FAILED("wanted 05 xx, got %02x %02x".printf(read_buffer[0], read_buffer[1])); + } + int num_auth_methods = read_buffer[1]; + // ?? authentication method (num_auth_methods times) + yield conn.input_stream.read_all_async(read_buffer[0:num_auth_methods], GLib.Priority.DEFAULT, null, out read); + bool found_null_auth = false; + for (int i = 0; i < read; i++) { + if (read_buffer[i] == 0x00) { + found_null_auth = true; + break; + } + } + if (read != num_auth_methods || !found_null_auth) { + throw new IOError.PROXY_FAILED("peer didn't offer null auth"); + } + // 05 SOCKS version 5 + // 00 nop authentication + yield conn.output_stream.write_all_async({0x05, 0x00}, GLib.Priority.DEFAULT, null, out written); + + // 05 SOCKS version 5 + // 01 connect + // 00 reserved + // 03 address type: domain name + // ?? length of the domain + // .. domain + // 00 port 0 (upper half) + // 00 port 0 (lower half) + yield conn.input_stream.read_all_async(read_buffer[0:4], GLib.Priority.DEFAULT, null, out read); + if (read != 4) { + throw new IOError.PROXY_FAILED("wanted connect message consisting of 4 bytes, only got %d bytes".printf((int)read)); + } + if (read_buffer[0] != 0x05 || read_buffer[1] != 0x01 || read_buffer[3] != 0x03) { + throw new IOError.PROXY_FAILED("wanted 05 00 ?? 03, got %02x %02x %02x %02x".printf(read_buffer[0], read_buffer[1], read_buffer[2], read_buffer[3])); + } + yield conn.input_stream.read_all_async(read_buffer[0:1], GLib.Priority.DEFAULT, null, out read); + if (read != 1) { + throw new IOError.PROXY_FAILED("wanted length of dstaddr consisting of 1 byte, only got %d bytes".printf((int)read)); + } + int dstaddr_len = read_buffer[0]; + yield conn.input_stream.read_all_async(read_buffer[0:dstaddr_len+2], GLib.Priority.DEFAULT, null, out read); + if (read != dstaddr_len + 2) { + throw new IOError.PROXY_FAILED("wanted dstaddr and port consisting of %d bytes, got %d bytes".printf(dstaddr_len + 2, (int)read)); + } + if (!bytes_equal(read_buffer[0:dstaddr_len], dstaddr.data)) { + string repr = ((string)read_buffer[0:dstaddr.length]).make_valid().escape(); + throw new IOError.PROXY_FAILED(@"wanted dstaddr $(dstaddr), got $(repr)"); + } + if (read_buffer[dstaddr_len] != 0x00 || read_buffer[dstaddr_len + 1] != 0x00) { + throw new IOError.PROXY_FAILED("wanted 00 00, got %02x %02x".printf(read_buffer[dstaddr_len], read_buffer[dstaddr_len + 1])); + } + + // 05 SOCKS version 5 + // 00 success + // 00 reserved + // 03 address type: domain name + // ?? length of the domain + // .. domain + // 00 port 0 (upper half) + // 00 port 0 (lower half) + write_buffer.append({0x05, 0x00, 0x00, 0x03}); + write_buffer.append({(uint8)dstaddr.length}); + write_buffer.append(dstaddr.data); + write_buffer.append({0x00, 0x00}); + yield conn.output_stream.write_all_async(write_buffer.data, GLib.Priority.DEFAULT, null, out written); + + conn.socket.timeout = 0; + if (!connections.has_key(cid)) { + connections[cid] = conn; + } + } catch (Error e) { + } + } + + public SocketConnection? get_connection(string cid) { + if (!connections.has_key(cid)) { + return null; + } + return connections[cid]; + } +} + class Parameters : Jingle.TransportParameters, Object { public Jingle.Role role { get; private set; } public string sid { get; private set; } @@ -163,6 +341,7 @@ class Parameters : Jingle.TransportParameters, Object { public string local_dstaddr { get; private set; } public Gee.List local_candidates = new ArrayList(); public Gee.List remote_candidates = new ArrayList(); + public LocalListener? listener = null; Jid local_full_jid; Jid peer_full_jid; @@ -179,10 +358,6 @@ class Parameters : Jingle.TransportParameters, Object { SourceFunc waiting_for_activation_callback; bool waiting_for_activation_error = false; - private static string calculate_dstaddr(string sid, Jid first_jid, Jid second_jid) { - string hashed = sid + first_jid.to_string() + second_jid.to_string(); - return Checksum.compute_for_string(ChecksumType.SHA1, hashed); - } private Parameters(Jingle.Role role, string sid, Jid local_full_jid, Jid peer_full_jid, string? remote_dstaddr) { this.role = role; this.sid = sid; @@ -355,7 +530,21 @@ class Parameters : Jingle.TransportParameters, Object { wait_for_remote_activation.begin(local_selected_candidate, local_selected_candidate_conn); } } else { - connect_to_local_candidate.begin(remote_selected_candidate); + if (remote_selected_candidate.type_ == CandidateType.DIRECT) { + Jingle.Session? strong = session; + if (strong == null) { + return; + } + SocketConnection? conn = listener.get_connection(remote_selected_candidate.cid); + if (conn == null) { + // Remote hasn't actually connected to us?! + strong.set_transport_connection(hack, null); + return; + } + strong.set_transport_connection(hack, conn); + } else { + connect_to_local_candidate.begin(remote_selected_candidate); + } } } public async void wait_for_remote_activation(Candidate candidate, SocketConnection conn) { @@ -425,7 +614,7 @@ class Parameters : Jingle.TransportParameters, Object { } } public async SocketConnection connect_to_socks5(Candidate candidate, string dstaddr) throws Error { - SocketClient socket_client = new SocketClient() { timeout=3 }; + SocketClient socket_client = new SocketClient() { timeout=NEGOTIATION_TIMEOUT }; string address = @"[$(candidate.host)]:$(candidate.port)"; debug("Connecting to SOCKS5 server at %s", address); @@ -444,7 +633,10 @@ class Parameters : Jingle.TransportParameters, Object { yield conn.input_stream.read_all_async(read_buffer[0:2], GLib.Priority.DEFAULT, null, out read); // 05 SOCKS version 5 - // 01 success + // 00 nop authentication + if (read != 2) { + throw new IOError.PROXY_FAILED("wanted 05 00, only got %d bytes".printf((int)read)); + } if (read_buffer[0] != 0x05 || read_buffer[1] != 0x00) { throw new IOError.PROXY_FAILED("wanted 05 00, got %02x %02x".printf(read_buffer[0], read_buffer[1])); } @@ -472,6 +664,9 @@ class Parameters : Jingle.TransportParameters, Object { // .. domain // 00 port 0 (upper half) // 00 port 0 (lower half) + if (read != write_buffer.len) { + throw new IOError.PROXY_FAILED("wanted server success response consisting of %d bytes, only got %d bytes".printf((int)write_buffer.len, (int)read)); + } if (read_buffer[0] != 0x05 || read_buffer[1] != 0x00 || read_buffer[3] != 0x03) { throw new IOError.PROXY_FAILED("wanted 05 00 ?? 03, got %02x %02x %02x %02x".printf(read_buffer[0], read_buffer[1], read_buffer[2], read_buffer[3])); } @@ -486,7 +681,7 @@ class Parameters : Jingle.TransportParameters, Object { throw new IOError.PROXY_FAILED("wanted port 00 00, got %02x %02x".printf(read_buffer[5+dstaddr.length], read_buffer[5+dstaddr.length+1])); } - conn.get_socket().set_timeout(0); + conn.socket.timeout = 0; return conn; } From 2b90fcc39a1079346d6c5e2bfff8987104da737a Mon Sep 17 00:00:00 2001 From: fiaxh Date: Fri, 19 Mar 2021 22:46:39 +0100 Subject: [PATCH 02/56] Improve & refactor Jingle base implementation Co-authored-by: Marvin W --- xmpp-vala/CMakeLists.txt | 29 +- xmpp-vala/src/core/xmpp_log.vala | 4 +- xmpp-vala/src/module/iq/module.vala | 2 + xmpp-vala/src/module/xep/0166_jingle.vala | 1061 ----------------- .../src/module/xep/0166_jingle/component.vala | 52 + .../src/module/xep/0166_jingle/content.vala | 236 ++++ .../xep/0166_jingle/content_description.vala | 27 + .../module/xep/0166_jingle/content_node.vala | 112 ++ .../xep/0166_jingle/content_security.vala | 18 + .../xep/0166_jingle/content_transport.vala | 29 + .../module/xep/0166_jingle/jingle_flag.vala | 38 + .../module/xep/0166_jingle/jingle_module.vala | 235 ++++ .../xep/0166_jingle/jingle_structs.vala | 73 ++ .../xep/0166_jingle/reason_element.vala | 29 + .../src/module/xep/0166_jingle/session.vala | 559 +++++++++ .../module/xep/0166_jingle/session_info.vala | 12 + .../module/xep/0234_jingle_file_transfer.vala | 162 ++- .../xep/0260_jingle_socks5_bytestreams.vala | 100 +- .../xep/0261_jingle_in_band_bytestreams.vala | 50 +- 19 files changed, 1671 insertions(+), 1157 deletions(-) delete mode 100644 xmpp-vala/src/module/xep/0166_jingle.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/component.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/content.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/content_description.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/content_node.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/content_security.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/content_transport.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/jingle_flag.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/jingle_structs.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/reason_element.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/session.vala create mode 100644 xmpp-vala/src/module/xep/0166_jingle/session_info.vala diff --git a/xmpp-vala/CMakeLists.txt b/xmpp-vala/CMakeLists.txt index fcc74fdc..48f411c7 100644 --- a/xmpp-vala/CMakeLists.txt +++ b/xmpp-vala/CMakeLists.txt @@ -61,15 +61,18 @@ SOURCES "src/module/xep/0048_conference.vala" "src/module/xep/0402_bookmarks2.vala" "src/module/xep/0004_data_forms.vala" + "src/module/xep/0030_service_discovery/flag.vala" "src/module/xep/0030_service_discovery/identity.vala" "src/module/xep/0030_service_discovery/info_result.vala" "src/module/xep/0030_service_discovery/item.vala" "src/module/xep/0030_service_discovery/items_result.vala" "src/module/xep/0030_service_discovery/module.vala" + "src/module/xep/0045_muc/flag.vala" "src/module/xep/0045_muc/module.vala" "src/module/xep/0045_muc/status_code.vala" + "src/module/xep/0047_in_band_bytestreams.vala" "src/module/xep/0049_private_xml_storage.vala" "src/module/xep/0054_vcard/module.vala" @@ -81,7 +84,31 @@ SOURCES "src/module/xep/0084_user_avatars.vala" "src/module/xep/0085_chat_state_notifications.vala" "src/module/xep/0115_entity_capabilities.vala" - "src/module/xep/0166_jingle.vala" + + "src/module/xep/0166_jingle/content.vala" + "src/module/xep/0166_jingle/content_description.vala" + "src/module/xep/0166_jingle/content_node.vala" + "src/module/xep/0166_jingle/content_security.vala" + "src/module/xep/0166_jingle/content_transport.vala" + "src/module/xep/0166_jingle/component.vala" + "src/module/xep/0166_jingle/jingle_flag.vala" + "src/module/xep/0166_jingle/jingle_module.vala" + "src/module/xep/0166_jingle/jingle_structs.vala" + "src/module/xep/0166_jingle/reason_element.vala" + "src/module/xep/0166_jingle/session.vala" + "src/module/xep/0166_jingle/session_info.vala" + + "src/module/xep/0167_jingle_rtp/content_parameters.vala" + "src/module/xep/0167_jingle_rtp/content_type.vala" + "src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala" + "src/module/xep/0167_jingle_rtp/payload_type.vala" + "src/module/xep/0167_jingle_rtp/session_info_type.vala" + "src/module/xep/0167_jingle_rtp/stream.vala" + + "src/module/xep/0176_jingle_ice_udp/candidate.vala" + "src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala" + "src/module/xep/0176_jingle_ice_udp/transport_parameters.vala" + "src/module/xep/0184_message_delivery_receipts.vala" "src/module/xep/0191_blocking_command.vala" "src/module/xep/0198_stream_management.vala" diff --git a/xmpp-vala/src/core/xmpp_log.vala b/xmpp-vala/src/core/xmpp_log.vala index 4790a8ab..3d5693ef 100644 --- a/xmpp-vala/src/core/xmpp_log.vala +++ b/xmpp-vala/src/core/xmpp_log.vala @@ -110,13 +110,13 @@ public class XmppLog { public void node(string what, StanzaNode node, XmppStream stream) { if (should_log_node(node)) { - stderr.printf("%sXMPP %s [%s %p %s]%s\n%s\n", use_ansi ? ANSI_COLOR_WHITE : "", what, ident, stream, new DateTime.now_local().to_string(), use_ansi ? ANSI_COLOR_END : "", use_ansi ? node.to_ansi_string(hide_ns) : node.to_string()); + stderr.printf("%sXMPP %s [%s stream:%p thread:%p %s]%s\n%s\n", use_ansi ? ANSI_COLOR_WHITE : "", what, ident, stream, Thread.self(), new DateTime.now_local().to_string(), use_ansi ? ANSI_COLOR_END : "", use_ansi ? node.to_ansi_string(hide_ns) : node.to_string()); } } public void str(string what, string str, XmppStream stream) { if (should_log_str(str)) { - stderr.printf("%sXMPP %s [%s %p %s]%s\n%s\n", use_ansi ? ANSI_COLOR_WHITE : "", what, ident, stream, new DateTime.now_local().to_string(), use_ansi ? ANSI_COLOR_END : "", str); + stderr.printf("%sXMPP %s [%s stream:%p thread:%p %s]%s\n%s\n", use_ansi ? ANSI_COLOR_WHITE : "", what, ident, stream, Thread.self(), new DateTime.now_local().to_string(), use_ansi ? ANSI_COLOR_END : "", str); } } diff --git a/xmpp-vala/src/module/iq/module.vala b/xmpp-vala/src/module/iq/module.vala index 9deb0422..56605d01 100644 --- a/xmpp-vala/src/module/iq/module.vala +++ b/xmpp-vala/src/module/iq/module.vala @@ -10,6 +10,8 @@ namespace Xmpp.Iq { private HashMap> namespaceRegistrants = new HashMap>(); public async Iq.Stanza send_iq_async(XmppStream stream, Iq.Stanza iq) { + assert(iq.type_ == Iq.Stanza.TYPE_GET || iq.type_ == Iq.Stanza.TYPE_SET); + Iq.Stanza? return_stanza = null; send_iq(stream, iq, (_, result_iq) => { return_stanza = result_iq; diff --git a/xmpp-vala/src/module/xep/0166_jingle.vala b/xmpp-vala/src/module/xep/0166_jingle.vala deleted file mode 100644 index 3a634222..00000000 --- a/xmpp-vala/src/module/xep/0166_jingle.vala +++ /dev/null @@ -1,1061 +0,0 @@ -using Gee; -using Xmpp.Xep; -using Xmpp; - -namespace Xmpp.Xep.Jingle { - -private const string NS_URI = "urn:xmpp:jingle:1"; -private const string ERROR_NS_URI = "urn:xmpp:jingle:errors:1"; - -public errordomain IqError { - BAD_REQUEST, - NOT_ACCEPTABLE, - NOT_IMPLEMENTED, - UNSUPPORTED_INFO, - OUT_OF_ORDER, - RESOURCE_CONSTRAINT, -} - -void send_iq_error(IqError iq_error, XmppStream stream, Iq.Stanza iq) { - ErrorStanza error; - if (iq_error is IqError.BAD_REQUEST) { - error = new ErrorStanza.bad_request(iq_error.message); - } else if (iq_error is IqError.NOT_ACCEPTABLE) { - error = new ErrorStanza.not_acceptable(iq_error.message); - } else if (iq_error is IqError.NOT_IMPLEMENTED) { - error = new ErrorStanza.feature_not_implemented(iq_error.message); - } else if (iq_error is IqError.UNSUPPORTED_INFO) { - StanzaNode unsupported_info = new StanzaNode.build("unsupported-info", ERROR_NS_URI).add_self_xmlns(); - error = new ErrorStanza.build(ErrorStanza.TYPE_CANCEL, ErrorStanza.CONDITION_FEATURE_NOT_IMPLEMENTED, iq_error.message, unsupported_info); - } else if (iq_error is IqError.OUT_OF_ORDER) { - StanzaNode out_of_order = new StanzaNode.build("out-of-order", ERROR_NS_URI).add_self_xmlns(); - error = new ErrorStanza.build(ErrorStanza.TYPE_MODIFY, ErrorStanza.CONDITION_UNEXPECTED_REQUEST, iq_error.message, out_of_order); - } else if (iq_error is IqError.RESOURCE_CONSTRAINT) { - error = new ErrorStanza.resource_constraint(iq_error.message); - } else { - assert_not_reached(); - } - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.error(iq, error) { to=iq.from }); -} - -public errordomain Error { - GENERAL, - BAD_REQUEST, - INVALID_PARAMETERS, - UNSUPPORTED_TRANSPORT, - UNSUPPORTED_SECURITY, - NO_SHARED_PROTOCOLS, - TRANSPORT_ERROR, -} - -StanzaNode? get_single_node_anyns(StanzaNode parent, string? node_name = null) throws IqError { - StanzaNode? result = null; - foreach (StanzaNode child in parent.get_all_subnodes()) { - if (node_name == null || child.name == node_name) { - if (result != null) { - if (node_name != null) { - throw new IqError.BAD_REQUEST(@"multiple $(node_name) nodes"); - } else { - throw new IqError.BAD_REQUEST(@"expected single subnode"); - } - } - result = child; - } - } - return result; -} - -class ContentNode { - public Role creator; - public string name; - public StanzaNode? description; - public StanzaNode? transport; - public StanzaNode? security; -} - -ContentNode get_single_content_node(StanzaNode jingle) throws IqError { - Gee.List contents = jingle.get_subnodes("content"); - if (contents.size == 0) { - throw new IqError.BAD_REQUEST("missing content node"); - } - if (contents.size > 1) { - throw new IqError.NOT_IMPLEMENTED("can't process multiple content nodes"); - } - StanzaNode content = contents[0]; - string? creator_str = content.get_attribute("creator"); - // Vala can't typecheck the ternary operator here. - Role? creator = null; - if (creator_str != null) { - creator = Role.parse(creator_str); - } else { - // TODO(hrxi): now, is the creator attribute optional or not (XEP-0166 - // Jingle)? - creator = Role.INITIATOR; - } - - string? name = content.get_attribute("name"); - StanzaNode? description = get_single_node_anyns(content, "description"); - StanzaNode? transport = get_single_node_anyns(content, "transport"); - StanzaNode? security = get_single_node_anyns(content, "security"); - if (name == null || creator == null) { - throw new IqError.BAD_REQUEST("missing name or creator"); - } - - return new ContentNode() { - creator=creator, - name=name, - description=description, - transport=transport, - security=security - }; -} - -// This module can only be attached to one stream at a time. -public class Module : XmppStreamModule, Iq.Handler { - public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "0166_jingle"); - - private HashMap content_types = new HashMap(); - private HashMap transports = new HashMap(); - private HashMap security_preconditions = new HashMap(); - - private XmppStream? current_stream = null; - - public override void attach(XmppStream stream) { - stream.add_flag(new Flag()); - stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); - stream.get_module(Iq.Module.IDENTITY).register_for_namespace(NS_URI, this); - current_stream = stream; - } - public override void detach(XmppStream stream) { - stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI); - stream.get_module(Iq.Module.IDENTITY).unregister_from_namespace(NS_URI, this); - } - - public void register_content_type(ContentType content_type) { - content_types[content_type.content_type_ns_uri()] = content_type; - } - public ContentType? get_content_type(string ns_uri) { - if (!content_types.has_key(ns_uri)) { - return null; - } - return content_types[ns_uri]; - } - public void register_transport(Transport transport) { - transports[transport.transport_ns_uri()] = transport; - } - public Transport? get_transport(string ns_uri) { - if (!transports.has_key(ns_uri)) { - return null; - } - return transports[ns_uri]; - } - public async Transport? select_transport(XmppStream stream, TransportType type, Jid receiver_full_jid, Set blacklist) { - Transport? result = null; - foreach (Transport transport in transports.values) { - if (transport.transport_type() != type) { - continue; - } - if (transport.transport_ns_uri() in blacklist) { - continue; - } - if (yield transport.is_transport_available(stream, receiver_full_jid)) { - if (result != null) { - if (result.transport_priority() >= transport.transport_priority()) { - continue; - } - } - result = transport; - } - } - return result; - } - public void register_security_precondition(SecurityPrecondition precondition) { - security_preconditions[precondition.security_ns_uri()] = precondition; - } - public SecurityPrecondition? get_security_precondition(string? ns_uri) { - if (ns_uri == null) return null; - if (!security_preconditions.has_key(ns_uri)) { - return null; - } - return security_preconditions[ns_uri]; - } - - private async bool is_jingle_available(XmppStream stream, Jid full_jid) { - bool? has_jingle = yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI); - return has_jingle != null && has_jingle; - } - - public async bool is_available(XmppStream stream, TransportType type, Jid full_jid) { - return (yield is_jingle_available(stream, full_jid)) && (yield select_transport(stream, type, full_jid, Set.empty())) != null; - } - - public async Session create_session(XmppStream stream, TransportType type, Jid receiver_full_jid, Senders senders, string content_name, StanzaNode description, string? precondition_name = null, Object? precondation_options = null) throws Error { - if (!yield is_jingle_available(stream, receiver_full_jid)) { - throw new Error.NO_SHARED_PROTOCOLS("No Jingle support"); - } - Transport? transport = yield select_transport(stream, type, receiver_full_jid, Set.empty()); - if (transport == null) { - throw new Error.NO_SHARED_PROTOCOLS("No suitable transports"); - } - SecurityPrecondition? precondition = get_security_precondition(precondition_name); - if (precondition_name != null && precondition == null) { - throw new Error.UNSUPPORTED_SECURITY("No suitable security precondiiton found"); - } - Jid? my_jid = stream.get_flag(Bind.Flag.IDENTITY).my_jid; - if (my_jid == null) { - throw new Error.GENERAL("Couldn't determine own JID"); - } - TransportParameters transport_params = transport.create_transport_parameters(stream, my_jid, receiver_full_jid); - SecurityParameters? security_params = precondition != null ? precondition.create_security_parameters(stream, my_jid, receiver_full_jid, precondation_options) : null; - Session session = new Session.initiate_sent(random_uuid(), type, transport_params, security_params, my_jid, receiver_full_jid, content_name, send_terminate_and_remove_session); - StanzaNode content = new StanzaNode.build("content", NS_URI) - .put_attribute("creator", "initiator") - .put_attribute("name", content_name) - .put_attribute("senders", senders.to_string()) - .put_node(description) - .put_node(transport_params.to_transport_stanza_node()); - if (security_params != null) { - content.put_node(security_params.to_security_stanza_node(stream, my_jid, receiver_full_jid)); - } - StanzaNode jingle = new StanzaNode.build("jingle", NS_URI) - .add_self_xmlns() - .put_attribute("action", "session-initiate") - .put_attribute("initiator", my_jid.to_string()) - .put_attribute("sid", session.sid) - .put_node(content); - Iq.Stanza iq = new Iq.Stanza.set(jingle) { to=receiver_full_jid }; - - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq, (stream, iq) => { - // TODO(hrxi): handle errors - stream.get_flag(Flag.IDENTITY).add_session(session); - }); - - return session; - } - - public void handle_session_initiate(XmppStream stream, string sid, StanzaNode jingle, Iq.Stanza iq) throws IqError { - ContentNode content = get_single_content_node(jingle); - if (content.description == null || content.transport == null) { - throw new IqError.BAD_REQUEST("missing description or transport node"); - } - Jid? my_jid = stream.get_flag(Bind.Flag.IDENTITY).my_jid; - if (my_jid == null) { - throw new IqError.RESOURCE_CONSTRAINT("Couldn't determine own JID"); - } - Transport? transport = get_transport(content.transport.ns_uri); - TransportParameters? transport_params = null; - if (transport != null) { - transport_params = transport.parse_transport_parameters(stream, my_jid, iq.from, content.transport); - } else { - // terminate the session below - } - - ContentType? content_type = get_content_type(content.description.ns_uri); - if (content_type == null) { - // TODO(hrxi): how do we signal an unknown content type? - throw new IqError.NOT_IMPLEMENTED("unknown content type"); - } - ContentParameters content_params = content_type.parse_content_parameters(content.description); - - SecurityPrecondition? precondition = content.security != null ? get_security_precondition(content.security.ns_uri) : null; - SecurityParameters? security_params = null; - if (precondition != null) { - debug("Using precondition %s", precondition.security_ns_uri()); - security_params = precondition.parse_security_parameters(stream, my_jid, iq.from, content.security); - } else if (content.security != null) { - throw new IqError.NOT_IMPLEMENTED("unknown security precondition"); - } - - TransportType type = content_type.content_type_transport_type(); - Session session = new Session.initiate_received(sid, type, transport_params, security_params, my_jid, iq.from, content.name, send_terminate_and_remove_session); - stream.get_flag(Flag.IDENTITY).add_session(session); - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); - - if (transport == null || transport.transport_type() != type) { - StanzaNode reason = new StanzaNode.build("reason", NS_URI) - .put_node(new StanzaNode.build("unsupported-transports", NS_URI)); - session.terminate(reason, "unsupported transports"); - return; - } - - content_params.on_session_initiate(stream, session); - } - - private void send_terminate_and_remove_session(Jid to, string sid, StanzaNode reason) { - StanzaNode jingle = new StanzaNode.build("jingle", NS_URI) - .add_self_xmlns() - .put_attribute("action", "session-terminate") - .put_attribute("sid", sid) - .put_node(reason); - Iq.Stanza iq = new Iq.Stanza.set(jingle) { to=to }; - current_stream.get_module(Iq.Module.IDENTITY).send_iq(current_stream, iq); - - // Immediately remove the session from the open sessions as per the - // XEP, don't wait for confirmation. - current_stream.get_flag(Flag.IDENTITY).remove_session(sid); - } - - public async void on_iq_set(XmppStream stream, Iq.Stanza iq) { - try { - handle_iq_set(stream, iq); - } catch (IqError e) { - send_iq_error(e, stream, iq); - } - } - - public void handle_iq_set(XmppStream stream, Iq.Stanza iq) throws IqError { - StanzaNode? jingle = iq.stanza.get_subnode("jingle", NS_URI); - string? sid = jingle != null ? jingle.get_attribute("sid") : null; - string? action = jingle != null ? jingle.get_attribute("action") : null; - if (jingle == null || sid == null || action == null) { - throw new IqError.BAD_REQUEST("missing jingle node, sid or action"); - } - Session? session = stream.get_flag(Flag.IDENTITY).get_session(sid); - if (action == "session-initiate") { - if (session != null) { - // TODO(hrxi): Info leak if other clients use predictable session IDs? - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.error(iq, new ErrorStanza.build(ErrorStanza.TYPE_MODIFY, ErrorStanza.CONDITION_CONFLICT, "session ID already in use", null)) { to=iq.from }); - return; - } - handle_session_initiate(stream, sid, jingle, iq); - return; - } - if (session == null) { - StanzaNode unknown_session = new StanzaNode.build("unknown-session", ERROR_NS_URI).add_self_xmlns(); - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.error(iq, new ErrorStanza.item_not_found(unknown_session)) { to=iq.from }); - return; - } - session.handle_iq_set(stream, action, jingle, iq); - } - - public override string get_ns() { return NS_URI; } - public override string get_id() { return IDENTITY.id; } -} - -public enum TransportType { - DATAGRAM, - STREAMING, -} - -public enum Senders { - BOTH, - INITIATOR, - NONE, - RESPONDER; - - public string to_string() { - switch (this) { - case BOTH: return "both"; - case INITIATOR: return "initiator"; - case NONE: return "none"; - case RESPONDER: return "responder"; - } - assert_not_reached(); - } -} - -public delegate void SessionTerminate(Jid to, string sid, StanzaNode reason); - -public interface Transport : Object { - public abstract string transport_ns_uri(); - public async abstract bool is_transport_available(XmppStream stream, Jid full_jid); - public abstract TransportType transport_type(); - public abstract int transport_priority(); - public abstract TransportParameters create_transport_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid) throws Error; - public abstract TransportParameters parse_transport_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws IqError; -} - - -// Gets a null `stream` if connection setup was unsuccessful and another -// transport method should be tried. -public interface TransportParameters : Object { - public abstract string transport_ns_uri(); - public abstract StanzaNode to_transport_stanza_node(); - public abstract void on_transport_accept(StanzaNode transport) throws IqError; - public abstract void on_transport_info(StanzaNode transport) throws IqError; - public abstract void create_transport_connection(XmppStream stream, Session session); -} - -public enum Role { - INITIATOR, - RESPONDER; - - public string to_string() { - switch (this) { - case INITIATOR: return "initiator"; - case RESPONDER: return "responder"; - } - assert_not_reached(); - } - - public static Role parse(string role) throws IqError { - switch (role) { - case "initiator": return INITIATOR; - case "responder": return RESPONDER; - } - throw new IqError.BAD_REQUEST(@"invalid role $(role)"); - } -} - -public interface ContentType : Object { - public abstract string content_type_ns_uri(); - public abstract TransportType content_type_transport_type(); - public abstract ContentParameters parse_content_parameters(StanzaNode description) throws IqError; - public abstract void handle_content_session_info(XmppStream stream, Session session, StanzaNode info, Iq.Stanza iq) throws IqError; -} - -public interface ContentParameters : Object { - public abstract void on_session_initiate(XmppStream stream, Session session); -} - -public interface SecurityPrecondition : Object { - public abstract string security_ns_uri(); - public abstract SecurityParameters? create_security_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, Object options) throws Jingle.Error; - public abstract SecurityParameters? parse_security_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, StanzaNode security) throws IqError; -} - -public interface SecurityParameters : Object { - public abstract string security_ns_uri(); - public abstract StanzaNode to_security_stanza_node(XmppStream stream, Jid local_full_jid, Jid peer_full_jid); - public abstract IOStream wrap_stream(IOStream stream); -} - -public class Session { - // INITIATE_SENT -> CONNECTING -> [REPLACING_TRANSPORT -> CONNECTING ->]... ACTIVE -> ENDED - // INITIATE_RECEIVED -> CONNECTING -> [WAITING_FOR_TRANSPORT_REPLACE -> CONNECTING ->].. ACTIVE -> ENDED - public enum State { - INITIATE_SENT, - REPLACING_TRANSPORT, - INITIATE_RECEIVED, - WAITING_FOR_TRANSPORT_REPLACE, - CONNECTING, - ACTIVE, - ENDED, - } - - public State state { get; private set; } - - public Role role { get; private set; } - public string sid { get; private set; } - public TransportType type_ { get; private set; } - public Jid local_full_jid { get; private set; } - public Jid peer_full_jid { get; private set; } - public Role content_creator { get; private set; } - public string content_name { get; private set; } - public SecurityParameters? security { get; private set; } - - private Connection connection; - public IOStream conn { get { return connection; } } - - public bool terminate_on_connection_close { get; set; } - - // INITIATE_SENT | INITIATE_RECEIVED | CONNECTING - Set tried_transport_methods = new HashSet(); - TransportParameters? transport = null; - - SessionTerminate session_terminate_handler; - - public Session.initiate_sent(string sid, TransportType type, TransportParameters transport, SecurityParameters? security, Jid local_full_jid, Jid peer_full_jid, string content_name, owned SessionTerminate session_terminate_handler) { - this.state = State.INITIATE_SENT; - this.role = Role.INITIATOR; - this.sid = sid; - this.type_ = type; - this.local_full_jid = local_full_jid; - this.peer_full_jid = peer_full_jid; - this.content_creator = Role.INITIATOR; - this.content_name = content_name; - this.tried_transport_methods = new HashSet(); - this.tried_transport_methods.add(transport.transport_ns_uri()); - this.transport = transport; - this.security = security; - this.connection = new Connection(this); - this.session_terminate_handler = (owned)session_terminate_handler; - this.terminate_on_connection_close = true; - } - - public Session.initiate_received(string sid, TransportType type, TransportParameters? transport, SecurityParameters? security, Jid local_full_jid, Jid peer_full_jid, string content_name, owned SessionTerminate session_terminate_handler) { - this.state = State.INITIATE_RECEIVED; - this.role = Role.RESPONDER; - this.sid = sid; - this.type_ = type; - this.local_full_jid = local_full_jid; - this.peer_full_jid = peer_full_jid; - this.content_creator = Role.INITIATOR; - this.content_name = content_name; - this.transport = transport; - this.security = security; - this.tried_transport_methods = new HashSet(); - if (transport != null) { - this.tried_transport_methods.add(transport.transport_ns_uri()); - } - this.connection = new Connection(this); - this.session_terminate_handler = (owned)session_terminate_handler; - this.terminate_on_connection_close = true; - } - - public void handle_iq_set(XmppStream stream, string action, StanzaNode jingle, Iq.Stanza iq) throws IqError { - // Validate action. - switch (action) { - case "session-accept": - case "session-info": - case "session-terminate": - case "transport-accept": - case "transport-info": - case "transport-reject": - case "transport-replace": - break; - case "content-accept": - case "content-add": - case "content-modify": - case "content-reject": - case "content-remove": - case "description-info": - case "security-info": - throw new IqError.NOT_IMPLEMENTED(@"$(action) is not implemented"); - default: - throw new IqError.BAD_REQUEST("invalid action"); - } - ContentNode? content = null; - StanzaNode? transport = null; - // Do some pre-processing. - if (action != "session-info" && action != "session-terminate") { - content = get_single_content_node(jingle); - verify_content(content); - switch (action) { - case "transport-accept": - case "transport-reject": - case "transport-replace": - case "transport-info": - switch (state) { - case State.INITIATE_SENT: - case State.REPLACING_TRANSPORT: - case State.INITIATE_RECEIVED: - case State.WAITING_FOR_TRANSPORT_REPLACE: - case State.CONNECTING: - break; - default: - throw new IqError.OUT_OF_ORDER("transport-* unsupported after connection setup"); - } - // TODO(hrxi): What to do with description nodes? - if (content.transport == null) { - throw new IqError.BAD_REQUEST("missing transport node"); - } - transport = content.transport; - break; - } - } - switch (action) { - case "session-accept": - if (state != State.INITIATE_SENT) { - throw new IqError.OUT_OF_ORDER("got session-accept while not waiting for one"); - } - handle_session_accept(stream, content, jingle, iq); - break; - case "session-info": - handle_session_info(stream, jingle, iq); - break; - case "session-terminate": - handle_session_terminate(stream, jingle, iq); - break; - case "transport-accept": - handle_transport_accept(stream, transport, jingle, iq); - break; - case "transport-reject": - handle_transport_reject(stream, jingle, iq); - break; - case "transport-replace": - handle_transport_replace(stream, transport, jingle, iq); - break; - case "transport-info": - handle_transport_info(stream, transport, jingle, iq); - break; - } - } - void handle_session_accept(XmppStream stream, ContentNode content, StanzaNode jingle, Iq.Stanza iq) throws IqError { - string? responder_str = jingle.get_attribute("responder"); - Jid responder = iq.from; - if (responder_str != null) { - try { - responder = new Jid(responder_str); - } catch (InvalidJidError e) { - warning("Received invalid session accept: %s", e.message); - } - } - // TODO(hrxi): more sanity checking, perhaps replace who we're talking to - if (!responder.is_full()) { - throw new IqError.BAD_REQUEST("invalid responder JID"); - } - if (content.description == null || content.transport == null) { - throw new IqError.BAD_REQUEST("missing description or transport node"); - } - if (content.transport.ns_uri != transport.transport_ns_uri()) { - throw new IqError.BAD_REQUEST("session-accept with unnegotiated transport method"); - } - transport.on_transport_accept(content.transport); - // TODO(hrxi): handle content.description :) - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); - - state = State.CONNECTING; - transport.create_transport_connection(stream, this); - } - void connection_created(XmppStream stream, IOStream? conn) { - if (state != State.CONNECTING) { - return; - } - if (conn != null) { - state = State.ACTIVE; - tried_transport_methods.clear(); - if (security != null) { - connection.set_inner(security.wrap_stream(conn)); - } else { - connection.set_inner(conn); - } - transport = null; - } else { - if (role == Role.INITIATOR) { - select_new_transport.begin(stream); - } else { - state = State.WAITING_FOR_TRANSPORT_REPLACE; - } - } - } - void handle_session_terminate(XmppStream stream, StanzaNode jingle, Iq.Stanza iq) throws IqError { - connection.on_terminated_by_jingle("remote terminated jingle session"); - state = State.ENDED; - stream.get_flag(Flag.IDENTITY).remove_session(sid); - - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); - // TODO(hrxi): also handle presence type=unavailable - } - void handle_session_info(XmppStream stream, StanzaNode jingle, Iq.Stanza iq) throws IqError { - StanzaNode? info = get_single_node_anyns(jingle); - if (info == null) { - // Jingle session ping - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); - return; - } - ContentType? content_type = stream.get_module(Module.IDENTITY).get_content_type(info.ns_uri); - if (content_type == null) { - throw new IqError.UNSUPPORTED_INFO("unknown session-info namespace"); - } - content_type.handle_content_session_info(stream, this, info, iq); - } - async void select_new_transport(XmppStream stream) { - Transport? new_transport = yield stream.get_module(Module.IDENTITY).select_transport(stream, type_, peer_full_jid, tried_transport_methods); - if (new_transport == null) { - StanzaNode reason = new StanzaNode.build("reason", NS_URI) - .put_node(new StanzaNode.build("failed-transport", NS_URI)); - terminate(reason, "failed transport"); - return; - } - tried_transport_methods.add(new_transport.transport_ns_uri()); - transport = new_transport.create_transport_parameters(stream, local_full_jid, peer_full_jid); - StanzaNode jingle = new StanzaNode.build("jingle", NS_URI) - .add_self_xmlns() - .put_attribute("action", "transport-replace") - .put_attribute("sid", sid) - .put_node(new StanzaNode.build("content", NS_URI) - .put_attribute("creator", "initiator") - .put_attribute("name", content_name) - .put_node(transport.to_transport_stanza_node()) - ); - Iq.Stanza iq = new Iq.Stanza.set(jingle) { to=peer_full_jid }; - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); - state = State.REPLACING_TRANSPORT; - } - void handle_transport_accept(XmppStream stream, StanzaNode transport_node, StanzaNode jingle, Iq.Stanza iq) throws IqError { - if (state != State.REPLACING_TRANSPORT) { - throw new IqError.OUT_OF_ORDER("no outstanding transport-replace request"); - } - if (transport_node.ns_uri != transport.transport_ns_uri()) { - throw new IqError.BAD_REQUEST("transport-accept with unnegotiated transport method"); - } - transport.on_transport_accept(transport_node); - state = State.CONNECTING; - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); - transport.create_transport_connection(stream, this); - } - void handle_transport_reject(XmppStream stream, StanzaNode jingle, Iq.Stanza iq) throws IqError { - if (state != State.REPLACING_TRANSPORT) { - throw new IqError.OUT_OF_ORDER("no outstanding transport-replace request"); - } - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); - select_new_transport.begin(stream); - } - void handle_transport_replace(XmppStream stream, StanzaNode transport_node, StanzaNode jingle, Iq.Stanza iq) throws IqError { - Transport? transport = stream.get_module(Module.IDENTITY).get_transport(transport_node.ns_uri); - TransportParameters? parameters = null; - if (transport != null) { - // Just parse the transport info for the errors. - parameters = transport.parse_transport_parameters(stream, local_full_jid, peer_full_jid, transport_node); - } - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); - if (state != State.WAITING_FOR_TRANSPORT_REPLACE || transport == null) { - StanzaNode jingle_response = new StanzaNode.build("jingle", NS_URI) - .add_self_xmlns() - .put_attribute("action", "transport-reject") - .put_attribute("sid", sid) - .put_node(new StanzaNode.build("content", NS_URI) - .put_attribute("creator", "initiator") - .put_attribute("name", content_name) - .put_node(transport_node) - ); - Iq.Stanza iq_response = new Iq.Stanza.set(jingle_response) { to=peer_full_jid }; - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq_response); - return; - } - this.transport = parameters; - StanzaNode jingle_response = new StanzaNode.build("jingle", NS_URI) - .add_self_xmlns() - .put_attribute("action", "transport-accept") - .put_attribute("sid", sid) - .put_node(new StanzaNode.build("content", NS_URI) - .put_attribute("creator", "initiator") - .put_attribute("name", content_name) - .put_node(this.transport.to_transport_stanza_node()) - ); - Iq.Stanza iq_response = new Iq.Stanza.set(jingle_response) { to=peer_full_jid }; - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq_response); - - state = State.CONNECTING; - this.transport.create_transport_connection(stream, this); - } - void handle_transport_info(XmppStream stream, StanzaNode transport, StanzaNode jingle, Iq.Stanza iq) throws IqError { - this.transport.on_transport_info(transport); - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); - } - void verify_content(ContentNode content) throws IqError { - if (content.name != content_name || content.creator != content_creator) { - throw new IqError.BAD_REQUEST("unknown content"); - } - } - public void set_transport_connection(XmppStream stream, IOStream? conn) { - if (state != State.CONNECTING) { - return; - } - connection_created(stream, conn); - } - public void send_transport_info(XmppStream stream, StanzaNode transport) { - if (state != State.CONNECTING) { - return; - } - StanzaNode jingle = new StanzaNode.build("jingle", NS_URI) - .add_self_xmlns() - .put_attribute("action", "transport-info") - .put_attribute("sid", sid) - .put_node(new StanzaNode.build("content", NS_URI) - .put_attribute("creator", "initiator") - .put_attribute("name", content_name) - .put_node(transport) - ); - Iq.Stanza iq = new Iq.Stanza.set(jingle) { to=peer_full_jid }; - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); - } - public void accept(XmppStream stream, StanzaNode description) { - if (state != State.INITIATE_RECEIVED) { - return; // TODO(hrxi): what to do? - } - StanzaNode jingle = new StanzaNode.build("jingle", NS_URI) - .add_self_xmlns() - .put_attribute("action", "session-accept") - .put_attribute("sid", sid) - .put_node(new StanzaNode.build("content", NS_URI) - .put_attribute("creator", "initiator") - .put_attribute("name", content_name) - .put_node(description) - .put_node(transport.to_transport_stanza_node()) - ); - Iq.Stanza iq = new Iq.Stanza.set(jingle) { to=peer_full_jid }; - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); - - state = State.CONNECTING; - transport.create_transport_connection(stream, this); - } - - public void reject(XmppStream stream) { - if (state != State.INITIATE_RECEIVED) { - return; // TODO(hrxi): what to do? - } - StanzaNode reason = new StanzaNode.build("reason", NS_URI) - .put_node(new StanzaNode.build("decline", NS_URI)); - terminate(reason, "declined"); - } - - public void set_application_error(XmppStream stream, StanzaNode? application_reason = null) { - StanzaNode reason = new StanzaNode.build("reason", NS_URI) - .put_node(new StanzaNode.build("failed-application", NS_URI)); - if (application_reason != null) { - reason.put_node(application_reason); - } - terminate(reason, "application error"); - } - - public void on_connection_error(IOError error) { - // TODO(hrxi): where can we get an XmppStream from? - StanzaNode reason = new StanzaNode.build("reason", NS_URI) - .put_node(new StanzaNode.build("failed-transport", NS_URI)) - .put_node(new StanzaNode.build("text", NS_URI) - .put_node(new StanzaNode.text(error.message)) - ); - terminate(reason, @"transport error: $(error.message)"); - } - public void on_connection_close() { - if (terminate_on_connection_close) { - StanzaNode reason = new StanzaNode.build("reason", NS_URI) - .put_node(new StanzaNode.build("success", NS_URI)); - terminate(reason, "success"); - } - } - - public void terminate(StanzaNode reason, string? local_reason) { - if (state == State.ENDED) { - return; - } - if (state == State.ACTIVE) { - if (local_reason != null) { - connection.on_terminated_by_jingle(@"local session-terminate: $(local_reason)"); - } else { - connection.on_terminated_by_jingle("local session-terminate"); - } - } - - session_terminate_handler(peer_full_jid, sid, reason); - state = State.ENDED; - } -} - -public class Connection : IOStream { - public class Input : InputStream { - private weak Connection connection; - public Input(Connection connection) { - this.connection = connection; - } - public override ssize_t read(uint8[] buffer, Cancellable? cancellable = null) throws IOError { - throw new IOError.NOT_SUPPORTED("can't do non-async reads on jingle connections"); - } - public override async ssize_t read_async(uint8[]? buffer, int io_priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) throws IOError { - return yield connection.read_async(buffer, io_priority, cancellable); - } - public override bool close(Cancellable? cancellable = null) throws IOError { - return connection.close_read(cancellable); - } - public override async bool close_async(int io_priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) throws IOError { - return yield connection.close_read_async(io_priority, cancellable); - } - } - public class Output : OutputStream { - private weak Connection connection; - public Output(Connection connection) { - this.connection = connection; - } - public override ssize_t write(uint8[] buffer, Cancellable? cancellable = null) throws IOError { - throw new IOError.NOT_SUPPORTED("can't do non-async writes on jingle connections"); - } - public override async ssize_t write_async(uint8[]? buffer, int io_priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) throws IOError { - return yield connection.write_async(buffer, io_priority, cancellable); - } - public override bool close(Cancellable? cancellable = null) throws IOError { - return connection.close_write(cancellable); - } - public override async bool close_async(int io_priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) throws IOError { - return yield connection.close_write_async(io_priority, cancellable); - } - } - - private Input input; - private Output output; - public override InputStream input_stream { get { return input; } } - public override OutputStream output_stream { get { return output; } } - - private weak Session session; - private IOStream? inner = null; - private string? error = null; - - private bool read_closed = false; - private bool write_closed = false; - - private class OnSetInnerCallback { - public SourceFunc callback; - public int io_priority; - } - - Gee.List callbacks = new ArrayList(); - - public Connection(Session session) { - this.input = new Input(this); - this.output = new Output(this); - this.session = session; - } - - public void set_inner(IOStream inner) { - assert(this.inner == null); - this.inner = inner; - foreach (OnSetInnerCallback c in callbacks) { - Idle.add((owned) c.callback, c.io_priority); - } - callbacks = null; - } - - public void on_terminated_by_jingle(string reason) { - if (error == null) { - close_async.begin(); - error = reason; - } - } - - private void check_for_errors() throws IOError { - if (error != null) { - throw new IOError.CLOSED(error); - } - } - private async void wait_and_check_for_errors(int io_priority, Cancellable? cancellable = null) throws IOError { - while (true) { - check_for_errors(); - if (inner != null) { - return; - } - SourceFunc callback = wait_and_check_for_errors.callback; - ulong id = 0; - if (cancellable != null) { - id = cancellable.connect(() => callback()); - } - callbacks.add(new OnSetInnerCallback() { callback=(owned)callback, io_priority=io_priority}); - yield; - if (cancellable != null) { - cancellable.disconnect(id); - } - } - } - private void handle_connection_error(IOError error) { - Session? strong = session; - if (strong != null) { - strong.on_connection_error(error); - } - } - private void handle_connection_close() { - Session? strong = session; - if (strong != null) { - strong.on_connection_close(); - } - } - - public async ssize_t read_async(uint8[]? buffer, int io_priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) throws IOError { - yield wait_and_check_for_errors(io_priority, cancellable); - try { - return yield inner.input_stream.read_async(buffer, io_priority, cancellable); - } catch (IOError e) { - handle_connection_error(e); - throw e; - } - } - public async ssize_t write_async(uint8[]? buffer, int io_priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) throws IOError { - yield wait_and_check_for_errors(io_priority, cancellable); - try { - return yield inner.output_stream.write_async(buffer, io_priority, cancellable); - } catch (IOError e) { - handle_connection_error(e); - throw e; - } - } - public bool close_read(Cancellable? cancellable = null) throws IOError { - check_for_errors(); - if (read_closed) { - return true; - } - close_read_async.begin(GLib.Priority.DEFAULT, cancellable); - return true; - } - public async bool close_read_async(int io_priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) throws IOError { - debug("Closing Jingle input stream"); - yield wait_and_check_for_errors(io_priority, cancellable); - if (read_closed) { - return true; - } - read_closed = true; - IOError error = null; - bool result = true; - try { - result = yield inner.input_stream.close_async(io_priority, cancellable); - } catch (IOError e) { - if (error == null) { - error = e; - } - } - try { - result = (yield close_if_both_closed(io_priority, cancellable)) && result; - } catch (IOError e) { - if (error == null) { - error = e; - } - } - if (error != null) { - handle_connection_error(error); - throw error; - } - return result; - } - public bool close_write(Cancellable? cancellable = null) throws IOError { - check_for_errors(); - if (write_closed) { - return true; - } - close_write_async.begin(GLib.Priority.DEFAULT, cancellable); - return true; - } - public async bool close_write_async(int io_priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) throws IOError { - yield wait_and_check_for_errors(io_priority, cancellable); - if (write_closed) { - return true; - } - write_closed = true; - IOError error = null; - bool result = true; - try { - result = yield inner.output_stream.close_async(io_priority, cancellable); - } catch (IOError e) { - if (error == null) { - error = e; - } - } - try { - result = (yield close_if_both_closed(io_priority, cancellable)) && result; - } catch (IOError e) { - if (error == null) { - error = e; - } - } - if (error != null) { - handle_connection_error(error); - throw error; - } - return result; - } - private async bool close_if_both_closed(int io_priority, Cancellable? cancellable = null) throws IOError { - if (read_closed && write_closed) { - handle_connection_close(); - //return yield inner.close_async(io_priority, cancellable); - } - return true; - } -} - -public class Flag : XmppStreamFlag { - public static FlagIdentity IDENTITY = new FlagIdentity(NS_URI, "jingle"); - - private HashMap sessions = new HashMap(); - - public void add_session(Session session) { - sessions[session.sid] = session; - } - public Session? get_session(string sid) { - return sessions.has_key(sid) ? sessions[sid] : null; - } - public void remove_session(string sid) { - sessions.unset(sid); - } - - public override string get_ns() { return NS_URI; } - public override string get_id() { return IDENTITY.id; } -} - -} diff --git a/xmpp-vala/src/module/xep/0166_jingle/component.vala b/xmpp-vala/src/module/xep/0166_jingle/component.vala new file mode 100644 index 00000000..544bcd69 --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/component.vala @@ -0,0 +1,52 @@ +namespace Xmpp.Xep.Jingle { + + public abstract class ComponentConnection : Object { + public uint8 component_id { get; set; default = 0; } + public abstract async void terminate(bool we_terminated, string? reason_name = null, string? reason_text = null); + public signal void connection_closed(); + public signal void connection_error(IOError e); + } + + public abstract class DatagramConnection : ComponentConnection { + public bool ready { get; set; default = false; } + private string? terminate_reason_name = null; + private string? terminate_reason_text = null; + private bool terminated = false; + + public override async void terminate(bool we_terminated, string? reason_string = null, string? reason_text = null) { + if (!terminated) { + terminated = true; + terminate_reason_name = reason_string; + terminate_reason_text = reason_text; + connection_closed(); + } + } + + public signal void datagram_received(Bytes datagram); + public abstract void send_datagram(Bytes datagram); + } + + public class StreamingConnection : ComponentConnection { + public Gee.Future stream { get { return promise.future; } } + protected Gee.Promise promise = new Gee.Promise(); + private string? terminated = null; + + public async void init(IOStream stream) { + assert(!this.stream.ready); + promise.set_value(stream); + if (terminated != null) { + yield stream.close_async(); + } + } + + public override async void terminate(bool we_terminated, string? reason_name = null, string? reason_text = null) { + if (terminated == null) { + terminated = (reason_name ?? "") + " - " + (reason_text ?? "") + @"we terminated? $we_terminated"; + if (stream.ready) { + yield stream.value.close_async(); + } + } + } + } +} + diff --git a/xmpp-vala/src/module/xep/0166_jingle/content.vala b/xmpp-vala/src/module/xep/0166_jingle/content.vala new file mode 100644 index 00000000..67c13dd8 --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/content.vala @@ -0,0 +1,236 @@ +using Gee; +using Xmpp; + +public class Xmpp.Xep.Jingle.Content : Object { + + public signal void senders_modify_incoming(Senders proposed_senders); + + // INITIATE_SENT -> CONNECTING -> [REPLACING_TRANSPORT -> CONNECTING ->]... ACTIVE -> ENDED + // INITIATE_RECEIVED -> CONNECTING -> [WAITING_FOR_TRANSPORT_REPLACE -> CONNECTING ->].. ACTIVE -> ENDED + public enum State { + PENDING, + WANTS_TO_BE_ACCEPTED, + ACCEPTED, + REPLACING_TRANSPORT, + WAITING_FOR_TRANSPORT_REPLACE + } + + public State state { get; set; } + + public Role role { get; private set; } + public Jid local_full_jid { get; private set; } + public Jid peer_full_jid { get; private set; } + public Role content_creator { get; private set; } + public string content_name { get; private set; } + public Senders senders { get; private set; } + + public ContentType content_type; + public ContentParameters content_params; + public Transport transport; + public TransportParameters? transport_params; + public SecurityPrecondition security_precondition; + public SecurityParameters? security_params; + + public weak Session session; + public Map component_connections = new HashMap(); // TODO private + + // INITIATE_SENT | INITIATE_RECEIVED | CONNECTING + public Set tried_transport_methods = new HashSet(); + + + public Content.initiate_sent(string content_name, Senders senders, + ContentType content_type, ContentParameters content_params, + Transport transport, TransportParameters? transport_params, + SecurityPrecondition? security_precondition, SecurityParameters? security_params, + Jid local_full_jid, Jid peer_full_jid) { + this.content_name = content_name; + this.senders = senders; + this.role = Role.INITIATOR; + this.local_full_jid = local_full_jid; + this.peer_full_jid = peer_full_jid; + this.content_creator = Role.INITIATOR; + + this.content_type = content_type; + this.content_params = content_params; + this.transport = transport; + this.transport_params = transport_params; + this.security_precondition = security_precondition; + this.security_params = security_params; + + this.tried_transport_methods.add(transport.ns_uri); + + state = State.PENDING; + } + + public Content.initiate_received(string content_name, Senders senders, + ContentType content_type, ContentParameters content_params, + Transport transport, TransportParameters? transport_params, + SecurityPrecondition? security_precondition, SecurityParameters? security_params, + Jid local_full_jid, Jid peer_full_jid) throws Error { + this.content_name = content_name; + this.senders = senders; + this.role = Role.RESPONDER; + this.local_full_jid = local_full_jid; + this.peer_full_jid = peer_full_jid; + this.content_creator = Role.INITIATOR; + + this.content_type = content_type; + this.content_params = content_params; + this.transport = transport; + this.transport_params = transport_params; + this.security_precondition = security_precondition; + this.security_params = security_params; + + if (transport != null) { + this.tried_transport_methods.add(transport.ns_uri); + } + + state = State.PENDING; + } + + public void set_session(Session session) { + this.session = session; + this.transport_params.set_content(this); + } + + public void accept() { + state = State.WANTS_TO_BE_ACCEPTED; + + session.accept_content(this); + } + + public void reject() { + session.reject_content(this); + } + + public void terminate(bool we_terminated, string? reason_name, string? reason_text) { + content_params.terminate(we_terminated, reason_name, reason_text); + + foreach (ComponentConnection connection in component_connections.values) { + connection.terminate(we_terminated, reason_name, reason_text); + } + } + + public void modify(Senders new_sender) { + session.send_content_modify(this, new_sender); + this.senders = new_sender; + } + + public void accept_content_modify(Senders senders) { + this.senders = senders; + } + + internal void handle_content_modify(XmppStream stream, Senders proposed_senders) { + senders_modify_incoming(proposed_senders); + } + + internal void on_accept(XmppStream stream) { + this.transport_params.create_transport_connection(stream, this); + this.content_params.accept(stream, session, this); + } + + internal void handle_accept(XmppStream stream, ContentNode content_node) { + this.transport_params.handle_transport_accept(content_node.transport); + this.transport_params.create_transport_connection(stream, this); + this.content_params.handle_accept(stream, this.session, this, content_node.description); + } + + private async void select_new_transport() { + XmppStream stream = session.stream; + Transport? new_transport = yield stream.get_module(Module.IDENTITY).select_transport(stream, transport.type_, transport_params.components, peer_full_jid, tried_transport_methods); + if (new_transport == null) { + session.terminate(ReasonElement.FAILED_TRANSPORT, null, "failed transport"); + // TODO should we only terminate this content or really the whole session? + return; + } + tried_transport_methods.add(new_transport.ns_uri); + transport_params = new_transport.create_transport_parameters(stream, transport_params.components, local_full_jid, peer_full_jid); + set_transport_params(transport_params); + session.send_transport_replace(this, transport_params); + state = State.REPLACING_TRANSPORT; + } + + public void handle_transport_accept(XmppStream stream, StanzaNode transport_node, StanzaNode jingle, Iq.Stanza iq) throws IqError { + if (state != State.REPLACING_TRANSPORT) { + throw new IqError.OUT_OF_ORDER("no outstanding transport-replace request"); + } + if (transport_node.ns_uri != transport.ns_uri) { + throw new IqError.BAD_REQUEST("transport-accept with unnegotiated transport method"); + } + transport_params.handle_transport_accept(transport_node); + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); + transport_params.create_transport_connection(stream, this); + } + + public void handle_transport_reject(XmppStream stream, StanzaNode jingle, Iq.Stanza iq) throws IqError { + if (state != State.REPLACING_TRANSPORT) { + throw new IqError.OUT_OF_ORDER("no outstanding transport-replace request"); + } + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); + select_new_transport.begin(); + } + + public void handle_transport_replace(XmppStream stream, StanzaNode transport_node, StanzaNode jingle, Iq.Stanza iq) throws IqError { + Transport? transport = stream.get_module(Module.IDENTITY).get_transport(transport_node.ns_uri); + TransportParameters? parameters = null; + if (transport != null) { + // Just parse the transport info for the errors. + parameters = transport.parse_transport_parameters(stream, content_type.required_components, local_full_jid, peer_full_jid, transport_node); + } + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); + if (state != State.WAITING_FOR_TRANSPORT_REPLACE || transport == null) { + session.send_transport_reject(this, transport_node); + return; + } + set_transport_params(parameters); + session.send_transport_accept(this, parameters); + + this.transport_params.create_transport_connection(stream, this); + } + + public void handle_transport_info(XmppStream stream, StanzaNode transport, StanzaNode jingle, Iq.Stanza iq) throws IqError { + this.transport_params.handle_transport_info(transport); + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); + } + + public void on_description_info(XmppStream stream, StanzaNode description, StanzaNode jinglq, Iq.Stanza iq) throws IqError { + // TODO: do something. + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); + } + + void verify_content(ContentNode content) throws IqError { + if (content.name != content_name || content.creator != content_creator) { + throw new IqError.BAD_REQUEST("unknown content"); + } + } + + public void set_transport_connection(ComponentConnection? conn, uint8 component = 1) { + debug(@"set_transport_connection: %s, %s, %i, %s, overwrites: %s", this.content_name, this.state.to_string(), component, (conn != null).to_string(), component_connections.has_key(component).to_string()); + + if (conn != null) { + component_connections[component] = conn; + if (transport_params.components == component) { + state = State.ACCEPTED; + tried_transport_methods.clear(); + } + } else { + if (role == Role.INITIATOR) { + select_new_transport.begin(); + } else { + state = State.WAITING_FOR_TRANSPORT_REPLACE; + } + } + } + + private void set_transport_params(TransportParameters transport_params) { + this.transport_params = transport_params; + } + + public ComponentConnection? get_transport_connection(uint8 component = 1) { + return component_connections[component]; + } + + public void send_transport_info(StanzaNode transport) { + session.send_transport_info(this, transport); + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/content_description.vala b/xmpp-vala/src/module/xep/0166_jingle/content_description.vala new file mode 100644 index 00000000..1a24e52e --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/content_description.vala @@ -0,0 +1,27 @@ +using Gee; +using Xmpp.Xep; +using Xmpp; + +namespace Xmpp.Xep.Jingle { + + public interface ContentType : Object { + public abstract string ns_uri { get; } + public abstract TransportType required_transport_type { get; } + public abstract uint8 required_components { get; } + public abstract ContentParameters parse_content_parameters(StanzaNode description) throws IqError; + } + + public interface ContentParameters : Object { + /** Called when the counterpart proposes the content */ + public abstract async void handle_proposed_content(XmppStream stream, Jingle.Session session, Content content); + + /** Called when we accept the content that was proposed by the counterpart */ + public abstract void accept(XmppStream stream, Jingle.Session session, Jingle.Content content); + /** Called when the counterpart accepts the content that was proposed by us*/ + public abstract void handle_accept(XmppStream stream, Jingle.Session session, Jingle.Content content, StanzaNode description_node); + + public abstract void terminate(bool we_terminated, string? reason_name, string? reason_text); + + public abstract StanzaNode get_description_node(); + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/content_node.vala b/xmpp-vala/src/module/xep/0166_jingle/content_node.vala new file mode 100644 index 00000000..7d8d56c8 --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/content_node.vala @@ -0,0 +1,112 @@ +using Gee; +using Xmpp.Xep; +using Xmpp; + +namespace Xmpp.Xep.Jingle { + + class ContentNode { + public Role creator; + public string name; + public Senders senders; + public StanzaNode? description; + public StanzaNode? transport; + public StanzaNode? security; + } + + [Version(deprecated = true)] + ContentNode get_single_content_node(StanzaNode jingle) throws IqError { + Gee.List contents = jingle.get_subnodes("content"); + if (contents.size == 0) { + throw new IqError.BAD_REQUEST("missing content node"); + } + if (contents.size > 1) { + throw new IqError.NOT_IMPLEMENTED("can't process multiple content nodes"); + } + StanzaNode content = contents[0]; + string? creator_str = content.get_attribute("creator"); + // Vala can't typecheck the ternary operator here. + Role? creator = null; + if (creator_str != null) { + creator = Role.parse(creator_str); + } else { + // TODO(hrxi): now, is the creator attribute optional or not (XEP-0166 + // Jingle)? + creator = Role.INITIATOR; + } + + string? name = content.get_attribute("name"); + + Senders senders = Senders.parse(content.get_attribute("senders")); + + StanzaNode? description = get_single_node_anyns(content, "description"); + StanzaNode? transport = get_single_node_anyns(content, "transport"); + StanzaNode? security = get_single_node_anyns(content, "security"); + if (name == null || creator == null) { + throw new IqError.BAD_REQUEST("missing name or creator"); + } + + return new ContentNode() { + creator=creator, + name=name, + senders=senders, + description=description, + transport=transport, + security=security + }; + } + + Gee.List get_content_nodes(StanzaNode jingle) throws IqError { + Gee.List contents = jingle.get_subnodes("content"); + if (contents.size == 0) { + throw new IqError.BAD_REQUEST("missing content node"); + } + Gee.List list = new ArrayList(); + foreach (StanzaNode content in contents) { + string? creator_str = content.get_attribute("creator"); + // Vala can't typecheck the ternary operator here. + Role? creator = null; + if (creator_str != null) { + creator = Role.parse(creator_str); + } else { + // TODO(hrxi): now, is the creator attribute optional or not (XEP-0166 + // Jingle)? + creator = Role.INITIATOR; + } + + string? name = content.get_attribute("name"); + Senders senders = Senders.parse(content.get_attribute("senders")); + StanzaNode? description = get_single_node_anyns(content, "description"); + StanzaNode? transport = get_single_node_anyns(content, "transport"); + StanzaNode? security = get_single_node_anyns(content, "security"); + if (name == null || creator == null) { + throw new IqError.BAD_REQUEST("missing name or creator"); + } + list.add(new ContentNode() { + creator=creator, + name=name, + senders=senders, + description=description, + transport=transport, + security=security + }); + } + return list; + } + + StanzaNode? get_single_node_anyns(StanzaNode parent, string? node_name = null) throws IqError { + StanzaNode? result = null; + foreach (StanzaNode child in parent.get_all_subnodes()) { + if (node_name == null || child.name == node_name) { + if (result != null) { + if (node_name != null) { + throw new IqError.BAD_REQUEST(@"multiple $(node_name) nodes"); + } else { + throw new IqError.BAD_REQUEST(@"expected single subnode"); + } + } + result = child; + } + } + return result; + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/content_security.vala b/xmpp-vala/src/module/xep/0166_jingle/content_security.vala new file mode 100644 index 00000000..0e10311d --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/content_security.vala @@ -0,0 +1,18 @@ +using Gee; +using Xmpp.Xep; +using Xmpp; + +namespace Xmpp.Xep.Jingle { + + public interface SecurityPrecondition : Object { + public abstract string security_ns_uri(); + public abstract SecurityParameters? create_security_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, Object options) throws Jingle.Error; + public abstract SecurityParameters? parse_security_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, StanzaNode security) throws IqError; + } + + public interface SecurityParameters : Object { + public abstract string security_ns_uri(); + public abstract StanzaNode to_security_stanza_node(XmppStream stream, Jid local_full_jid, Jid peer_full_jid); + public abstract IOStream wrap_stream(IOStream stream); + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/content_transport.vala b/xmpp-vala/src/module/xep/0166_jingle/content_transport.vala new file mode 100644 index 00000000..cd74c836 --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/content_transport.vala @@ -0,0 +1,29 @@ +namespace Xmpp.Xep.Jingle { + + public interface Transport : Object { + public abstract string ns_uri { get; } + public async abstract bool is_transport_available(XmppStream stream, uint8 components, Jid full_jid); + public abstract TransportType type_ { get; } + public abstract int priority { get; } + public abstract TransportParameters create_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid) throws Error; + public abstract TransportParameters parse_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws IqError; + } + + public enum TransportType { + DATAGRAM, + STREAMING, + } + + // Gets a null `stream` if connection setup was unsuccessful and another + // transport method should be tried. + public interface TransportParameters : Object { + public abstract string ns_uri { get; } + public abstract uint8 components { get; } + + public abstract void set_content(Content content); + public abstract StanzaNode to_transport_stanza_node(); + public abstract void handle_transport_accept(StanzaNode transport) throws IqError; + public abstract void handle_transport_info(StanzaNode transport) throws IqError; + public abstract void create_transport_connection(XmppStream stream, Content content); + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/jingle_flag.vala b/xmpp-vala/src/module/xep/0166_jingle/jingle_flag.vala new file mode 100644 index 00000000..9f0acd27 --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/jingle_flag.vala @@ -0,0 +1,38 @@ +using Gee; +using Xmpp; + +public class Xmpp.Xep.Jingle.Flag : XmppStreamFlag { + public static FlagIdentity IDENTITY = new FlagIdentity(NS_URI, "jingle"); + + public HashMap sessions = new HashMap(); + public HashMap> promises = new HashMap>(); + + // We might get transport-infos about a session before we finished fully creating the session. (e.g. telepathy outgoing calls) + // Thus, we "pre add" the session as soon as possible and can then await it. + public void pre_add_session(string sid) { + var promise = new Promise(); + promises[sid] = promise; + } + + public void add_session(Session session) { + if (promises.has_key(session.sid)) { + promises[session.sid].set_value(session); + promises.unset(session.sid); + } + sessions[session.sid] = session; + } + + public async Session? get_session(string sid) { + if (promises.has_key(sid)) { + return yield promises[sid].future.wait_async(); + } + return sessions[sid]; + } + + public void remove_session(string sid) { + sessions.unset(sid); + } + + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala b/xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala new file mode 100644 index 00000000..1e8a36d1 --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala @@ -0,0 +1,235 @@ +using Gee; +using Xmpp; + +namespace Xmpp.Xep.Jingle { + + internal const string NS_URI = "urn:xmpp:jingle:1"; + private const string ERROR_NS_URI = "urn:xmpp:jingle:errors:1"; + + // This module can only be attached to one stream at a time. + public class Module : XmppStreamModule, Iq.Handler { + public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "0166_jingle"); + + public signal void session_initiate_received(XmppStream stream, Session session); + + private HashMap content_types = new HashMap(); + private HashMap session_info_types = new HashMap(); + private HashMap transports = new HashMap(); + private HashMap security_preconditions = new HashMap(); + + public override void attach(XmppStream stream) { + stream.add_flag(new Flag()); + stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); + stream.get_module(Iq.Module.IDENTITY).register_for_namespace(NS_URI, this); + + // TODO update stream in all sessions + } + + public override void detach(XmppStream stream) { + stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI); + stream.get_module(Iq.Module.IDENTITY).unregister_from_namespace(NS_URI, this); + } + + public void register_content_type(ContentType content_type) { + content_types[content_type.ns_uri] = content_type; + } + + public void register_session_info_type(SessionInfoNs info_ns) { + session_info_types[info_ns.ns_uri] = info_ns; + } + + public ContentType? get_content_type(string ns_uri) { + if (!content_types.has_key(ns_uri)) { + return null; + } + return content_types[ns_uri]; + } + + public SessionInfoNs? get_session_info_type(string ns_uri) { + return session_info_types[ns_uri]; + } + + public void register_transport(Transport transport) { + transports[transport.ns_uri] = transport; + } + + public Transport? get_transport(string ns_uri) { + if (!transports.has_key(ns_uri)) { + return null; + } + return transports[ns_uri]; + } + + public async Transport? select_transport(XmppStream stream, TransportType type, uint8 components, Jid receiver_full_jid, Set blacklist) { + Transport? result = null; + foreach (Transport transport in transports.values) { + if (transport.type_ != type) { + continue; + } + if (transport.ns_uri in blacklist) { + continue; + } + if (yield transport.is_transport_available(stream, components, receiver_full_jid)) { + if (result != null) { + if (result.priority >= transport.priority) { + continue; + } + } + result = transport; + } + } + return result; + } + + public void register_security_precondition(SecurityPrecondition precondition) { + security_preconditions[precondition.security_ns_uri()] = precondition; + } + + public SecurityPrecondition? get_security_precondition(string? ns_uri) { + if (ns_uri == null) return null; + if (!security_preconditions.has_key(ns_uri)) { + return null; + } + return security_preconditions[ns_uri]; + } + + private async bool is_jingle_available(XmppStream stream, Jid full_jid) { + bool? has_jingle = yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI); + return has_jingle != null && has_jingle; + } + + public async bool is_available(XmppStream stream, TransportType type, uint8 components, Jid full_jid) { + return (yield is_jingle_available(stream, full_jid)) && (yield select_transport(stream, type, components, full_jid, Set.empty())) != null; + } + + public async Session create_session(XmppStream stream, Gee.List contents, Jid receiver_full_jid, string sid = random_uuid()) throws Error { + if (!yield is_jingle_available(stream, receiver_full_jid)) { + throw new Error.NO_SHARED_PROTOCOLS("No Jingle support"); + } + Jid? my_jid = stream.get_flag(Bind.Flag.IDENTITY).my_jid; + if (my_jid == null) { + throw new Error.GENERAL("Couldn't determine own JID"); + } + + Session session = new Session.initiate_sent(stream, sid, my_jid, receiver_full_jid); + session.terminated.connect((session, stream, _1, _2, _3) => { stream.get_flag(Flag.IDENTITY).remove_session(session.sid); }); + + foreach (Content content in contents) { + session.insert_content(content); + } + + // Build & send session-initiate iq stanza + StanzaNode initiate_jingle_iq = new StanzaNode.build("jingle", NS_URI) + .add_self_xmlns() + .put_attribute("action", "session-initiate") + .put_attribute("initiator", my_jid.to_string()) + .put_attribute("sid", session.sid); + + foreach (Content content in contents) { + StanzaNode content_node = new StanzaNode.build("content", NS_URI) + .put_attribute("creator", "initiator") + .put_attribute("name", content.content_name) + .put_attribute("senders", content.senders.to_string()) + .put_node(content.content_params.get_description_node()) + .put_node(content.transport_params.to_transport_stanza_node()); + if (content.security_params != null) { + content_node.put_node(content.security_params.to_security_stanza_node(stream, my_jid, receiver_full_jid)); + } + initiate_jingle_iq.put_node(content_node); + } + + Iq.Stanza iq = new Iq.Stanza.set(initiate_jingle_iq) { to=receiver_full_jid }; + + stream.get_flag(Flag.IDENTITY).add_session(session); + // We might get a follow-up before the ack => add_session before send_iq returns + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq, (stream, iq) => { + if (iq.is_error()) warning("Jingle session-initiate got error: %s", iq.stanza.to_string()); + }); + + return session; + } + + public async void handle_session_initiate(XmppStream stream, string sid, StanzaNode jingle, Iq.Stanza iq) throws IqError { + Jid? my_jid = stream.get_flag(Bind.Flag.IDENTITY).my_jid; + if (my_jid == null) { + throw new IqError.RESOURCE_CONSTRAINT("Couldn't determine own JID"); + } + + Session session = new Session.initiate_received(stream, sid, my_jid, iq.from); + session.terminated.connect((stream) => { stream.get_flag(Flag.IDENTITY).remove_session(sid); }); + + stream.get_flag(Flag.IDENTITY).pre_add_session(session.sid); + + foreach (ContentNode content_node in get_content_nodes(jingle)) { + yield session.insert_content_node(content_node, iq.from); + } + + stream.get_flag(Flag.IDENTITY).add_session(session); + + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); + + session_initiate_received(stream, session); + } + + public async void on_iq_set(XmppStream stream, Iq.Stanza iq) { + try { + yield handle_iq_set(stream, iq); + } catch (IqError e) { + send_iq_error(e, stream, iq); + } + } + + public async void handle_iq_set(XmppStream stream, Iq.Stanza iq) throws IqError { + StanzaNode? jingle_node = iq.stanza.get_subnode("jingle", NS_URI); + if (jingle_node == null) { + throw new IqError.BAD_REQUEST("missing jingle node"); + } + string? sid = jingle_node.get_attribute("sid"); + string? action = jingle_node.get_attribute("action"); + if (sid == null || action == null) { + throw new IqError.BAD_REQUEST("missing jingle node, sid or action"); + } + + Session? session = yield stream.get_flag(Flag.IDENTITY).get_session(sid); + if (action == "session-initiate") { + if (session != null) { + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.error(iq, new ErrorStanza.build(ErrorStanza.TYPE_MODIFY, ErrorStanza.CONDITION_CONFLICT, "session ID already in use", null)) { to=iq.from }); + return; + } + yield handle_session_initiate(stream, sid, jingle_node, iq); + return; + } + if (session == null) { + StanzaNode unknown_session = new StanzaNode.build("unknown-session", ERROR_NS_URI).add_self_xmlns(); + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.error(iq, new ErrorStanza.item_not_found(unknown_session)) { to=iq.from }); + return; + } + session.handle_iq_set(action, jingle_node, iq); + } + + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } + } + + void send_iq_error(IqError iq_error, XmppStream stream, Iq.Stanza iq) { + ErrorStanza error; + if (iq_error is IqError.BAD_REQUEST) { + error = new ErrorStanza.bad_request(iq_error.message); + } else if (iq_error is IqError.NOT_ACCEPTABLE) { + error = new ErrorStanza.not_acceptable(iq_error.message); + } else if (iq_error is IqError.NOT_IMPLEMENTED) { + error = new ErrorStanza.feature_not_implemented(iq_error.message); + } else if (iq_error is IqError.UNSUPPORTED_INFO) { + StanzaNode unsupported_info = new StanzaNode.build("unsupported-info", ERROR_NS_URI).add_self_xmlns(); + error = new ErrorStanza.build(ErrorStanza.TYPE_CANCEL, ErrorStanza.CONDITION_FEATURE_NOT_IMPLEMENTED, iq_error.message, unsupported_info); + } else if (iq_error is IqError.OUT_OF_ORDER) { + StanzaNode out_of_order = new StanzaNode.build("out-of-order", ERROR_NS_URI).add_self_xmlns(); + error = new ErrorStanza.build(ErrorStanza.TYPE_MODIFY, ErrorStanza.CONDITION_UNEXPECTED_REQUEST, iq_error.message, out_of_order); + } else if (iq_error is IqError.RESOURCE_CONSTRAINT) { + error = new ErrorStanza.resource_constraint(iq_error.message); + } else { + assert_not_reached(); + } + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.error(iq, error) { to=iq.from }); + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/jingle_structs.vala b/xmpp-vala/src/module/xep/0166_jingle/jingle_structs.vala new file mode 100644 index 00000000..0f283e0e --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/jingle_structs.vala @@ -0,0 +1,73 @@ +using Gee; +using Xmpp; + +namespace Xmpp.Xep.Jingle { + + public errordomain IqError { + BAD_REQUEST, + NOT_ACCEPTABLE, + NOT_IMPLEMENTED, + UNSUPPORTED_INFO, + OUT_OF_ORDER, + RESOURCE_CONSTRAINT, + } + + public errordomain Error { + GENERAL, + BAD_REQUEST, + INVALID_PARAMETERS, + UNSUPPORTED_TRANSPORT, + UNSUPPORTED_SECURITY, + NO_SHARED_PROTOCOLS, + TRANSPORT_ERROR, + } + + public enum Senders { + BOTH, + INITIATOR, + NONE, + RESPONDER; + + public string to_string() { + switch (this) { + case BOTH: return "both"; + case INITIATOR: return "initiator"; + case NONE: return "none"; + case RESPONDER: return "responder"; + } + assert_not_reached(); + } + + public static Senders parse(string? senders) throws IqError { + if (senders == null) return Senders.BOTH; + switch (senders) { + case "initiator": return Senders.INITIATOR; + case "responder": return Senders.RESPONDER; + case "both": return Senders.BOTH; + } + throw new IqError.BAD_REQUEST(@"invalid role $(senders)"); + } + } + + public enum Role { + INITIATOR, + RESPONDER; + + public string to_string() { + switch (this) { + case INITIATOR: return "initiator"; + case RESPONDER: return "responder"; + } + assert_not_reached(); + } + + public static Role parse(string role) throws IqError { + switch (role) { + case "initiator": return INITIATOR; + case "responder": return RESPONDER; + } + throw new IqError.BAD_REQUEST(@"invalid role $(role)"); + } + } + +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/reason_element.vala b/xmpp-vala/src/module/xep/0166_jingle/reason_element.vala new file mode 100644 index 00000000..1cbdf936 --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/reason_element.vala @@ -0,0 +1,29 @@ +using Gee; +using Xmpp; + +namespace Xmpp.Xep.Jingle.ReasonElement { + public const string ALTERNATIVE_SESSION = "alternative-session"; + public const string BUSY = "busy"; + public const string CANCEL = "cancel"; + public const string CONNECTIVITY_ERROR = "connectivity-error"; + public const string DECLINE = "decline"; + public const string EXPIRED = "expired"; + public const string FAILED_APPLICATION = "failed_application"; + public const string FAILED_TRANSPORT = "failed_transport"; + public const string GENERAL_ERROR = "general-error"; + public const string GONE = "gone"; + public const string INCOMPATIBLE_PARAMETERS = "incompatible-parameters"; + public const string MEDIA_ERROR = "media-error"; + public const string SECURITY_ERROR = "security-error"; + public const string SUCCESS = "success"; + public const string TIMEOUT = "timeout"; + public const string UNSUPPORTED_APPLICATIONS = "unsupported-applications"; + public const string UNSUPPORTED_TRANSPORTS = "unsupported-transports"; + + public const string[] NORMAL_TERMINATE_REASONS = { + BUSY, + CANCEL, + DECLINE, + SUCCESS + }; +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/session.vala b/xmpp-vala/src/module/xep/0166_jingle/session.vala new file mode 100644 index 00000000..e9ad9169 --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/session.vala @@ -0,0 +1,559 @@ +using Gee; +using Xmpp; + + +public delegate void Xmpp.Xep.Jingle.SessionTerminate(Jid to, string sid, StanzaNode reason); + +public class Xmpp.Xep.Jingle.Session : Object { + + public signal void terminated(XmppStream stream, bool we_terminated, string? reason_name, string? reason_text); + public signal void additional_content_add_incoming(XmppStream stream, Content content); + + // INITIATE_SENT/INITIATE_RECEIVED -> CONNECTING -> PENDING -> ACTIVE -> ENDED + public enum State { + INITIATE_SENT, + INITIATE_RECEIVED, + ACTIVE, + ENDED, + } + + public XmppStream stream { get; set; } + public State state { get; set; } + public string sid { get; private set; } + public Jid local_full_jid { get; private set; } + public Jid peer_full_jid { get; private set; } + public bool we_initiated { get; private set; } + + public HashMap contents = new HashMap(); + + public SecurityParameters? security { get { return contents.values.to_array()[0].security_params; } } + + public Session.initiate_sent(XmppStream stream, string sid, Jid local_full_jid, Jid peer_full_jid) { + this.stream = stream; + this.sid = sid; + this.local_full_jid = local_full_jid; + this.peer_full_jid = peer_full_jid; + this.state = State.INITIATE_SENT; + this.we_initiated = true; + } + + public Session.initiate_received(XmppStream stream, string sid, Jid local_full_jid, Jid peer_full_jid) { + this.stream = stream; + this.sid = sid; + this.local_full_jid = local_full_jid; + this.peer_full_jid = peer_full_jid; + this.state = State.INITIATE_RECEIVED; + this.we_initiated = false; + } + + public void handle_iq_set(string action, StanzaNode jingle, Iq.Stanza iq) throws IqError { + + if (action.has_prefix("session-")) { + switch (action) { + case "session-accept": + Gee.List content_nodes = get_content_nodes(jingle); + + if (state != State.INITIATE_SENT) { + throw new IqError.OUT_OF_ORDER("got session-accept while not waiting for one"); + } + handle_session_accept(content_nodes, jingle, iq); + break; + case "session-info": + handle_session_info.begin(jingle, iq); + break; + case "session-terminate": + handle_session_terminate(jingle, iq); + break; + default: + throw new IqError.BAD_REQUEST("invalid action"); + } + + + } else if (action.has_prefix("content-")) { + switch (action) { + case "content-accept": + ContentNode content_node = get_single_content_node(jingle); + handle_content_accept(content_node); + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); + break; + case "content-add": + ContentNode content_node = get_single_content_node(jingle); + insert_content_node.begin(content_node, peer_full_jid); + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); + break; + case "content-modify": + handle_content_modify(stream, jingle, iq); + break; + case "content-reject": + case "content-remove": + throw new IqError.NOT_IMPLEMENTED(@"$(action) is not implemented"); + default: + throw new IqError.BAD_REQUEST("invalid action"); + } + + + } else if (action.has_prefix("transport-")) { + ContentNode content_node = get_single_content_node(jingle); + if (!contents.has_key(content_node.name)) { + throw new IqError.BAD_REQUEST("unknown content"); + } + + if (content_node.transport == null) { + throw new IqError.BAD_REQUEST("missing transport node"); + } + + Content content = contents[content_node.name]; + + if (content_node.creator != content.content_creator) { + throw new IqError.BAD_REQUEST("unknown content; creator"); + } + + switch (action) { + case "transport-accept": + content.handle_transport_accept(stream, content_node.transport, jingle, iq); + break; + case "transport-info": + content.handle_transport_info(stream, content_node.transport, jingle, iq); + break; + case "transport-reject": + content.handle_transport_reject(stream, jingle, iq); + break; + case "transport-replace": + content.handle_transport_replace(stream, content_node.transport, jingle, iq); + break; + default: + throw new IqError.BAD_REQUEST("invalid action"); + } + + + } else if (action == "description-info") { + ContentNode content_node = get_single_content_node(jingle); + if (!contents.has_key(content_node.name)) { + throw new IqError.BAD_REQUEST("unknown content"); + } + + Content content = contents[content_node.name]; + + if (content_node.creator != content.content_creator) { + throw new IqError.BAD_REQUEST("unknown content; creator"); + } + + content.on_description_info(stream, content_node.description, jingle, iq); + } else if (action == "security-info") { + throw new IqError.NOT_IMPLEMENTED(@"$(action) is not implemented"); + + + } else { + throw new IqError.BAD_REQUEST("invalid action"); + } + } + + internal void insert_content(Content content) { + this.contents[content.content_name] = content; + content.set_session(this); + } + + internal async void insert_content_node(ContentNode content_node, Jid peer_full_jid) throws IqError { + if (content_node.description == null || content_node.transport == null) { + throw new IqError.BAD_REQUEST("missing description or transport node"); + } + + Jid? my_jid = stream.get_flag(Bind.Flag.IDENTITY).my_jid; + + Transport? transport = stream.get_module(Jingle.Module.IDENTITY).get_transport(content_node.transport.ns_uri); + ContentType? content_type = stream.get_module(Jingle.Module.IDENTITY).get_content_type(content_node.description.ns_uri); + + if (content_type == null) { + // TODO(hrxi): how do we signal an unknown content type? + throw new IqError.NOT_IMPLEMENTED("unknown content type"); + } + + TransportParameters? transport_params = null; + if (transport != null) { + transport_params = transport.parse_transport_parameters(stream, content_type.required_components, my_jid, peer_full_jid, content_node.transport); + } else { + // terminate the session below + } + + ContentParameters content_params = content_type.parse_content_parameters(content_node.description); + + SecurityPrecondition? precondition = content_node.security != null ? stream.get_module(Jingle.Module.IDENTITY).get_security_precondition(content_node.security.ns_uri) : null; + SecurityParameters? security_params = null; + if (precondition != null) { + debug("Using precondition %s", precondition.security_ns_uri()); + security_params = precondition.parse_security_parameters(stream, my_jid, peer_full_jid, content_node.security); + } else if (content_node.security != null) { + throw new IqError.NOT_IMPLEMENTED("unknown security precondition"); + } + + TransportType type = content_type.required_transport_type; + + if (transport == null || transport.type_ != type) { + terminate(ReasonElement.UNSUPPORTED_TRANSPORTS, null, "unsupported transports"); + throw new IqError.NOT_IMPLEMENTED("unsupported transports"); + } + + Content content = new Content.initiate_received(content_node.name, content_node.senders, + content_type, content_params, + transport, transport_params, + precondition, security_params, + my_jid, peer_full_jid); + insert_content(content); + + yield content_params.handle_proposed_content(stream, this, content); + + if (this.state == State.ACTIVE) { + additional_content_add_incoming(stream, content); + } + } + + public async void add_content(Content content) { + content.session = this; + this.contents[content.content_name] = content; + + StanzaNode content_add_node = new StanzaNode.build("jingle", NS_URI) + .add_self_xmlns() + .put_attribute("action", "content-add") + .put_attribute("sid", sid) + .put_node(new StanzaNode.build("content", NS_URI) + .put_attribute("creator", "initiator") + .put_attribute("name", content.content_name) + .put_attribute("senders", content.senders.to_string()) + .put_node(content.content_params.get_description_node()) + .put_node(content.transport_params.to_transport_stanza_node())); + + Iq.Stanza iq = new Iq.Stanza.set(content_add_node) { to=peer_full_jid }; + yield stream.get_module(Iq.Module.IDENTITY).send_iq_async(stream, iq); + } + + private void handle_content_accept(ContentNode content_node) throws IqError { + if (content_node.description == null || content_node.transport == null) throw new IqError.BAD_REQUEST("missing description or transport node"); + if (!contents.has_key(content_node.name)) throw new IqError.BAD_REQUEST("unknown content"); + + Content content = contents[content_node.name]; + + if (content_node.creator != content.content_creator) warning("Counterpart accepts content with an unexpected `creator`"); + if (content_node.senders != content.senders) warning("Counterpart accepts content with an unexpected `senders`"); + if (content_node.transport.ns_uri != content.transport_params.ns_uri) throw new IqError.BAD_REQUEST("session-accept with unnegotiated transport method"); + + content.handle_accept(stream, content_node); + } + + private void handle_content_modify(XmppStream stream, StanzaNode jingle_node, Iq.Stanza iq) throws IqError { + ContentNode content_node = get_single_content_node(jingle_node); + + Content? content = contents[content_node.name]; + + if (content == null) throw new IqError.BAD_REQUEST("no such content"); + if (content_node.creator != content.content_creator) throw new IqError.BAD_REQUEST("mismatching creator"); + + Iq.Stanza result_iq = new Iq.Stanza.result(iq) { to=peer_full_jid }; + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, result_iq); + + content.handle_content_modify(stream, content_node.senders); + } + + private void handle_session_accept(Gee.List content_nodes, StanzaNode jingle, Iq.Stanza iq) throws IqError { + string? responder_str = jingle.get_attribute("responder"); + Jid responder = iq.from; + if (responder_str != null) { + try { + responder = new Jid(responder_str); + } catch (InvalidJidError e) { + warning("Received invalid session accept: %s", e.message); + } + } + // TODO(hrxi): more sanity checking, perhaps replace who we're talking to + if (!responder.is_full()) { + throw new IqError.BAD_REQUEST("invalid responder JID"); + } + foreach (ContentNode content_node in content_nodes) { + handle_content_accept(content_node); + } + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); + + state = State.ACTIVE; + } + + private void handle_session_terminate(StanzaNode jingle, Iq.Stanza iq) throws IqError { + string? reason_text = null; + string? reason_name = null; + StanzaNode? reason_node = iq.stanza.get_deep_subnode(NS_URI + ":jingle", NS_URI + ":reason"); + if (reason_node != null) { + if (reason_node.sub_nodes.size > 2) warning("Jingle session-terminate reason node w/ >2 subnodes: %s", iq.stanza.to_string()); + + StanzaNode? specific_reason_node = null; + StanzaNode? text_node = null; + foreach (StanzaNode node in reason_node.sub_nodes) { + if (node.name == "text") { + text_node = node; + } else if (node.ns_uri == NS_URI) { + specific_reason_node = node; + } + } + reason_name = specific_reason_node != null ? specific_reason_node.name : null; + reason_text = text_node != null ? text_node.get_string_content() : null; + + if (reason_name != null && !(specific_reason_node.name in ReasonElement.NORMAL_TERMINATE_REASONS)) { + warning("Jingle session terminated: %s : %s", reason_name ?? "", reason_text ?? ""); + } else { + debug("Jingle session terminated: %s : %s", reason_name ?? "", reason_text ?? ""); + } + } + + foreach (Content content in contents.values) { + content.terminate(false, reason_name, reason_text); + } + + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); + // TODO(hrxi): also handle presence type=unavailable + + state = State.ENDED; + terminated(stream, false, reason_name, reason_text); + } + + private async void handle_session_info(StanzaNode jingle, Iq.Stanza iq) throws IqError { + StanzaNode? info = get_single_node_anyns(jingle); + if (info == null) { + // Jingle session ping + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); + return; + } + SessionInfoNs? info_ns = stream.get_module(Module.IDENTITY).get_session_info_type(info.ns_uri); + if (info_ns == null) { + throw new IqError.UNSUPPORTED_INFO("unknown session-info namespace"); + } + info_ns.handle_content_session_info(stream, this, info, iq); + + Iq.Stanza result_iq = new Iq.Stanza.result(iq) { to=peer_full_jid }; + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, result_iq); + } + + private void accept() { + if (state != State.INITIATE_RECEIVED) critical("Accepting a stream, but we're the initiator"); + + StanzaNode jingle = new StanzaNode.build("jingle", NS_URI) + .add_self_xmlns() + .put_attribute("action", "session-accept") + .put_attribute("sid", sid); + foreach (Content content in contents.values) { + StanzaNode content_node = new StanzaNode.build("content", NS_URI) + .put_attribute("creator", "initiator") + .put_attribute("name", content.content_name) + .put_attribute("senders", content.senders.to_string()) + .put_node(content.content_params.get_description_node()) + .put_node(content.transport_params.to_transport_stanza_node()); + jingle.put_node(content_node); + } + Iq.Stanza iq = new Iq.Stanza.set(jingle) { to=peer_full_jid }; + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); + + + foreach (Content content in contents.values) { + content.on_accept(stream); + } + + state = State.ACTIVE; + } + + internal void accept_content(Content content) { + if (state == State.INITIATE_RECEIVED) { + bool all_accepted = true; + foreach (Content c in contents.values) { + if (c.state != Content.State.WANTS_TO_BE_ACCEPTED) { + all_accepted = false; + } + } + if (all_accepted) { + accept(); + } + } else if (state == State.ACTIVE) { + StanzaNode content_accept_node = new StanzaNode.build("jingle", NS_URI) + .add_self_xmlns() + .put_attribute("action", "content-accept") + .put_attribute("sid", sid) + .put_node(new StanzaNode.build("content", NS_URI) + .put_attribute("creator", "initiator") + .put_attribute("name", content.content_name) + .put_attribute("senders", content.senders.to_string()) + .put_node(content.content_params.get_description_node()) + .put_node(content.transport_params.to_transport_stanza_node())); + + Iq.Stanza iq = new Iq.Stanza.set(content_accept_node) { to=peer_full_jid }; + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); + + content.on_accept(stream); + } + } + + private void reject() { + if (state != State.INITIATE_RECEIVED) critical("Accepting a stream, but we're the initiator"); + terminate(ReasonElement.DECLINE, null, "declined"); + } + + internal void reject_content(Content content) { + if (state == State.INITIATE_RECEIVED) { + reject(); + } else { + warning("not really handeling content rejects"); + } + } + + public void set_application_error(StanzaNode? application_reason = null) { + terminate(ReasonElement.FAILED_APPLICATION, null, "application error"); + } + + public void terminate(string? reason_name, string? reason_text, string? local_reason) { + if (state == State.ENDED) return; + + if (state == State.ACTIVE) { + string reason_str; + if (local_reason != null) { + reason_str = @"local session-terminate: $(local_reason)"; + } else { + reason_str = "local session-terminate"; + } + foreach (Content content in contents.values) { + content.terminate(true, reason_name, reason_text); + } + } + + StanzaNode terminate_iq = new StanzaNode.build("jingle", NS_URI) + .add_self_xmlns() + .put_attribute("action", "session-terminate") + .put_attribute("sid", sid); + if (reason_name != null || reason_text != null) { + StanzaNode reason_node = new StanzaNode.build("reason", NS_URI); + if (reason_name != null) { + reason_node.put_node(new StanzaNode.build(reason_name, NS_URI)); + } + if (reason_text != null) { + reason_node.put_node(new StanzaNode.text(reason_text)); + } + terminate_iq.put_node(reason_node); + } + Iq.Stanza iq = new Iq.Stanza.set(terminate_iq) { to=peer_full_jid }; + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); + + state = State.ENDED; + terminated(stream, true, reason_name, reason_text); + } + + internal void send_session_info(StanzaNode child_node) { + if (state == State.ENDED) return; + + StanzaNode node = new StanzaNode.build("jingle", NS_URI).add_self_xmlns() + .put_attribute("action", "session-info") + .put_attribute("sid", sid) + // TODO put `initiator`? + .put_node(child_node); + Iq.Stanza iq = new Iq.Stanza.set(node) { to=peer_full_jid }; + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); + } + + internal void send_content_modify(Content content, Senders senders) { + if (state == State.ENDED) return; + + StanzaNode node = new StanzaNode.build("jingle", NS_URI).add_self_xmlns() + .put_attribute("action", "content-modify") + .put_attribute("sid", sid) + .put_node(new StanzaNode.build("content", NS_URI) + .put_attribute("creator", content.content_creator.to_string()) + .put_attribute("name", content.content_name) + .put_attribute("senders", senders.to_string())); + Iq.Stanza iq = new Iq.Stanza.set(node) { to=peer_full_jid }; + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); + } + + internal void send_transport_accept(Content content, TransportParameters transport_params) { + if (state == State.ENDED) return; + + StanzaNode jingle_response = new StanzaNode.build("jingle", NS_URI) + .add_self_xmlns() + .put_attribute("action", "transport-accept") + .put_attribute("sid", sid) + .put_node(new StanzaNode.build("content", NS_URI) + .put_attribute("creator", "initiator") + .put_attribute("name", content.content_name) + .put_node(transport_params.to_transport_stanza_node()) + ); + Iq.Stanza iq_response = new Iq.Stanza.set(jingle_response) { to=peer_full_jid }; + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq_response); + } + + internal void send_transport_replace(Content content, TransportParameters transport_params) { + if (state == State.ENDED) return; + + StanzaNode jingle = new StanzaNode.build("jingle", NS_URI) + .add_self_xmlns() + .put_attribute("action", "transport-replace") + .put_attribute("sid", sid) + .put_node(new StanzaNode.build("content", NS_URI) + .put_attribute("creator", "initiator") + .put_attribute("name", content.content_name) + .put_node(transport_params.to_transport_stanza_node()) + ); + Iq.Stanza iq = new Iq.Stanza.set(jingle) { to=peer_full_jid }; + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); + } + + internal void send_transport_reject(Content content, StanzaNode transport_node) { + if (state == State.ENDED) return; + + StanzaNode jingle_response = new StanzaNode.build("jingle", NS_URI) + .add_self_xmlns() + .put_attribute("action", "transport-reject") + .put_attribute("sid", sid) + .put_node(new StanzaNode.build("content", NS_URI) + .put_attribute("creator", "initiator") + .put_attribute("name", content.content_name) + .put_node(transport_node) + ); + Iq.Stanza iq_response = new Iq.Stanza.set(jingle_response) { to=peer_full_jid }; + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq_response); + } + + internal void send_transport_info(Content content, StanzaNode transport) { + if (state == State.ENDED) return; + + StanzaNode jingle = new StanzaNode.build("jingle", NS_URI) + .add_self_xmlns() + .put_attribute("action", "transport-info") + .put_attribute("sid", sid) + .put_node(new StanzaNode.build("content", NS_URI) + .put_attribute("creator", "initiator") + .put_attribute("name", content.content_name) + .put_node(transport) + ); + Iq.Stanza iq = new Iq.Stanza.set(jingle) { to=peer_full_jid }; + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); + } + + public bool senders_include_us(Senders senders) { + switch (senders) { + case Senders.BOTH: + return true; + case Senders.NONE: + return false; + case Senders.INITIATOR: + return we_initiated; + case Senders.RESPONDER: + return !we_initiated; + } + assert_not_reached(); + } + + public bool senders_include_counterpart(Senders senders) { + switch (senders) { + case Senders.BOTH: + return true; + case Senders.NONE: + return false; + case Senders.INITIATOR: + return !we_initiated; + case Senders.RESPONDER: + return we_initiated; + } + assert_not_reached(); + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/session_info.vala b/xmpp-vala/src/module/xep/0166_jingle/session_info.vala new file mode 100644 index 00000000..fcf7584f --- /dev/null +++ b/xmpp-vala/src/module/xep/0166_jingle/session_info.vala @@ -0,0 +1,12 @@ +using Gee; +using Xmpp.Xep; +using Xmpp; + +namespace Xmpp.Xep.Jingle { + + public interface SessionInfoNs : Object { + public abstract string ns_uri { get; } + + public abstract void handle_content_session_info(XmppStream stream, Session session, StanzaNode info, Iq.Stanza iq) throws IqError; + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala b/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala index 1c0323be..07b158bc 100644 --- a/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala +++ b/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala @@ -7,50 +7,42 @@ namespace Xmpp.Xep.JingleFileTransfer { private const string NS_URI = "urn:xmpp:jingle:apps:file-transfer:5"; public class Module : Jingle.ContentType, XmppStreamModule { + + public signal void file_incoming(XmppStream stream, FileTransfer file_transfer); + public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "0234_jingle_file_transfer"); + public SessionInfoType session_info_type = new SessionInfoType(); public override void attach(XmppStream stream) { stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); stream.get_module(Jingle.Module.IDENTITY).register_content_type(this); + stream.get_module(Jingle.Module.IDENTITY).register_session_info_type(session_info_type); } public override void detach(XmppStream stream) { stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI); } - public string content_type_ns_uri() { - return NS_URI; - } - public Jingle.TransportType content_type_transport_type() { - return Jingle.TransportType.STREAMING; - } + public string ns_uri { get { return NS_URI; } } + public Jingle.TransportType required_transport_type { get { return Jingle.TransportType.STREAMING; } } + public uint8 required_components { get { return 1; } } + public Jingle.ContentParameters parse_content_parameters(StanzaNode description) throws Jingle.IqError { return Parameters.parse(this, description); } - public void handle_content_session_info(XmppStream stream, Jingle.Session session, StanzaNode info, Iq.Stanza iq) throws Jingle.IqError { - switch (info.name) { - case "received": - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); - break; - case "checksum": - // TODO(hrxi): handle hash - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); - break; - default: - throw new Jingle.IqError.UNSUPPORTED_INFO(@"unsupported file transfer info $(info.name)"); - } - } - public signal void file_incoming(XmppStream stream, FileTransfer file_transfer); + public Jingle.ContentParameters create_content_parameters(Object object) throws Jingle.IqError { + assert_not_reached(); + } public async bool is_available(XmppStream stream, Jid full_jid) { bool? has_feature = yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI); if (has_feature == null || !(!)has_feature) { return false; } - return yield stream.get_module(Jingle.Module.IDENTITY).is_available(stream, Jingle.TransportType.STREAMING, full_jid); + return yield stream.get_module(Jingle.Module.IDENTITY).is_available(stream, required_transport_type, required_components, full_jid); } - public async void offer_file_stream(XmppStream stream, Jid receiver_full_jid, InputStream input_stream, string basename, int64 size, string? precondition_name = null, Object? precondition_options = null) throws IOError { + public async void offer_file_stream(XmppStream stream, Jid receiver_full_jid, InputStream input_stream, string basename, int64 size, string? precondition_name = null, Object? precondition_options = null) throws Jingle.Error { StanzaNode file_node; StanzaNode description = new StanzaNode.build("description", NS_URI) .add_self_xmlns() @@ -64,25 +56,83 @@ public class Module : Jingle.ContentType, XmppStreamModule { warning("Sending file %s without size, likely going to cause problems down the road...", basename); } - Jingle.Session session; - try { - session = yield stream.get_module(Jingle.Module.IDENTITY) - .create_session(stream, Jingle.TransportType.STREAMING, receiver_full_jid, Jingle.Senders.INITIATOR, "a-file-offer", description, precondition_name, precondition_options); // TODO(hrxi): Why "a-file-offer"? - } catch (Jingle.Error e) { - throw new IOError.FAILED(@"couldn't create Jingle session: $(e.message)"); + Parameters parameters = Parameters.parse(this, description); + + Jingle.Module jingle_module = stream.get_module(Jingle.Module.IDENTITY); + + Jingle.Transport? transport = yield jingle_module.select_transport(stream, required_transport_type, required_components, receiver_full_jid, Set.empty()); + if (transport == null) { + throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable transports"); } - session.terminate_on_connection_close = false; + Jingle.SecurityPrecondition? precondition = jingle_module.get_security_precondition(precondition_name); + if (precondition_name != null && precondition == null) { + throw new Jingle.Error.UNSUPPORTED_SECURITY("No suitable security precondiiton found"); + } + Jid? my_jid = stream.get_flag(Bind.Flag.IDENTITY).my_jid; + if (my_jid == null) { + throw new Jingle.Error.GENERAL("Couldn't determine own JID"); + } + Jingle.TransportParameters transport_params = transport.create_transport_parameters(stream, required_components, my_jid, receiver_full_jid); + Jingle.SecurityParameters? security_params = precondition != null ? precondition.create_security_parameters(stream, my_jid, receiver_full_jid, precondition_options) : null; - yield session.conn.input_stream.close_async(); + Jingle.Content content = new Jingle.Content.initiate_sent("a-file-offer", Jingle.Senders.INITIATOR, + this, parameters, + transport, transport_params, + precondition, security_params, + my_jid, receiver_full_jid); - // TODO(hrxi): catch errors - yield session.conn.output_stream.splice_async(input_stream, OutputStreamSpliceFlags.CLOSE_SOURCE|OutputStreamSpliceFlags.CLOSE_TARGET); + ArrayList contents = new ArrayList(); + contents.add(content); + + + Jingle.Session? session = null; + try { + session = yield jingle_module.create_session(stream, contents, receiver_full_jid); + + // Wait for the counterpart to accept our offer + ulong content_notify_id = 0; + content_notify_id = content.notify["state"].connect(() => { + if (content.state == Jingle.Content.State.ACCEPTED) { + Idle.add(offer_file_stream.callback); + content.disconnect(content_notify_id); + } + }); + yield; + + // Send the file data + Jingle.StreamingConnection connection = content.component_connections.values.to_array()[0] as Jingle.StreamingConnection; + IOStream io_stream = yield connection.stream.wait_async(); + yield io_stream.input_stream.close_async(); + yield io_stream.output_stream.splice_async(input_stream, OutputStreamSpliceFlags.CLOSE_SOURCE|OutputStreamSpliceFlags.CLOSE_TARGET); + yield connection.terminate(true); + } catch (Jingle.Error e) { + session.terminate(Jingle.ReasonElement.FAILED_TRANSPORT, e.message, e.message); + throw new Jingle.Error.GENERAL(@"couldn't create Jingle session: $(e.message)"); + } } public override string get_ns() { return NS_URI; } public override string get_id() { return IDENTITY.id; } } +public class SessionInfoType : Jingle.SessionInfoNs, Object { + + public string ns_uri { get { return NS_URI; } } + + public void handle_content_session_info(XmppStream stream, Jingle.Session session, StanzaNode info, Iq.Stanza iq) throws Jingle.IqError { + switch (info.name) { + case "received": + break; + case "checksum": + // TODO(hrxi): handle hash + break; + default: + throw new Jingle.IqError.UNSUPPORTED_INFO(@"unsupported file transfer info $(info.name)"); + } + } + +} + public class Parameters : Jingle.ContentParameters, Object { Module parent; @@ -127,24 +177,42 @@ public class Parameters : Jingle.ContentParameters, Object { return new Parameters(parent, description, media_type, name, size); } - public void on_session_initiate(XmppStream stream, Jingle.Session session) { - parent.file_incoming(stream, new FileTransfer(session, this)); + public StanzaNode get_description_node() { + return original_description; } + + public async void handle_proposed_content(XmppStream stream, Jingle.Session session, Jingle.Content content) { + parent.file_incoming(stream, new FileTransfer(session, content, this)); + } + + public void modify(XmppStream stream, Jingle.Session session, Jingle.Content content, Jingle.Senders senders) { } + + public void accept(XmppStream stream, Jingle.Session session, Jingle.Content content) { } + + public void handle_accept(XmppStream stream, Jingle.Session session, Jingle.Content content, StanzaNode description_node) { } + + public void terminate(bool we_terminated, string? reason_name, string? reason_text) { } } // Does nothing except wrapping an input stream to signal EOF after reading // `max_size` bytes. private class FileTransferInputStream : InputStream { + + public signal void closed(); + InputStream inner; int64 remaining_size; + public FileTransferInputStream(InputStream inner, int64 max_size) { this.inner = inner; this.remaining_size = max_size; } + private ssize_t update_remaining(ssize_t read) { this.remaining_size -= read; return read; } + public override ssize_t read(uint8[] buffer_, Cancellable? cancellable = null) throws IOError { unowned uint8[] buffer = buffer_; if (remaining_size <= 0) { @@ -155,6 +223,7 @@ private class FileTransferInputStream : InputStream { } return update_remaining(inner.read(buffer, cancellable)); } + public override async ssize_t read_async(uint8[]? buffer_, int io_priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) throws IOError { unowned uint8[] buffer = buffer_; if (remaining_size <= 0) { @@ -165,16 +234,21 @@ private class FileTransferInputStream : InputStream { } return update_remaining(yield inner.read_async(buffer, io_priority, cancellable)); } + public override bool close(Cancellable? cancellable = null) throws IOError { + closed(); return inner.close(cancellable); } + public override async bool close_async(int io_priority = GLib.Priority.DEFAULT, Cancellable? cancellable = null) throws IOError { + closed(); return yield inner.close_async(io_priority, cancellable); } } public class FileTransfer : Object { Jingle.Session session; + Jingle.Content content; Parameters parameters; public Jid peer { get { return session.peer_full_jid; } } @@ -184,19 +258,27 @@ public class FileTransfer : Object { public InputStream? stream { get; private set; } - public FileTransfer(Jingle.Session session, Parameters parameters) { + public FileTransfer(Jingle.Session session, Jingle.Content content, Parameters parameters) { this.session = session; + this.content = content; this.parameters = parameters; - this.stream = new FileTransferInputStream(session.conn.input_stream, size); } - public void accept(XmppStream stream) throws IOError { - session.accept(stream, parameters.original_description); - session.conn.output_stream.close(); + public async void accept(XmppStream stream) throws IOError { + content.accept(); + + Jingle.StreamingConnection connection = content.component_connections.values.to_array()[0] as Jingle.StreamingConnection; + IOStream? io_stream = yield connection.stream.wait_async(); + FileTransferInputStream ft_stream = new FileTransferInputStream(io_stream.input_stream, size); + io_stream.output_stream.close(); + ft_stream.closed.connect(() => { + session.terminate(Jingle.ReasonElement.SUCCESS, null, null); + }); + this.stream = ft_stream; } public void reject(XmppStream stream) { - session.reject(stream); + content.reject(); } } diff --git a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala index bb31cadc..1c4e0d38 100644 --- a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala +++ b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala @@ -21,19 +21,13 @@ public class Module : Jingle.Transport, XmppStreamModule { public override string get_ns() { return NS_URI; } public override string get_id() { return IDENTITY.id; } - public async bool is_transport_available(XmppStream stream, Jid full_jid) { - return yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI); + public async bool is_transport_available(XmppStream stream, uint8 components, Jid full_jid) { + return components == 1 && yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI); } - public string transport_ns_uri() { - return NS_URI; - } - public Jingle.TransportType transport_type() { - return Jingle.TransportType.STREAMING; - } - public int transport_priority() { - return 1; - } + public string ns_uri { get { return NS_URI; } } + public Jingle.TransportType type_ { get { return Jingle.TransportType.STREAMING; } } + public int priority { get { return 1; } } private Gee.List get_proxies(XmppStream stream) { Gee.List result = new ArrayList(); int i = 1 << 15; @@ -70,20 +64,21 @@ public class Module : Jingle.Transport, XmppStreamModule { } private void select_candidates(XmppStream stream, Jid local_full_jid, string dstaddr, Parameters result) { result.local_candidates.add_all(get_proxies(stream)); - result.local_candidates.add_all(start_local_listeners(stream, local_full_jid, dstaddr, out result.listener)); + //result.local_candidates.add_all(start_local_listeners(stream, local_full_jid, dstaddr, out result.listener)); result.local_candidates.sort((c1, c2) => { if (c1.priority < c2.priority) { return 1; } if (c1.priority > c2.priority) { return -1; } return 0; }); } - public Jingle.TransportParameters create_transport_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid) { + public Jingle.TransportParameters create_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid) { + assert(components == 1); Parameters result = new Parameters.create(local_full_jid, peer_full_jid, random_uuid()); string dstaddr = calculate_dstaddr(result.sid, local_full_jid, peer_full_jid); select_candidates(stream, local_full_jid, dstaddr, result); return result; } - public Jingle.TransportParameters parse_transport_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { + public Jingle.TransportParameters parse_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { Parameters result = Parameters.parse(local_full_jid, peer_full_jid, transport); string dstaddr = calculate_dstaddr(result.sid, local_full_jid, peer_full_jid); select_candidates(stream, local_full_jid, dstaddr, result); @@ -335,6 +330,8 @@ class LocalListener { } class Parameters : Jingle.TransportParameters, Object { + public string ns_uri { get { return NS_URI; } } + public uint8 components { get { return 1; } } public Jingle.Role role { get; private set; } public string sid { get; private set; } public string remote_dstaddr { get; private set; } @@ -352,6 +349,7 @@ class Parameters : Jingle.TransportParameters, Object { Candidate? local_selected_candidate = null; SocketConnection? local_selected_candidate_conn = null; weak Jingle.Session? session = null; + weak Jingle.Content? content = null; XmppStream? hack = null; string? waiting_for_activation_cid = null; @@ -367,9 +365,11 @@ class Parameters : Jingle.TransportParameters, Object { this.local_full_jid = local_full_jid; this.peer_full_jid = peer_full_jid; } + public Parameters.create(Jid local_full_jid, Jid peer_full_jid, string sid) { this(Jingle.Role.INITIATOR, sid, local_full_jid, peer_full_jid, null); } + public static Parameters parse(Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { string? dstaddr = transport.get_attribute("dstaddr"); string? mode = transport.get_attribute("mode"); @@ -386,9 +386,11 @@ class Parameters : Jingle.TransportParameters, Object { } return result; } - public string transport_ns_uri() { - return NS_URI; + + public void set_content(Jingle.Content content) { + } + public StanzaNode to_transport_stanza_node() { StanzaNode transport = new StanzaNode.build("transport", NS_URI) .add_self_xmlns() @@ -405,7 +407,8 @@ class Parameters : Jingle.TransportParameters, Object { } return transport; } - public void on_transport_accept(StanzaNode transport) throws Jingle.IqError { + + public void handle_transport_accept(StanzaNode transport) throws Jingle.IqError { Parameters other = Parameters.parse(local_full_jid, peer_full_jid, transport); if (other.sid != sid) { throw new Jingle.IqError.BAD_REQUEST("invalid sid"); @@ -413,7 +416,8 @@ class Parameters : Jingle.TransportParameters, Object { remote_candidates = other.remote_candidates; remote_dstaddr = other.remote_dstaddr; } - public void on_transport_info(StanzaNode transport) throws Jingle.IqError { + + public void handle_transport_info(StanzaNode transport) throws Jingle.IqError { StanzaNode? candidate_error = transport.get_subnode("candidate-error", NS_URI); StanzaNode? candidate_used = transport.get_subnode("candidate-used", NS_URI); StanzaNode? activated = transport.get_subnode("activated", NS_URI); @@ -449,6 +453,7 @@ class Parameters : Jingle.TransportParameters, Object { handle_proxy_error(); } } + private void handle_remote_candidate(string? cid) throws Jingle.IqError { if (remote_sent_selected_candidate) { throw new Jingle.IqError.BAD_REQUEST("remote candidate already specified"); @@ -470,6 +475,7 @@ class Parameters : Jingle.TransportParameters, Object { debug("Remote selected candidate %s", candidate != null ? candidate.cid : "(null)"); try_completing_negotiation(); } + private void handle_activated(string cid) throws Jingle.IqError { if (waiting_for_activation_cid == null || cid != waiting_for_activation_cid) { throw new Jingle.IqError.BAD_REQUEST("unexpected proxy activation message"); @@ -477,6 +483,7 @@ class Parameters : Jingle.TransportParameters, Object { Idle.add((owned)waiting_for_activation_callback); waiting_for_activation_cid = null; } + private void handle_proxy_error() throws Jingle.IqError { if (waiting_for_activation_cid == null) { throw new Jingle.IqError.BAD_REQUEST("unexpected proxy error message"); @@ -486,6 +493,7 @@ class Parameters : Jingle.TransportParameters, Object { waiting_for_activation_error = true; } + private void try_completing_negotiation() { if (!remote_sent_selected_candidate || !local_determined_selected_candidate) { return; @@ -500,7 +508,7 @@ class Parameters : Jingle.TransportParameters, Object { if (num_candidates == 0) { // Notify Jingle of the failed transport. - session.set_transport_connection(hack, null); + content_set_transport_connection(null); return; } @@ -525,7 +533,7 @@ class Parameters : Jingle.TransportParameters, Object { if (strong == null) { return; } - strong.set_transport_connection(hack, local_selected_candidate_conn); + content_set_transport_connection(local_selected_candidate_conn); } else { wait_for_remote_activation.begin(local_selected_candidate, local_selected_candidate_conn); } @@ -538,15 +546,16 @@ class Parameters : Jingle.TransportParameters, Object { SocketConnection? conn = listener.get_connection(remote_selected_candidate.cid); if (conn == null) { // Remote hasn't actually connected to us?! - strong.set_transport_connection(hack, null); + content_set_transport_connection(null); return; } - strong.set_transport_connection(hack, conn); + content_set_transport_connection(conn); } else { connect_to_local_candidate.begin(remote_selected_candidate); } } } + public async void wait_for_remote_activation(Candidate candidate, SocketConnection conn) { debug("Waiting for remote activation of %s", candidate.cid); waiting_for_activation_cid = candidate.cid; @@ -558,11 +567,12 @@ class Parameters : Jingle.TransportParameters, Object { return; } if (!waiting_for_activation_error) { - strong.set_transport_connection(hack, conn); + content_set_transport_connection(conn); } else { - strong.set_transport_connection(hack, null); + content_set_transport_connection(null); } } + public async void connect_to_local_candidate(Candidate candidate) { debug("Connecting to candidate %s", candidate.cid); try { @@ -587,11 +597,11 @@ class Parameters : Jingle.TransportParameters, Object { throw new IOError.PROXY_FAILED("activation iq error"); } - Jingle.Session? strong = session; - if (strong == null) { + Jingle.Content? strong_content = content; + if (strong_content == null) { return; } - strong.send_transport_info(hack, new StanzaNode.build("transport", NS_URI) + strong_content.send_transport_info(new StanzaNode.build("transport", NS_URI) .add_self_xmlns() .put_attribute("sid", sid) .put_node(new StanzaNode.build("activated", NS_URI) @@ -599,20 +609,21 @@ class Parameters : Jingle.TransportParameters, Object { ) ); - strong.set_transport_connection(hack, conn); + content_set_transport_connection(conn); } catch (Error e) { - Jingle.Session? strong = session; - if (strong == null) { + Jingle.Content? strong_content = content; + if (strong_content == null) { return; } - strong.send_transport_info(hack, new StanzaNode.build("transport", NS_URI) + strong_content.send_transport_info(new StanzaNode.build("transport", NS_URI) .add_self_xmlns() .put_attribute("sid", sid) .put_node(new StanzaNode.build("proxy-error", NS_URI)) ); - strong.set_transport_connection(hack, null); + content_set_transport_connection(null); } } + public async SocketConnection connect_to_socks5(Candidate candidate, string dstaddr) throws Error { SocketClient socket_client = new SocketClient() { timeout=NEGOTIATION_TIMEOUT }; @@ -685,6 +696,7 @@ class Parameters : Jingle.TransportParameters, Object { return conn; } + public async void try_connecting_to_candidates(XmppStream stream, Jingle.Session session) throws Error { remote_candidates.sort((c1, c2) => { // sort from priorities from high to low @@ -705,7 +717,7 @@ class Parameters : Jingle.TransportParameters, Object { local_selected_candidate = candidate; local_selected_candidate_conn = conn; debug("Selected candidate %s", candidate.cid); - session.send_transport_info(stream, new StanzaNode.build("transport", NS_URI) + content.send_transport_info(new StanzaNode.build("transport", NS_URI) .add_self_xmlns() .put_attribute("sid", sid) .put_node(new StanzaNode.build("candidate-used", NS_URI) @@ -722,7 +734,7 @@ class Parameters : Jingle.TransportParameters, Object { } local_determined_selected_candidate = true; local_selected_candidate = null; - session.send_transport_info(stream, new StanzaNode.build("transport", NS_URI) + content.send_transport_info(new StanzaNode.build("transport", NS_URI) .add_self_xmlns() .put_attribute("sid", sid) .put_node(new StanzaNode.build("candidate-error", NS_URI)) @@ -730,10 +742,26 @@ class Parameters : Jingle.TransportParameters, Object { // Try remote candidates try_completing_negotiation(); } - public void create_transport_connection(XmppStream stream, Jingle.Session session) { - this.session = session; + + private Jingle.StreamingConnection connection = new Jingle.StreamingConnection(); + + private void content_set_transport_connection(IOStream? ios) { + IOStream? iostream = ios; + Jingle.Content? strong_content = content; + if (strong_content == null) return; + + if (strong_content.security_params != null) { + iostream = strong_content.security_params.wrap_stream(iostream); + } + connection.init.begin(iostream); + } + + public void create_transport_connection(XmppStream stream, Jingle.Content content) { + this.session = content.session; + this.content = content; this.hack = stream; try_connecting_to_candidates.begin(stream, session); + this.content.set_transport_connection(connection, 1); } } diff --git a/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala b/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala index e26d63b7..5bb71831 100644 --- a/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala +++ b/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala @@ -21,41 +21,41 @@ public class Module : Jingle.Transport, XmppStreamModule { public override string get_ns() { return NS_URI; } public override string get_id() { return IDENTITY.id; } - public async bool is_transport_available(XmppStream stream, Jid full_jid) { - return yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI); + public async bool is_transport_available(XmppStream stream, uint8 components, Jid full_jid) { + return components == 1 && yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI); } - public string transport_ns_uri() { - return NS_URI; - } - public Jingle.TransportType transport_type() { - return Jingle.TransportType.STREAMING; - } - public int transport_priority() { - return 0; - } - public Jingle.TransportParameters create_transport_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid) { + public string ns_uri { get { return NS_URI; } } + public Jingle.TransportType type_ { get { return Jingle.TransportType.STREAMING; } } + public int priority { get { return 0; } } + public Jingle.TransportParameters create_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid) { + assert(components == 1); return new Parameters.create(peer_full_jid, random_uuid()); } - public Jingle.TransportParameters parse_transport_parameters(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { + public Jingle.TransportParameters parse_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { return Parameters.parse(peer_full_jid, transport); } } class Parameters : Jingle.TransportParameters, Object { + public string ns_uri { get { return NS_URI; } } + public uint8 components { get { return 1; } } public Jingle.Role role { get; private set; } public Jid peer_full_jid { get; private set; } public string sid { get; private set; } public int block_size { get; private set; } + private Parameters(Jingle.Role role, Jid peer_full_jid, string sid, int block_size) { this.role = role; this.peer_full_jid = peer_full_jid; this.sid = sid; this.block_size = block_size; } + public Parameters.create(Jid peer_full_jid, string sid) { this(Jingle.Role.INITIATOR, peer_full_jid, sid, DEFAULT_BLOCKSIZE); } + public static Parameters parse(Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { string? sid = transport.get_attribute("sid"); int block_size = transport.get_attribute_int("block-size"); @@ -64,27 +64,43 @@ class Parameters : Jingle.TransportParameters, Object { } return new Parameters(Jingle.Role.RESPONDER, peer_full_jid, sid, block_size); } + public string transport_ns_uri() { return NS_URI; } + + public void set_content(Jingle.Content content) { + + } + public StanzaNode to_transport_stanza_node() { return new StanzaNode.build("transport", NS_URI) .add_self_xmlns() .put_attribute("block-size", block_size.to_string()) .put_attribute("sid", sid); } - public void on_transport_accept(StanzaNode transport) throws Jingle.IqError { + + public void handle_transport_accept(StanzaNode transport) throws Jingle.IqError { Parameters other = Parameters.parse(peer_full_jid, transport); if (other.sid != sid || other.block_size > block_size) { throw new Jingle.IqError.NOT_ACCEPTABLE("invalid IBB sid or block_size"); } block_size = other.block_size; } - public void on_transport_info(StanzaNode transport) throws Jingle.IqError { + + public void handle_transport_info(StanzaNode transport) throws Jingle.IqError { throw new Jingle.IqError.UNSUPPORTED_INFO("transport-info not supported for IBBs"); } - public void create_transport_connection(XmppStream stream, Jingle.Session session) { - session.set_transport_connection(stream, InBandBytestreams.Connection.create(stream, peer_full_jid, sid, block_size, role == Jingle.Role.INITIATOR)); + + public void create_transport_connection(XmppStream stream, Jingle.Content content) { + IOStream iostream = InBandBytestreams.Connection.create(stream, peer_full_jid, sid, block_size, role == Jingle.Role.INITIATOR); + Jingle.StreamingConnection connection = new Jingle.StreamingConnection(); + if (content.security_params != null) { + iostream = content.security_params.wrap_stream(iostream); + } + connection.init.begin(iostream); + debug("set transport conn ibb"); + content.set_transport_connection(connection, 1); } } From 5bd719a919eea613bd800bfca994156a4e9bb968 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 21 Mar 2021 12:41:06 +0100 Subject: [PATCH 03/56] Add ICE-UDP Jingle transport (XEP-0176) to xmpp-vala Co-authored-by: fiaxh --- .../xep/0176_jingle_ice_udp/candidate.vala | 93 ++++++++++++++ .../jingle_ice_udp_module.vala | 36 ++++++ .../transport_parameters.vala | 114 ++++++++++++++++++ 3 files changed, 243 insertions(+) create mode 100644 xmpp-vala/src/module/xep/0176_jingle_ice_udp/candidate.vala create mode 100644 xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala create mode 100644 xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/candidate.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/candidate.vala new file mode 100644 index 00000000..a2988d90 --- /dev/null +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/candidate.vala @@ -0,0 +1,93 @@ +using Gee; +using Xmpp.Xep; +using Xmpp; + +public class Xmpp.Xep.JingleIceUdp.Candidate { + public uint8 component; + public uint8 foundation; + public uint8 generation; + public string id; + public string ip; + public uint8 network; + public uint16 port; + public uint32 priority; + public string protocol; + public string? rel_addr; + public uint16 rel_port; + public Type type_; + + public static Candidate parse(StanzaNode node) throws Jingle.IqError { + Candidate candidate = new Candidate(); + candidate.component = (uint8) node.get_attribute_uint("component"); + candidate.foundation = (uint8) node.get_attribute_uint("foundation"); + candidate.generation = (uint8) node.get_attribute_uint("generation"); + candidate.id = node.get_attribute("id"); + candidate.ip = node.get_attribute("ip"); + candidate.network = (uint8) node.get_attribute_uint("network"); + candidate.port = (uint16) node.get_attribute_uint("port"); + candidate.priority = (uint32) node.get_attribute_uint("priority"); + candidate.protocol = node.get_attribute("protocol"); + candidate.rel_addr = node.get_attribute("rel-addr"); + candidate.rel_port = (uint16) node.get_attribute_uint("rel-port"); + candidate.type_ = Type.parse(node.get_attribute("type")); + return candidate; + } + + public enum Type { + HOST, PRFLX, RELAY, SRFLX; + public static Type parse(string str) throws Jingle.IqError { + switch (str) { + case "host": return HOST; + case "prflx": return PRFLX; + case "relay": return RELAY; + case "srflx": return SRFLX; + default: throw new Jingle.IqError.BAD_REQUEST("Illegal ICE-UDP candidate type"); + } + } + public string to_string() { + switch (this) { + case HOST: return "host"; + case PRFLX: return "prflx"; + case RELAY: return "relay"; + case SRFLX: return "srflx"; + default: assert_not_reached(); + } + } + } + + public StanzaNode to_xml() { + StanzaNode node = new StanzaNode.build("candidate", NS_URI) + .put_attribute("component", component.to_string()) + .put_attribute("foundation", foundation.to_string()) + .put_attribute("generation", generation.to_string()) + .put_attribute("id", id) + .put_attribute("ip", ip) + .put_attribute("network", network.to_string()) + .put_attribute("port", port.to_string()) + .put_attribute("priority", priority.to_string()) + .put_attribute("protocol", protocol) + .put_attribute("type", type_.to_string()); + if (rel_addr != null) node.put_attribute("rel-addr", rel_addr); + if (rel_port != 0) node.put_attribute("rel-port", rel_port.to_string()); + return node; + } + + public bool equals(Candidate c) { + return equals_func(this, c); + } + + public static bool equals_func(Candidate c1, Candidate c2) { + return c1.component == c2.component && + c1.foundation == c2.foundation && + c1.generation == c2.generation && + c1.id == c2.id && + c1.ip == c2.ip && + c1.network == c2.network && + c1.port == c2.port && + c1.priority == c2.priority && + c1.protocol == c2.protocol && + c1.rel_addr == c2.rel_addr && + c1.rel_port == c2.rel_port && + c1.type_ == c2.type_; + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala new file mode 100644 index 00000000..9ed494ff --- /dev/null +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala @@ -0,0 +1,36 @@ +using Gee; +using Xmpp.Xep; +using Xmpp; + +namespace Xmpp.Xep.JingleIceUdp { + +private const string NS_URI = "urn:xmpp:jingle:transports:ice-udp:1"; + +public abstract class Module : XmppStreamModule, Jingle.Transport { + public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "0176_jingle_ice_udp"); + + public override void attach(XmppStream stream) { + stream.get_module(Jingle.Module.IDENTITY).register_transport(this); + stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); + } + public override void detach(XmppStream stream) { + stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI); + } + + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } + + public async bool is_transport_available(XmppStream stream, uint8 components, Jid full_jid) { + return yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI); + } + + public string ns_uri{ get { return NS_URI; } } + public Jingle.TransportType type_{ get { return Jingle.TransportType.DATAGRAM; } } + public int priority { get { return 1; } } + + public abstract Jingle.TransportParameters create_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid); + + public abstract Jingle.TransportParameters parse_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError; +} + +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala new file mode 100644 index 00000000..8b8aa07d --- /dev/null +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala @@ -0,0 +1,114 @@ +using Gee; +using Xmpp.Xep; +using Xmpp; + +public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.TransportParameters, Object { + public string ns_uri { get { return NS_URI; } } + public string remote_pwd { get; private set; } + public string remote_ufrag { get; private set; } + public string local_pwd { get; private set; } + public string local_ufrag { get; private set; } + + public ConcurrentList local_candidates = new ConcurrentList(Candidate.equals_func); + public ConcurrentList unsent_local_candidates = new ConcurrentList(Candidate.equals_func); + public Gee.List remote_candidates = new ArrayList(Candidate.equals_func); + + public Jid local_full_jid { get; private set; } + public Jid peer_full_jid { get; private set; } + private uint8 components_; + public uint8 components { get { return components_; } } + + public bool incoming { get; private set; default = false; } + private bool connection_created = false; + + private weak Jingle.Content? content = null; + + protected IceUdpTransportParameters(uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode? node = null) { + this.components_ = components; + this.local_full_jid = local_full_jid; + this.peer_full_jid = peer_full_jid; + if (node != null) { + incoming = true; + remote_pwd = node.get_attribute("pwd"); + remote_ufrag = node.get_attribute("ufrag"); + foreach (StanzaNode candidateNode in node.get_subnodes("candidate")) { + remote_candidates.add(Candidate.parse(candidateNode)); + } + } + } + + public void init(string ufrag, string pwd) { + this.local_ufrag = ufrag; + this.local_pwd = pwd; + debug("Initialized for %s", pwd); + } + + public void set_content(Jingle.Content content) { + this.content = content; + this.content.weak_ref(unset_content); + } + + public void unset_content() { + this.content = null; + } + + public StanzaNode to_transport_stanza_node() { + var node = new StanzaNode.build("transport", NS_URI) + .add_self_xmlns() + .put_attribute("ufrag", local_ufrag) + .put_attribute("pwd", local_pwd); + foreach (Candidate candidate in unsent_local_candidates) { + node.put_node(candidate.to_xml()); + } + unsent_local_candidates.clear(); + return node; + } + + public virtual void handle_transport_accept(StanzaNode node) throws Jingle.IqError { + string? pwd = node.get_attribute("pwd"); + string? ufrag = node.get_attribute("ufrag"); + if (pwd != null) remote_pwd = pwd; + if (ufrag != null) remote_ufrag = ufrag; + foreach (StanzaNode candidateNode in node.get_subnodes("candidate")) { + remote_candidates.add(Candidate.parse(candidateNode)); + } + } + + public virtual void handle_transport_info(StanzaNode node) throws Jingle.IqError { + string? pwd = node.get_attribute("pwd"); + string? ufrag = node.get_attribute("ufrag"); + if (pwd != null) remote_pwd = pwd; + if (ufrag != null) remote_ufrag = ufrag; + uint8 components = 0; + foreach (StanzaNode candidateNode in node.get_subnodes("candidate")) { + remote_candidates.add(Candidate.parse(candidateNode)); + } + } + + public virtual void create_transport_connection(XmppStream stream, Jingle.Content content) { + connection_created = true; + + check_send_transport_info(); + } + + public void add_local_candidate_threadsafe(Candidate candidate) { + if (local_candidates.contains(candidate)) return; + + debug("New local candidate %u %s %s:%u", candidate.component, candidate.type_.to_string(), candidate.ip, candidate.port); + unsent_local_candidates.add(candidate); + local_candidates.add(candidate); + + if (this.content != null && (this.connection_created || !this.incoming)) { + Timeout.add(50, () => { + check_send_transport_info(); + return false; + }); + } + } + + private void check_send_transport_info() { + if (this.content != null && unsent_local_candidates.size > 0) { + content.send_transport_info(to_transport_stanza_node()); + } + } +} \ No newline at end of file From f328bf93fbdd8e99a2fa27757a07223473fff4a5 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Sat, 20 Mar 2021 12:41:59 +0100 Subject: [PATCH 04/56] Add External Service Discovery (XEP-0215) support --- .../xep/0215_external_service_discovery.vala | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 xmpp-vala/src/module/xep/0215_external_service_discovery.vala diff --git a/xmpp-vala/src/module/xep/0215_external_service_discovery.vala b/xmpp-vala/src/module/xep/0215_external_service_discovery.vala new file mode 100644 index 00000000..07c3f71c --- /dev/null +++ b/xmpp-vala/src/module/xep/0215_external_service_discovery.vala @@ -0,0 +1,49 @@ +using Gee; + +namespace Xmpp.Xep.ExternalServiceDiscovery { + + private const string NS_URI = "urn:xmpp:extdisco:2"; + + public static async Gee.List request_services(XmppStream stream) { + Iq.Stanza request_iq = new Iq.Stanza.get((new StanzaNode.build("services", NS_URI)).add_self_xmlns()) { to=stream.remote_name }; + Iq.Stanza response_iq = yield stream.get_module(Iq.Module.IDENTITY).send_iq_async(stream, request_iq); + + ArrayList ret = new ArrayList(); + if (response_iq.is_error()) return ret; + StanzaNode? services_node = response_iq.stanza.get_subnode("services", NS_URI); + if (services_node == null) return ret; + + Gee.List service_nodes = services_node.get_subnodes("service", NS_URI); + foreach (StanzaNode service_node in service_nodes) { + Service service = new Service(); + service.host = service_node.get_attribute("host", NS_URI); + string? port_str = service_node.get_attribute("port", NS_URI); + if (port_str != null) service.port = int.parse(port_str); + service.ty = service_node.get_attribute("type", NS_URI); + + if (service.host == null || service.ty == null || port_str == null) continue; + + service.username = service_node.get_attribute("username", NS_URI); + service.password = service_node.get_attribute("password", NS_URI); + service.transport = service_node.get_attribute("transport", NS_URI); + service.name = service_node.get_attribute("name", NS_URI); + string? restricted_str = service_node.get_attribute("restricted", NS_URI); + if (restricted_str != null) service.restricted = bool.parse(restricted_str); + ret.add(service); + } + return ret; + } + + public class Service { + public string host { get; set; } + public uint port { get; set; } + public string ty { get; set; } + + public string username { get; set; } + public string password { get; set; } + + public string transport { get; set; } + public string name { get; set; } + public bool restricted { get; set; } + } +} From d703b7c09d5eea81ec383fd09c9d320199e6d577 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 21 Mar 2021 12:41:27 +0100 Subject: [PATCH 05/56] Add libnice-based ICE-UDP implementation as plugin --- plugins/ice/CMakeLists.txt | 5 +- plugins/ice/src/module.vala | 42 ++++ plugins/ice/src/plugin.vala | 65 ++++- plugins/ice/src/transport_parameters.vala | 261 ++++++++++++++++++++ plugins/ice/src/util.vala | 18 ++ plugins/ice/vapi/metadata/Nice-0.1.metadata | 4 + plugins/ice/vapi/nice.vapi | 9 +- 7 files changed, 387 insertions(+), 17 deletions(-) create mode 100644 plugins/ice/src/module.vala create mode 100644 plugins/ice/src/transport_parameters.vala create mode 100644 plugins/ice/src/util.vala diff --git a/plugins/ice/CMakeLists.txt b/plugins/ice/CMakeLists.txt index 76dba28f..90fe5b7d 100644 --- a/plugins/ice/CMakeLists.txt +++ b/plugins/ice/CMakeLists.txt @@ -10,6 +10,9 @@ find_packages(ICE_PACKAGES REQUIRED vala_precompile(ICE_VALA_C SOURCES src/plugin.vala + src/module.vala + src/transport_parameters.vala + src/util.vala src/register_plugin.vala CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi @@ -21,7 +24,7 @@ OPTIONS --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi ) -add_definitions(${VALA_CFLAGS}) +add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="ice") add_library(ice SHARED ${ICE_VALA_C}) target_link_libraries(ice libdino ${ICE_PACKAGES}) set_target_properties(ice PROPERTIES PREFIX "") diff --git a/plugins/ice/src/module.vala b/plugins/ice/src/module.vala new file mode 100644 index 00000000..e961ffb6 --- /dev/null +++ b/plugins/ice/src/module.vala @@ -0,0 +1,42 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Ice.Module : JingleIceUdp.Module { + + public string? stun_ip = null; + public uint stun_port = 0; + public string? turn_ip = null; + public Xep.ExternalServiceDiscovery.Service? turn_service = null; + + private weak Nice.Agent? agent; + + private Nice.Agent get_agent() { + Nice.Agent? agent = this.agent; + if (agent == null) { + agent = new Nice.Agent(MainContext.@default(), Nice.Compatibility.RFC5245); + if (stun_ip != null) { + agent.stun_server = stun_ip; + agent.stun_server_port = stun_port; + } + agent.ice_tcp = false; + agent.set_software("Dino"); + agent.weak_ref(agent_unweak); + this.agent = agent; + debug("STUN server for libnice %s %u", agent.stun_server, agent.stun_server_port); + } + return agent; + } + + public override Jingle.TransportParameters create_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid) { + return new TransportParameters(get_agent(), turn_service, turn_ip, components, local_full_jid, peer_full_jid); + } + + public override Jingle.TransportParameters parse_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { + return new TransportParameters(get_agent(), turn_service, turn_ip, components, local_full_jid, peer_full_jid, transport); + } + + private void agent_unweak() { + this.agent = null; + } +} \ No newline at end of file diff --git a/plugins/ice/src/plugin.vala b/plugins/ice/src/plugin.vala index f1c41a27..3ee8a72a 100644 --- a/plugins/ice/src/plugin.vala +++ b/plugins/ice/src/plugin.vala @@ -1,30 +1,71 @@ using Gee; -using Nice; +using Dino.Entities; using Xmpp; +using Xmpp.Xep; -namespace Dino.Plugins.Ice { +private extern const size_t NICE_ADDRESS_STRING_LEN; -public class Plugin : RootInterface, Object { +public class Dino.Plugins.Ice.Plugin : RootInterface, Object { public Dino.Application app; public void registered(Dino.Application app) { + Nice.debug_enable(true); this.app = app; - app.stream_interactor.stream_attached_modules.connect((account, stream) => { - stream.get_module(Xmpp.Xep.Socks5Bytestreams.Module.IDENTITY).set_local_ip_address_handler(get_local_ip_addresses); + app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => { + list.add(new Module()); }); + app.stream_interactor.stream_attached_modules.connect((account, stream) => { + stream.get_module(Socks5Bytestreams.Module.IDENTITY).set_local_ip_address_handler(get_local_ip_addresses); + }); + app.stream_interactor.stream_negotiated.connect(on_stream_negotiated); } - private Gee.List get_local_ip_addresses() { - Gee.List result = new ArrayList(); - foreach (string ip_address in Nice.interfaces_get_local_ips(false)) { - result.add(ip_address); + private async void on_stream_negotiated(Account account, XmppStream stream) { + Module? ice_udp_module = stream.get_module(JingleIceUdp.Module.IDENTITY) as Module; + if (ice_udp_module == null) return; + Gee.List services = yield ExternalServiceDiscovery.request_services(stream); + foreach (Xep.ExternalServiceDiscovery.Service service in services) { + if (service.transport == "udp" && (service.ty == "stun" || service.ty == "turn")) { + InetAddress ip = yield lookup_ipv4_addess(service.host); + if (ip == null) continue; + + if (service.ty == "stun") { + debug("Server offers STUN server: %s:%u, resolved to %s", service.host, service.port, ip.to_string()); + ice_udp_module.stun_ip = ip.to_string(); + ice_udp_module.stun_port = service.port; + } else if (service.ty == "turn") { + debug("Server offers TURN server: %s:%u, resolved to %s", service.host, service.port, ip.to_string()); + ice_udp_module.turn_ip = ip.to_string(); + ice_udp_module.turn_service = service; + } + } + } + if (ice_udp_module.stun_ip == null) { + InetAddress ip = yield lookup_ipv4_addess("stun.l.google.com"); + if (ip == null) return; + + debug("Using fallback STUN server: stun.l.google.com:19302, resolved to %s", ip.to_string()); + + ice_udp_module.stun_ip = ip.to_string(); + ice_udp_module.stun_port = 19302; } - return result; } public void shutdown() { // Nothing to do } -} -} + private async InetAddress? lookup_ipv4_addess(string host) { + try { + Resolver resolver = Resolver.get_default(); + GLib.List? ips = yield resolver.lookup_by_name_async(host); + foreach (GLib.InetAddress ina in ips) { + if (ina.get_family() != SocketFamily.IPV4) continue; + return ina; + } + } catch (Error e) { + warning("Failed looking up IP address of %s", host); + } + return null; + } +} \ No newline at end of file diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala new file mode 100644 index 00000000..acb44852 --- /dev/null +++ b/plugins/ice/src/transport_parameters.vala @@ -0,0 +1,261 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + + +public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransportParameters { + private Nice.Agent agent; + private uint stream_id; + private bool we_want_connection; + private bool remote_credentials_set; + private Map connections = new HashMap(); + + private class DatagramConnection : Jingle.DatagramConnection { + private Nice.Agent agent; + private uint stream_id; + private string? error; + private ulong sent; + private ulong sent_reported; + private ulong recv; + private ulong recv_reported; + private ulong datagram_received_id; + + public DatagramConnection(Nice.Agent agent, uint stream_id, uint8 component_id) { + this.agent = agent; + this.stream_id = stream_id; + this.component_id = component_id; + this.datagram_received_id = this.datagram_received.connect((datagram) => { + recv += datagram.length; + if (recv > recv_reported + 100000) { + debug("Received %lu bytes via stream %u component %u", recv, stream_id, component_id); + recv_reported = recv; + } + }); + } + + public override async void terminate(bool we_terminated, string? reason_string = null, string? reason_text = null) { + yield base.terminate(we_terminated, reason_string, reason_text); + this.disconnect(datagram_received_id); + agent = null; + } + + public override void send_datagram(Bytes datagram) { + if (this.agent != null && is_component_ready(agent, stream_id, component_id)) { + agent.send(stream_id, component_id, datagram.get_data()); + sent += datagram.length; + if (sent > sent_reported + 100000) { + debug("Sent %lu bytes via stream %u component %u", sent, stream_id, component_id); + sent_reported = sent; + } + } + } + } + + public TransportParameters(Nice.Agent agent, Xep.ExternalServiceDiscovery.Service? turn_service, string? turn_ip, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode? node = null) { + base(components, local_full_jid, peer_full_jid, node); + this.we_want_connection = (node == null); + this.agent = agent; + agent.candidate_gathering_done.connect(on_candidate_gathering_done); + agent.initial_binding_request_received.connect(on_initial_binding_request_received); + agent.component_state_changed.connect(on_component_state_changed); + agent.new_selected_pair_full.connect(on_new_selected_pair_full); + agent.new_candidate_full.connect(on_new_candidate); + + agent.controlling_mode = !incoming; + stream_id = agent.add_stream(components); + + if (turn_ip != null) { + for (uint8 component_id = 1; component_id <= components; component_id++) { + agent.set_relay_info(stream_id, component_id, turn_ip, turn_service.port, turn_service.username, turn_service.password, Nice.RelayType.UDP); + debug("TURN info (component %i) %s:%u", component_id, turn_ip, turn_service.port); + } + } + string ufrag; + string pwd; + agent.get_local_credentials(stream_id, out ufrag, out pwd); + init(ufrag, pwd); + + for (uint8 component_id = 1; component_id <= components; component_id++) { + // We don't properly get local candidates before this call + agent.attach_recv(stream_id, component_id, MainContext.@default(), on_recv); + } + + agent.gather_candidates(stream_id); + } + + private void on_candidate_gathering_done(uint stream_id) { + if (stream_id != this.stream_id) return; + debug("on_candidate_gathering_done in %u", stream_id); + + for (uint8 i = 1; i <= components; i++) { + foreach (unowned Nice.Candidate nc in agent.get_local_candidates(stream_id, i)) { + if (nc.transport == Nice.CandidateTransport.UDP) { + JingleIceUdp.Candidate? candidate = candidate_to_jingle(nc); + if (candidate == null) continue; + debug("Local candidate summary: %s", agent.generate_local_candidate_sdp(nc)); + } + } + } + } + + private void on_new_candidate(Nice.Candidate nc) { + if (nc.stream_id != stream_id) return; + JingleIceUdp.Candidate? candidate = candidate_to_jingle(nc); + if (candidate == null) return; + + if (nc.transport == Nice.CandidateTransport.UDP) { + // Execution was in the agent thread before + add_local_candidate_threadsafe(candidate); + } + } + + public override void handle_transport_accept(StanzaNode transport) throws Jingle.IqError { + debug("on_transport_accept from %s", peer_full_jid.to_string()); + base.handle_transport_accept(transport); + } + + public override void handle_transport_info(StanzaNode transport) throws Jingle.IqError { + debug("on_transport_info from %s", peer_full_jid.to_string()); + base.handle_transport_info(transport); + + if (!we_want_connection) return; + + if (remote_ufrag != null && remote_pwd != null && !remote_credentials_set) { + agent.set_remote_credentials(stream_id, remote_ufrag, remote_pwd); + remote_credentials_set = true; + } + for (uint8 i = 1; i <= components; i++) { + SList candidates = new SList(); + foreach (JingleIceUdp.Candidate candidate in remote_candidates) { + if (candidate.component == i) { + Nice.Candidate nc = candidate_to_nice(candidate); + candidates.append(nc); + } + } + int new_candidates = agent.set_remote_candidates(stream_id, i, candidates); + debug("Updated to %i remote candidates for candidate %u via transport info", new_candidates, i); + } + } + + public override void create_transport_connection(XmppStream stream, Jingle.Content content) { + debug("create_transport_connection: %s", content.session.sid); + debug("local_credentials: %s %s", local_ufrag, local_pwd); + debug("remote_credentials: %s %s", remote_ufrag, remote_pwd); + debug("expected incoming credentials: %s %s", local_ufrag + ":" + remote_ufrag, local_pwd); + debug("expected outgoing credentials: %s %s", remote_ufrag + ":" + local_ufrag, remote_pwd); + + we_want_connection = true; + + if (remote_ufrag != null && remote_pwd != null && !remote_credentials_set) { + agent.set_remote_credentials(stream_id, remote_ufrag, remote_pwd); + remote_credentials_set = true; + } + for (uint8 i = 1; i <= components; i++) { + SList candidates = new SList(); + foreach (JingleIceUdp.Candidate candidate in remote_candidates) { + if (candidate.ip.has_prefix("fe80::")) continue; + if (candidate.component == i) { + Nice.Candidate nc = candidate_to_nice(candidate); + candidates.append(nc); + debug("remote candidate: %s", agent.generate_local_candidate_sdp(nc)); + } + } + int new_candidates = agent.set_remote_candidates(stream_id, i, candidates); + debug("Initiated component %u with %i remote candidates", i, new_candidates); + + connections[i] = new DatagramConnection(agent, stream_id, i); + content.set_transport_connection(connections[i], i); + } + base.create_transport_connection(stream, content); + } + + private void on_component_state_changed(uint stream_id, uint component_id, uint state) { + if (stream_id != this.stream_id) return; + debug("stream %u component %u state changed to %s", stream_id, component_id, agent.get_component_state(stream_id, component_id).to_string()); + if (is_component_ready(agent, stream_id, component_id) && connections.has_key((uint8) component_id) && !connections[(uint8)component_id].ready) { + connections[(uint8)component_id].ready = true; + } + } + + private void on_initial_binding_request_received(uint stream_id) { + if (stream_id != this.stream_id) return; + debug("initial_binding_request_received"); + } + + private void on_new_selected_pair_full(uint stream_id, uint component_id, Nice.Candidate p1, Nice.Candidate p2) { + if (stream_id != this.stream_id) return; + debug("new_selected_pair_full %u [%s, %s]", component_id, agent.generate_local_candidate_sdp(p1), agent.generate_local_candidate_sdp(p2)); + } + + private void on_recv(Nice.Agent agent, uint stream_id, uint component_id, uint8[] data) { + if (stream_id != this.stream_id) return; + if (is_component_ready(agent, stream_id, component_id) && connections.has_key((uint8) component_id)) { + connections[(uint8) component_id].datagram_received(new Bytes(data)); + } else { + debug("on_recv stream %u component %u length %u", stream_id, component_id, data.length); + } + } + + private static Nice.Candidate candidate_to_nice(JingleIceUdp.Candidate c) { + Nice.CandidateType type; + switch (c.type_) { + case JingleIceUdp.Candidate.Type.HOST: type = Nice.CandidateType.HOST; break; + case JingleIceUdp.Candidate.Type.PRFLX: type = Nice.CandidateType.PEER_REFLEXIVE; break; + case JingleIceUdp.Candidate.Type.RELAY: type = Nice.CandidateType.RELAYED; break; + case JingleIceUdp.Candidate.Type.SRFLX: type = Nice.CandidateType.SERVER_REFLEXIVE; break; + default: assert_not_reached(); + } + + Nice.Candidate candidate = new Nice.Candidate(type); + candidate.component_id = c.component; + char[] foundation = new char[Nice.CANDIDATE_MAX_FOUNDATION]; + string foundation_str = c.foundation.to_string(); + Memory.copy(foundation, foundation_str.data, foundation_str.length); + candidate.foundation = foundation; + candidate.addr = Nice.Address(); + candidate.addr.init(); + candidate.addr.set_from_string(c.ip); + candidate.addr.set_port(c.port); + candidate.priority = c.priority; + if (c.rel_addr != null) { + candidate.base_addr = Nice.Address(); + candidate.base_addr.init(); + candidate.base_addr.set_from_string(c.rel_addr); + candidate.base_addr.set_port(c.rel_port); + } + candidate.transport = Nice.CandidateTransport.UDP; + return candidate; + } + + private static JingleIceUdp.Candidate? candidate_to_jingle(Nice.Candidate nc) { + JingleIceUdp.Candidate candidate = new JingleIceUdp.Candidate(); + switch (nc.type) { + case Nice.CandidateType.HOST: candidate.type_ = JingleIceUdp.Candidate.Type.HOST; break; + case Nice.CandidateType.PEER_REFLEXIVE: candidate.type_ = JingleIceUdp.Candidate.Type.PRFLX; break; + case Nice.CandidateType.RELAYED: candidate.type_ = JingleIceUdp.Candidate.Type.RELAY; break; + case Nice.CandidateType.SERVER_REFLEXIVE: candidate.type_ = JingleIceUdp.Candidate.Type.SRFLX; break; + default: assert_not_reached(); + } + candidate.component = (uint8) nc.component_id; + candidate.foundation = (uint8) int.parse((string)nc.foundation); + candidate.generation = 0; + candidate.id = Random.next_int().to_string("%08x"); // TODO + + char[] res = new char[NICE_ADDRESS_STRING_LEN]; + nc.addr.to_string(res); + candidate.ip = (string) res; + candidate.network = 0; // TODO + candidate.port = (uint16) nc.addr.get_port(); + candidate.priority = nc.priority; + candidate.protocol = "udp"; + if (nc.base_addr.is_valid() && !nc.base_addr.equal(nc.addr)) { + res = new char[NICE_ADDRESS_STRING_LEN]; + nc.base_addr.to_string(res); + candidate.rel_addr = (string) res; + candidate.rel_port = (uint16) nc.base_addr.get_port(); + } + if (candidate.ip.has_prefix("fe80::")) return null; + + return candidate; + } +} \ No newline at end of file diff --git a/plugins/ice/src/util.vala b/plugins/ice/src/util.vala new file mode 100644 index 00000000..dd89d2f4 --- /dev/null +++ b/plugins/ice/src/util.vala @@ -0,0 +1,18 @@ +using Gee; + +namespace Dino.Plugins.Ice { + +internal static bool is_component_ready(Nice.Agent agent, uint stream_id, uint component_id) { + var state = agent.get_component_state(stream_id, component_id); + return state == Nice.ComponentState.CONNECTED || state == Nice.ComponentState.READY; +} + +internal Gee.List get_local_ip_addresses() { + Gee.List result = new ArrayList(); + foreach (string ip_address in Nice.interfaces_get_local_ips(false)) { + result.add(ip_address); + } + return result; +} + +} \ No newline at end of file diff --git a/plugins/ice/vapi/metadata/Nice-0.1.metadata b/plugins/ice/vapi/metadata/Nice-0.1.metadata index 437da816..d6899f87 100644 --- a/plugins/ice/vapi/metadata/Nice-0.1.metadata +++ b/plugins/ice/vapi/metadata/Nice-0.1.metadata @@ -1,4 +1,8 @@ Nice cheader_filename="nice.h" +Address.to_string.dst type="char[]" Agent.new_reliable#constructor name="create_reliable" +Agent.attach_recv skip=false +Agent.send.buf type="uint8[]" array_length_idx=2 +AgentRecvFunc.buf type="uint8[]" array_length_idx=3 PseudoTcpCallbacks#record skip PseudoTcpSocket#class skip diff --git a/plugins/ice/vapi/nice.vapi b/plugins/ice/vapi/nice.vapi index aa45cf08..39768b9b 100644 --- a/plugins/ice/vapi/nice.vapi +++ b/plugins/ice/vapi/nice.vapi @@ -8,6 +8,7 @@ namespace Nice { public Agent (GLib.MainContext ctx, Nice.Compatibility compat); public bool add_local_address (Nice.Address addr); public uint add_stream (uint n_components); + public bool attach_recv (uint stream_id, uint component_id, GLib.MainContext ctx, Nice.AgentRecvFunc func); [Version (since = "0.1.16")] public async void close_async (); [CCode (cname = "nice_agent_new_reliable", has_construct_function = false)] @@ -58,7 +59,7 @@ namespace Nice { public bool restart (); [Version (since = "0.1.6")] public bool restart_stream (uint stream_id); - public int send (uint stream_id, uint component_id, uint len, string buf); + public int send (uint stream_id, uint component_id, [CCode (array_length_cname = "len", array_length_pos = 2.5, array_length_type = "guint", type = "const gchar*")] uint8[] buf); [Version (since = "0.1.5")] public int send_messages_nonblocking (uint stream_id, uint component_id, [CCode (array_length_cname = "n_messages", array_length_pos = 3.5, array_length_type = "guint")] Nice.OutputMessage[] messages, GLib.Cancellable? cancellable = null) throws GLib.Error; public bool set_local_credentials (uint stream_id, string ufrag, string pwd); @@ -209,7 +210,7 @@ namespace Nice { public void set_ipv4 (uint32 addr_ipv4); public void set_ipv6 (uint8 addr_ipv6); public void set_port (uint port); - public void to_string (string dst); + public void to_string ([CCode (array_length = false, type = "gchar*")] char[] dst); } [CCode (cheader_filename = "nice.h", has_type_id = false)] [Version (since = "0.1.5")] @@ -343,8 +344,8 @@ namespace Nice { TCP, TLS } - [CCode (cheader_filename = "nice.h", instance_pos = 5.9)] - public delegate void AgentRecvFunc (Nice.Agent agent, uint stream_id, uint component_id, uint len, string buf); + [CCode (cheader_filename = "nice.h", instance_pos = 4.9)] + public delegate void AgentRecvFunc (Nice.Agent agent, uint stream_id, uint component_id, [CCode (array_length_cname = "len", array_length_pos = 3.5, array_length_type = "guint", type = "gchar*")] uint8[] buf); [CCode (cheader_filename = "nice.h", cname = "NICE_AGENT_MAX_REMOTE_CANDIDATES")] public const int AGENT_MAX_REMOTE_CANDIDATES; [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_DIRECTION_MS_PREF_ACTIVE")] From dfd79401044834b164c50f5948986719eabf8127 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 21 Mar 2021 12:41:32 +0100 Subject: [PATCH 06/56] Add support for Jingle RTP sessions (XEP-0167) to xmpp-vala Co-authored-by: fiaxh --- xmpp-vala/CMakeLists.txt | 2 + .../0167_jingle_rtp/content_parameters.vala | 142 ++++++++++++++ .../xep/0167_jingle_rtp/content_type.vala | 23 +++ .../0167_jingle_rtp/jingle_rtp_module.vala | 175 ++++++++++++++++++ .../xep/0167_jingle_rtp/payload_type.vala | 52 ++++++ .../0167_jingle_rtp/session_info_type.vala | 67 +++++++ .../module/xep/0167_jingle_rtp/stream.vala | 46 +++++ 7 files changed, 507 insertions(+) create mode 100644 xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala create mode 100644 xmpp-vala/src/module/xep/0167_jingle_rtp/content_type.vala create mode 100644 xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala create mode 100644 xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala create mode 100644 xmpp-vala/src/module/xep/0167_jingle_rtp/session_info_type.vala create mode 100644 xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala diff --git a/xmpp-vala/CMakeLists.txt b/xmpp-vala/CMakeLists.txt index 48f411c7..3aa10caf 100644 --- a/xmpp-vala/CMakeLists.txt +++ b/xmpp-vala/CMakeLists.txt @@ -114,6 +114,7 @@ SOURCES "src/module/xep/0198_stream_management.vala" "src/module/xep/0199_ping.vala" "src/module/xep/0203_delayed_delivery.vala" + "src/module/xep/0215_external_service_discovery.vala" "src/module/xep/0234_jingle_file_transfer.vala" "src/module/xep/0249_direct_muc_invitations.vala" "src/module/xep/0260_jingle_socks5_bytestreams.vala" @@ -123,6 +124,7 @@ SOURCES "src/module/xep/0313_message_archive_management.vala" "src/module/xep/0333_chat_markers.vala" "src/module/xep/0334_message_processing_hints.vala" + "src/module/xep/0353_jingle_message_initiation.vala" "src/module/xep/0359_unique_stable_stanza_ids.vala" "src/module/xep/0363_http_file_upload.vala" "src/module/xep/0380_explicit_encryption.vala" diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala new file mode 100644 index 00000000..8a3668b2 --- /dev/null +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -0,0 +1,142 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { + + public signal void stream_created(Stream stream); + public signal void connection_ready(); + + public string media { get; private set; } + public string? ssrc { get; private set; } + public bool rtcp_mux { get; private set; } + + public string? bandwidth { get; private set; } + public string? bandwidth_type { get; private set; } + + public bool encryption_required { get; private set; default = false; } + public PayloadType? agreed_payload_type { get; private set; } + public Gee.List payload_types = new ArrayList(PayloadType.equals_func); + public Gee.List cryptos = new ArrayList(); + + public weak Stream? stream { get; private set; } + + private Module parent; + + public Parameters(Module parent, + string media, Gee.List payload_types, + string? ssrc = null, bool rtcp_mux = false, + string? bandwidth = null, string? bandwidth_type = null, + bool encryption_required = false, Gee.List cryptos = new ArrayList() + ) { + this.parent = parent; + this.media = media; + this.ssrc = ssrc; + this.rtcp_mux = rtcp_mux; + this.bandwidth = bandwidth; + this.bandwidth_type = bandwidth_type; + this.encryption_required = encryption_required; + this.payload_types = payload_types; + this.cryptos = cryptos; + } + + public Parameters.from_node(Module parent, StanzaNode node) throws Jingle.IqError { + this.parent = parent; + this.media = node.get_attribute("media"); + this.ssrc = node.get_attribute("ssrc"); + this.rtcp_mux = node.get_subnode("rtcp-mux") != null; + StanzaNode? encryption = node.get_subnode("encryption"); + if (encryption != null) { + this.encryption_required = encryption.get_attribute_bool("required", this.encryption_required); + foreach (StanzaNode crypto in encryption.get_subnodes("crypto")) { + this.cryptos.add(Crypto.parse(crypto)); + } + } + foreach (StanzaNode payloadType in node.get_subnodes("payload-type")) { + this.payload_types.add(PayloadType.parse(payloadType)); + } + } + + public async void handle_proposed_content(XmppStream stream, Jingle.Session session, Jingle.Content content) { + agreed_payload_type = yield parent.pick_payload_type(media, payload_types); + if (agreed_payload_type == null) { + debug("no usable payload type"); + content.reject(); + return; + } + } + + public void accept(XmppStream stream, Jingle.Session session, Jingle.Content content) { + debug("[%p] Jingle RTP on_accept", stream); + + Jingle.DatagramConnection rtp_datagram = (Jingle.DatagramConnection) content.get_transport_connection(1); + Jingle.DatagramConnection rtcp_datagram = (Jingle.DatagramConnection) content.get_transport_connection(2); + + ulong rtcp_ready_handler_id = 0; + rtcp_ready_handler_id = rtcp_datagram.notify["ready"].connect(() => { + this.stream.on_rtcp_ready(); + + rtcp_datagram.disconnect(rtcp_ready_handler_id); + rtcp_ready_handler_id = 0; + }); + + ulong rtp_ready_handler_id = 0; + rtp_ready_handler_id = rtp_datagram.notify["ready"].connect(() => { + this.stream.on_rtp_ready(); + connection_ready(); + + rtp_datagram.disconnect(rtp_ready_handler_id); + rtp_ready_handler_id = 0; + }); + + session.notify["state"].connect((obj, _) => { + Jingle.Session session2 = (Jingle.Session) obj; + if (session2.state == Jingle.Session.State.ENDED) { + if (rtcp_ready_handler_id != 0) rtcp_datagram.disconnect(rtcp_ready_handler_id); + if (rtp_ready_handler_id != 0) rtp_datagram.disconnect(rtp_ready_handler_id); + } + }); + + this.stream = parent.create_stream(content); + rtp_datagram.datagram_received.connect(this.stream.on_recv_rtp_data); + rtcp_datagram.datagram_received.connect(this.stream.on_recv_rtcp_data); + this.stream.on_send_rtp_data.connect(rtp_datagram.send_datagram); + this.stream.on_send_rtcp_data.connect(rtcp_datagram.send_datagram); + this.stream_created(this.stream); + this.stream.create(); + } + + public void handle_accept(XmppStream stream, Jingle.Session session, Jingle.Content content, StanzaNode description_node) { + Gee.List payload_type_nodes = description_node.get_subnodes("payload-type"); + if (payload_type_nodes.size == 0) { + warning("Counterpart didn't include any payload types"); + return; + } + PayloadType preferred_payload_type = PayloadType.parse(payload_type_nodes[0]); + if (!payload_types.contains(preferred_payload_type)) { + warning("Counterpart's preferred content type doesn't match any of our sent ones"); + } + agreed_payload_type = preferred_payload_type; + + accept(stream, session, content); + } + + public void terminate(bool we_terminated, string? reason_name, string? reason_text) { + if (stream != null) parent.close_stream(stream); + } + + public StanzaNode get_description_node() { + StanzaNode ret = new StanzaNode.build("description", NS_URI) + .add_self_xmlns() + .put_attribute("media", media); + + if (agreed_payload_type != null) { + ret.put_node(agreed_payload_type.to_xml()); + } else { + foreach (PayloadType payload_type in payload_types) { + ret.put_node(payload_type.to_xml()); + } + } + return ret; + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_type.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_type.vala new file mode 100644 index 00000000..5a8ed1b6 --- /dev/null +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_type.vala @@ -0,0 +1,23 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Xmpp.Xep.JingleRtp.ContentType : Jingle.ContentType, Object { + public string ns_uri { get { return NS_URI; } } + public Jingle.TransportType required_transport_type { get { return Jingle.TransportType.DATAGRAM; } } + public uint8 required_components { get { return 2; /* RTP + RTCP */ } } + + private Module module; + + public ContentType(Module module) { + this.module = module; + } + + public Jingle.ContentParameters parse_content_parameters(StanzaNode description) throws Jingle.IqError { + return new Parameters.from_node(module, description); + } + + public Jingle.ContentParameters create_content_parameters(Object object) throws Jingle.IqError { + assert_not_reached(); + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala new file mode 100644 index 00000000..35e03168 --- /dev/null +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala @@ -0,0 +1,175 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +namespace Xmpp.Xep.JingleRtp { + +public const string NS_URI = "urn:xmpp:jingle:apps:rtp:1"; +public const string NS_URI_AUDIO = "urn:xmpp:jingle:apps:rtp:audio"; +public const string NS_URI_VIDEO = "urn:xmpp:jingle:apps:rtp:video"; + +public abstract class Module : XmppStreamModule { + public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "0167_jingle_rtp"); + + private ContentType content_type; + public SessionInfoType session_info_type = new SessionInfoType(); + + protected Module() { + content_type = new ContentType(this); + } + + public abstract async Gee.List get_supported_payloads(string media); + public abstract async PayloadType? pick_payload_type(string media, Gee.List payloads); + public abstract Stream create_stream(Jingle.Content content); + public abstract void close_stream(Stream stream); + + public async Jingle.Session start_call(XmppStream stream, Jid receiver_full_jid, bool video) throws Jingle.Error { + + Jingle.Module jingle_module = stream.get_module(Jingle.Module.IDENTITY); + + Jid? my_jid = stream.get_flag(Bind.Flag.IDENTITY).my_jid; + if (my_jid == null) { + throw new Jingle.Error.GENERAL("Couldn't determine own JID"); + } + + ArrayList contents = new ArrayList(); + + // Create audio content + Parameters audio_content_parameters = new Parameters(this, "audio", yield get_supported_payloads("audio")); + Jingle.Transport? audio_transport = yield jingle_module.select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); + if (audio_transport == null) { + throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable audio transports"); + } + Jingle.TransportParameters audio_transport_params = audio_transport.create_transport_parameters(stream, content_type.required_components, my_jid, receiver_full_jid); + Jingle.Content audio_content = new Jingle.Content.initiate_sent("voice", Jingle.Senders.BOTH, + content_type, audio_content_parameters, + audio_transport, audio_transport_params, + null, null, + my_jid, receiver_full_jid); + contents.add(audio_content); + + Jingle.Content? video_content = null; + if (video) { + // Create video content + Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); + Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); + if (video_transport == null) { + throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); + } + Jingle.TransportParameters video_transport_params = video_transport.create_transport_parameters(stream, content_type.required_components, my_jid, receiver_full_jid); + video_content = new Jingle.Content.initiate_sent("webcam", Jingle.Senders.BOTH, + content_type, video_content_parameters, + video_transport, video_transport_params, + null, null, + my_jid, receiver_full_jid); + contents.add(video_content); + } + + // Create session + try { + Jingle.Session session = yield jingle_module.create_session(stream, contents, receiver_full_jid); + return session; + } catch (Jingle.Error e) { + throw new Jingle.Error.GENERAL(@"Couldn't create Jingle session: $(e.message)"); + } + } + + public async Jingle.Content add_outgoing_video_content(XmppStream stream, Jingle.Session session) { + Jid my_jid = session.local_full_jid; + Jid receiver_full_jid = session.peer_full_jid; + + Jingle.Content? content = null; + foreach (Jingle.Content c in session.contents.values) { + Parameters? parameters = c.content_params as Parameters; + if (parameters == null) continue; + + if (parameters.media == "video") { + content = c; + break; + } + } + + if (content == null) { + // Content for video does not yet exist -> create it + Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); + Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); + if (video_transport == null) { + throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); + } + Jingle.TransportParameters video_transport_params = video_transport.create_transport_parameters(stream, content_type.required_components, my_jid, receiver_full_jid); + content = new Jingle.Content.initiate_sent("webcam", + session.we_initiated ? Jingle.Senders.INITIATOR : Jingle.Senders.RESPONDER, + content_type, video_content_parameters, + video_transport, video_transport_params, + null, null, + my_jid, receiver_full_jid); + + session.add_content.begin(content); + } else { + // Content for video already exists -> modify senders + bool we_initiated = session.we_initiated; + Jingle.Senders want_sender = we_initiated ? Jingle.Senders.INITIATOR : Jingle.Senders.RESPONDER; + if (content.senders == Jingle.Senders.BOTH || content.senders == want_sender) { + warning("want to add video but senders is already both/target"); + } else if (content.senders == Jingle.Senders.NONE) { + content.modify(want_sender); + } else { + content.modify(Jingle.Senders.BOTH); + } + } + + return content; + } + + public override void attach(XmppStream stream) { + stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); + stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI_AUDIO); + stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI_VIDEO); + stream.get_module(Jingle.Module.IDENTITY).register_content_type(content_type); + stream.get_module(Jingle.Module.IDENTITY).register_session_info_type(session_info_type); + } + + public override void detach(XmppStream stream) { + stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI); + stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI_AUDIO); + stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI_VIDEO); + } + + public async bool is_available(XmppStream stream, Jid full_jid) { + bool? has_feature = yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI); + if (has_feature == null || !(!)has_feature) { + return false; + } + return yield stream.get_module(Jingle.Module.IDENTITY).is_available(stream, content_type.required_transport_type, content_type.required_components, full_jid); + } + + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } +} + +public class Crypto { + public string cryptoSuite { get; private set; } + public string keyParams { get; private set; } + public string? sessionParams { get; private set; } + public string? tag { get; private set; } + + public static Crypto parse(StanzaNode node) { + Crypto crypto = new Crypto(); + crypto.cryptoSuite = node.get_attribute("crypto-suite"); + crypto.keyParams = node.get_attribute("key-params"); + crypto.sessionParams = node.get_attribute("session-params"); + crypto.tag = node.get_attribute("tag"); + return crypto; + } + + public StanzaNode to_xml() { + StanzaNode node = new StanzaNode.build("crypto", NS_URI) + .put_attribute("crypto-suite", cryptoSuite) + .put_attribute("key-params", keyParams); + if (sessionParams != null) node.put_attribute("session-params", sessionParams); + if (tag != null) node.put_attribute("tag", tag); + return node; + } +} + +} diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala new file mode 100644 index 00000000..452f1d65 --- /dev/null +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala @@ -0,0 +1,52 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Xmpp.Xep.JingleRtp.PayloadType { + public uint8 id { get; set; } + public string? name { get; set; } + public uint8 channels { get; set; default = 1; } + public uint32 clockrate { get; set; } + public uint32 maxptime { get; set; } + public uint32 ptime { get; set; } + public Map parameters = new HashMap(); + + public static PayloadType parse(StanzaNode node) { + PayloadType payloadType = new PayloadType(); + payloadType.channels = (uint8) node.get_attribute_uint("channels", payloadType.channels); + payloadType.clockrate = node.get_attribute_uint("clockrate"); + payloadType.id = (uint8) node.get_attribute_uint("id"); + payloadType.maxptime = node.get_attribute_uint("maxptime"); + payloadType.name = node.get_attribute("name"); + payloadType.ptime = node.get_attribute_uint("ptime"); + foreach (StanzaNode parameter in node.get_subnodes("parameter")) { + payloadType.parameters[parameter.get_attribute("name")] = parameter.get_attribute("value"); + } + return payloadType; + } + + public StanzaNode to_xml() { + StanzaNode node = new StanzaNode.build("payload-type", NS_URI) + .put_attribute("id", id.to_string()); + if (channels != 1) node.put_attribute("channels", channels.to_string()); + if (clockrate != 0) node.put_attribute("clockrate", clockrate.to_string()); + if (maxptime != 0) node.put_attribute("maxptime", maxptime.to_string()); + if (name != null) node.put_attribute("name", name); + if (ptime != 0) node.put_attribute("ptime", ptime.to_string()); + foreach (string parameter in parameters.keys) { + node.put_node(new StanzaNode.build("parameter", NS_URI) + .put_attribute("name", parameter) + .put_attribute("value", parameters[parameter])); + } + return node; + } + + public static bool equals_func(PayloadType a, PayloadType b) { + return a.id == b.id && + a.name == b.name && + a.channels == b.channels && + a.clockrate == b.clockrate && + a.maxptime == b.maxptime && + a.ptime == b.ptime; + } +} \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/session_info_type.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/session_info_type.vala new file mode 100644 index 00000000..d36255f0 --- /dev/null +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/session_info_type.vala @@ -0,0 +1,67 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +namespace Xmpp.Xep.JingleRtp { + + public enum CallSessionInfo { + ACTIVE, + HOLD, + UNHOLD, + MUTE, + UNMUTE, + RINGING + } + + public class SessionInfoType : Jingle.SessionInfoNs, Object { + public const string NS_URI = "urn:xmpp:jingle:apps:rtp:info:1"; + public string ns_uri { get { return NS_URI; } } + + public signal void info_received(Jingle.Session session, CallSessionInfo info); + public signal void mute_update_received(Jingle.Session session, bool mute, string name); + + public void handle_content_session_info(XmppStream stream, Jingle.Session session, StanzaNode info, Iq.Stanza iq) throws Jingle.IqError { + switch (info.name) { + case "active": + info_received(session, CallSessionInfo.ACTIVE); + break; + case "hold": + info_received(session, CallSessionInfo.HOLD); + break; + case "unhold": + info_received(session, CallSessionInfo.UNHOLD); + break; + case "mute": + string? name = info.get_attribute("name"); + mute_update_received(session, true, name); + info_received(session, CallSessionInfo.MUTE); + break; + case "unmute": + string? name = info.get_attribute("name"); + mute_update_received(session, false, name); + info_received(session, CallSessionInfo.UNMUTE); + break; + case "ringing": + info_received(session, CallSessionInfo.RINGING); + break; + } + } + + public void send_mute(Jingle.Session session, bool mute, string media) { + string node_name = mute ? "mute" : "unmute"; + + foreach (Jingle.Content content in session.contents.values) { + Parameters? parameters = content.content_params as Parameters; + if (parameters != null && parameters.media == media) { + StanzaNode session_info_content = new StanzaNode.build(node_name, NS_URI).add_self_xmlns().put_attribute("name", content.content_name); + session.send_session_info(session_info_content); + } + } + } + + public void send_ringing(Jingle.Session session) { + StanzaNode session_info_content = new StanzaNode.build("ringing", NS_URI).add_self_xmlns(); + session.send_session_info(session_info_content); + } + } +} diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala new file mode 100644 index 00000000..62d85dec --- /dev/null +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala @@ -0,0 +1,46 @@ +public abstract class Xmpp.Xep.JingleRtp.Stream : Object { + public Jingle.Content content { get; protected set; } + public string name { get { + return content.content_name; + }} + public string? media { get { + var content_params = content.content_params; + if (content_params is Parameters) { + return ((Parameters)content_params).media; + } + return null; + }} + public JingleRtp.PayloadType? payload_type { get { + var content_params = content.content_params; + if (content_params is Parameters) { + return ((Parameters)content_params).agreed_payload_type; + } + return null; + }} + public bool sending { get { + return content.session.senders_include_us(content.senders); + }} + public bool receiving { get { + return content.session.senders_include_counterpart(content.senders); + }} + + protected Stream(Jingle.Content content) { + this.content = content; + } + + public signal void on_send_rtp_data(Bytes bytes); + public signal void on_send_rtcp_data(Bytes bytes); + + public abstract void on_recv_rtp_data(Bytes bytes); + public abstract void on_recv_rtcp_data(Bytes bytes); + + public abstract void on_rtp_ready(); + public abstract void on_rtcp_ready(); + + public abstract void create(); + public abstract void destroy(); + + public string to_string() { + return @"$name/$media stream in $(content.session.sid)"; + } +} \ No newline at end of file From 8a95f9dd1dcc3e9038f0ef48d8ec10692fe08a4e Mon Sep 17 00:00:00 2001 From: fiaxh Date: Fri, 19 Mar 2021 23:05:12 +0100 Subject: [PATCH 07/56] Add initial Jingle Message Initiation (XEP-0353) support Co-authored-by: Marvin W --- xmpp-vala/src/module/bind.vala | 4 + .../xep/0353_jingle_message_initiation.vala | 97 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala diff --git a/xmpp-vala/src/module/bind.vala b/xmpp-vala/src/module/bind.vala index 89398bfb..4df8881a 100644 --- a/xmpp-vala/src/module/bind.vala +++ b/xmpp-vala/src/module/bind.vala @@ -69,6 +69,10 @@ namespace Xmpp.Bind { public Jid? my_jid; public bool finished = false; + public static Jid? get_my_jid(XmppStream stream) { + return stream.get_flag(IDENTITY).my_jid; + } + public override string get_ns() { return NS_URI; } public override string get_id() { return IDENTITY.id; } } diff --git a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala new file mode 100644 index 00000000..acb2ba2e --- /dev/null +++ b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala @@ -0,0 +1,97 @@ +using Gee; + +namespace Xmpp.Xep.JingleMessageInitiation { + private const string NS_URI = "urn:xmpp:jingle-message:0"; + + public class Module : XmppStreamModule { + public static ModuleIdentity IDENTITY = new ModuleIdentity(NS_URI, "0353_jingle_message_initiation"); + + public signal void session_proposed(Jid from, Jid to, string sid, Gee.List descriptions); + public signal void session_retracted(Jid from, Jid to, string sid); + public signal void session_accepted(Jid from, string sid); + public signal void session_rejected(Jid from, Jid to, string sid); + + public void send_session_accept_to_self(XmppStream stream, string sid) { + MessageStanza accepted_message = new MessageStanza() { to=Bind.Flag.get_my_jid(stream).bare_jid }; + accepted_message.stanza.put_node( + new StanzaNode.build("accept", NS_URI).add_self_xmlns() + .put_attribute("id", sid, NS_URI)); + stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); + } + + public void send_session_reject_to_self(XmppStream stream, string sid) { + MessageStanza accepted_message = new MessageStanza() { to=Bind.Flag.get_my_jid(stream).bare_jid }; + accepted_message.stanza.put_node( + new StanzaNode.build("reject", NS_URI).add_self_xmlns() + .put_attribute("id", sid, NS_URI)); + stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); + } + + public void send_session_proceed_to_peer(XmppStream stream, Jid to, string sid) { + MessageStanza accepted_message = new MessageStanza() { to=to }; + accepted_message.stanza.put_node( + new StanzaNode.build("proceed", NS_URI).add_self_xmlns() + .put_attribute("id", sid, NS_URI)); + stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); + } + + public void send_session_reject_to_peer(XmppStream stream, Jid to, string sid) { + MessageStanza accepted_message = new MessageStanza() { to=to }; + accepted_message.stanza.put_node( + new StanzaNode.build("reject", NS_URI).add_self_xmlns() + .put_attribute("id", sid, NS_URI)); + stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); + } + + private void on_received_message(XmppStream stream, MessageStanza message) { + Xep.MessageArchiveManagement.MessageFlag? mam_flag = Xep.MessageArchiveManagement.MessageFlag.get_flag(message); + if (mam_flag != null) return; + + StanzaNode? mi_node = null; + foreach (StanzaNode node in message.stanza.sub_nodes) { + if (node.ns_uri == NS_URI) { + mi_node = node; + } + } + if (mi_node == null) return; + + switch (mi_node.name) { + case "accept": + case "proceed": + if (!message.from.equals_bare(Bind.Flag.get_my_jid(stream))) return; + session_accepted(message.from, mi_node.get_attribute("id")); + break; + case "propose": + ArrayList descriptions = new ArrayList(); + + foreach (StanzaNode node in mi_node.sub_nodes) { + if (node.name != "description") continue; + descriptions.add(node); + } + + if (descriptions.size > 0) { + session_proposed(message.from, message.to, mi_node.get_attribute("id"), descriptions); + } + break; + case "retract": + session_retracted(message.from, message.to, mi_node.get_attribute("id")); + break; + case "reject": + if (!message.from.equals_bare(Bind.Flag.get_my_jid(stream))) return; + session_rejected(message.from, message.to, mi_node.get_attribute("id")); + break; + } + } + + public override void attach(XmppStream stream) { + stream.get_module(MessageModule.IDENTITY).received_message.connect(on_received_message); + } + + public override void detach(XmppStream stream) { + stream.get_module(MessageModule.IDENTITY).received_message.disconnect(on_received_message); + } + + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } + } +} From e6a933ad307116952d3202c36d0a8d6e7f4b0946 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 21 Mar 2021 12:41:36 +0100 Subject: [PATCH 08/56] Add gstreamer .cmake instructions --- cmake/FindGst.cmake | 12 ++++++++++++ cmake/FindGstApp.cmake | 14 ++++++++++++++ cmake/FindGstVideo.cmake | 14 ++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 cmake/FindGst.cmake create mode 100644 cmake/FindGstApp.cmake create mode 100644 cmake/FindGstVideo.cmake diff --git a/cmake/FindGst.cmake b/cmake/FindGst.cmake new file mode 100644 index 00000000..942d0129 --- /dev/null +++ b/cmake/FindGst.cmake @@ -0,0 +1,12 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(Gst + PKG_CONFIG_NAME gstreamer-1.0 + LIB_NAMES gstreamer-1.0 + INCLUDE_NAMES gst/gst.h + INCLUDE_DIR_SUFFIXES gstreamer-1.0 gstreamer-1.0/include +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Gst + REQUIRED_VARS Gst_LIBRARY + VERSION_VAR Gst_VERSION) diff --git a/cmake/FindGstApp.cmake b/cmake/FindGstApp.cmake new file mode 100644 index 00000000..834b8e8e --- /dev/null +++ b/cmake/FindGstApp.cmake @@ -0,0 +1,14 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(GstApp + PKG_CONFIG_NAME gstreamer-app-1.0 + LIB_NAMES gstapp + LIB_DIR_HINTS gstreamer-1.0 + INCLUDE_NAMES gst/app/app.h + INCLUDE_DIR_SUFFIXES gstreamer-1.0 gstreamer-1.0/include gstreamer-app-1.0 gstreamer-app-1.0/include + DEPENDS Gst +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GstApp + REQUIRED_VARS GstApp_LIBRARY + VERSION_VAR GstApp_VERSION) diff --git a/cmake/FindGstVideo.cmake b/cmake/FindGstVideo.cmake new file mode 100644 index 00000000..7d529391 --- /dev/null +++ b/cmake/FindGstVideo.cmake @@ -0,0 +1,14 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(GstVideo + PKG_CONFIG_NAME gstreamer-video-1.0 + LIB_NAMES gstvideo + LIB_DIR_HINTS gstreamer-1.0 + INCLUDE_NAMES gst/video/video.h + INCLUDE_DIR_SUFFIXES gstreamer-1.0 gstreamer-1.0/include gstreamer-video-1.0 gstreamer-video-1.0/include + DEPENDS Gst +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GstVideo + REQUIRED_VARS GstVideo_LIBRARY + VERSION_VAR GstVideo_VERSION) From ef2e3c774cab82a94a5e34399f2013d64c3cf03b Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 21 Mar 2021 12:41:38 +0100 Subject: [PATCH 09/56] Add RTP implementation as plugin --- plugins/CMakeLists.txt | 4 + plugins/http-files/src/file_sender.vala | 6 - plugins/rtp/CMakeLists.txt | 36 ++ plugins/rtp/src/codec_util.vala | 245 ++++++++++++++ plugins/rtp/src/device.vala | 206 +++++++++++ plugins/rtp/src/module.vala | 264 +++++++++++++++ plugins/rtp/src/participant.vala | 39 +++ plugins/rtp/src/plugin.vala | 413 ++++++++++++++++++++++ plugins/rtp/src/register_plugin.vala | 3 + plugins/rtp/src/stream.vala | 432 ++++++++++++++++++++++++ plugins/rtp/src/video_widget.vala | 110 ++++++ 11 files changed, 1752 insertions(+), 6 deletions(-) create mode 100644 plugins/rtp/CMakeLists.txt create mode 100644 plugins/rtp/src/codec_util.vala create mode 100644 plugins/rtp/src/device.vala create mode 100644 plugins/rtp/src/module.vala create mode 100644 plugins/rtp/src/participant.vala create mode 100644 plugins/rtp/src/plugin.vala create mode 100644 plugins/rtp/src/register_plugin.vala create mode 100644 plugins/rtp/src/stream.vala create mode 100644 plugins/rtp/src/video_widget.vala diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 48593c7a..00bb6509 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -6,6 +6,10 @@ if(DINO_PLUGIN_ENABLED_ice) add_subdirectory(ice) endif(DINO_PLUGIN_ENABLED_ice) +if(DINO_PLUGIN_ENABLED_rtp) + add_subdirectory(rtp) +endif(DINO_PLUGIN_ENABLED_rtp) + if(DINO_PLUGIN_ENABLED_openpgp) add_subdirectory(gpgme-vala) add_subdirectory(openpgp) diff --git a/plugins/http-files/src/file_sender.vala b/plugins/http-files/src/file_sender.vala index 25db49b9..a038e70f 100644 --- a/plugins/http-files/src/file_sender.vala +++ b/plugins/http-files/src/file_sender.vala @@ -81,12 +81,6 @@ public class HttpFileSender : FileSender, Object { } } - public async long get_max_file_size(Account account) { - lock (max_file_sizes) { - return max_file_sizes[account]; - } - } - private static void transfer_more_bytes(InputStream stream, Soup.MessageBody body) { uint8[] bytes = new uint8[4096]; ssize_t read = stream.read(bytes); diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt new file mode 100644 index 00000000..2b66f6ff --- /dev/null +++ b/plugins/rtp/CMakeLists.txt @@ -0,0 +1,36 @@ +find_packages(RTP_PACKAGES REQUIRED + Gee + GLib + GModule + GObject + GTK3 + Gst + GstApp +) + +vala_precompile(RTP_VALA_C +SOURCES + src/codec_util.vala + src/device.vala + src/module.vala + src/plugin.vala + src/stream.vala + src/video_widget.vala + src/register_plugin.vala +CUSTOM_VAPIS + ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi + ${CMAKE_BINARY_DIR}/exports/dino.vapi + ${CMAKE_BINARY_DIR}/exports/qlite.vapi +PACKAGES + ${RTP_PACKAGES} +OPTIONS + --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi +) + +add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp") +add_library(rtp SHARED ${RTP_VALA_C}) +target_link_libraries(rtp libdino ${RTP_PACKAGES}) +set_target_properties(rtp PROPERTIES PREFIX "") +set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) + +install(TARGETS rtp ${PLUGIN_INSTALL}) diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala new file mode 100644 index 00000000..e419b5ad --- /dev/null +++ b/plugins/rtp/src/codec_util.vala @@ -0,0 +1,245 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.CodecUtil { + private Set supported_elements = new HashSet(); + private Set unsupported_elements = new HashSet(); + + public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type) { + Gst.Caps caps = new Gst.Caps.simple("application/x-rtp", + "media", typeof(string), media, + "payload", typeof(int), payload_type.id); + //"channels", typeof(int), payloadType.channels, + //"max-ptime", typeof(int), payloadType.maxptime); + unowned Gst.Structure s = caps.get_structure(0); + if (payload_type.clockrate != 0) { + s.set("clock-rate", typeof(int), payload_type.clockrate); + } + if (payload_type.name != null) { + s.set("encoding-name", typeof(string), payload_type.name.up()); + } + return caps; + } + + public static string? get_codec_from_payload(string media, JingleRtp.PayloadType payload_type) { + if (payload_type.name != null) return payload_type.name.down(); + if (media == "audio") { + switch (payload_type.id) { + case 0: + return "pcmu"; + case 8: + return "pcma"; + } + } + return null; + } + + public static string? get_media_type_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_media_type(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_media_type(string media, string? codec) { + if (codec == null) return null; + if (media == "audio") { + switch (codec) { + case "pcma": + return "audio/x-alaw"; + case "pcmu": + return "audio/x-mulaw"; + } + } + return @"$media/x-$codec"; + } + + public static string? get_rtp_pay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_pay_candidate(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_pay_candidate(string media, string? codec) { + if (codec == null) return null; + return @"rtp$(codec)pay"; + } + + public static string? get_rtp_depay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_depay_candidate(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_depay_candidate(string media, string? codec) { + if (codec == null) return null; + return @"rtp$(codec)depay"; + } + + public static string[] get_encode_candidates(string media, string? codec) { + if (codec == null) return new string[0]; + if (media == "audio") { + switch (codec) { + case "opus": + return new string[] {"opusenc"}; + case "speex": + return new string[] {"speexenc"}; + case "pcma": + return new string[] {"alawenc"}; + case "pcmu": + return new string[] {"mulawenc"}; + } + } else if (media == "video") { + switch (codec) { + case "h264": + return new string[] {/*"msdkh264enc", */"vaapih264enc", "x264enc"}; + case "vp9": + return new string[] {/*"msdkvp9enc", */"vaapivp9enc" /*, "vp9enc" */}; + case "vp8": + return new string[] {/*"msdkvp8enc", */"vaapivp8enc", "vp8enc"}; + } + } + return new string[0]; + } + + public static string[] get_decode_candidates(string media, string? codec) { + if (codec == null) return new string[0]; + if (media == "audio") { + switch (codec) { + case "opus": + return new string[] {"opusdec"}; + case "speex": + return new string[] {"speexdec"}; + case "pcma": + return new string[] {"alawdec"}; + case "pcmu": + return new string[] {"mulawdec"}; + } + } else if (media == "video") { + switch (codec) { + case "h264": + return new string[] {/*"msdkh264dec", */"vaapih264dec"}; + case "vp9": + return new string[] {/*"msdkvp9dec", */"vaapivp9dec", "vp9dec"}; + case "vp8": + return new string[] {/*"msdkvp8dec", */"vaapivp8dec", "vp8dec"}; + } + } + return new string[0]; + } + + public static string? get_encode_prefix(string media, string codec, string encode) { + if (encode == "msdkh264enc") return "video/x-raw,format=NV12 ! "; + if (encode == "vaapih264enc") return "video/x-raw,format=NV12 ! "; + return null; + } + + public static string? get_encode_suffix(string media, string codec, string encode) { + // H264 + const string h264_suffix = " ! video/x-h264,profile=constrained-baseline ! h264parse"; + if (encode == "msdkh264enc") return @" bitrate=256 rate-control=vbr target-usage=7$h264_suffix"; + if (encode == "vaapih264enc") return @" bitrate=256 quality-level=7 tune=low-power$h264_suffix"; + if (encode == "x264enc") return @" byte-stream=1 bitrate=256 profile=baseline speed-preset=ultrafast tune=zerolatency$h264_suffix"; + if (media == "video" && codec == "h264") return h264_suffix; + + // VP8 + if (encode == "msdkvp8enc") return " bitrate=256 rate-control=vbr target-usage=7"; + if (encode == "vaapivp8enc") return " bitrate=256 rate-control=vbr quality-level=7"; + if (encode == "vp8enc") return " target-bitrate=256000 deadline=1 error-resilient=1"; + + // OPUS + if (encode == "opusenc") return " audio-type=voice"; + + return null; + } + + public static string? get_decode_prefix(string media, string codec, string decode) { + return null; + } + + public bool is_element_supported(string element_name) { + if (unsupported_elements.contains(element_name)) return false; + if (supported_elements.contains(element_name)) return true; + var test_element = Gst.ElementFactory.make(element_name, @"test-$element_name"); + if (test_element != null) { + supported_elements.add(element_name); + return true; + } else { + debug("%s is not supported on this platform", element_name); + unsupported_elements.add(element_name); + return false; + } + } + + public string? get_encode_element_name(string media, string? codec) { + foreach (string candidate in get_encode_candidates(media, codec)) { + if (is_element_supported(candidate)) return candidate; + } + return null; + } + + public string? get_pay_element_name(string media, string? codec) { + string candidate = get_pay_candidate(media, codec); + if (is_element_supported(candidate)) return candidate; + return null; + } + + public string? get_decode_element_name(string media, string? codec) { + foreach (string candidate in get_decode_candidates(media, codec)) { + if (is_element_supported(candidate)) return candidate; + } + return null; + } + + public string? get_depay_element_name(string media, string? codec) { + string candidate = get_depay_candidate(media, codec); + if (is_element_supported(candidate)) return candidate; + return null; + } + + public void mark_element_unsupported(string element_name) { + unsupported_elements.add(element_name); + } + + public string? get_decode_bin_description(string media, string? codec, string? element_name = null, string? name = null) { + if (codec == null) return null; + string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string depay = get_depay_element_name(media, codec); + string decode = element_name ?? get_decode_element_name(media, codec); + if (depay == null || decode == null) return null; + string decode_prefix = get_decode_prefix(media, codec, decode) ?? ""; + return @"$depay name=$base_name-rtp-depay ! $decode_prefix$decode name=$base_name-decode ! $(media)convert name=$base_name-convert"; + } + + public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string? desc = get_decode_bin_description(media, codec, null, base_name); + if (desc == null) return null; + debug("Pipeline to decode %s %s: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + + public string? get_encode_bin_description(string media, string? codec, string? element_name = null, uint pt = 96, string? name = null) { + if (codec == null) return null; + string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string pay = get_pay_element_name(media, codec); + string encode = element_name ?? get_encode_element_name(media, codec); + if (pay == null || encode == null) return null; + string encode_prefix = get_encode_prefix(media, codec, encode) ?? ""; + string encode_suffix = get_encode_suffix(media, codec, encode) ?? ""; + if (media == "audio") { + return @"audioconvert name=$base_name-convert ! audioresample name=$base_name-resample ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; + } else { + return @"$(media)convert name=$base_name-convert ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; + } + } + + public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string? desc = get_encode_bin_description(media, codec, null, payload_type.id, base_name); + if (desc == null) return null; + debug("Pipeline to encode %s %s: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + +} \ No newline at end of file diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala new file mode 100644 index 00000000..796be203 --- /dev/null +++ b/plugins/rtp/src/device.vala @@ -0,0 +1,206 @@ +public class Dino.Plugins.Rtp.Device : MediaDevice, Object { + public Plugin plugin { get; private set; } + public Gst.Device device { get; private set; } + + private string device_name; + public string id { get { + return device_name; + }} + private string device_display_name; + public string display_name { get { + return device_display_name; + }} + public string detail_name { get { + return device.properties.get_string("alsa.card_name") ?? device.properties.get_string("alsa.id") ?? id; + }} + public Gst.Pipeline pipe { get { + return plugin.pipe; + }} + public string? media { get { + if (device.device_class.has_prefix("Audio/")) { + return "audio"; + } else if (device.device_class.has_prefix("Video/")) { + return "video"; + } else { + return null; + } + }} + public bool is_source { get { + return device.device_class.has_suffix("/Source"); + }} + public bool is_sink { get { + return device.device_class.has_suffix("/Sink"); + }} + + private Gst.Element element; + private Gst.Element tee; + private Gst.Element dsp; + private Gst.Element mixer; + private Gst.Element filter; + private int links = 0; + + public Device(Plugin plugin, Gst.Device device) { + this.plugin = plugin; + update(device); + } + + public bool matches(Gst.Device device) { + if (this.device.name == device.name) return true; + return false; + } + + public void update(Gst.Device device) { + this.device = device; + this.device_name = device.name; + this.device_display_name = device.display_name; + } + + public Gst.Element? link_sink() { + if (element == null) create(); + links++; + if (mixer != null) return mixer; + if (is_sink && media == "audio") return plugin.echoprobe; + return element; + } + + public Gst.Element? link_source() { + if (element == null) create(); + links++; + if (tee != null) return tee; + return element; + } + + public void unlink() { + if (links <= 0) { + critical("Link count below zero."); + return; + } + links--; + if (links == 0) { + destroy(); + } + } + + private Gst.Caps get_best_caps() { + if (media == "audio") { + return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1"); + } else if (media == "video" && device.caps.get_size() > 0) { + int best_index = 0; + int best_fps = 0; + int best_width = 0; + int best_height = 0; + for (int i = 0; i < device.caps.get_size(); i++) { + unowned Gst.Structure? that = device.caps.get_structure(i); + if (!that.has_name("video/x-raw")) continue; + int num = 0, den = 0, width = 0, height = 0; + if (!that.has_field("framerate") || !that.get_fraction("framerate", out num, out den)) continue; + if (!that.has_field("width") || !that.get_int("width", out width)) continue; + if (!that.has_field("height") || !that.get_int("height", out height)) continue; + int fps = num/den; + if (best_fps < fps || best_fps == fps && best_width < width || best_fps == fps && best_width == width && best_height < height) { + best_fps = fps; + best_width = width; + best_height = height; + best_index = i; + } + } + return device.caps.copy_nth(best_index); + } else if (device.caps.get_size() > 0) { + return device.caps.copy_nth(0); + } else { + return new Gst.Caps.any(); + } + } + + private void create() { + debug("Creating device %s", id); + plugin.pause(); + element = device.create_element(id); + pipe.add(element); + if (is_source) { + filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); + filter.@set("caps", get_best_caps()); + pipe.add(filter); + element.link(filter); + if (media == "audio") { + dsp = Gst.ElementFactory.make("webrtcdsp", @"$id-dsp"); + dsp.@set("probe", plugin.echoprobe.name); + pipe.add(dsp); + filter.link(dsp); + } + tee = Gst.ElementFactory.make("tee", @"$id-tee"); + tee.@set("allow-not-linked", true); + pipe.add(tee); + (dsp ?? filter).link(tee); + } + if (is_sink) { + element.@set("async", false); + element.@set("sync", false); + } + if (is_sink && media == "audio") { +// mixer = Gst.ElementFactory.make("audiomixer", @"$id-mixer"); +// pipe.add(mixer); +// mixer.link(plugin.echoprobe); + plugin.echoprobe.link(element); + } + plugin.unpause(); + } + + private void destroy() { + if (mixer != null) { + if (is_sink && media == "audio") { + plugin.echoprobe.unlink(mixer); + } + int linked_sink_pads = 0; + mixer.foreach_sink_pad((_, pad) => { + if (pad.is_linked()) linked_sink_pads++; + return true; + }); + if (linked_sink_pads > 0) { + warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads); + } + mixer.set_locked_state(true); + mixer.set_state(Gst.State.NULL); + mixer.unlink(element); + pipe.remove(mixer); + mixer = null; + } else if (is_sink && media == "audio") { + plugin.echoprobe.unlink(element); + } + element.set_locked_state(true); + element.set_state(Gst.State.NULL); + if (filter != null) element.unlink(filter); + else if (is_source) element.unlink(tee); + pipe.remove(element); + element = null; + if (filter != null) { + filter.set_locked_state(true); + filter.set_state(Gst.State.NULL); + filter.unlink(dsp ?? tee); + pipe.remove(filter); + filter = null; + } + if (dsp != null) { + dsp.set_locked_state(true); + dsp.set_state(Gst.State.NULL); + dsp.unlink(tee); + pipe.remove(dsp); + dsp = null; + } + if (tee != null) { + int linked_src_pads = 0; + tee.foreach_src_pad((_, pad) => { + if (pad.is_linked()) linked_src_pads++; + return true; + }); + if (linked_src_pads != 0) { + warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); + } + tee.set_locked_state(true); + tee.set_state(Gst.State.NULL); + pipe.remove(tee); + tee = null; + } + debug("Destroyed device %s", id); + } +} \ No newline at end of file diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala new file mode 100644 index 00000000..577e9f53 --- /dev/null +++ b/plugins/rtp/src/module.vala @@ -0,0 +1,264 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.Module : JingleRtp.Module { + private Set supported_codecs = new HashSet(); + private Set unsupported_codecs = new HashSet(); + public Plugin plugin { get; private set; } + public CodecUtil codec_util { get { + return plugin.codec_util; + }} + + public Module(Plugin plugin) { + base(); + this.plugin = plugin; + } + + private async bool pipeline_works(string media, string element_desc) { + var supported = false; + string pipeline_desc = @"$(media)testsrc is-live=true ! $element_desc ! appsink name=output"; + try { + var pipeline = Gst.parse_launch(pipeline_desc); + var output = (pipeline as Gst.Bin).get_by_name("output") as Gst.App.Sink; + SourceFunc callback = pipeline_works.callback; + var finished = false; + output.emit_signals = true; + output.new_sample.connect(() => { + if (!finished) { + finished = true; + supported = true; + Idle.add(() => { + callback(); + return Source.REMOVE; + }); + } + return Gst.FlowReturn.EOS; + }); + pipeline.bus.add_watch(Priority.DEFAULT, (_, message) => { + if (message.type == Gst.MessageType.ERROR && !finished) { + Error e; + string d; + message.parse_error(out e, out d); + debug("pipeline [%s] failed: %s", pipeline_desc, e.message); + debug(d); + finished = true; + callback(); + } + return true; + }); + Timeout.add(2000, () => { + if (!finished) { + finished = true; + callback(); + } + return Source.REMOVE; + }); + pipeline.set_state(Gst.State.PLAYING); + yield; + pipeline.set_state(Gst.State.NULL); + } catch (Error e) { + debug("pipeline [%s] failed: %s", pipeline_desc, e.message); + } + return supported; + } + + private async bool supports(string media, JingleRtp.PayloadType payload_type) { + string codec = CodecUtil.get_codec_from_payload(media, payload_type); + if (codec == null) return false; + if (unsupported_codecs.contains(codec)) return false; + if (supported_codecs.contains(codec)) return true; + + string encode_element = codec_util.get_encode_element_name(media, codec); + string decode_element = codec_util.get_decode_element_name(media, codec); + if (encode_element == null || decode_element == null) { + debug("No suitable encoder or decoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + + string encode_bin = codec_util.get_encode_bin_description(media, codec, encode_element); + while (!(yield pipeline_works(media, encode_bin))) { + debug("%s not suited for encoding %s", encode_element, codec); + codec_util.mark_element_unsupported(encode_element); + encode_element = codec_util.get_encode_element_name(media, codec); + if (encode_element == null) { + debug("No suitable encoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + encode_bin = codec_util.get_encode_bin_description(media, codec, encode_element); + } + debug("using %s to encode %s", encode_element, codec); + + string decode_bin = codec_util.get_decode_bin_description(media, codec, decode_element); + while (!(yield pipeline_works(media, @"$encode_bin ! $decode_bin"))) { + debug("%s not suited for decoding %s", decode_element, codec); + codec_util.mark_element_unsupported(decode_element); + decode_element = codec_util.get_decode_element_name(media, codec); + if (decode_element == null) { + debug("No suitable decoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + decode_bin = codec_util.get_decode_bin_description(media, codec, decode_element); + } + debug("using %s to decode %s", decode_element, codec); + + supported_codecs.add(codec); + return true; + } + + public async void add_if_supported(Gee.List list, string media, JingleRtp.PayloadType payload_type) { + if (yield supports(media, payload_type)) { + list.add(payload_type); + } + } + + public override async Gee.List get_supported_payloads(string media) { + Gee.List list = new ArrayList(JingleRtp.PayloadType.equals_func); + if (media == "audio") { + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 2, + clockrate = 48000, + name = "opus", + id = 96 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 1, + clockrate = 32000, + name = "speex", + id = 97 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 1, + clockrate = 16000, + name = "speex", + id = 98 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 1, + clockrate = 8000, + name = "speex", + id = 99 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 1, + clockrate = 8000, + name = "PCMU", + id = 0 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 1, + clockrate = 8000, + name = "PCMA", + id = 8 + }); + } else if (media == "video") { + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + clockrate = 90000, + name = "H264", + id = 96 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + clockrate = 90000, + name = "VP9", + id = 97 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + clockrate = 90000, + name = "VP8", + id = 98 + }); + } else { + warning("Unsupported media type: %s", media); + } + return list; + } + + public override async JingleRtp.PayloadType? pick_payload_type(string media, Gee.List payloads) { + if (media == "audio") { + foreach (JingleRtp.PayloadType type in payloads) { + if (yield supports(media, type)) return type; + } + } else if (media == "video") { + foreach (JingleRtp.PayloadType type in payloads) { + if (yield supports(media, type)) return type; + } + } else { + warning("Unsupported media type: %s", media); + } + return null; + } + + public override JingleRtp.Stream create_stream(Jingle.Content content) { + return plugin.open_stream(content); + } + + public override void close_stream(JingleRtp.Stream stream) { + var rtp_stream = stream as Rtp.Stream; + plugin.close_stream(rtp_stream); + } + +// public uint32 get_session_id(string id) { +// return (uint32) id.split("-")[0].to_int(); +// } +// +// public string create_feed(string media, bool incoming) { +// init(); +// string id = random_uuid(); +// if (media == "audio") { +// id = "0-" + id; +// } else { +// id = "1-" + id; +// } +// MediaDevice? device = plugin.get_preferred_device(media, incoming); +// Feed feed; +// if (incoming) { +// if (media == "audio") { +// feed = new IncomingAudioFeed(id, this, device); +// } else if (media == "video") { +// feed = new IncomingVideoFeed(id, this, device); +// } else { +// critical("Incoming feed of media '%s' not supported", media); +// return id; +// } +// } else { +// if (media == "audio") { +// string? matching_incoming_feed_id = null; +// foreach (Feed match in plugin.feeds.values) { +// if (match is IncomingAudioFeed) { +// matching_incoming_feed_id = match.id; +// } +// } +// feed = new OutgoingAudioFeed(id, this, device); +// } else if (media == "video") { +// feed = new OutgoingVideoFeed(id, this, device); +// } else { +// critical("Outgoing feed of media '%s' not supported", media); +// return id; +// } +// } +// plugin.add_feed(id, feed); +// return id; +// } +// +// public void connect_feed(string id, JingleRtp.PayloadType payload, Jingle.DatagramConnection connection) { +// if (!plugin.feeds.has_key(id)) { +// critical("Tried to connect feed with id %s, but no such feed found", id); +// return; +// } +// Feed feed = plugin.feeds[id]; +// feed.connect(payload, connection); +// } +// +// public void destroy_feed(string id) { +// if (!plugin.feeds.has_key(id)) { +// critical("Tried to destroy feed with id %s, but no such feed found", id); +// return; +// } +// Feed feed = plugin.feeds[id]; +// feed.destroy(); +// plugin.feeds.remove(id); +// } +} \ No newline at end of file diff --git a/plugins/rtp/src/participant.vala b/plugins/rtp/src/participant.vala new file mode 100644 index 00000000..1ca13191 --- /dev/null +++ b/plugins/rtp/src/participant.vala @@ -0,0 +1,39 @@ +using Gee; +using Xmpp; + +public class Dino.Plugins.Rtp.Participant { + public Jid full_jid { get; private set; } + + protected Gst.Pipeline pipe; + private Map ssrcs = new HashMap(); + + public Participant(Gst.Pipeline pipe, Jid full_jid) { + this.pipe = pipe; + this.full_jid = full_jid; + } + + public uint32 get_ssrc(Stream stream) { + if (ssrcs.has_key(stream)) { + return ssrcs[stream]; + } + return 0; + } + + public void set_ssrc(Stream stream, uint32 ssrc) { + if (ssrcs.has_key(stream)) { + warning("Learning ssrc %ul for %s in %s when it is already known as %ul", ssrc, full_jid.to_string(), stream.to_string(), ssrcs[stream]); + } else { + stream.on_destroy.connect(unset_ssrc); + } + ssrcs[stream] = ssrc; + } + + public void unset_ssrc(Stream stream) { + ssrcs.unset(stream); + stream.on_destroy.disconnect(unset_ssrc); + } + + public string to_string() { + return @"participant $full_jid"; + } +} \ No newline at end of file diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala new file mode 100644 index 00000000..69b0f37a --- /dev/null +++ b/plugins/rtp/src/plugin.vala @@ -0,0 +1,413 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { + public Dino.Application app { get; private set; } + public CodecUtil codec_util { get; private set; } + public Gst.DeviceMonitor device_monitor { get; private set; } + public Gst.Pipeline pipe { get; private set; } + public Gst.Bin rtpbin { get; private set; } + public Gst.Element echoprobe { get; private set; } + + private Gee.List streams = new ArrayList(); + private Gee.List devices = new ArrayList(); + // private Gee.List participants = new ArrayList(); + + public void registered(Dino.Application app) { + this.app = app; + this.codec_util = new CodecUtil(); + app.startup.connect(startup); + app.add_option_group(Gst.init_get_option_group()); + app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => { + list.add(new Module(this)); + }); + app.plugin_registry.video_call_plugin = this; + } + + private int pause_count = 0; + public void pause() { +// if (pause_count == 0) { +// debug("Pausing pipe for modifications"); +// pipe.set_state(Gst.State.PAUSED); +// } + pause_count++; + } + public void unpause() { + pause_count--; + if (pause_count == 0) { + debug("Continue pipe after modifications"); + pipe.set_state(Gst.State.PLAYING); + } + if (pause_count < 0) warning("Pause count below zero!"); + } + + public void startup() { + device_monitor = new Gst.DeviceMonitor(); + device_monitor.show_all = true; + device_monitor.get_bus().add_watch(Priority.DEFAULT, on_device_monitor_message); + device_monitor.start(); + + pipe = new Gst.Pipeline(null); + + // RTP + rtpbin = Gst.ElementFactory.make("rtpbin", null) as Gst.Bin; + if (rtpbin == null) { + warning("RTP not supported"); + pipe = null; + return; + } + rtpbin.pad_added.connect(on_rtp_pad_added); + rtpbin.@set("latency", 100); + rtpbin.connect("signal::request-pt-map", request_pt_map, this); + pipe.add(rtpbin); + + // Audio echo probe + echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); + pipe.add(echoprobe); + + // Pipeline + pipe.auto_flush_bus = true; + pipe.bus.add_watch(GLib.Priority.DEFAULT, (_, message) => { + on_pipe_bus_message(message); + return true; + }); + pipe.set_state(Gst.State.PLAYING); + } + + private static Gst.Caps? request_pt_map(Gst.Element rtpbin, uint session, uint pt, Plugin plugin) { + debug("request-pt-map"); + return null; + } + + private void on_rtp_pad_added(Gst.Pad pad) { + debug("pad added: %s", pad.name); + if (pad.name.has_prefix("recv_rtp_src_")) { + string[] split = pad.name.split("_"); + uint8 rtpid = (uint8)int.parse(split[3]); + foreach (Stream stream in streams) { + if (stream.rtpid == rtpid) { + stream.on_ssrc_pad_added(split[4], pad); + } + } + } + if (pad.name.has_prefix("send_rtp_src_")) { + string[] split = pad.name.split("_"); + uint8 rtpid = (uint8)int.parse(split[3]); + debug("pad %s for stream %hhu", pad.name, rtpid); + foreach (Stream stream in streams) { + if (stream.rtpid == rtpid) { + stream.on_send_rtp_src_added(pad); + } + } + } + } + + private void on_pipe_bus_message(Gst.Message message) { + switch (message.type) { + case Gst.MessageType.ERROR: + Error error; + string str; + message.parse_error(out error, out str); + warning("Error in pipeline: %s", error.message); + debug(str); + break; + case Gst.MessageType.WARNING: + Error error; + string str; + message.parse_warning(out error, out str); + warning("Warning in pipeline: %s", error.message); + debug(str); + break; + case Gst.MessageType.CLOCK_LOST: + debug("Clock lost. Restarting"); + pipe.set_state(Gst.State.READY); + pipe.set_state(Gst.State.PLAYING); + break; + case Gst.MessageType.STATE_CHANGED: + Gst.State new_state; + message.parse_state_changed(null, out new_state, null); + if (message.src is Gst.Element) { + debug("%s changed state to %s", ((Gst.Element)message.src).name, new_state.to_string()); + } + break; + case Gst.MessageType.STREAM_STATUS: + Gst.StreamStatusType status; + Gst.Element owner; + message.parse_stream_status(out status, out owner); + if (owner != null) { + debug("%s stream changed status to %s", owner.name, status.to_string()); + } + break; + case Gst.MessageType.ELEMENT: + unowned Gst.Structure struc = message.get_structure(); + if (struc != null && message.src is Gst.Element) { + debug("Message from %s in pipeline: %s", ((Gst.Element)message.src).name, struc.to_string()); + } + break; + case Gst.MessageType.NEW_CLOCK: + debug("New clock."); + break; + case Gst.MessageType.TAG: + // Ignore + break; + case Gst.MessageType.QOS: + // Ignore + break; + default: + debug("Pipe bus message: %s", message.type.to_string()); + break; + } + } + + private bool on_device_monitor_message(Gst.Bus bus, Gst.Message message) { + Gst.Device old_device = null; + Gst.Device device = null; + switch (message.type) { + case Gst.MessageType.DEVICE_ADDED: + message.parse_device_added(out device); + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; + if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + devices.add(new Device(this, device)); + break; + case Gst.MessageType.DEVICE_CHANGED: + message.parse_device_changed(out device, out old_device); + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; + if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + devices.first_match((it) => it.matches(old_device)).update(device); + break; + case Gst.MessageType.DEVICE_REMOVED: + message.parse_device_removed(out device); + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; + if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + devices.remove(devices.first_match((it) => it.matches(device))); + break; + } + if (device != null) { + switch (device.device_class) { + case "Audio/Source": + devices_changed("audio", false); + break; + case "Audio/Sink": + devices_changed("audio", true); + break; + case "Video/Source": + devices_changed("video", false); + break; + case "Video/Sink": + devices_changed("video", true); + break; + } + } + return Source.CONTINUE; + } + + public uint8 next_free_id() { + uint8 rtpid = 0; + while (streams.size < 100 && streams.any_match((stream) => stream.rtpid == rtpid)) { + rtpid++; + } + return rtpid; + } + + // public Participant get_participant(Jid full_jid, bool self) { +// foreach (Participant participant in participants) { +// if (participant.full_jid.equals(full_jid)) { +// return participant; +// } +// } +// Participant participant; +// if (self) { +// participant = new SelfParticipant(pipe, full_jid); +// } else { +// participant = new Participant(pipe, full_jid); +// } +// participants.add(participant); +// return participant; +// } + + public Stream open_stream(Xmpp.Xep.Jingle.Content content) { + var content_params = content.content_params as Xmpp.Xep.JingleRtp.Parameters; + if (content_params == null) return null; + Stream stream; + if (content_params.media == "video") { + stream = new VideoStream(this, content); + } else { + stream = new Stream(this, content); + } + streams.add(stream); + return stream; + } + + public void close_stream(Stream stream) { + streams.remove(stream); + stream.destroy(); + } + + public void shutdown() { + device_monitor.stop(); + pipe.set_state(Gst.State.NULL); + rtpbin = null; + pipe = null; + Gst.deinit(); + } + + public VideoCallWidget? create_widget(WidgetType type) { + if (type == WidgetType.GTK) { + return new VideoWidget(this); + } + return null; + } + + public Gee.List get_devices(string media, bool incoming) { + if (media == "video" && !incoming) { + return get_video_sources(); + } + + ArrayList result = new ArrayList(); + foreach (Device device in devices) { + if (device.media == media && (incoming && device.is_sink || !incoming && device.is_source)) { + result.add(device); + } + } + if (media == "audio") { + // Reorder sources + result.sort((media_left, media_right) => { + Device left = media_left as Device; + Device right = media_right as Device; + if (left == null) return 1; + if (right == null) return -1; + + bool left_is_pipewire = left.device.properties.has_name("pipewire-proplist"); + bool right_is_pipewire = right.device.properties.has_name("pipewire-proplist"); + + bool left_is_default = false; + left.device.properties.get_boolean("is-default", out left_is_default); + bool right_is_default = false; + right.device.properties.get_boolean("is-default", out right_is_default); + + // Prefer pipewire + if (left_is_pipewire && !right_is_pipewire) return -1; + if (right_is_pipewire && !left_is_pipewire) return 1; + + // Prefer pulse audio default device + if (left_is_default && !right_is_default) return -1; + if (right_is_default && !left_is_default) return 1; + + + return 0; + }); + } + return result; + } + + public Gee.List get_video_sources() { + ArrayList pipewire_devices = new ArrayList(); + ArrayList other_devices = new ArrayList(); + + foreach (Device device in devices) { + if (device.media != "video") continue; + if (device.is_sink) continue; + + bool is_color = false; + for (int i = 0; i < device.device.caps.get_size(); i++) { + unowned Gst.Structure structure = device.device.caps.get_structure(i); + if (structure.has_field("format") && !structure.get_string("format").has_prefix("GRAY")) { + is_color = true; + } + } + + // Don't allow grey-scale devices + if (!is_color) continue; + + if (device.device.properties.has_name("pipewire-proplist")) { + pipewire_devices.add(device); + } else { + other_devices.add(device); + } + } + + // If we have any pipewire devices, present only those. Don't want duplicated devices from pipewire and video for linux. + ArrayList devices = pipewire_devices.size > 0 ? pipewire_devices : other_devices; + + // Reorder sources + devices.sort((media_left, media_right) => { + Device left = media_left as Device; + Device right = media_right as Device; + if (left == null) return 1; + if (right == null) return -1; + + int left_fps = 0; + for (int i = 0; i < left.device.caps.get_size(); i++) { + unowned Gst.Structure structure = left.device.caps.get_structure(i); + int num = 0, den = 0; + if (structure.has_field("framerate") && structure.get_fraction("framerate", out num, out den)) left_fps = int.max(left_fps, num / den); + } + + int right_fps = 0; + for (int i = 0; i < left.device.caps.get_size(); i++) { + unowned Gst.Structure structure = left.device.caps.get_structure(i); + int num = 0, den = 0; + if (structure.has_field("framerate") && structure.get_fraction("framerate", out num, out den)) right_fps = int.max(right_fps, num / den); + } + + // More FPS is better + if (left_fps > right_fps) return -1; + if (right_fps > left_fps) return 1; + + return 0; + }); + + return devices; + } + + public Device? get_preferred_device(string media, bool incoming) { + foreach (MediaDevice media_device in get_devices(media, incoming)) { + Device? device = media_device as Device; + if (device != null) return device; + } + warning("No preferred device for %s %s. Media will not be processed.", incoming ? "incoming" : "outgoing", media); + return null; + } + + public MediaDevice? get_device(Xmpp.Xep.JingleRtp.Stream stream, bool incoming) { + Stream plugin_stream = stream as Stream; + if (plugin_stream == null) return null; + if (incoming) { + return plugin_stream.output_device ?? get_preferred_device(stream.media, incoming); + } else { + return plugin_stream.input_device ?? get_preferred_device(stream.media, incoming); + } + } + + private void dump_dot() { + string name = @"pipe-$(pipe.clock.get_time())-$(pipe.current_state)"; + Gst.Debug.bin_to_dot_file(pipe, Gst.DebugGraphDetails.ALL, name); + debug("Stored pipe details as %s", name); + } + + public void set_pause(Xmpp.Xep.JingleRtp.Stream stream, bool pause) { + Stream plugin_stream = stream as Stream; + if (plugin_stream == null) return; + if (pause) { + plugin_stream.pause(); + } else { + plugin_stream.unpause(); + Timeout.add_seconds(3, () => { + dump_dot(); + return false; + }); + } + } + + public void set_device(Xmpp.Xep.JingleRtp.Stream stream, MediaDevice? device) { + Device real_device = device as Device; + Stream plugin_stream = stream as Stream; + if (real_device == null || plugin_stream == null) return; + if (real_device.is_source) { + plugin_stream.input_device = real_device; + } else if (real_device.is_sink) { + plugin_stream.output_device = real_device; + } + } +} diff --git a/plugins/rtp/src/register_plugin.vala b/plugins/rtp/src/register_plugin.vala new file mode 100644 index 00000000..a80137ea --- /dev/null +++ b/plugins/rtp/src/register_plugin.vala @@ -0,0 +1,3 @@ +public Type register_plugin(Module module) { + return typeof (Dino.Plugins.Rtp.Plugin); +} diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala new file mode 100644 index 00000000..aea2fe85 --- /dev/null +++ b/plugins/rtp/src/stream.vala @@ -0,0 +1,432 @@ +using Gee; +using Xmpp; + +public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { + public uint8 rtpid { get; private set; } + + public Plugin plugin { get; private set; } + public Gst.Pipeline pipe { get { + return plugin.pipe; + }} + public Gst.Element rtpbin { get { + return plugin.rtpbin; + }} + public CodecUtil codec_util { get { + return plugin.codec_util; + }} + private Gst.App.Sink send_rtp; + private Gst.App.Sink send_rtcp; + private Gst.App.Src recv_rtp; + private Gst.App.Src recv_rtcp; + private Gst.Element encode; + private Gst.Element decode; + private Gst.Element input; + private Gst.Element output; + + private Device _input_device; + public Device input_device { get { return _input_device; } set { + if (!paused) { + if (this._input_device != null) { + this._input_device.unlink(); + this._input_device = null; + } + set_input(value != null ? value.link_source() : null); + } + this._input_device = value; + }} + private Device _output_device; + public Device output_device { get { return _output_device; } set { + if (output != null) remove_output(output); + if (value != null) add_output(value.link_sink()); + this._output_device = value; + }} + + public bool created { get; private set; default = false; } + public bool paused { get; private set; default = false; } + private bool push_recv_data = false; + private string participant_ssrc = null; + + private Gst.Pad recv_rtcp_sink_pad; + private Gst.Pad recv_rtp_sink_pad; + private Gst.Pad recv_rtp_src_pad; + private Gst.Pad send_rtcp_src_pad; + private Gst.Pad send_rtp_sink_pad; + private Gst.Pad send_rtp_src_pad; + + public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { + base(content); + this.plugin = plugin; + this.rtpid = plugin.next_free_id(); + + content.notify["senders"].connect_after(on_senders_changed); + } + + public void on_senders_changed() { + if (sending && input == null) { + input_device = plugin.get_preferred_device(media, false); + } + if (receiving && output == null) { + output_device = plugin.get_preferred_device(media, true); + } + } + + public override void create() { + plugin.pause(); + + // Create i/o if needed + + if (input == null && input_device == null && sending) { + input_device = plugin.get_preferred_device(media, false); + } + if (output == null && output_device == null && receiving && media == "audio") { + output_device = plugin.get_preferred_device(media, true); + } + + // Create app elements + send_rtp = Gst.ElementFactory.make("appsink", @"rtp-sink-$rtpid") as Gst.App.Sink; + send_rtp.async = false; + send_rtp.caps = CodecUtil.get_caps(media, payload_type); + send_rtp.emit_signals = true; + send_rtp.sync = false; + send_rtp.new_sample.connect(on_new_sample); + pipe.add(send_rtp); + + send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp-sink-$rtpid") as Gst.App.Sink; + send_rtcp.async = false; + send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); + send_rtcp.emit_signals = true; + send_rtcp.sync = false; + send_rtcp.new_sample.connect(on_new_sample); + pipe.add(send_rtcp); + + recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp-src-$rtpid") as Gst.App.Src; + recv_rtp.caps = CodecUtil.get_caps(media, payload_type); + recv_rtp.do_timestamp = true; + recv_rtp.format = Gst.Format.TIME; + recv_rtp.is_live = true; + pipe.add(recv_rtp); + + recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp-src-$rtpid") as Gst.App.Src; + recv_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); + recv_rtcp.do_timestamp = true; + recv_rtcp.format = Gst.Format.TIME; + recv_rtcp.is_live = true; + pipe.add(recv_rtcp); + + // Connect RTCP + send_rtcp_src_pad = rtpbin.get_request_pad(@"send_rtcp_src_$rtpid"); + send_rtcp_src_pad.link(send_rtcp.get_static_pad("sink")); + recv_rtcp_sink_pad = rtpbin.get_request_pad(@"recv_rtcp_sink_$rtpid"); + recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad); + + // Connect input + encode = codec_util.get_encode_bin(media, payload_type, @"encode-$rtpid"); + pipe.add(encode); + send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid"); + encode.get_static_pad("src").link(send_rtp_sink_pad); + if (input != null) { + input.link(encode); + } + + // Connect output + decode = codec_util.get_decode_bin(media, payload_type, @"decode-$rtpid"); + pipe.add(decode); + if (output != null) { + decode.link(output); + } + + // Connect RTP + recv_rtp_sink_pad = rtpbin.get_request_pad(@"recv_rtp_sink_$rtpid"); + recv_rtp.get_static_pad("src").link(recv_rtp_sink_pad); + + created = true; + push_recv_data = true; + plugin.unpause(); + } + + private Gst.FlowReturn on_new_sample(Gst.App.Sink sink) { + if (sink == null) { + debug("Sink is null"); + return Gst.FlowReturn.EOS; + } + Gst.Sample sample = sink.pull_sample(); + Gst.Buffer buffer = sample.get_buffer(); + uint8[] data; + buffer.extract_dup(0, buffer.get_size(), out data); + if (sink == send_rtp) { + on_send_rtp_data(new Bytes.take(data)); + } else if (sink == send_rtcp) { + on_send_rtcp_data(new Bytes.take(data)); + } else { + warning("unknown sample"); + } + return Gst.FlowReturn.OK; + } + + private static Gst.PadProbeReturn drop_probe() { + return Gst.PadProbeReturn.DROP; + } + + public override void destroy() { + // Stop network communication + push_recv_data = false; + recv_rtp.end_of_stream(); + recv_rtcp.end_of_stream(); + send_rtp.new_sample.disconnect(on_new_sample); + send_rtcp.new_sample.disconnect(on_new_sample); + + // Disconnect input device + if (input != null) { + input.unlink(encode); + input = null; + } + if (this._input_device != null) { + if (!paused) this._input_device.unlink(); + this._input_device = null; + } + + // Disconnect encode + encode.set_locked_state(true); + encode.set_state(Gst.State.NULL); + encode.get_static_pad("src").unlink(send_rtp_sink_pad); + pipe.remove(encode); + encode = null; + + // Disconnect RTP sending + if (send_rtp_src_pad != null) { + send_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe); + send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink")); + } + send_rtp.set_locked_state(true); + send_rtp.set_state(Gst.State.NULL); + pipe.remove(send_rtp); + send_rtp = null; + + // Disconnect decode + if (recv_rtp_src_pad != null) { + recv_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe); + recv_rtp_src_pad.unlink(decode.get_static_pad("sink")); + } + + // Disconnect RTP receiving + recv_rtp.set_locked_state(true); + recv_rtp.set_state(Gst.State.NULL); + recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad); + pipe.remove(recv_rtp); + recv_rtp = null; + + // Disconnect output + if (output != null) { + decode.unlink(output); + } + decode.set_locked_state(true); + decode.set_state(Gst.State.NULL); + pipe.remove(decode); + decode = null; + output = null; + + // Disconnect output device + if (this._output_device != null) { + this._output_device.unlink(); + this._output_device = null; + } + + // Disconnect RTCP receiving + recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad); + recv_rtcp.set_locked_state(true); + recv_rtcp.set_state(Gst.State.NULL); + pipe.remove(recv_rtcp); + recv_rtcp = null; + + // Disconnect RTCP sending + send_rtcp_src_pad.unlink(send_rtcp.get_static_pad("sink")); + send_rtcp.set_locked_state(true); + send_rtcp.set_state(Gst.State.NULL); + pipe.remove(send_rtcp); + send_rtcp = null; + + // Release rtp pads + rtpbin.release_request_pad(send_rtp_sink_pad); + send_rtp_sink_pad = null; + rtpbin.release_request_pad(recv_rtp_sink_pad); + recv_rtp_sink_pad = null; + rtpbin.release_request_pad(recv_rtcp_sink_pad); + recv_rtcp_sink_pad = null; + rtpbin.release_request_pad(send_rtcp_src_pad); + send_rtcp_src_pad = null; + send_rtp_src_pad = null; + recv_rtp_src_pad = null; + } + + public override void on_recv_rtp_data(Bytes bytes) { + if (push_recv_data) { + recv_rtp.push_buffer(new Gst.Buffer.wrapped_bytes(bytes)); + } + } + + public override void on_recv_rtcp_data(Bytes bytes) { + if (push_recv_data) { + recv_rtcp.push_buffer(new Gst.Buffer.wrapped_bytes(bytes)); + } + } + + public override void on_rtp_ready() { + // If full frame has been sent before the connection was ready, the counterpart would only display our video after the next full frame. + // Send a full frame to let the counterpart display our video asap + rtpbin.send_event(new Gst.Event.custom( + Gst.EventType.CUSTOM_UPSTREAM, + new Gst.Structure("GstForceKeyUnit", "all-headers", typeof(bool), true, null)) + ); + } + + public override void on_rtcp_ready() { + int rtp_session_id = (int) rtpid; + uint64 max_delay = int.MAX; + Object rtp_session; + bool rtp_sent; + GLib.Signal.emit_by_name(rtpbin, "get-internal-session", rtp_session_id, out rtp_session); + GLib.Signal.emit_by_name(rtp_session, "send-rtcp-full", max_delay, out rtp_sent); + debug("RTCP is ready, resending rtcp: %s", rtp_sent.to_string()); + } + + public void on_ssrc_pad_added(string ssrc, Gst.Pad pad) { + participant_ssrc = ssrc; + recv_rtp_src_pad = pad; + if (decode != null) { + plugin.pause(); + debug("Link %s to %s decode for %s", recv_rtp_src_pad.name, media, name); + recv_rtp_src_pad.link(decode.get_static_pad("sink")); + plugin.unpause(); + } + } + + public void on_send_rtp_src_added(Gst.Pad pad) { + send_rtp_src_pad = pad; + if (send_rtp != null) { + plugin.pause(); + debug("Link %s to %s send_rtp for %s", send_rtp_src_pad.name, media, name); + send_rtp_src_pad.link(send_rtp.get_static_pad("sink")); + plugin.unpause(); + } + } + + public void set_input(Gst.Element? input) { + set_input_and_pause(input, paused); + } + + private void set_input_and_pause(Gst.Element? input, bool paused) { + if (created && this.input != null) { + this.input.unlink(encode); + this.input = null; + } + + this.input = input; + this.paused = paused; + + if (created && sending && !paused && input != null) { + plugin.pause(); + input.link(encode); + plugin.unpause(); + } + } + + public void pause() { + if (paused) return; + set_input_and_pause(null, true); + if (input_device != null) input_device.unlink(); + } + + public void unpause() { + if (!paused) return; + set_input_and_pause(input_device != null ? input_device.link_source() : null, false); + } + + ulong block_probe_handler_id = 0; + public virtual void add_output(Gst.Element element) { + if (output != null) { + critical("add_output() invoked more than once"); + return; + } + this.output = element; + if (created) { + plugin.pause(); + decode.link(element); + if (block_probe_handler_id != 0) { + decode.get_static_pad("src").remove_probe(block_probe_handler_id); + } + plugin.unpause(); + } + } + + public virtual void remove_output(Gst.Element element) { + if (output != element) { + critical("remove_output() invoked without prior add_output()"); + return; + } + if (created) { + block_probe_handler_id = decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe); + decode.unlink(element); + } + if (this._output_device != null) { + this._output_device.unlink(); + this._output_device = null; + } + this.output = null; + } +} + +public class Dino.Plugins.Rtp.VideoStream : Stream { + private Gee.List outputs = new ArrayList(); + private Gst.Element output_tee; + + public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { + base(plugin, content); + if (media != "video") critical("VideoStream created for non-video media"); + } + + public override void create() { + plugin.pause(); + output_tee = Gst.ElementFactory.make("tee", null); + output_tee.@set("allow-not-linked", true); + pipe.add(output_tee); + add_output(output_tee); + base.create(); + foreach (Gst.Element output in outputs) { + output_tee.link(output); + } + plugin.unpause(); + } + + public override void destroy() { + foreach (Gst.Element output in outputs) { + output_tee.unlink(output); + } + base.destroy(); + output_tee.set_locked_state(true); + output_tee.set_state(Gst.State.NULL); + pipe.remove(output_tee); + output_tee = null; + } + + public override void add_output(Gst.Element element) { + if (element == output_tee) { + base.add_output(element); + return; + } + outputs.add(element); + if (output_tee != null) { + output_tee.link(element); + } + } + + public override void remove_output(Gst.Element element) { + if (element == output_tee) { + base.remove_output(element); + return; + } + outputs.remove(element); + if (output_tee != null) { + output_tee.unlink(element); + } + } +} \ No newline at end of file diff --git a/plugins/rtp/src/video_widget.vala b/plugins/rtp/src/video_widget.vala new file mode 100644 index 00000000..fa5ba138 --- /dev/null +++ b/plugins/rtp/src/video_widget.vala @@ -0,0 +1,110 @@ +public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidget { + private static uint last_id = 0; + + public uint id { get; private set; } + public Gst.Element element { get; private set; } + public Gtk.Widget widget { get; private set; } + + public Plugin plugin { get; private set; } + public Gst.Pipeline pipe { get { + return plugin.pipe; + }} + + private bool attached; + private Device? connected_device; + private Stream? connected_stream; + private Gst.Element convert; + + public VideoWidget(Plugin plugin) { + this.plugin = plugin; + + id = last_id++; + element = Gst.ElementFactory.make("gtksink", @"video-widget-$id"); + if (element != null) { + Gtk.Widget widget; + element.@get("widget", out widget); + element.@set("async", false); + element.@set("sync", false); + this.widget = widget; + add(widget); + widget.visible = true; + + // Listen for resolution changes + element.get_static_pad("sink").notify["caps"].connect(() => { + if (element.get_static_pad("sink").caps == null) return; + + int width, height; + element.get_static_pad("sink").caps.get_structure(0).get_int("width", out width); + element.get_static_pad("sink").caps.get_structure(0).get_int("height", out height); + resolution_changed(width, height); + }); + } else { + warning("Could not create GTK video sink. Won't display videos."); + } + } + + public void display_stream(Xmpp.Xep.JingleRtp.Stream stream) { + if (element == null) return; + detach(); + if (stream.media != "video") return; + connected_stream = stream as Stream; + if (connected_stream == null) return; + plugin.pause(); + pipe.add(element); + convert = Gst.parse_bin_from_description(@"videoconvert name=video-widget-$id-convert", true); + convert.name = @"video-widget-$id-prepare"; + pipe.add(convert); + convert.link(element); + connected_stream.add_output(convert); + element.set_locked_state(false); + plugin.unpause(); + attached = true; + } + + public void display_device(MediaDevice media_device) { + if (element == null) return; + detach(); + connected_device = media_device as Device; + if (connected_device == null) return; + plugin.pause(); + pipe.add(element); + convert = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video-widget-$id-flip ! videoconvert name=video-widget-$id-convert", true); + convert.name = @"video-widget-$id-prepare"; + pipe.add(convert); + convert.link(element); + connected_device.link_source().link(convert); + element.set_locked_state(false); + plugin.unpause(); + attached = true; + } + + public void detach() { + if (element == null) return; + if (attached) { + if (connected_stream != null) { + connected_stream.remove_output(convert); + connected_stream = null; + } + if (connected_device != null) { + connected_device.link_source().unlink(element); + connected_device.unlink(); // We get a new ref to recover the element, so unlink twice + connected_device.unlink(); + connected_device = null; + } + convert.set_locked_state(true); + convert.set_state(Gst.State.NULL); + pipe.remove(convert); + convert = null; + element.set_locked_state(true); + element.set_state(Gst.State.NULL); + pipe.remove(element); + attached = false; + } + } + + public override void dispose() { + detach(); + widget = null; + element = null; + } +} \ No newline at end of file From cdb4d77259e6c361aaca64a483a43d7441f4803d Mon Sep 17 00:00:00 2001 From: fiaxh Date: Fri, 19 Mar 2021 23:07:40 +0100 Subject: [PATCH 10/56] Add support for unencrypted RTP calls to libdino Co-authored-by: Marvin W --- libdino/CMakeLists.txt | 3 + libdino/src/application.vala | 2 + libdino/src/entity/call.vala | 126 +++++ libdino/src/plugin/interfaces.vala | 25 + libdino/src/plugin/loader.vala | 2 +- libdino/src/plugin/registry.vala | 1 + libdino/src/service/call_store.vala | 61 +++ libdino/src/service/calls.vala | 514 ++++++++++++++++++ libdino/src/service/content_item_store.vala | 33 +- libdino/src/service/database.vala | 24 +- .../src/service/jingle_file_transfers.vala | 7 +- libdino/src/service/message_processor.vala | 2 +- libdino/src/service/module_manager.vala | 1 + libdino/src/service/notification_events.vala | 20 + 14 files changed, 814 insertions(+), 7 deletions(-) create mode 100644 libdino/src/entity/call.vala create mode 100644 libdino/src/service/call_store.vala create mode 100644 libdino/src/service/calls.vala diff --git a/libdino/CMakeLists.txt b/libdino/CMakeLists.txt index 90efcc73..d7f7583c 100644 --- a/libdino/CMakeLists.txt +++ b/libdino/CMakeLists.txt @@ -15,6 +15,7 @@ SOURCES src/dbus/upower.vala src/entity/account.vala + src/entity/call.vala src/entity/conversation.vala src/entity/encryption.vala src/entity/file_transfer.vala @@ -27,6 +28,8 @@ SOURCES src/service/avatar_manager.vala src/service/blocking_manager.vala + src/service/call_store.vala + src/service/calls.vala src/service/chat_interaction.vala src/service/connection_manager.vala src/service/content_item_store.vala diff --git a/libdino/src/application.vala b/libdino/src/application.vala index c1fd7e39..f381c21d 100644 --- a/libdino/src/application.vala +++ b/libdino/src/application.vala @@ -39,6 +39,8 @@ public interface Application : GLib.Application { AvatarManager.start(stream_interactor, db); RosterManager.start(stream_interactor, db); FileManager.start(stream_interactor, db); + Calls.start(stream_interactor, db); + CallStore.start(stream_interactor, db); ContentItemStore.start(stream_interactor, db); ChatInteraction.start(stream_interactor); NotificationEvents.start(stream_interactor); diff --git a/libdino/src/entity/call.vala b/libdino/src/entity/call.vala new file mode 100644 index 00000000..b836e2cd --- /dev/null +++ b/libdino/src/entity/call.vala @@ -0,0 +1,126 @@ +using Xmpp; + +namespace Dino.Entities { + + public class Call : Object { + + public const bool DIRECTION_OUTGOING = true; + public const bool DIRECTION_INCOMING = false; + + public enum State { + RINGING, + ESTABLISHING, + IN_PROGRESS, + OTHER_DEVICE_ACCEPTED, + ENDED, + DECLINED, + MISSED, + FAILED + } + + public int id { get; set; default=-1; } + public Account account { get; set; } + public Jid counterpart { get; set; } + public Jid ourpart { get; set; } + public Jid? from { + get { return direction == DIRECTION_OUTGOING ? ourpart : counterpart; } + } + public Jid? to { + get { return direction == DIRECTION_OUTGOING ? counterpart : ourpart; } + } + public bool direction { get; set; } + public DateTime time { get; set; } + public DateTime local_time { get; set; } + public DateTime end_time { get; set; } + + public State state { get; set; } + + private Database? db; + + public Call.from_row(Database db, Qlite.Row row) throws InvalidJidError { + this.db = db; + + id = row[db.call.id]; + account = db.get_account_by_id(row[db.call.account_id]); + + counterpart = db.get_jid_by_id(row[db.call.counterpart_id]); + string counterpart_resource = row[db.call.counterpart_resource]; + if (counterpart_resource != null) counterpart = counterpart.with_resource(counterpart_resource); + + string our_resource = row[db.call.our_resource]; + if (our_resource != null) { + ourpart = account.bare_jid.with_resource(our_resource); + } else { + ourpart = account.bare_jid; + } + direction = row[db.call.direction]; + time = new DateTime.from_unix_utc(row[db.call.time]); + local_time = new DateTime.from_unix_utc(row[db.call.local_time]); + end_time = new DateTime.from_unix_utc(row[db.call.end_time]); + state = (State) row[db.call.state]; + + notify.connect(on_update); + } + + public void persist(Database db) { + if (id != -1) return; + + this.db = db; + Qlite.InsertBuilder builder = db.call.insert() + .value(db.call.account_id, account.id) + .value(db.call.counterpart_id, db.get_jid_id(counterpart)) + .value(db.call.counterpart_resource, counterpart.resourcepart) + .value(db.call.our_resource, ourpart.resourcepart) + .value(db.call.direction, direction) + .value(db.call.time, (long) time.to_unix()) + .value(db.call.local_time, (long) local_time.to_unix()) + .value(db.call.state, State.ENDED); // No point in persisting states that can't survive a restart + if (end_time != null) { + builder.value(db.call.end_time, (long) end_time.to_unix()); + } + id = (int) builder.perform(); + + notify.connect(on_update); + } + + public bool equals(Call c) { + return equals_func(this, c); + } + + public static bool equals_func(Call c1, Call c2) { + if (c1.id == c2.id) { + return true; + } + return false; + } + + public static uint hash_func(Call call) { + return (uint)call.id; + } + + private void on_update(Object o, ParamSpec sp) { + Qlite.UpdateBuilder update_builder = db.call.update().with(db.call.id, "=", id); + switch (sp.name) { + case "counterpart": + update_builder.set(db.call.counterpart_id, db.get_jid_id(counterpart)); + update_builder.set(db.call.counterpart_resource, counterpart.resourcepart); break; + case "ourpart": + update_builder.set(db.call.our_resource, ourpart.resourcepart); break; + case "direction": + update_builder.set(db.call.direction, direction); break; + case "time": + update_builder.set(db.call.time, (long) time.to_unix()); break; + case "local-time": + update_builder.set(db.call.local_time, (long) local_time.to_unix()); break; + case "end-time": + update_builder.set(db.call.end_time, (long) end_time.to_unix()); break; + case "state": + // No point in persisting states that can't survive a restart + if (state == State.RINGING || state == State.ESTABLISHING || state == State.IN_PROGRESS) return; + update_builder.set(db.call.state, state); + break; + } + update_builder.perform(); + } + } +} diff --git a/libdino/src/plugin/interfaces.vala b/libdino/src/plugin/interfaces.vala index dab058af..8be77895 100644 --- a/libdino/src/plugin/interfaces.vala +++ b/libdino/src/plugin/interfaces.vala @@ -84,6 +84,31 @@ public abstract interface ConversationAdditionPopulator : ConversationItemPopula public virtual void populate_timespan(Conversation conversation, DateTime from, DateTime to) { } } +public abstract interface VideoCallPlugin : Object { + // Video widget + public abstract VideoCallWidget? create_widget(WidgetType type); + + // Devices + public signal void devices_changed(string media, bool incoming); + public abstract Gee.List get_devices(string media, bool incoming); + public abstract MediaDevice? get_device(Xmpp.Xep.JingleRtp.Stream stream, bool incoming); + public abstract void set_pause(Xmpp.Xep.JingleRtp.Stream stream, bool pause); + public abstract void set_device(Xmpp.Xep.JingleRtp.Stream stream, MediaDevice? device); +} + +public abstract interface VideoCallWidget : Object { + public signal void resolution_changed(uint width, uint height); + public abstract void display_stream(Xmpp.Xep.JingleRtp.Stream stream); // TODO: Multi participant + public abstract void display_device(MediaDevice device); + public abstract void detach(); +} + +public abstract interface MediaDevice : Object { + public abstract string id { get; } + public abstract string display_name { get; } + public abstract string detail_name { get; } +} + public abstract interface NotificationPopulator : Object { public abstract string id { get; } public abstract void init(Conversation conversation, NotificationCollection summary, WidgetType type); diff --git a/libdino/src/plugin/loader.vala b/libdino/src/plugin/loader.vala index 102bf3f9..8b0d93ad 100644 --- a/libdino/src/plugin/loader.vala +++ b/libdino/src/plugin/loader.vala @@ -26,7 +26,7 @@ public class Loader : Object { this.search_paths = app.search_path_generator.get_plugin_paths(); } - public void loadAll() throws Error { + public void load_all() throws Error { if (Module.supported() == false) { throw new Error(-1, 0, "Plugins are not supported"); } diff --git a/libdino/src/plugin/registry.vala b/libdino/src/plugin/registry.vala index e3f73855..27d72b80 100644 --- a/libdino/src/plugin/registry.vala +++ b/libdino/src/plugin/registry.vala @@ -12,6 +12,7 @@ public class Registry { internal Gee.Collection conversation_titlebar_entries = new Gee.TreeSet((a, b) => { return (int)(a.order - b.order); }); + public VideoCallPlugin? video_call_plugin; public bool register_encryption_list_entry(EncryptionListEntry entry) { lock(encryption_list_entries) { diff --git a/libdino/src/service/call_store.vala b/libdino/src/service/call_store.vala new file mode 100644 index 00000000..fa6e63ee --- /dev/null +++ b/libdino/src/service/call_store.vala @@ -0,0 +1,61 @@ +using Xmpp; +using Gee; +using Qlite; + +using Dino.Entities; + +namespace Dino { + + public class CallStore : StreamInteractionModule, Object { + public static ModuleIdentity IDENTITY = new ModuleIdentity("call_store"); + public string id { get { return IDENTITY.id; } } + + private StreamInteractor stream_interactor; + private Database db; + + private WeakMap calls_by_db_id = new WeakMap(); + + public static void start(StreamInteractor stream_interactor, Database db) { + CallStore m = new CallStore(stream_interactor, db); + stream_interactor.add_module(m); + } + + private CallStore(StreamInteractor stream_interactor, Database db) { + this.stream_interactor = stream_interactor; + this.db = db; + } + + public void add_call(Call call, Conversation conversation) { + call.persist(db); + cache_call(call); + } + + public Call? get_call_by_id(int id) { + Call? call = calls_by_db_id[id]; + if (call != null) { + return call; + } + + RowOption row_option = db.call.select().with(db.call.id, "=", id).row(); + + return create_call_from_row_opt(row_option); + } + + private Call? create_call_from_row_opt(RowOption row_opt) { + if (!row_opt.is_present()) return null; + + try { + Call call = new Call.from_row(db, row_opt.inner); + cache_call(call); + return call; + } catch (InvalidJidError e) { + warning("Got message with invalid Jid: %s", e.message); + } + return null; + } + + private void cache_call(Call call) { + calls_by_db_id[call.id] = call; + } + } +} \ No newline at end of file diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala new file mode 100644 index 00000000..5224bdd1 --- /dev/null +++ b/libdino/src/service/calls.vala @@ -0,0 +1,514 @@ +using Gee; + +using Xmpp; +using Dino.Entities; + +namespace Dino { + + public class Calls : StreamInteractionModule, Object { + + public signal void call_incoming(Call call, Conversation conversation, bool video); + public signal void call_outgoing(Call call, Conversation conversation); + + public signal void call_terminated(Call call, string? reason_name, string? reason_text); + public signal void counterpart_ringing(Call call); + public signal void counterpart_sends_video_updated(Call call, bool mute); + public signal void info_received(Call call, Xep.JingleRtp.CallSessionInfo session_info); + + public signal void stream_created(Call call, string media); + + public static ModuleIdentity IDENTITY = new ModuleIdentity("calls"); + public string id { get { return IDENTITY.id; } } + + private StreamInteractor stream_interactor; + private Xep.JingleRtp.SessionInfoType session_info_type; + + private HashMap> sid_by_call = new HashMap>(Account.hash_func, Account.equals_func); + private HashMap> call_by_sid = new HashMap>(Account.hash_func, Account.equals_func); + public HashMap sessions = new HashMap(Call.hash_func, Call.equals_func); + + public Call? mi_accepted_call = null; + public string? mi_accepted_sid = null; + public bool mi_accepted_video = false; + + private HashMap counterpart_sends_video = new HashMap(Call.hash_func, Call.equals_func); + private HashMap we_should_send_video = new HashMap(Call.hash_func, Call.equals_func); + private HashMap we_should_send_audio = new HashMap(Call.hash_func, Call.equals_func); + + private HashMap audio_content_parameter = new HashMap(Call.hash_func, Call.equals_func); + private HashMap video_content_parameter = new HashMap(Call.hash_func, Call.equals_func); + private HashMap video_content = new HashMap(Call.hash_func, Call.equals_func); + + public static void start(StreamInteractor stream_interactor, Database db) { + Calls m = new Calls(stream_interactor, db); + stream_interactor.add_module(m); + } + + private Calls(StreamInteractor stream_interactor, Database db) { + this.stream_interactor = stream_interactor; + + stream_interactor.account_added.connect(on_account_added); + } + + public Xep.JingleRtp.Stream? get_video_stream(Call call) { + if (video_content_parameter.has_key(call)) { + return video_content_parameter[call].stream; + } + return null; + } + + public Xep.JingleRtp.Stream? get_audio_stream(Call call) { + if (audio_content_parameter.has_key(call)) { + return audio_content_parameter[call].stream; + } + return null; + } + + public async Call? initiate_call(Conversation conversation, bool video) { + Call call = new Call(); + call.direction = Call.DIRECTION_OUTGOING; + call.account = conversation.account; + call.counterpart = conversation.counterpart; + call.ourpart = conversation.account.full_jid; + call.time = call.local_time = new DateTime.now_utc(); + call.state = Call.State.RINGING; + + stream_interactor.get_module(CallStore.IDENTITY).add_call(call, conversation); + + XmppStream? stream = stream_interactor.get_stream(conversation.account); + if (stream == null) return null; + + Gee.List call_resources = yield get_call_resources(conversation); + if (call_resources.size > 0) { + Jid full_jid = call_resources[0]; + Xep.Jingle.Session session = yield stream.get_module(Xep.JingleRtp.Module.IDENTITY).start_call(stream, full_jid, video); + sessions[call] = session; + call_by_sid[call.account][session.sid] = call; + sid_by_call[call.account][call] = session.sid; + + connect_session_signals(call, session); + } + + we_should_send_video[call] = video; + we_should_send_audio[call] = true; + + conversation.last_active = call.time; + call_outgoing(call, conversation); + + return call; + } + + public void end_call(Conversation conversation, Call call) { + XmppStream? stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + if (call.state == Call.State.IN_PROGRESS || call.state == Call.State.ESTABLISHING) { + sessions[call].terminate(Xep.Jingle.ReasonElement.SUCCESS, null, "success"); + call.state = Call.State.ENDED; + } else if (call.state == Call.State.RINGING) { + if (sessions.has_key(call)) { + sessions[call].terminate(Xep.Jingle.ReasonElement.CANCEL, null, "cancel"); + } else { + // Only a JMI so far + } + call.state = Call.State.MISSED; + } else { + return; + } + + call.end_time = new DateTime.now_utc(); + + remove_call_from_datastructures(call); + } + + public void accept_call(Call call) { + call.state = Call.State.ESTABLISHING; + + if (sessions.has_key(call)) { + foreach (Xep.Jingle.Content content in sessions[call].contents.values) { + content.accept(); + } + } else { + // Only a JMI so far + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + mi_accepted_call = call; + mi_accepted_sid = sid_by_call[call.account][call]; + mi_accepted_video = we_should_send_video[call]; + + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_accept_to_self(stream, mi_accepted_sid); + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_proceed_to_peer(stream, call.counterpart, mi_accepted_sid); + } + } + + public void reject_call(Call call) { + call.state = Call.State.DECLINED; + + if (sessions.has_key(call)) { + foreach (Xep.Jingle.Content content in sessions[call].contents.values) { + content.reject(); + } + remove_call_from_datastructures(call); + } else { + // Only a JMI so far + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + string sid = sid_by_call[call.account][call]; + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_reject_to_self(stream, sid); + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_reject_to_peer(stream, call.counterpart, sid); + remove_call_from_datastructures(call); + } + } + + public void mute_own_audio(Call call, bool mute) { + we_should_send_audio[call] = !mute; + + Xep.JingleRtp.Stream stream = audio_content_parameter[call].stream; + // The user might mute audio before a feed was created. The feed will be muted as soon as it has been created. + if (stream == null) return; + + // Inform our counterpart that we (un)muted our audio + stream_interactor.module_manager.get_module(call.account, Xep.JingleRtp.Module.IDENTITY).session_info_type.send_mute(sessions[call], mute, "audio"); + + // Start/Stop sending audio data + Application.get_default().plugin_registry.video_call_plugin.set_pause(stream, mute); + } + + public void mute_own_video(Call call, bool mute) { + we_should_send_video[call] = !mute; + + Xep.JingleRtp.Module rtp_module = stream_interactor.module_manager.get_module(call.account, Xep.JingleRtp.Module.IDENTITY); + + if (video_content_parameter.has_key(call) && + video_content_parameter[call].stream != null && + sessions[call].senders_include_us(video_content[call].senders)) { + // A video feed has already been established + + // Start/Stop sending video data + Xep.JingleRtp.Stream stream = video_content_parameter[call].stream; + if (stream != null) { + // TODO maybe the user muted video before the feed was created... + Application.get_default().plugin_registry.video_call_plugin.set_pause(stream, mute); + } + + // Inform our counterpart that we started/stopped our video + rtp_module.session_info_type.send_mute(sessions[call], mute, "video"); + } else if (!mute) { + // Need to start a new video feed + XmppStream stream = stream_interactor.get_stream(call.account); + rtp_module.add_outgoing_video_content.begin(stream, sessions[call], (_, res) => { + if (video_content_parameter[call] == null) { + Xep.Jingle.Content content = rtp_module.add_outgoing_video_content.end(res); + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + if (rtp_content_parameter != null) { + connect_content_signals(call, content, rtp_content_parameter); + } + } + }); + } + // If video_feed == null && !mute we're trying to mute a non-existant feed. It will be muted as soon as it is created. + } + + public async Gee.List get_call_resources(Conversation conversation) { + ArrayList ret = new ArrayList(); + + XmppStream? stream = stream_interactor.get_stream(conversation.account); + if (stream == null) return ret; + + Gee.List? full_jids = stream.get_flag(Presence.Flag.IDENTITY).get_resources(conversation.counterpart); + if (full_jids == null) return ret; + + foreach (Jid full_jid in full_jids) { + bool supports_rtc = yield stream.get_module(Xep.JingleRtp.Module.IDENTITY).is_available(stream, full_jid); + if (!supports_rtc) continue; + + // dtls support indicates webRTC support. Clients tend to not do normal ice udp in that case. Except Dino. + bool supports_dtls = yield stream_interactor.get_module(EntityInfo.IDENTITY).has_feature(conversation.account, full_jid, "urn:xmpp:jingle:apps:dtls:0"); + if (supports_dtls) { + Xep.ServiceDiscovery.Identity? identity = yield stream_interactor.get_module(EntityInfo.IDENTITY).get_identity(conversation.account, full_jid); + bool is_dino = identity != null && identity.name == "Dino"; + + if (!is_dino) continue; + } + + ret.add(full_jid); + } + return ret; + } + + public bool should_we_send_video(Call call) { + return we_should_send_video[call]; + } + + public Jid? is_call_in_progress() { + foreach (Call call in sessions.keys) { + if (call.state == Call.State.IN_PROGRESS || call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { + return call.counterpart; + } + } + return null; + } + + private void on_incoming_call(Account account, Xep.Jingle.Session session) { + bool counterpart_wants_video = false; + foreach (Xep.Jingle.Content content in session.contents.values) { + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + if (rtp_content_parameter == null) continue; + if (rtp_content_parameter.media == "video" && session.senders_include_us(content.senders)) { + counterpart_wants_video = true; + } + } + + // Session might have already been accepted via Jingle Message Initiation + bool already_accepted = mi_accepted_sid == session.sid && mi_accepted_call.account.equals(account) && + mi_accepted_call.counterpart.equals_bare(session.peer_full_jid) && + mi_accepted_video == counterpart_wants_video; + + Call? call = null; + if (already_accepted) { + call = mi_accepted_call; + } else { + call = create_received_call(account, session.peer_full_jid, account.full_jid, counterpart_wants_video); + } + sessions[call] = session; + + call_by_sid[account][session.sid] = call; + sid_by_call[account][call] = session.sid; + + connect_session_signals(call, session); + + if (already_accepted) { + accept_call(call); + } else { + stream_interactor.module_manager.get_module(account, Xep.JingleRtp.Module.IDENTITY).session_info_type.send_ringing(session); + } + } + + private Call create_received_call(Account account, Jid from, Jid to, bool video_requested) { + Call call = new Call(); + if (from.equals_bare(account.bare_jid)) { + // Call requested by another of our devices + call.direction = Call.DIRECTION_OUTGOING; + call.ourpart = from; + call.counterpart = to; + } else { + call.direction = Call.DIRECTION_INCOMING; + call.ourpart = account.full_jid; + call.counterpart = from; + } + call.account = account; + call.time = call.local_time = new DateTime.now_utc(); + call.state = Call.State.RINGING; + + Conversation conversation = stream_interactor.get_module(ConversationManager.IDENTITY).create_conversation(call.counterpart.bare_jid, account, Conversation.Type.CHAT); + + stream_interactor.get_module(CallStore.IDENTITY).add_call(call, conversation); + + conversation.last_active = call.time; + + we_should_send_video[call] = video_requested; + we_should_send_audio[call] = true; + + if (call.direction == Call.DIRECTION_INCOMING) { + call_incoming(call, conversation, video_requested); + } else { + call_outgoing(call, conversation); + } + + return call; + } + + private void on_incoming_content_add(XmppStream stream, Call call, Xep.Jingle.Session session, Xep.Jingle.Content content) { + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + + if (rtp_content_parameter == null || session.senders_include_us(content.senders)) { + content.reject(); + return; + } + + connect_content_signals(call, content, rtp_content_parameter); + content.accept(); + } + + private void on_connection_ready(Call call) { + if (call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { + call.state = Call.State.IN_PROGRESS; + } + } + + private void on_call_terminated(Call call, bool we_terminated, string? reason_name, string? reason_text) { + if (call.state == Call.State.RINGING || call.state == Call.State.IN_PROGRESS || call.state == Call.State.ESTABLISHING) { + call.end_time = new DateTime.now_utc(); + } + if (call.state == Call.State.IN_PROGRESS) { + call.state = Call.State.ENDED; + call_terminated(call, reason_name, reason_text); + } else if (call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { + if (reason_name == Xep.Jingle.ReasonElement.DECLINE) { + call.state = Call.State.DECLINED; + } else { + call.state = Call.State.FAILED; + } + call_terminated(call, reason_name, reason_text); + } + + remove_call_from_datastructures(call); + } + + private void on_stream_created(Call call, string media, Xep.JingleRtp.Stream stream) { + if (media == "video" && stream.receiving) { + counterpart_sends_video[call] = true; + video_content_parameter[call].connection_ready.connect((status) => { + counterpart_sends_video_updated(call, false); + }); + } + stream_created(call, media); + + // Outgoing audio/video might have been muted in the meanwhile. + if (media == "video" && !we_should_send_video[call]) { + mute_own_video(call, true); + } else if (media == "audio" && !we_should_send_audio[call]) { + mute_own_audio(call, true); + } + } + + private void on_counterpart_mute_update(Call call, bool mute, string? media) { + if (!call.equals(call)) return; + + if (media == "video") { + counterpart_sends_video[call] = !mute; + counterpart_sends_video_updated(call, mute); + } + } + + private void connect_session_signals(Call call, Xep.Jingle.Session session) { + session.terminated.connect((stream, we_terminated, reason_name, reason_text) => + on_call_terminated(call, we_terminated, reason_name, reason_text) + ); + session.additional_content_add_incoming.connect((session,stream, content) => + on_incoming_content_add(stream, call, session, content) + ); + + foreach (Xep.Jingle.Content content in session.contents.values) { + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + if (rtp_content_parameter == null) continue; + + connect_content_signals(call, content, rtp_content_parameter); + } + } + + private void connect_content_signals(Call call, Xep.Jingle.Content content, Xep.JingleRtp.Parameters rtp_content_parameter) { + if (rtp_content_parameter.media == "audio") { + audio_content_parameter[call] = rtp_content_parameter; + } else if (rtp_content_parameter.media == "video") { + video_content[call] = content; + video_content_parameter[call] = rtp_content_parameter; + } + + rtp_content_parameter.stream_created.connect((stream) => on_stream_created(call, rtp_content_parameter.media, stream)); + rtp_content_parameter.connection_ready.connect((status) => on_connection_ready(call)); + + content.senders_modify_incoming.connect((content, proposed_senders) => { + if (content.session.senders_include_us(content.senders) != content.session.senders_include_us(proposed_senders)) { + warning("counterpart set us to (not)sending %s. ignoring", content.content_name); + return; + } + + if (!content.session.senders_include_counterpart(content.senders) && content.session.senders_include_counterpart(proposed_senders)) { + // Counterpart wants to start sending. Ok. + content.accept_content_modify(proposed_senders); + on_counterpart_mute_update(call, false, "video"); + } + }); + } + + private void remove_call_from_datastructures(Call call) { + string? sid = sid_by_call[call.account][call]; + sid_by_call[call.account].unset(call); + if (sid != null) call_by_sid[call.account].unset(sid); + + sessions.unset(call); + + counterpart_sends_video.unset(call); + we_should_send_video.unset(call); + we_should_send_audio.unset(call); + + audio_content_parameter.unset(call); + video_content_parameter.unset(call); + video_content.unset(call); + } + + private void on_account_added(Account account) { + call_by_sid[account] = new HashMap(); + sid_by_call[account] = new HashMap(); + + Xep.Jingle.Module jingle_module = stream_interactor.module_manager.get_module(account, Xep.Jingle.Module.IDENTITY); + jingle_module.session_initiate_received.connect((stream, session) => { + foreach (Xep.Jingle.Content content in session.contents.values) { + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + if (rtp_content_parameter != null) { + on_incoming_call(account, session); + break; + } + } + }); + + var session_info_type = stream_interactor.module_manager.get_module(account, Xep.JingleRtp.Module.IDENTITY).session_info_type; + session_info_type.mute_update_received.connect((session,mute, name) => { + if (!call_by_sid[account].has_key(session.sid)) return; + Call call = call_by_sid[account][session.sid]; + + foreach (Xep.Jingle.Content content in session.contents.values) { + if (name == null || content.content_name == name) { + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + if (rtp_content_parameter != null) { + on_counterpart_mute_update(call, mute, rtp_content_parameter.media); + } + } + } + }); + session_info_type.info_received.connect((session, session_info) => { + if (!call_by_sid[account].has_key(session.sid)) return; + Call call = call_by_sid[account][session.sid]; + + info_received(call, session_info); + }); + + Xep.JingleMessageInitiation.Module mi_module = stream_interactor.module_manager.get_module(account, Xep.JingleMessageInitiation.Module.IDENTITY); + mi_module.session_proposed.connect((from, to, sid, descriptions) => { + bool audio_requested = descriptions.any_match((description) => description.ns_uri == Xep.JingleRtp.NS_URI && description.get_attribute("media") == "audio"); + bool video_requested = descriptions.any_match((description) => description.ns_uri == Xep.JingleRtp.NS_URI && description.get_attribute("media") == "video"); + if (!audio_requested && !video_requested) return; + Call call = create_received_call(account, from, to, video_requested); + call_by_sid[account][sid] = call; + sid_by_call[account][call] = sid; + }); + mi_module.session_accepted.connect((from, sid) => { + if (!call_by_sid[account].has_key(sid)) return; + + // Ignore session-accepted from ourselves + if (!from.equals(account.full_jid)) { + Call call = call_by_sid[account][sid]; + call.state = Call.State.OTHER_DEVICE_ACCEPTED; + remove_call_from_datastructures(call); + } + }); + mi_module.session_rejected.connect((from, to, sid) => { + if (!call_by_sid[account].has_key(sid)) return; + Call call = call_by_sid[account][sid]; + call.state = Call.State.DECLINED; + remove_call_from_datastructures(call); + call_terminated(call, null, null); + }); + mi_module.session_retracted.connect((from, to, sid) => { + if (!call_by_sid[account].has_key(sid)) return; + Call call = call_by_sid[account][sid]; + call.state = Call.State.MISSED; + remove_call_from_datastructures(call); + call_terminated(call, null, null); + }); + } + } +} \ No newline at end of file diff --git a/libdino/src/service/content_item_store.vala b/libdino/src/service/content_item_store.vala index 2de804a1..cde8dd10 100644 --- a/libdino/src/service/content_item_store.vala +++ b/libdino/src/service/content_item_store.vala @@ -29,6 +29,8 @@ public class ContentItemStore : StreamInteractionModule, Object { stream_interactor.get_module(FileManager.IDENTITY).received_file.connect(insert_file_transfer); stream_interactor.get_module(MessageProcessor.IDENTITY).message_received.connect(announce_message); stream_interactor.get_module(MessageProcessor.IDENTITY).message_sent.connect(announce_message); + stream_interactor.get_module(Calls.IDENTITY).call_incoming.connect(insert_call); + stream_interactor.get_module(Calls.IDENTITY).call_outgoing.connect(insert_call); } public void init(Conversation conversation, ContentItemCollection item_collection) { @@ -51,7 +53,6 @@ public class ContentItemStore : StreamInteractionModule, Object { Message? message = stream_interactor.get_module(MessageStorage.IDENTITY).get_message_by_id(foreign_id, conversation); if (message != null) { var message_item = new MessageItem(message, conversation, row[db.content_item.id]); - message_item.time = time; items.add(message_item); } break; @@ -62,6 +63,13 @@ public class ContentItemStore : StreamInteractionModule, Object { items.add(file_item); } break; + case 3: + Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(foreign_id); + if (call != null) { + var call_item = new CallItem(call, conversation, row[db.content_item.id]); + items.add(call_item); + } + break; } } @@ -173,6 +181,15 @@ public class ContentItemStore : StreamInteractionModule, Object { } } + private void insert_call(Call call, Conversation conversation) { + CallItem item = new CallItem(call, conversation, -1); + item.id = db.add_content_item(conversation, call.time, call.local_time, 3, call.id, false); + if (collection_conversations.has_key(conversation)) { + collection_conversations.get(conversation).insert_item(item); + } + new_item(item, conversation); + } + public bool get_item_hide(ContentItem content_item) { return db.content_item.row_with(db.content_item.id, content_item.id)[db.content_item.hide, false]; } @@ -292,4 +309,18 @@ public class FileItem : ContentItem { } } +public class CallItem : ContentItem { + public const string TYPE = "call"; + + public Call call; + public Conversation conversation; + + public CallItem(Call call, Conversation conversation, int id) { + base(id, TYPE, call.from, call.time, Encryption.NONE, Message.Marked.NONE); + + this.call = call; + this.conversation = conversation; + } +} + } diff --git a/libdino/src/service/database.vala b/libdino/src/service/database.vala index 17499404..98e18d16 100644 --- a/libdino/src/service/database.vala +++ b/libdino/src/service/database.vala @@ -7,7 +7,7 @@ using Dino.Entities; namespace Dino { public class Database : Qlite.Database { - private const int VERSION = 19; + private const int VERSION = 20; public class AccountTable : Table { public Column id = new Column.Integer("id") { primary_key = true, auto_increment = true }; @@ -155,6 +155,24 @@ public class Database : Qlite.Database { } } + public class CallTable : Table { + public Column id = new Column.Integer("id") { primary_key = true, auto_increment = true }; + public Column account_id = new Column.Integer("account_id") { not_null = true }; + public Column counterpart_id = new Column.Integer("counterpart_id") { not_null = true }; + public Column counterpart_resource = new Column.Text("counterpart_resource"); + public Column our_resource = new Column.Text("our_resource"); + public Column direction = new Column.BoolInt("direction") { not_null = true }; + public Column time = new Column.Long("time") { not_null = true }; + public Column local_time = new Column.Long("local_time") { not_null = true }; + public Column end_time = new Column.Long("end_time"); + public Column state = new Column.Integer("state"); + + internal CallTable(Database db) { + base(db, "call"); + init({id, account_id, counterpart_id, counterpart_resource, our_resource, direction, time, local_time, end_time, state}); + } + } + public class ConversationTable : Table { public Column id = new Column.Integer("id") { primary_key = true, auto_increment = true }; public Column account_id = new Column.Integer("account_id") { not_null = true }; @@ -275,6 +293,7 @@ public class Database : Qlite.Database { public MessageCorrectionTable message_correction { get; private set; } public RealJidTable real_jid { get; private set; } public FileTransferTable file_transfer { get; private set; } + public CallTable call { get; private set; } public ConversationTable conversation { get; private set; } public AvatarTable avatar { get; private set; } public EntityIdentityTable entity_identity { get; private set; } @@ -298,6 +317,7 @@ public class Database : Qlite.Database { message_correction = new MessageCorrectionTable(this); real_jid = new RealJidTable(this); file_transfer = new FileTransferTable(this); + call = new CallTable(this); conversation = new ConversationTable(this); avatar = new AvatarTable(this); entity_identity = new EntityIdentityTable(this); @@ -306,7 +326,7 @@ public class Database : Qlite.Database { mam_catchup = new MamCatchupTable(this); settings = new SettingsTable(this); conversation_settings = new ConversationSettingsTable(this); - init({ account, jid, entity, content_item, message, message_correction, real_jid, file_transfer, conversation, avatar, entity_identity, entity_feature, roster, mam_catchup, settings, conversation_settings }); + init({ account, jid, entity, content_item, message, message_correction, real_jid, file_transfer, call, conversation, avatar, entity_identity, entity_feature, roster, mam_catchup, settings, conversation_settings }); try { exec("PRAGMA journal_mode = WAL"); diff --git a/libdino/src/service/jingle_file_transfers.vala b/libdino/src/service/jingle_file_transfers.vala index a96c716a..e86f923c 100644 --- a/libdino/src/service/jingle_file_transfers.vala +++ b/libdino/src/service/jingle_file_transfers.vala @@ -103,7 +103,7 @@ public class JingleFileProvider : FileProvider, Object { throw new FileReceiveError.DOWNLOAD_FAILED("Transfer data not available anymore"); } try { - jingle_file_transfer.accept(stream); + yield jingle_file_transfer.accept(stream); } catch (IOError e) { throw new FileReceiveError.DOWNLOAD_FAILED("Establishing connection did not work"); } @@ -202,8 +202,11 @@ public class JingleFileSender : FileSender, Object { if (stream == null) throw new FileSendError.UPLOAD_FAILED("No stream available"); JingleFileEncryptionHelper? helper = JingleFileHelperRegistry.instance.get_encryption_helper(file_transfer.encryption); bool must_encrypt = helper != null && yield helper.can_encrypt(conversation, file_transfer); + // TODO(hrxi): Prioritization of transports (and resources?). foreach (Jid full_jid in stream.get_flag(Presence.Flag.IDENTITY).get_resources(conversation.counterpart)) { - // TODO(hrxi): Prioritization of transports (and resources?). + if (full_jid.equals(stream.get_flag(Bind.Flag.IDENTITY).my_jid)) { + continue; + } if (!yield stream.get_module(Xep.JingleFileTransfer.Module.IDENTITY).is_available(stream, full_jid)) { continue; } diff --git a/libdino/src/service/message_processor.vala b/libdino/src/service/message_processor.vala index 98f14945..669aa193 100644 --- a/libdino/src/service/message_processor.vala +++ b/libdino/src/service/message_processor.vala @@ -331,7 +331,7 @@ public class MessageProcessor : StreamInteractionModule, Object { if (conversation == null) return; // MAM state database update - Xep.MessageArchiveManagement.MessageFlag mam_flag = Xep.MessageArchiveManagement.MessageFlag.get_flag(message_stanza); + Xep.MessageArchiveManagement.MessageFlag? mam_flag = Xep.MessageArchiveManagement.MessageFlag.get_flag(message_stanza); if (mam_flag == null) { if (current_catchup_id.has_key(account)) { string? stanza_id = UniqueStableStanzaIDs.get_stanza_id(message_stanza, account.bare_jid); diff --git a/libdino/src/service/module_manager.vala b/libdino/src/service/module_manager.vala index c3f524df..a6165392 100644 --- a/libdino/src/service/module_manager.vala +++ b/libdino/src/service/module_manager.vala @@ -79,6 +79,7 @@ public class ModuleManager { module_map[account].add(new Xep.Jet.Module()); module_map[account].add(new Xep.LastMessageCorrection.Module()); module_map[account].add(new Xep.DirectMucInvitations.Module()); + module_map[account].add(new Xep.JingleMessageInitiation.Module()); initialize_account_modules(account, module_map[account]); } } diff --git a/libdino/src/service/notification_events.vala b/libdino/src/service/notification_events.vala index 7e99dcf9..0243dbe5 100644 --- a/libdino/src/service/notification_events.vala +++ b/libdino/src/service/notification_events.vala @@ -24,12 +24,15 @@ public class NotificationEvents : StreamInteractionModule, Object { stream_interactor.get_module(ContentItemStore.IDENTITY).new_item.connect(on_content_item_received); stream_interactor.get_module(PresenceManager.IDENTITY).received_subscription_request.connect(on_received_subscription_request); + stream_interactor.get_module(MucManager.IDENTITY).invite_received.connect(on_invite_received); stream_interactor.get_module(MucManager.IDENTITY).voice_request_received.connect((account, room_jid, from_jid, nick) => { Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation(room_jid, account, Conversation.Type.GROUPCHAT); if (conversation == null) return; notifier.notify_voice_request.begin(conversation, from_jid); }); + + stream_interactor.get_module(Calls.IDENTITY).call_incoming.connect(on_call_incoming); stream_interactor.connection_manager.connection_error.connect((account, error) => notifier.notify_connection_error.begin(account, error)); stream_interactor.get_module(ChatInteraction.IDENTITY).focused_in.connect((conversation) => { notifier.retract_content_item_notifications.begin(); @@ -91,6 +94,9 @@ public class NotificationEvents : StreamInteractionModule, Object { notifier.notify_file.begin(file_transfer, conversation, is_image, conversation_display_name, participant_display_name); } break; + case CallItem.TYPE: + // handled in `on_call_incoming` + break; } } @@ -101,6 +107,18 @@ public class NotificationEvents : StreamInteractionModule, Object { notifier.notify_subscription_request.begin(conversation); } + private void on_call_incoming(Call call, Conversation conversation, bool video) { + string conversation_display_name = get_conversation_display_name(stream_interactor, conversation, null); + + notifier.notify_call.begin(call, conversation, video, conversation_display_name); + call.notify["state"].connect(() => { + + if (call.state != Call.State.RINGING) { + notifier.retract_call_notification(call, conversation); + } + }); + } + private void on_invite_received(Account account, Jid room_jid, Jid from_jid, string? password, string? reason) { string inviter_display_name; if (room_jid.equals_bare(from_jid)) { @@ -119,6 +137,8 @@ public interface NotificationProvider : Object { public abstract async void notify_message(Message message, Conversation conversation, string conversation_display_name, string? participant_display_name); public abstract async void notify_file(FileTransfer file_transfer, Conversation conversation, bool is_image, string conversation_display_name, string? participant_display_name); + public abstract async void notify_call(Call call, Conversation conversation, bool video, string conversation_display_name); + public abstract async void retract_call_notification(Call call, Conversation conversation); public abstract async void notify_subscription_request(Conversation conversation); public abstract async void notify_connection_error(Account account, ConnectionManager.ConnectionError error); public abstract async void notify_muc_invite(Account account, Jid room_jid, Jid from_jid, string inviter_display_name); From 0f46facecd558786631c2ad4cf66d27331f16a86 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Fri, 19 Mar 2021 23:09:56 +0100 Subject: [PATCH 11/56] Add UI for audio/video calls --- CMakeLists.txt | 4 +- main/CMakeLists.txt | 21 ++ main/data/call_widget.ui | 111 ++++++++ .../icons/dino-microphone-off-symbolic.svg | 1 + main/data/icons/dino-microphone-symbolic.svg | 1 + .../data/icons/dino-phone-hangup-symbolic.svg | 1 + .../icons/dino-phone-in-talk-symbolic.svg | 1 + .../data/icons/dino-phone-missed-symbolic.svg | 1 + main/data/icons/dino-phone-ring-symbolic.svg | 1 + main/data/icons/dino-phone-symbolic.svg | 1 + main/data/icons/dino-video-off-symbolic.svg | 1 + main/data/icons/dino-video-symbolic.svg | 1 + main/data/theme.css | 97 ++++++- main/src/main.vala | 2 +- main/src/ui/application.vala | 18 ++ .../call_window/audio_settings_popover.vala | 127 +++++++++ main/src/ui/call_window/call_bottom_bar.vala | 165 +++++++++++ main/src/ui/call_window/call_window.vala | 260 ++++++++++++++++++ .../call_window/call_window_controller.vala | 208 ++++++++++++++ .../call_window/video_settings_popover.vala | 73 +++++ .../call_widget.vala | 215 +++++++++++++++ .../content_populator.vala | 3 + .../file_widget.vala | 3 - .../conversation_selector_row.vala | 8 + .../ui/conversation_titlebar/call_entry.vala | 130 +++++++++ main/src/ui/conversation_view_controller.vala | 1 + main/src/ui/notifier_freedesktop.vala | 38 +++ main/src/ui/notifier_gnotifications.vala | 19 ++ main/src/ui/util/helper.vala | 9 + 29 files changed, 1514 insertions(+), 7 deletions(-) create mode 100644 main/data/call_widget.ui create mode 100644 main/data/icons/dino-microphone-off-symbolic.svg create mode 100644 main/data/icons/dino-microphone-symbolic.svg create mode 100644 main/data/icons/dino-phone-hangup-symbolic.svg create mode 100644 main/data/icons/dino-phone-in-talk-symbolic.svg create mode 100644 main/data/icons/dino-phone-missed-symbolic.svg create mode 100644 main/data/icons/dino-phone-ring-symbolic.svg create mode 100644 main/data/icons/dino-phone-symbolic.svg create mode 100644 main/data/icons/dino-video-off-symbolic.svg create mode 100644 main/data/icons/dino-video-symbolic.svg create mode 100644 main/src/ui/call_window/audio_settings_popover.vala create mode 100644 main/src/ui/call_window/call_bottom_bar.vala create mode 100644 main/src/ui/call_window/call_window.vala create mode 100644 main/src/ui/call_window/call_window_controller.vala create mode 100644 main/src/ui/call_window/video_settings_popover.vala create mode 100644 main/src/ui/conversation_content_view/call_widget.vala create mode 100644 main/src/ui/conversation_titlebar/call_entry.vala diff --git a/CMakeLists.txt b/CMakeLists.txt index 2af0719b..f480b0b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ else () endif () # Prepare Plugins -set(DEFAULT_PLUGINS omemo;openpgp;http-files;ice) +set(DEFAULT_PLUGINS omemo;openpgp;http-files;ice;rtp) foreach (plugin ${DEFAULT_PLUGINS}) if ("$CACHE{DINO_PLUGIN_ENABLED_${plugin}}" STREQUAL "") if (NOT DEFINED DINO_PLUGIN_ENABLED_${plugin}}) @@ -96,6 +96,7 @@ macro(AddCFlagIfSupported list flag) endif () endmacro() + if ("Ninja" STREQUAL ${CMAKE_GENERATOR}) AddCFlagIfSupported(CMAKE_C_FLAGS -fdiagnostics-color) endif () @@ -105,6 +106,7 @@ AddCFlagIfSupported(CMAKE_C_FLAGS -Wall) AddCFlagIfSupported(CMAKE_C_FLAGS -Wextra) AddCFlagIfSupported(CMAKE_C_FLAGS -Werror=format-security) AddCFlagIfSupported(CMAKE_C_FLAGS -Wno-duplicate-decl-specifier) +AddCFlagIfSupported(CMAKE_C_FLAGS -fno-omit-frame-pointer) if (NOT VALA_WARN) set(VALA_WARN "conversion") diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 5169e8ae..69992f06 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -5,6 +5,8 @@ gettext_compile(${GETTEXT_PACKAGE} SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/po TAR find_packages(MAIN_PACKAGES REQUIRED Gee + Gst + GstVideo GLib GModule GObject @@ -21,7 +23,14 @@ set(RESOURCE_LIST icons/dino-emoticon-symbolic.svg icons/dino-qr-code-symbolic.svg icons/dino-security-high-symbolic.svg + icons/dino-microphone-off-symbolic.svg + icons/dino-microphone-symbolic.svg icons/dino-party-popper-symbolic.svg + icons/dino-phone-hangup-symbolic.svg + icons/dino-phone-in-talk-symbolic.svg + icons/dino-phone-missed-symbolic.svg + icons/dino-phone-ring-symbolic.svg + icons/dino-phone-symbolic.svg icons/dino-status-away.svg icons/dino-status-chat.svg icons/dino-status-dnd.svg @@ -29,6 +38,8 @@ set(RESOURCE_LIST icons/im.dino.Dino.svg icons/im.dino.Dino-symbolic.svg icons/dino-tick-symbolic.svg + icons/dino-video-off-symbolic.svg + icons/dino-video-symbolic.svg icons/dino-device-desktop-symbolic.svg icons/dino-device-phone-symbolic.svg @@ -46,6 +57,8 @@ set(RESOURCE_LIST add_conversation/conference_details_fragment.ui add_conversation/list_row.ui add_conversation/select_jid_fragment.ui + + call_widget.ui chat_input.ui contact_details_dialog.ui conversation_list_titlebar.ui @@ -124,6 +137,12 @@ SOURCES src/ui/add_conversation/select_contact_dialog.vala src/ui/add_conversation/select_jid_fragment.vala + src/ui/call_window/audio_settings_popover.vala + src/ui/call_window/call_bottom_bar.vala + src/ui/call_window/call_window.vala + src/ui/call_window/call_window_controller.vala + src/ui/call_window/video_settings_popover.vala + src/ui/chat_input/chat_input_controller.vala src/ui/chat_input/chat_text_view.vala src/ui/chat_input/edit_history.vala @@ -142,6 +161,7 @@ SOURCES src/ui/conversation_selector/conversation_selector_row.vala src/ui/conversation_selector/conversation_selector.vala + src/ui/conversation_content_view/call_widget.vala src/ui/conversation_content_view/chat_state_populator.vala src/ui/conversation_content_view/content_populator.vala src/ui/conversation_content_view/conversation_item_skeleton.vala @@ -153,6 +173,7 @@ SOURCES src/ui/conversation_content_view/message_widget.vala src/ui/conversation_content_view/subscription_notification.vala + src/ui/conversation_titlebar/call_entry.vala src/ui/conversation_titlebar/menu_entry.vala src/ui/conversation_titlebar/occupants_entry.vala src/ui/conversation_titlebar/search_entry.vala diff --git a/main/data/call_widget.ui b/main/data/call_widget.ui new file mode 100644 index 00000000..47fb0046 --- /dev/null +++ b/main/data/call_widget.ui @@ -0,0 +1,111 @@ + + + + diff --git a/main/data/icons/dino-microphone-off-symbolic.svg b/main/data/icons/dino-microphone-off-symbolic.svg new file mode 100644 index 00000000..7e5b853d --- /dev/null +++ b/main/data/icons/dino-microphone-off-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-microphone-symbolic.svg b/main/data/icons/dino-microphone-symbolic.svg new file mode 100644 index 00000000..fbf0784a --- /dev/null +++ b/main/data/icons/dino-microphone-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-phone-hangup-symbolic.svg b/main/data/icons/dino-phone-hangup-symbolic.svg new file mode 100644 index 00000000..ecd230ac --- /dev/null +++ b/main/data/icons/dino-phone-hangup-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-phone-in-talk-symbolic.svg b/main/data/icons/dino-phone-in-talk-symbolic.svg new file mode 100644 index 00000000..351035da --- /dev/null +++ b/main/data/icons/dino-phone-in-talk-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-phone-missed-symbolic.svg b/main/data/icons/dino-phone-missed-symbolic.svg new file mode 100644 index 00000000..228f073e --- /dev/null +++ b/main/data/icons/dino-phone-missed-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-phone-ring-symbolic.svg b/main/data/icons/dino-phone-ring-symbolic.svg new file mode 100644 index 00000000..06b8dcbf --- /dev/null +++ b/main/data/icons/dino-phone-ring-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-phone-symbolic.svg b/main/data/icons/dino-phone-symbolic.svg new file mode 100644 index 00000000..0020dddc --- /dev/null +++ b/main/data/icons/dino-phone-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-video-off-symbolic.svg b/main/data/icons/dino-video-off-symbolic.svg new file mode 100644 index 00000000..d438e065 --- /dev/null +++ b/main/data/icons/dino-video-off-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-video-symbolic.svg b/main/data/icons/dino-video-symbolic.svg new file mode 100644 index 00000000..60a1c742 --- /dev/null +++ b/main/data/icons/dino-video-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/theme.css b/main/data/theme.css index 6bacee30..423cbf68 100644 --- a/main/data/theme.css +++ b/main/data/theme.css @@ -86,16 +86,26 @@ window.dino-main .dino-conversation .message-box.edit-mode:hover { background: alpha(@theme_selected_bg_color, 0.12); } -window.dino-main .file-box-outer { +window.dino-main .file-box-outer, +window.dino-main .call-box-outer { background: @theme_base_color; border-radius: 3px; border: 1px solid alpha(@theme_fg_color, 0.1); } -window.dino-main .file-box { +window.dino-main .file-box, +window.dino-main .call-box { margin: 12px 16px 12px 12px; } +window.dino-main .call-box-outer.incoming { + border-color: alpha(@theme_selected_bg_color, 0.3); +} + +window.dino-main .incoming-call-box { + background: alpha(@theme_selected_bg_color, 0.1); +} + window.dino-main .dino-sidebar > frame.collapsed { border-bottom: 1px solid @borders; } @@ -204,3 +214,86 @@ box.dino-input-error label.input-status-highlight-once { animation-iteration-count: 1; animation-name: input-error-highlight; } + +/* Call window */ + +.dino-call-window .titlebar { + min-height: 0; +} + +.dino-call-window headerbar, .call-button { + box-shadow: none; +} + +.dino-call-window .titlebutton.close:hover { + background: rgba(255,255,255,0.15); + border-color: rgba(255,255,255,0); + box-shadow: none; +} + +.dino-call-window button.call-button { + outline: 0; + border-radius: 1000px; +} +.dino-call-window button.white-button { + color: #1d1c1d; + background: rgba(255,255,255,0.9); + border: lightgrey; +} + +.dino-call-window button.transparent-white-button { + color: white; + background: rgba(255,255,255,0.15); + border: none; +} + +.dino-call-window button.call-mediadevice-settings-button { + border-radius: 1000px; + min-height: 0; + min-width: 0; + padding: 3px; + margin: 2px; + transition-duration: 0; +} + +.dino-call-window button.call-mediadevice-settings-button:hover, +.dino-call-window button.call-mediadevice-settings-button:checked { /* Effect that makes the button slightly larger on hover :) */ + border-radius: 1000px; + min-height: 0; + min-width: 0; + padding: 5px; + margin: 0; +} + +.dino-call-window .unencrypted-box { + color: @error_color; + padding: 10px; + border-radius: 5px; + background: rgba(0,0,0,0.5); +} + +.dino-call-window .call-header-bar { + background: linear-gradient(rgba(0,0,0,0.4), rgba(0,0,0,0)); + border: 0; + border-radius: 0; +} + +.dino-call-window .call-header-bar, +.dino-call-window .call-header-bar image { + color: #ededec; +} + +.dino-call-window .call-bottom-bar { + background: linear-gradient(rgba(0,0,0,0), rgba(0,0,0,0.3)); + border: 0; +} + +.dino-call-window .video-placeholder-box { + background-color: #212121; +} + +.dino-call-window .text-no-controls { + background: white; + border-radius: 5px; + padding: 5px 10px; +} \ No newline at end of file diff --git a/main/src/main.vala b/main/src/main.vala index 6274dcdd..afa1f52b 100644 --- a/main/src/main.vala +++ b/main/src/main.vala @@ -17,7 +17,7 @@ void main(string[] args) { Gtk.init(ref args); Dino.Ui.Application app = new Dino.Ui.Application() { search_path_generator=search_path_generator }; Plugins.Loader loader = new Plugins.Loader(app); - loader.loadAll(); + loader.load_all(); app.run(args); loader.shutdown(); diff --git a/main/src/ui/application.vala b/main/src/ui/application.vala index 358097e3..780c37fd 100644 --- a/main/src/ui/application.vala +++ b/main/src/ui/application.vala @@ -199,6 +199,24 @@ public class Dino.Ui.Application : Gtk.Application, Dino.Application { dialog.present(); }); add_action(open_shortcuts_action); + + SimpleAction accept_call_action = new SimpleAction("accept-call", VariantType.INT32); + accept_call_action.activate.connect((variant) => { + Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(variant.get_int32()); + stream_interactor.get_module(Calls.IDENTITY).accept_call(call); + + var call_window = new CallWindow(); + call_window.controller = new CallWindowController(call_window, call, stream_interactor); + call_window.present(); + }); + add_action(accept_call_action); + + SimpleAction deny_call_action = new SimpleAction("deny-call", VariantType.INT32); + deny_call_action.activate.connect((variant) => { + Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(variant.get_int32()); + stream_interactor.get_module(Calls.IDENTITY).reject_call(call); + }); + add_action(deny_call_action); } public bool use_csd() { diff --git a/main/src/ui/call_window/audio_settings_popover.vala b/main/src/ui/call_window/audio_settings_popover.vala new file mode 100644 index 00000000..7d1f39b0 --- /dev/null +++ b/main/src/ui/call_window/audio_settings_popover.vala @@ -0,0 +1,127 @@ +using Gee; +using Gtk; +using Dino.Entities; + +public class Dino.Ui.AudioSettingsPopover : Gtk.Popover { + + public signal void microphone_selected(Plugins.MediaDevice device); + public signal void speaker_selected(Plugins.MediaDevice device); + + public Plugins.MediaDevice? current_microphone_device { get; set; } + public Plugins.MediaDevice? current_speaker_device { get; set; } + + private HashMap row_microphone_device = new HashMap(); + private HashMap row_speaker_device = new HashMap(); + + public AudioSettingsPopover() { + Box box = new Box(Orientation.VERTICAL, 15) { margin=18, visible=true }; + box.add(create_microphone_box()); + box.add(create_speaker_box()); + + this.add(box); + } + + private Widget create_microphone_box() { + Plugins.VideoCallPlugin call_plugin = Dino.Application.get_default().plugin_registry.video_call_plugin; + Gee.List devices = call_plugin.get_devices("audio", false); + + Box micro_box = new Box(Orientation.VERTICAL, 10) { visible=true }; + micro_box.add(new Label("" + _("Microphones") + "") { use_markup=true, xalign=0, visible=true, can_focus=true /* grab initial focus*/ }); + + if (devices.size == 0) { + micro_box.add(new Label("No microphones found.")); + } else { + ListBox micro_list_box = new ListBox() { activate_on_single_click=true, selection_mode=SelectionMode.SINGLE, visible=true }; + micro_list_box.set_header_func(listbox_header_func); + Frame micro_frame = new Frame(null) { visible=true }; + micro_frame.add(micro_list_box); + foreach (Plugins.MediaDevice device in devices) { + Label label = new Label(device.display_name) { xalign=0, visible=true }; + Image image = new Image.from_icon_name("object-select-symbolic", IconSize.BUTTON) { visible=true }; + if (current_microphone_device == null || current_microphone_device.id != device.id) { + image.opacity = 0; + } + this.notify["current-microphone-device"].connect(() => { + if (current_microphone_device == null || current_microphone_device.id != device.id) { + image.opacity = 0; + } else { + image.opacity = 1; + } + }); + Box device_box = new Box(Orientation.HORIZONTAL, 0) { spacing=7, margin=7, visible=true }; + device_box.add(image); + device_box.add(label); + ListBoxRow list_box_row = new ListBoxRow() { visible=true }; + list_box_row.add(device_box); + micro_list_box.add(list_box_row); + + row_microphone_device[list_box_row] = device; + } + micro_list_box.row_activated.connect((row) => { + if (!row_microphone_device.has_key(row)) return; + microphone_selected(row_microphone_device[row]); + micro_list_box.unselect_row(row); + }); + micro_box.add(micro_frame); + } + + return micro_box; + } + + private Widget create_speaker_box() { + Plugins.VideoCallPlugin call_plugin = Dino.Application.get_default().plugin_registry.video_call_plugin; + Gee.List devices = call_plugin.get_devices("audio", true); + + Box speaker_box = new Box(Orientation.VERTICAL, 10) { visible=true }; + speaker_box.add(new Label("" + _("Speakers") +"") { use_markup=true, xalign=0, visible=true }); + + if (devices.size == 0) { + speaker_box.add(new Label("No speakers found.")); + } else { + ListBox speaker_list_box = new ListBox() { activate_on_single_click=true, selection_mode=SelectionMode.SINGLE, visible=true }; + speaker_list_box.set_header_func(listbox_header_func); + speaker_list_box.row_selected.connect((row) => { + + }); + Frame speaker_frame = new Frame(null) { visible=true }; + speaker_frame.add(speaker_list_box); + foreach (Plugins.MediaDevice device in devices) { + Label label = new Label(device.display_name) { xalign=0, visible=true }; + Image image = new Image.from_icon_name("object-select-symbolic", IconSize.BUTTON) { visible=true }; + if (current_speaker_device == null || current_speaker_device.id != device.id) { + image.opacity = 0; + } + this.notify["current-speaker-device"].connect(() => { + if (current_speaker_device == null || current_speaker_device.id != device.id) { + image.opacity = 0; + } else { + image.opacity = 1; + } + }); + Box device_box = new Box(Orientation.HORIZONTAL, 0) { spacing=7, margin=7, visible=true }; + device_box.add(image); + device_box.add(label); + ListBoxRow list_box_row = new ListBoxRow() { visible=true }; + list_box_row.add(device_box); + speaker_list_box.add(list_box_row); + + row_speaker_device[list_box_row] = device; + } + speaker_list_box.row_activated.connect((row) => { + if (!row_speaker_device.has_key(row)) return; + speaker_selected(row_speaker_device[row]); + speaker_list_box.unselect_row(row); + }); + speaker_box.add(speaker_frame); + } + + return speaker_box; + } + + private void listbox_header_func(ListBoxRow row, ListBoxRow? before_row) { + if (row.get_header() == null && before_row != null) { + row.set_header(new Separator(Orientation.HORIZONTAL)); + } + } + +} \ No newline at end of file diff --git a/main/src/ui/call_window/call_bottom_bar.vala b/main/src/ui/call_window/call_bottom_bar.vala new file mode 100644 index 00000000..bc800485 --- /dev/null +++ b/main/src/ui/call_window/call_bottom_bar.vala @@ -0,0 +1,165 @@ +using Dino.Entities; +using Gtk; + +public class Dino.Ui.CallBottomBar : Gtk.Box { + + public signal void hang_up(); + + public bool audio_enabled { get; set; } + public bool video_enabled { get; set; } + + public static IconSize ICON_SIZE_MEDIADEVICE_BUTTON = Gtk.icon_size_register("im.dino.Dino.CALL_MEDIADEVICE_BUTTON", 10, 10); + + public string counterpart_display_name { get; set; } + + private Button audio_button = new Button() { height_request=45, width_request=45, halign=Align.START, valign=Align.START, visible=true }; + private Overlay audio_button_overlay = new Overlay() { visible=true }; + private Image audio_image = new Image() { visible=true }; + private MenuButton audio_settings_button = new MenuButton() { halign=Align.END, valign=Align.END }; + public AudioSettingsPopover? audio_settings_popover; + + private Button video_button = new Button() { height_request=45, width_request=45, halign=Align.START, valign=Align.START, visible=true }; + private Overlay video_button_overlay = new Overlay() { visible=true }; + private Image video_image = new Image() { visible=true }; + private MenuButton video_settings_button = new MenuButton() { halign=Align.END, valign=Align.END }; + public VideoSettingsPopover? video_settings_popover; + + private Label label = new Label("") { margin=20, halign=Align.CENTER, valign=Align.CENTER, wrap=true, wrap_mode=Pango.WrapMode.WORD_CHAR, hexpand=true, visible=true }; + private Stack stack = new Stack() { visible=true }; + + public CallBottomBar() { + Object(orientation:Orientation.HORIZONTAL, spacing:0); + + Overlay default_control = new Overlay() { visible=true }; + Image encryption_image = new Image.from_icon_name("changes-allow-symbolic", IconSize.BUTTON) { margin_start=20, margin_bottom=25, halign=Align.START, valign=Align.END, visible=true }; + encryption_image.tooltip_text = _("Unencrypted"); + encryption_image.get_style_context().add_class("unencrypted-box"); + + default_control.add_overlay(encryption_image); + + Box main_buttons = new Box(Orientation.HORIZONTAL, 20) { margin_start=40, margin_end=40, margin=20, halign=Align.CENTER, hexpand=true, visible=true }; + + audio_button.add(audio_image); + audio_button.get_style_context().add_class("call-button"); + audio_button.clicked.connect(() => { audio_enabled = !audio_enabled; }); + audio_button.margin_end = audio_button.margin_bottom = 5; // space for the small settings button + audio_button_overlay.add(audio_button); + audio_button_overlay.add_overlay(audio_settings_button); + audio_settings_button.set_image(new Image.from_icon_name("go-up-symbolic", ICON_SIZE_MEDIADEVICE_BUTTON) { visible=true }); + audio_settings_button.get_style_context().add_class("call-mediadevice-settings-button"); + audio_settings_button.use_popover = true; + main_buttons.add(audio_button_overlay); + + video_button.add(video_image); + video_button.get_style_context().add_class("call-button"); + video_button.clicked.connect(() => { video_enabled = !video_enabled; }); + video_button.margin_end = video_button.margin_bottom = 5; + video_button_overlay.add(video_button); + video_button_overlay.add_overlay(video_settings_button); + video_settings_button.set_image(new Image.from_icon_name("go-up-symbolic", ICON_SIZE_MEDIADEVICE_BUTTON) { visible=true }); + video_settings_button.get_style_context().add_class("call-mediadevice-settings-button"); + video_settings_button.use_popover = true; + main_buttons.add(video_button_overlay); + + Button button_hang = new Button.from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR) { height_request=45, width_request=45, halign=Align.START, valign=Align.START, visible=true }; + button_hang.get_style_context().add_class("call-button"); + button_hang.get_style_context().add_class("destructive-action"); + button_hang.clicked.connect(() => hang_up()); + main_buttons.add(button_hang); + + default_control.add(main_buttons); + + label.get_style_context().add_class("text-no-controls"); + + stack.add_named(default_control, "control-buttons"); + stack.add_named(label, "label"); + this.add(stack); + + this.notify["audio-enabled"].connect(on_audio_enabled_changed); + this.notify["video-enabled"].connect(on_video_enabled_changed); + + audio_enabled = true; + video_enabled = false; + + on_audio_enabled_changed(); + on_video_enabled_changed(); + + this.get_style_context().add_class("call-bottom-bar"); + } + + public AudioSettingsPopover? show_audio_device_choices(bool show) { + audio_settings_button.visible = show; + if (audio_settings_popover != null) audio_settings_popover.visible = false; + if (!show) return null; + + audio_settings_popover = new AudioSettingsPopover(); + + audio_settings_button.popover = audio_settings_popover; + + audio_settings_popover.set_relative_to(audio_settings_button); + audio_settings_popover.microphone_selected.connect(() => { audio_settings_button.active = false; }); + audio_settings_popover.speaker_selected.connect(() => { audio_settings_button.active = false; }); + + return audio_settings_popover; + } + + public void show_audio_device_error() { + audio_settings_button.set_image(new Image.from_icon_name("dialog-warning-symbolic", IconSize.BUTTON) { visible=true }); + Util.force_error_color(audio_settings_button); + } + + public VideoSettingsPopover? show_video_device_choices(bool show) { + video_settings_button.visible = show; + if (video_settings_popover != null) video_settings_popover.visible = false; + if (!show) return null; + + video_settings_popover = new VideoSettingsPopover(); + + + video_settings_button.popover = video_settings_popover; + + video_settings_popover.set_relative_to(video_settings_button); + video_settings_popover.camera_selected.connect(() => { video_settings_button.active = false; }); + + return video_settings_popover; + } + + public void show_video_device_error() { + video_settings_button.set_image(new Image.from_icon_name("dialog-warning-symbolic", IconSize.BUTTON) { visible=true }); + Util.force_error_color(video_settings_button); + } + + public void on_audio_enabled_changed() { + if (audio_enabled) { + audio_image.set_from_icon_name("dino-microphone-symbolic", IconSize.LARGE_TOOLBAR); + audio_button.get_style_context().add_class("white-button"); + audio_button.get_style_context().remove_class("transparent-white-button"); + } else { + audio_image.set_from_icon_name("dino-microphone-off-symbolic", IconSize.LARGE_TOOLBAR); + audio_button.get_style_context().remove_class("white-button"); + audio_button.get_style_context().add_class("transparent-white-button"); + } + } + + public void on_video_enabled_changed() { + if (video_enabled) { + video_image.set_from_icon_name("dino-video-symbolic", IconSize.LARGE_TOOLBAR); + video_button.get_style_context().add_class("white-button"); + video_button.get_style_context().remove_class("transparent-white-button"); + + } else { + video_image.set_from_icon_name("dino-video-off-symbolic", IconSize.LARGE_TOOLBAR); + video_button.get_style_context().remove_class("white-button"); + video_button.get_style_context().add_class("transparent-white-button"); + } + } + + public void show_counterpart_ended(string text) { + stack.set_visible_child_name("label"); + label.label = text; + } + + public bool is_menu_active() { + return video_settings_button.active || audio_settings_button.active; + } +} \ No newline at end of file diff --git a/main/src/ui/call_window/call_window.vala b/main/src/ui/call_window/call_window.vala new file mode 100644 index 00000000..572f73b6 --- /dev/null +++ b/main/src/ui/call_window/call_window.vala @@ -0,0 +1,260 @@ +using Dino.Entities; +using Gtk; + +namespace Dino.Ui { + + public class CallWindow : Gtk.Window { + public string counterpart_display_name { get; set; } + + // TODO should find another place for this + public CallWindowController controller; + + public Overlay overlay = new Overlay() { visible=true }; + public EventBox event_box = new EventBox() { visible=true }; + public CallBottomBar bottom_bar = new CallBottomBar() { visible=true }; + public Revealer bottom_bar_revealer = new Revealer() { valign=Align.END, transition_type=RevealerTransitionType.CROSSFADE, transition_duration=200, visible=true }; + public HeaderBar header_bar = new HeaderBar() { show_close_button=true, visible=true }; + public Revealer header_bar_revealer = new Revealer() { valign=Align.START, transition_type=RevealerTransitionType.CROSSFADE, transition_duration=200, visible=true }; + public Stack stack = new Stack() { visible=true }; + public Box own_video_box = new Box(Orientation.HORIZONTAL, 0) { expand=true, visible=true }; + private Widget? own_video = null; + private Box? own_video_border = new Box(Orientation.HORIZONTAL, 0) { expand=true }; // hack to draw a border around our own video, since we apparently can't draw a border around the Gst widget + + private int own_video_width = 150; + private int own_video_height = 100; + + private bool hide_controll_elements = false; + private uint hide_controll_handler = 0; + private Widget? main_widget = null; + + construct { + header_bar.get_style_context().add_class("call-header-bar"); + header_bar_revealer.add(header_bar); + + this.get_style_context().add_class("dino-call-window"); + + bottom_bar_revealer.add(bottom_bar); + + overlay.add_overlay(own_video_box); + overlay.add_overlay(own_video_border); + overlay.add_overlay(bottom_bar_revealer); + overlay.add_overlay(header_bar_revealer); + + event_box.add(overlay); + add(event_box); + + Util.force_css(own_video_border, "* { border: 1px solid #616161; background-color: transparent; }"); + } + + public CallWindow() { + event_box.events |= Gdk.EventMask.POINTER_MOTION_MASK; + event_box.events |= Gdk.EventMask.ENTER_NOTIFY_MASK; + event_box.events |= Gdk.EventMask.LEAVE_NOTIFY_MASK; + + this.bind_property("counterpart-display-name", header_bar, "title", BindingFlags.SYNC_CREATE); + this.bind_property("counterpart-display-name", bottom_bar, "counterpart-display-name", BindingFlags.SYNC_CREATE); + + event_box.motion_notify_event.connect(reveal_control_elements); + event_box.enter_notify_event.connect(reveal_control_elements); + event_box.leave_notify_event.connect(reveal_control_elements); + this.configure_event.connect(reveal_control_elements); // upon resizing + this.configure_event.connect(update_own_video_position); + + this.set_titlebar(new OutsideHeaderBar(this.header_bar) { visible=true }); + + reveal_control_elements(); + } + + public void set_video_fallback(StreamInteractor stream_interactor, Conversation conversation) { + hide_controll_elements = false; + + Box box = new Box(Orientation.HORIZONTAL, 0) { visible=true }; + box.get_style_context().add_class("video-placeholder-box"); + AvatarImage avatar = new AvatarImage() { hexpand=true, vexpand=true, halign=Align.CENTER, valign=Align.CENTER, height=100, width=100, visible=true }; + avatar.set_conversation(stream_interactor, conversation); + box.add(avatar); + + set_new_main_widget(box); + } + + public void set_video(Widget widget) { + hide_controll_elements = true; + + widget.visible = true; + set_new_main_widget(widget); + } + + public void set_own_video(Widget? widget_) { + own_video_box.foreach((widget) => { own_video_box.remove(widget); }); + + own_video = widget_; + if (own_video == null) { + own_video = new Box(Orientation.HORIZONTAL, 0) { expand=true }; + } + own_video.visible = true; + own_video.width_request = 150; + own_video.height_request = 100; + own_video_box.add(own_video); + + own_video_border.visible = true; + + update_own_video_position(); + } + + public void set_own_video_ratio(int width, int height) { + if (width / height > 150 / 100) { + this.own_video_width = 150; + this.own_video_height = height * 150 / width; + } else { + this.own_video_width = width * 100 / height; + this.own_video_height = 100; + } + + own_video.width_request = own_video_width; + own_video.height_request = own_video_height; + + update_own_video_position(); + } + + public void unset_own_video() { + own_video_box.foreach((widget) => { own_video_box.remove(widget); }); + + own_video_border.visible = false; + } + + public void set_test_video() { + hide_controll_elements = true; + + var pipeline = new Gst.Pipeline(null); + var src = Gst.ElementFactory.make("videotestsrc", null); + pipeline.add(src); + Gst.Video.Sink sink = (Gst.Video.Sink) Gst.ElementFactory.make("gtksink", null); + Gtk.Widget widget; + sink.get("widget", out widget); + widget.unparent(); + pipeline.add(sink); + src.link(sink); + widget.visible = true; + + pipeline.set_state(Gst.State.PLAYING); + + sink.get_static_pad("sink").notify["caps"].connect(() => { + int width, height; + sink.get_static_pad("sink").caps.get_structure(0).get_int("width", out width); + sink.get_static_pad("sink").caps.get_structure(0).get_int("height", out height); + widget.width_request = width; + widget.height_request = height; + }); + + set_new_main_widget(widget); + } + + private void set_new_main_widget(Widget widget) { + if (main_widget != null) overlay.remove(main_widget); + overlay.add(widget); + main_widget = widget; + } + + public void set_status(string state) { + switch (state) { + case "requested": + header_bar.subtitle = _("Sending a call request…"); + break; + case "ringing": + header_bar.subtitle = _("Ringing…"); + break; + case "establishing": + header_bar.subtitle = _("Establishing a (peer-to-peer) connection…"); + break; + default: + header_bar.subtitle = null; + break; + } + } + + public void show_counterpart_ended(string? reason_name, string? reason_text) { + hide_controll_elements = false; + reveal_control_elements(); + + string text = ""; + if (reason_name == Xmpp.Xep.Jingle.ReasonElement.SUCCESS) { + text = _("%s ended the call").printf(counterpart_display_name); + } else if (reason_name == Xmpp.Xep.Jingle.ReasonElement.DECLINE || reason_name == Xmpp.Xep.Jingle.ReasonElement.BUSY) { + text = _("%s declined the call").printf(counterpart_display_name); + } else { + text = "The call has been terminated: " + (reason_name ?? "") + " " + (reason_text ?? ""); + } + + bottom_bar.show_counterpart_ended(text); + } + + public bool reveal_control_elements() { + if (!bottom_bar_revealer.child_revealed) { + bottom_bar_revealer.set_reveal_child(true); + header_bar_revealer.set_reveal_child(true); + } + + if (hide_controll_handler != 0) { + Source.remove(hide_controll_handler); + hide_controll_handler = 0; + } + + if (!hide_controll_elements) { + return false; + } + + hide_controll_handler = Timeout.add_seconds(3, () => { + if (!hide_controll_elements) { + return false; + } + + if (bottom_bar.is_menu_active()) { + return true; + } + + header_bar_revealer.set_reveal_child(false); + bottom_bar_revealer.set_reveal_child(false); + hide_controll_handler = 0; + return false; + }); + return false; + } + + private bool update_own_video_position() { + if (own_video == null) return false; + + int width, height; + this.get_size(out width,out height); + + own_video.margin_end = own_video.margin_bottom = own_video_border.margin_end = own_video_border.margin_bottom = 20; + own_video.margin_start = own_video_border.margin_start = width - own_video_width - 20; + own_video.margin_top = own_video_border.margin_top = height - own_video_height - 20; + + return false; + } + } + + /* Hack to make the CallHeaderBar feel like a HeaderBar (right click menu, double click, ..) although it isn't set as headerbar. + * OutsideHeaderBar is set as a headerbar and it doesn't take any space, but claims to take space (which is actually taken by CallHeaderBar). + */ + public class OutsideHeaderBar : Gtk.Box { + HeaderBar header_bar; + + public OutsideHeaderBar(HeaderBar header_bar) { + this.header_bar = header_bar; + + size_allocate.connect_after(on_header_bar_size_allocate); + header_bar.size_allocate.connect(on_header_bar_size_allocate); + } + + public void on_header_bar_size_allocate() { + Allocation header_bar_alloc; + header_bar.get_allocation(out header_bar_alloc); + + Allocation alloc; + get_allocation(out alloc); + alloc.height = header_bar_alloc.height; + set_allocation(alloc); + } + } +} \ No newline at end of file diff --git a/main/src/ui/call_window/call_window_controller.vala b/main/src/ui/call_window/call_window_controller.vala new file mode 100644 index 00000000..09c8f88c --- /dev/null +++ b/main/src/ui/call_window/call_window_controller.vala @@ -0,0 +1,208 @@ +using Dino.Entities; +using Gtk; + +public class Dino.Ui.CallWindowController : Object { + + public signal void terminated(); + + private CallWindow call_window; + private Call call; + private Conversation conversation; + private StreamInteractor stream_interactor; + private Calls calls; + private Plugins.VideoCallPlugin call_plugin = Dino.Application.get_default().plugin_registry.video_call_plugin; + + private Plugins.VideoCallWidget? own_video = null; + private Plugins.VideoCallWidget? counterpart_video = null; + + public CallWindowController(CallWindow call_window, Call call, StreamInteractor stream_interactor) { + this.call_window = call_window; + this.call = call; + this.stream_interactor = stream_interactor; + + this.calls = stream_interactor.get_module(Calls.IDENTITY); + this.conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation(call.counterpart.bare_jid, call.account, Conversation.Type.CHAT); + this.own_video = call_plugin.create_widget(Plugins.WidgetType.GTK); + this.counterpart_video = call_plugin.create_widget(Plugins.WidgetType.GTK); + + call_window.counterpart_display_name = Util.get_conversation_display_name(stream_interactor, conversation); + call_window.set_default_size(640, 480); + call_window.set_video_fallback(stream_interactor, conversation); + + this.call_window.bottom_bar.video_enabled = calls.should_we_send_video(call); + + if (call.direction == Call.DIRECTION_INCOMING) { + call_window.set_status("establishing"); + } else { + call_window.set_status("requested"); + } + + call_window.bottom_bar.hang_up.connect(end_call); + call_window.destroy.connect(end_call); + + call_window.bottom_bar.notify["audio-enabled"].connect(() => { + calls.mute_own_audio(call, !call_window.bottom_bar.audio_enabled); + }); + call_window.bottom_bar.notify["video-enabled"].connect(() => { + calls.mute_own_video(call, !call_window.bottom_bar.video_enabled); + update_own_video(); + }); + + calls.counterpart_sends_video_updated.connect((call, mute) => { + if (!this.call.equals(call)) return; + + if (mute) { + call_window.set_video_fallback(stream_interactor, conversation); + counterpart_video.detach(); + } else { + if (!(counterpart_video is Widget)) return; + Widget widget = (Widget) counterpart_video; + call_window.set_video(widget); + counterpart_video.display_stream(calls.get_video_stream(call)); + } + }); + calls.info_received.connect((call, session_info) => { + if (!this.call.equals(call)) return; + if (session_info == Xmpp.Xep.JingleRtp.CallSessionInfo.RINGING) { + call_window.set_status("ringing"); + } + }); + + own_video.resolution_changed.connect((width, height) => { + if (width == 0 || height == 0) return; + call_window.set_own_video_ratio((int)width, (int)height); + }); + counterpart_video.resolution_changed.connect((width, height) => { + if (width == 0 || height == 0) return; + if (width / height > 640 / 480) { + call_window.resize(640, (int) (height * 640 / width)); + } else { + call_window.resize((int) (width * 480 / height), 480); + } + }); + + call.notify["state"].connect(on_call_state_changed); + calls.call_terminated.connect(on_call_terminated); + + update_own_video(); + } + + private void end_call() { + call.notify["state"].disconnect(on_call_state_changed); + calls.call_terminated.disconnect(on_call_terminated); + + calls.end_call(conversation, call); + call_window.close(); + call_window.destroy(); + terminated(); + } + + private void on_call_state_changed() { + if (call.state == Call.State.IN_PROGRESS) { + call_window.set_status(""); + call_plugin.devices_changed.connect((media, incoming) => { + if (media == "audio") update_audio_device_choices(); + if (media == "video") update_video_device_choices(); + }); + + update_audio_device_choices(); + update_video_device_choices(); + } + } + + private void on_call_terminated(Call call, string? reason_name, string? reason_text) { + call_window.show_counterpart_ended(reason_name, reason_text); + Timeout.add_seconds(3, () => { + call.notify["state"].disconnect(on_call_state_changed); + calls.call_terminated.disconnect(on_call_terminated); + + + call_window.close(); + call_window.destroy(); + + return false; + }); + } + + private void update_audio_device_choices() { + if (call_plugin.get_devices("audio", true).size == 0 || call_plugin.get_devices("audio", false).size == 0) { + call_window.bottom_bar.show_audio_device_error(); + } /*else if (call_plugin.get_devices("audio", true).size == 1 && call_plugin.get_devices("audio", false).size == 1) { + call_window.bottom_bar.show_audio_device_choices(false); + return; + } + + AudioSettingsPopover? audio_settings_popover = call_window.bottom_bar.show_audio_device_choices(true); + update_current_audio_device(audio_settings_popover); + + audio_settings_popover.microphone_selected.connect((device) => { + call_plugin.set_device(calls.get_audio_stream(call), device); + update_current_audio_device(audio_settings_popover); + }); + audio_settings_popover.speaker_selected.connect((device) => { + call_plugin.set_device(calls.get_audio_stream(call), device); + update_current_audio_device(audio_settings_popover); + }); + calls.stream_created.connect((call, media) => { + if (media == "audio") { + update_current_audio_device(audio_settings_popover); + } + });*/ + } + + private void update_current_audio_device(AudioSettingsPopover audio_settings_popover) { + Xmpp.Xep.JingleRtp.Stream stream = calls.get_audio_stream(call); + if (stream != null) { + audio_settings_popover.current_microphone_device = call_plugin.get_device(stream, false); + audio_settings_popover.current_speaker_device = call_plugin.get_device(stream, true); + } + } + + private void update_video_device_choices() { + int device_count = call_plugin.get_devices("video", false).size; + + if (device_count == 0) { + call_window.bottom_bar.show_video_device_error(); + } /*else if (device_count == 1 || calls.get_video_stream(call) == null) { + call_window.bottom_bar.show_video_device_choices(false); + return; + } + + VideoSettingsPopover? video_settings_popover = call_window.bottom_bar.show_video_device_choices(true); + update_current_video_device(video_settings_popover); + + video_settings_popover.camera_selected.connect((device) => { + call_plugin.set_device(calls.get_video_stream(call), device); + update_current_video_device(video_settings_popover); + own_video.display_device(device); + }); + calls.stream_created.connect((call, media) => { + if (media == "video") { + update_current_video_device(video_settings_popover); + } + });*/ + } + + private void update_current_video_device(VideoSettingsPopover video_settings_popover) { + Xmpp.Xep.JingleRtp.Stream stream = calls.get_video_stream(call); + if (stream != null) { + video_settings_popover.current_device = call_plugin.get_device(stream, false); + } + } + + private void update_own_video() { + if (this.call_window.bottom_bar.video_enabled) { + Gee.List devices = call_plugin.get_devices("video", false); + if (!(own_video is Widget) || devices.is_empty) { + call_window.set_own_video(null); + } else { + Widget widget = (Widget) own_video; + call_window.set_own_video(widget); + own_video.display_device(devices.first()); + } + } else { + own_video.detach(); + call_window.unset_own_video(); + } + } +} \ No newline at end of file diff --git a/main/src/ui/call_window/video_settings_popover.vala b/main/src/ui/call_window/video_settings_popover.vala new file mode 100644 index 00000000..396c697c --- /dev/null +++ b/main/src/ui/call_window/video_settings_popover.vala @@ -0,0 +1,73 @@ +using Gee; +using Gtk; +using Dino.Entities; + +public class Dino.Ui.VideoSettingsPopover : Gtk.Popover { + + public signal void camera_selected(Plugins.MediaDevice device); + + public Plugins.MediaDevice? current_device { get; set; } + + private HashMap row_device = new HashMap(); + + public VideoSettingsPopover() { + Box box = new Box(Orientation.VERTICAL, 15) { margin=18, visible=true }; + box.add(create_camera_box()); + + this.add(box); + } + + private Widget create_camera_box() { + Plugins.VideoCallPlugin call_plugin = Dino.Application.get_default().plugin_registry.video_call_plugin; + Gee.List devices = call_plugin.get_devices("video", false); + + Box camera_box = new Box(Orientation.VERTICAL, 10) { visible=true }; + camera_box.add(new Label("" + _("Cameras") + "") { use_markup=true, xalign=0, visible=true, can_focus=true /* grab initial focus*/ }); + + if (devices.size == 0) { + camera_box.add(new Label("No cameras found.") { visible=true }); + } else { + ListBox list_box = new ListBox() { activate_on_single_click=true, selection_mode=SelectionMode.SINGLE, visible=true }; + list_box.set_header_func(listbox_header_func); + Frame frame = new Frame(null) { visible=true }; + frame.add(list_box); + foreach (Plugins.MediaDevice device in devices) { + Label label = new Label(device.display_name) { xalign=0, visible=true }; + Image image = new Image.from_icon_name("object-select-symbolic", IconSize.BUTTON) { visible=true }; + if (current_device == null || current_device.id != device.id) { + image.opacity = 0; + } + this.notify["current-device"].connect(() => { + if (current_device == null || current_device.id != device.id) { + image.opacity = 0; + } else { + image.opacity = 1; + } + }); + Box device_box = new Box(Orientation.HORIZONTAL, 0) { spacing=7, margin=7, visible=true }; + device_box.add(image); + device_box.add(label); + ListBoxRow list_box_row = new ListBoxRow() { visible=true }; + list_box_row.add(device_box); + list_box.add(list_box_row); + + row_device[list_box_row] = device; + } + list_box.row_activated.connect((row) => { + if (!row_device.has_key(row)) return; + camera_selected(row_device[row]); + list_box.unselect_row(row); + }); + camera_box.add(frame); + } + + return camera_box; + } + + private void listbox_header_func(ListBoxRow row, ListBoxRow? before_row) { + if (row.get_header() == null && before_row != null) { + row.set_header(new Separator(Orientation.HORIZONTAL)); + } + } + +} \ No newline at end of file diff --git a/main/src/ui/conversation_content_view/call_widget.vala b/main/src/ui/conversation_content_view/call_widget.vala new file mode 100644 index 00000000..66788e28 --- /dev/null +++ b/main/src/ui/conversation_content_view/call_widget.vala @@ -0,0 +1,215 @@ +using Gee; +using Gdk; +using Gtk; +using Pango; + +using Dino.Entities; + +namespace Dino.Ui { + + public class CallMetaItem : ConversationSummary.ContentMetaItem { + + private StreamInteractor stream_interactor; + + public CallMetaItem(ContentItem content_item, StreamInteractor stream_interactor) { + base(content_item); + this.stream_interactor = stream_interactor; + } + + public override Object? get_widget(Plugins.WidgetType type) { + CallItem call_item = content_item as CallItem; + return new CallWidget(stream_interactor, call_item.call, call_item.conversation) { visible=true }; + } + + public override Gee.List? get_item_actions(Plugins.WidgetType type) { return null; } + } + + [GtkTemplate (ui = "/im/dino/Dino/call_widget.ui")] + public class CallWidget : SizeRequestBox { + + [GtkChild] public Image image; + [GtkChild] public Label title_label; + [GtkChild] public Label subtitle_label; + [GtkChild] public Revealer incoming_call_revealer; + [GtkChild] public Button accept_call_button; + [GtkChild] public Button reject_call_button; + + private StreamInteractor stream_interactor; + private Call call; + private Conversation conversation; + public Call.State call_state { get; set; } // needs to be public for binding + private uint time_update_handler_id = 0; + + construct { + margin_top = 4; + size_request_mode = SizeRequestMode.HEIGHT_FOR_WIDTH; + } + + public CallWidget(StreamInteractor stream_interactor, Call call, Conversation conversation) { + this.stream_interactor = stream_interactor; + this.call = call; + this.conversation = conversation; + + size_allocate.connect((allocation) => { + if (allocation.height > parent.get_allocated_height()) { + Idle.add(() => { parent.queue_resize(); return false; }); + } + }); + + call.bind_property("state", this, "call-state"); + this.notify["call-state"].connect(update_widget); + + accept_call_button.clicked.connect(() => { + stream_interactor.get_module(Calls.IDENTITY).accept_call(call); + + var call_window = new CallWindow(); + call_window.controller = new CallWindowController(call_window, call, stream_interactor); + call_window.present(); + }); + + reject_call_button.clicked.connect(() => { + stream_interactor.get_module(Calls.IDENTITY).reject_call(call); + }); + + update_widget(); + } + + private void update_widget() { + incoming_call_revealer.reveal_child = false; + incoming_call_revealer.get_style_context().remove_class("incoming"); + + switch (call.state) { + case Call.State.RINGING: + image.set_from_icon_name("dino-phone-ring-symbolic", IconSize.LARGE_TOOLBAR); + if (call.direction == Call.DIRECTION_INCOMING) { + bool video = stream_interactor.get_module(Calls.IDENTITY).should_we_send_video(call); + title_label.label = video ? _("Video call incoming") : _("Call incoming"); + subtitle_label.label = "Ring ring…!"; + incoming_call_revealer.reveal_child = true; + incoming_call_revealer.get_style_context().add_class("incoming"); + } else { + title_label.label = _("Establishing call"); + subtitle_label.label = "Ring ring…?"; + } + break; + case Call.State.ESTABLISHING: + image.set_from_icon_name("dino-phone-ring-symbolic", IconSize.LARGE_TOOLBAR); + if (call.direction == Call.DIRECTION_INCOMING) { + bool video = stream_interactor.get_module(Calls.IDENTITY).should_we_send_video(call); + title_label.label = video ? _("Video call establishing") : _("Call establishing"); + subtitle_label.label = "Connecting…"; + } + break; + case Call.State.IN_PROGRESS: + image.set_from_icon_name("dino-phone-in-talk-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = _("Call in progress…"); + string duration = get_duration_string((new DateTime.now_utc()).difference(call.local_time)); + subtitle_label.label = _("Started %s ago").printf(duration); + + time_update_handler_id = Timeout.add_seconds(get_next_time_change() + 1, () => { + Source.remove(time_update_handler_id); + time_update_handler_id = 0; + update_widget(); + return true; + }); + + break; + case Call.State.OTHER_DEVICE_ACCEPTED: + image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = call.direction == Call.DIRECTION_INCOMING ? _("Incoming call") : _("Outgoing call"); + subtitle_label.label = _("You handled this call on another device"); + + break; + case Call.State.ENDED: + image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = _("Call ended"); + string formated_end = Util.format_time(call.end_time, _("%H∶%M"), _("%l∶%M %p")); + string duration = get_duration_string(call.end_time.difference(call.local_time)); + subtitle_label.label = _("Ended at %s").printf(formated_end) + + " · " + + _("Lasted for %s").printf(duration); + break; + case Call.State.MISSED: + image.set_from_icon_name("dino-phone-missed-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = _("Call missed"); + string who = null; + if (call.direction == Call.DIRECTION_INCOMING) { + who = "You"; + } else { + who = Util.get_participant_display_name(stream_interactor, conversation, call.to); + } + subtitle_label.label = "%s missed this call".printf(who); + break; + case Call.State.DECLINED: + image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = _("Call declined"); + string who = null; + if (call.direction == Call.DIRECTION_INCOMING) { + who = "You"; + } else { + who = Util.get_participant_display_name(stream_interactor, conversation, call.to); + } + subtitle_label.label = "%s declined this call".printf(who); + break; + case Call.State.FAILED: + image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = _("Call failed"); + subtitle_label.label = "This call failed to establish"; + break; + } + } + + private string get_duration_string(TimeSpan duration) { + DateTime a = new DateTime.now_utc(); + DateTime b = new DateTime.now_utc(); + a.difference(b); + + TimeSpan remainder_duration = duration; + + int hours = (int) Math.floor(remainder_duration / TimeSpan.HOUR); + remainder_duration -= hours * TimeSpan.HOUR; + + int minutes = (int) Math.floor(remainder_duration / TimeSpan.MINUTE); + remainder_duration -= minutes * TimeSpan.MINUTE; + + string ret = ""; + + if (hours > 0) { + ret += n("%i hour", "%i hours", hours).printf(hours); + } + + if (minutes > 0) { + if (ret.length > 0) { + ret += " "; + } + ret += n("%i minute", "%i minutes", minutes).printf(minutes); + } + + if (ret.length > 0) { + return ret; + } + + return _("seconds"); + } + + private int get_next_time_change() { + DateTime now = new DateTime.now_local(); + DateTime item_time = call.local_time; + + if (now.get_second() < item_time.get_second()) { + return item_time.get_second() - now.get_second(); + } else { + return 60 - (now.get_second() - item_time.get_second()); + } + } + + public override void dispose() { + base.dispose(); + + if (time_update_handler_id != 0) { + Source.remove(time_update_handler_id); + time_update_handler_id = 0; + } + } + } +} diff --git a/main/src/ui/conversation_content_view/content_populator.vala b/main/src/ui/conversation_content_view/content_populator.vala index 97f15bf9..d7ce9ce5 100644 --- a/main/src/ui/conversation_content_view/content_populator.vala +++ b/main/src/ui/conversation_content_view/content_populator.vala @@ -68,7 +68,10 @@ public class ContentProvider : ContentItemCollection, Object { return new MessageMetaItem(content_item, stream_interactor); } else if (content_item.type_ == FileItem.TYPE) { return new FileMetaItem(content_item, stream_interactor); + } else if (content_item.type_ == CallItem.TYPE) { + return new CallMetaItem(content_item, stream_interactor); } + critical("Got unknown content item type %s", content_item.type_); return null; } } diff --git a/main/src/ui/conversation_content_view/file_widget.vala b/main/src/ui/conversation_content_view/file_widget.vala index 9b748876..7d77ba11 100644 --- a/main/src/ui/conversation_content_view/file_widget.vala +++ b/main/src/ui/conversation_content_view/file_widget.vala @@ -32,9 +32,6 @@ public class FileWidget : SizeRequestBox { DEFAULT } - private const int MAX_HEIGHT = 300; - private const int MAX_WIDTH = 600; - private StreamInteractor stream_interactor; private FileTransfer file_transfer; public FileTransfer.State file_transfer_state { get; set; } diff --git a/main/src/ui/conversation_selector/conversation_selector_row.vala b/main/src/ui/conversation_selector/conversation_selector_row.vala index cd513d13..6f181a64 100644 --- a/main/src/ui/conversation_selector/conversation_selector_row.vala +++ b/main/src/ui/conversation_selector/conversation_selector_row.vala @@ -198,6 +198,14 @@ public class ConversationSelectorRow : ListBoxRow { message_label.label = (file_is_image ? _("Image received") : _("File received") ); } break; + case CallItem.TYPE: + CallItem call_item = (CallItem) last_content_item; + Call call = call_item.call; + + nick_label.label = call.direction == Call.DIRECTION_OUTGOING ? _("Me") + ": " : ""; + message_label.attributes.insert(attr_style_new(Pango.Style.ITALIC)); + message_label.label = call.direction == Call.DIRECTION_OUTGOING ? _("Outgoing call") : _("Incoming call"); + break; } nick_label.visible = true; message_label.visible = true; diff --git a/main/src/ui/conversation_titlebar/call_entry.vala b/main/src/ui/conversation_titlebar/call_entry.vala new file mode 100644 index 00000000..1ac4dd83 --- /dev/null +++ b/main/src/ui/conversation_titlebar/call_entry.vala @@ -0,0 +1,130 @@ +using Xmpp; +using Gtk; +using Gee; + +using Dino.Entities; + +namespace Dino.Ui { + + public class CallTitlebarEntry : Plugins.ConversationTitlebarEntry, Object { + public string id { get { return "call"; } } + + public CallButton call_button; + + private StreamInteractor stream_interactor; + + public CallTitlebarEntry(StreamInteractor stream_interactor) { + this.stream_interactor = stream_interactor; + + call_button = new CallButton(stream_interactor) { tooltip_text=_("Start call") }; + call_button.set_image(new Gtk.Image.from_icon_name("dino-phone-symbolic", Gtk.IconSize.MENU) { visible=true }); + } + + public double order { get { return 4; } } + public Plugins.ConversationTitlebarWidget? get_widget(Plugins.WidgetType type) { + if (type == Plugins.WidgetType.GTK) { + return call_button; + } + return null; + } + } + + public class CallButton : Plugins.ConversationTitlebarWidget, Gtk.MenuButton { + + private StreamInteractor stream_interactor; + private Conversation conversation; + + public CallButton(StreamInteractor stream_interactor) { + this.stream_interactor = stream_interactor; + + use_popover = true; + image = new Gtk.Image.from_icon_name("dino-phone-symbolic", Gtk.IconSize.MENU) { visible=true }; + + Gtk.PopoverMenu popover_menu = new Gtk.PopoverMenu(); + Box box = new Box(Orientation.VERTICAL, 0) { margin=10, visible=true }; + ModelButton audio_button = new ModelButton() { text="Audio call", visible=true }; + audio_button.clicked.connect(() => { + stream_interactor.get_module(Calls.IDENTITY).initiate_call.begin(conversation, false, (_, res) => { + Call call = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); + open_call_window(call); + }); + }); + box.add(audio_button); + ModelButton video_button = new ModelButton() { text="Video call", visible=true }; + video_button.clicked.connect(() => { + stream_interactor.get_module(Calls.IDENTITY).initiate_call.begin(conversation, true, (_, res) => { + Call call = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); + open_call_window(call); + }); + }); + box.add(video_button); + popover_menu.add(box); + + popover = popover_menu; + + clicked.connect(() => { + popover_menu.visible = true; + }); + + stream_interactor.get_module(Calls.IDENTITY).call_incoming.connect((call, conversation) => { + update_button_state(); + }); + + stream_interactor.get_module(Calls.IDENTITY).call_terminated.connect((call) => { + update_button_state(); + }); + stream_interactor.get_module(PresenceManager.IDENTITY).show_received.connect((jid, account) => { + if (this.conversation.counterpart.equals_bare(jid) && this.conversation.account.equals(account)) { + update_visibility.begin(); + } + }); + stream_interactor.connection_manager.connection_state_changed.connect((account, state) => { + update_visibility.begin(); + }); + } + + private void open_call_window(Call call) { + var call_window = new CallWindow(); + var call_controller = new CallWindowController(call_window, call, stream_interactor); + call_window.controller = call_controller; + call_window.present(); + + update_button_state(); + call_controller.terminated.connect(() => { + update_button_state(); + }); + } + + public new void set_conversation(Conversation conversation) { + this.conversation = conversation; + + update_visibility.begin(); + update_button_state(); + } + + private void update_button_state() { + Jid? call_counterpart = stream_interactor.get_module(Calls.IDENTITY).is_call_in_progress(); + this.sensitive = call_counterpart == null; + + if (call_counterpart != null && call_counterpart.equals_bare(conversation.counterpart)) { + this.set_image(new Gtk.Image.from_icon_name("dino-phone-in-talk-symbolic", Gtk.IconSize.MENU) { visible=true }); + } else { + this.set_image(new Gtk.Image.from_icon_name("dino-phone-symbolic", Gtk.IconSize.MENU) { visible=true }); + } + } + + private async void update_visibility() { + if (conversation.type_ == Conversation.Type.CHAT) { + Conversation conv_bak = conversation; + Gee.List? resources = yield stream_interactor.get_module(Calls.IDENTITY).get_call_resources(conversation); + if (conv_bak != conversation) return; + visible = resources != null && resources.size > 0; + } else { + visible = false; + } + } + + public new void unset_conversation() { } + } + +} diff --git a/main/src/ui/conversation_view_controller.vala b/main/src/ui/conversation_view_controller.vala index dcd3e1c7..a9a94738 100644 --- a/main/src/ui/conversation_view_controller.vala +++ b/main/src/ui/conversation_view_controller.vala @@ -87,6 +87,7 @@ public class ConversationViewController : Object { app.plugin_registry.register_contact_titlebar_entry(new MenuEntry(stream_interactor)); app.plugin_registry.register_contact_titlebar_entry(search_menu_entry); app.plugin_registry.register_contact_titlebar_entry(new OccupantsEntry(stream_interactor)); + app.plugin_registry.register_contact_titlebar_entry(new CallTitlebarEntry(stream_interactor)); foreach(var entry in app.plugin_registry.conversation_titlebar_entries) { titlebar.insert_entry(entry); } diff --git a/main/src/ui/notifier_freedesktop.vala b/main/src/ui/notifier_freedesktop.vala index 00ba0d06..35b95e3e 100644 --- a/main/src/ui/notifier_freedesktop.vala +++ b/main/src/ui/notifier_freedesktop.vala @@ -14,6 +14,7 @@ public class Dino.Ui.FreeDesktopNotifier : NotificationProvider, Object { private HashMap content_notifications = new HashMap(Conversation.hash_func, Conversation.equals_func); private HashMap> conversation_notifications = new HashMap>(Conversation.hash_func, Conversation.equals_func); private HashMap> action_listeners = new HashMap>(); + private HashMap call_notifications = new HashMap(Call.hash_func, Call.equals_func); private FreeDesktopNotifier(StreamInteractor stream_interactor) { this.stream_interactor = stream_interactor; @@ -109,6 +110,43 @@ public class Dino.Ui.FreeDesktopNotifier : NotificationProvider, Object { } } + public async void notify_call(Call call, Conversation conversation, bool video, string conversation_display_name) { + string summary = Markup.escape_text(conversation_display_name); + string body = video ? _("Incoming video call") : _("Incoming call"); + + HashTable hash_table = new HashTable(null, null); + hash_table["image-path"] = "call-start-symbolic"; + hash_table["sound-name"] = new Variant.string("phone-incoming-call"); + hash_table["urgency"] = new Variant.byte(2); + string[] actions = new string[] {"default", "Open conversation", "reject", _("Reject"), "accept", _("Accept")}; + try { + uint32 notification_id = dbus_notifications.notify("Dino", 0, "", summary, body, actions, hash_table, 0); + call_notifications[call] = notification_id; + + add_action_listener(notification_id, "default", () => { + GLib.Application.get_default().activate_action("open-conversation", new Variant.int32(conversation.id)); + }); + add_action_listener(notification_id, "reject", () => { + GLib.Application.get_default().activate_action("deny-call", new Variant.int32(call.id)); + }); + add_action_listener(notification_id, "accept", () => { + GLib.Application.get_default().activate_action("accept-call", new Variant.int32(call.id)); + }); + } catch (Error e) { + warning("Failed showing subscription request notification: %s", e.message); + } + } + + public async void retract_call_notification(Call call, Conversation conversation) { + if (!call_notifications.has_key(call)) return; + uint32 notification_id = call_notifications[call]; + try { + dbus_notifications.close_notification(notification_id); + action_listeners.unset(notification_id); + call_notifications.unset(call); + } catch (Error e) { } + } + public async void notify_subscription_request(Conversation conversation) { string summary = _("Subscription request"); string body = Markup.escape_text(conversation.counterpart.to_string()); diff --git a/main/src/ui/notifier_gnotifications.vala b/main/src/ui/notifier_gnotifications.vala index 31d1ffa3..5fd3be4b 100644 --- a/main/src/ui/notifier_gnotifications.vala +++ b/main/src/ui/notifier_gnotifications.vala @@ -65,6 +65,25 @@ namespace Dino.Ui { } } + public async void notify_call(Call call, Conversation conversation, bool video, string conversation_display_name) { + Notification notification = new Notification(conversation_display_name); + string body = _("Incoming call"); + notification.set_body(body); + notification.set_urgent(true); + + notification.set_icon(new ThemedIcon.from_names(new string[] {"call-start-symbolic"})); + + notification.set_default_action_and_target_value("app.open-conversation", new Variant.int32(conversation.id)); + notification.add_button_with_target_value(_("Deny"), "app.deny-call", new Variant.int32(call.id)); + notification.add_button_with_target_value(_("Accept"), "app.accept-call", new Variant.int32(call.id)); + + GLib.Application.get_default().send_notification(call.id.to_string(), notification); + } + + private async void retract_call_notification(Call call, Conversation conversation) { + GLib.Application.get_default().withdraw_notification(call.id.to_string()); + } + public async void notify_subscription_request(Conversation conversation) { Notification notification = new Notification(_("Subscription request")); notification.set_body(conversation.counterpart.to_string()); diff --git a/main/src/ui/util/helper.vala b/main/src/ui/util/helper.vala index b6c9cb5a..d3ca063b 100644 --- a/main/src/ui/util/helper.vala +++ b/main/src/ui/util/helper.vala @@ -194,6 +194,15 @@ public static bool is_24h_format() { return is24h == 1; } +public static string format_time(DateTime datetime, string format_24h, string format_12h) { + string format = Util.is_24h_format() ? format_24h : format_12h; + if (!get_charset(null)) { + // No UTF-8 support, use simple colon for time instead + format = format.replace("∶", ":"); + } + return datetime.format(format); +} + public static Regex get_url_regex() { if (URL_REGEX == null) { URL_REGEX = /\b(((http|ftp)s?:\/\/|(ircs?|xmpp|mailto|sms|smsto|mms|tel|geo|openpgp4fpr|im|news|nntp|sip|ssh|bitcoin|sftp|magnet|vnc|urn):)\S+)/; From c6c677d62da868953c3e5cba33ddee2f1f8047c7 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Sun, 21 Mar 2021 13:26:22 +0100 Subject: [PATCH 12/56] Adapt github actions for libnice & libgstreamer dependency --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0fb5e49..2ec82dd5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,7 @@ jobs: steps: - uses: actions/checkout@v2 - run: sudo apt-get update - - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-3-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libgspell-1-dev + - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-3-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libgspell-1-dev libnice-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev - run: ./configure --with-tests --with-libsignal-in-tree - run: make - run: build/xmpp-vala-test From cde1e38f5d56269addff93b36c57299cbf546279 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 21 Mar 2021 15:42:58 +0100 Subject: [PATCH 13/56] RTP: Backport gst_caps_copy_nth from GStreamer 1.16 --- plugins/rtp/src/device.vala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 796be203..570c6667 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -104,14 +104,22 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { best_index = i; } } - return device.caps.copy_nth(best_index); + return caps_copy_nth(device.caps, best_index); } else if (device.caps.get_size() > 0) { - return device.caps.copy_nth(0); + return caps_copy_nth(device.caps, 0); } else { return new Gst.Caps.any(); } } + // Backport from gst_caps_copy_nth added in GStreamer 1.16 + private static Gst.Caps caps_copy_nth(Gst.Caps source, uint index) { + Gst.Caps target = new Gst.Caps.empty(); + target.flags = source.flags; + target.append_structure_full(source.get_structure(index).copy(), source.get_features(index).copy()); + return target; + } + private void create() { debug("Creating device %s", id); plugin.pause(); From 9fed5ea8650d6e7735fca4b3fe2cf4fc29f81c33 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 21 Mar 2021 16:01:50 +0100 Subject: [PATCH 14/56] Don't wait for reply on XEP-0199 pongs. Nobody replies to pongs... --- xmpp-vala/src/module/xep/0199_ping.vala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmpp-vala/src/module/xep/0199_ping.vala b/xmpp-vala/src/module/xep/0199_ping.vala index f3e68660..0b31011f 100644 --- a/xmpp-vala/src/module/xep/0199_ping.vala +++ b/xmpp-vala/src/module/xep/0199_ping.vala @@ -23,7 +23,7 @@ namespace Xmpp.Xep.Ping { } public async void on_iq_get(XmppStream stream, Iq.Stanza iq) { - yield stream.get_module(Iq.Module.IDENTITY).send_iq_async(stream, new Iq.Stanza.result(iq)); + stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); } public override string get_ns() { return NS_URI; } From b393d4160182873ea2acd9fbc6421f7e1a3adb9e Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 23 Mar 2021 15:05:50 +0100 Subject: [PATCH 15/56] Add support for SRTP --- plugins/crypto-vala/CMakeLists.txt | 1 + plugins/crypto-vala/src/random.vala | 5 + plugins/rtp/CMakeLists.txt | 8 +- plugins/rtp/src/module.vala | 80 +- plugins/rtp/src/srtp.c | 836 ++++++++++++++++++ plugins/rtp/src/srtp.h | 82 ++ plugins/rtp/src/srtp.vapi | 103 +++ plugins/rtp/src/stream.vala | 60 +- .../0167_jingle_rtp/content_parameters.vala | 46 +- .../0167_jingle_rtp/jingle_rtp_module.vala | 132 ++- .../module/xep/0167_jingle_rtp/stream.vala | 14 + 11 files changed, 1286 insertions(+), 81 deletions(-) create mode 100644 plugins/crypto-vala/src/random.vala create mode 100644 plugins/rtp/src/srtp.c create mode 100644 plugins/rtp/src/srtp.h create mode 100644 plugins/rtp/src/srtp.vapi diff --git a/plugins/crypto-vala/CMakeLists.txt b/plugins/crypto-vala/CMakeLists.txt index 2c9f790a..036e45f6 100644 --- a/plugins/crypto-vala/CMakeLists.txt +++ b/plugins/crypto-vala/CMakeLists.txt @@ -10,6 +10,7 @@ SOURCES "src/cipher.vala" "src/cipher_converter.vala" "src/error.vala" + "src/random.vala" CUSTOM_VAPIS "${CMAKE_CURRENT_SOURCE_DIR}/vapi/gcrypt.vapi" PACKAGES diff --git a/plugins/crypto-vala/src/random.vala b/plugins/crypto-vala/src/random.vala new file mode 100644 index 00000000..3f5d3ba9 --- /dev/null +++ b/plugins/crypto-vala/src/random.vala @@ -0,0 +1,5 @@ +namespace Crypto { +public static void randomize(uint8[] buffer) { + GCrypt.Random.randomize(buffer); +} +} \ No newline at end of file diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 2b66f6ff..ef2f7698 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -18,18 +18,20 @@ SOURCES src/video_widget.vala src/register_plugin.vala CUSTOM_VAPIS + ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi PACKAGES ${RTP_PACKAGES} OPTIONS --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi ) -add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp") -add_library(rtp SHARED ${RTP_VALA_C}) -target_link_libraries(rtp libdino ${RTP_PACKAGES}) +add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) +add_library(rtp SHARED ${RTP_VALA_C} src/srtp.c) +target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES}) set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index 577e9f53..ecf7b658 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -200,65 +200,23 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { plugin.close_stream(rtp_stream); } -// public uint32 get_session_id(string id) { -// return (uint32) id.split("-")[0].to_int(); -// } -// -// public string create_feed(string media, bool incoming) { -// init(); -// string id = random_uuid(); -// if (media == "audio") { -// id = "0-" + id; -// } else { -// id = "1-" + id; -// } -// MediaDevice? device = plugin.get_preferred_device(media, incoming); -// Feed feed; -// if (incoming) { -// if (media == "audio") { -// feed = new IncomingAudioFeed(id, this, device); -// } else if (media == "video") { -// feed = new IncomingVideoFeed(id, this, device); -// } else { -// critical("Incoming feed of media '%s' not supported", media); -// return id; -// } -// } else { -// if (media == "audio") { -// string? matching_incoming_feed_id = null; -// foreach (Feed match in plugin.feeds.values) { -// if (match is IncomingAudioFeed) { -// matching_incoming_feed_id = match.id; -// } -// } -// feed = new OutgoingAudioFeed(id, this, device); -// } else if (media == "video") { -// feed = new OutgoingVideoFeed(id, this, device); -// } else { -// critical("Outgoing feed of media '%s' not supported", media); -// return id; -// } -// } -// plugin.add_feed(id, feed); -// return id; -// } -// -// public void connect_feed(string id, JingleRtp.PayloadType payload, Jingle.DatagramConnection connection) { -// if (!plugin.feeds.has_key(id)) { -// critical("Tried to connect feed with id %s, but no such feed found", id); -// return; -// } -// Feed feed = plugin.feeds[id]; -// feed.connect(payload, connection); -// } -// -// public void destroy_feed(string id) { -// if (!plugin.feeds.has_key(id)) { -// critical("Tried to destroy feed with id %s, but no such feed found", id); -// return; -// } -// Feed feed = plugin.feeds[id]; -// feed.destroy(); -// plugin.feeds.remove(id); -// } + public override JingleRtp.Crypto? generate_local_crypto() { + uint8[] keyAndSalt = new uint8[30]; + Crypto.randomize(keyAndSalt); + return JingleRtp.Crypto.create(JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_80, keyAndSalt); + } + + public override JingleRtp.Crypto? pick_remote_crypto(Gee.List cryptos) { + foreach (JingleRtp.Crypto crypto in cryptos) { + if (crypto.is_valid) return crypto; + } + return null; + } + + public override JingleRtp.Crypto? pick_local_crypto(JingleRtp.Crypto? remote) { + if (remote == null || !remote.is_valid) return null; + uint8[] keyAndSalt = new uint8[30]; + Crypto.randomize(keyAndSalt); + return remote.rekey(keyAndSalt); + } } \ No newline at end of file diff --git a/plugins/rtp/src/srtp.c b/plugins/rtp/src/srtp.c new file mode 100644 index 00000000..708244d9 --- /dev/null +++ b/plugins/rtp/src/srtp.c @@ -0,0 +1,836 @@ +/* + * Secure RTP with libgcrypt + * Copyright (C) 2007 Rémi Denis-Courmont + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* TODO: + * Useless stuff (because nothing depends on it): + * - non-nul key derivation rate + * - MKI payload + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include +#include + +#include "srtp.h" + +#include +#include +#include +#include + +#include + +#ifdef _WIN32 +# include +#else +# include +#endif + +#define debug( ... ) (void)0 + +typedef struct srtp_proto_t +{ + gcry_cipher_hd_t cipher; + gcry_md_hd_t mac; + uint64_t window; + uint32_t salt[4]; +} srtp_proto_t; + +struct srtp_session_t +{ + srtp_proto_t rtp; + srtp_proto_t rtcp; + unsigned flags; + unsigned kdr; + uint32_t rtcp_index; + uint32_t rtp_roc; + uint16_t rtp_seq; + uint16_t rtp_rcc; + uint8_t tag_len; +}; + +enum +{ + SRTP_CRYPT, + SRTP_AUTH, + SRTP_SALT, + SRTCP_CRYPT, + SRTCP_AUTH, + SRTCP_SALT +}; + + +static inline unsigned rcc_mode (const srtp_session_t *s) +{ + return (s->flags >> 4) & 3; +} + + +static void proto_destroy (srtp_proto_t *p) +{ + gcry_md_close (p->mac); + gcry_cipher_close (p->cipher); +} + + +/** + * Releases all resources associated with a Secure RTP session. + */ +void srtp_destroy (srtp_session_t *s) +{ + assert (s != NULL); + + proto_destroy (&s->rtcp); + proto_destroy (&s->rtp); + free (s); +} + + +static int proto_create (srtp_proto_t *p, int gcipher, int gmd) +{ + if (gcry_cipher_open (&p->cipher, gcipher, GCRY_CIPHER_MODE_CTR, 0) == 0) + { + if (gcry_md_open (&p->mac, gmd, GCRY_MD_FLAG_HMAC) == 0) + return 0; + gcry_cipher_close (p->cipher); + } + return -1; +} + + +/** + * Allocates a Secure RTP one-way session. + * The same session cannot be used both ways because this would confuse + * internal cryptographic counters; it is however of course feasible to open + * multiple simultaneous sessions with the same master key. + * + * @param encr encryption algorithm number + * @param auth authentication algortihm number + * @param tag_len authentication tag byte length (NOT including RCC) + * @param flags OR'ed optional flags. + * + * @return NULL in case of error + */ +srtp_session_t * +srtp_create (int encr, int auth, unsigned tag_len, int prf, unsigned flags) +{ + if ((flags & ~SRTP_FLAGS_MASK)) + return NULL; + + int cipher, md; + switch (encr) + { + case SRTP_ENCR_NULL: + cipher = GCRY_CIPHER_NONE; + break; + + case SRTP_ENCR_AES_CM: + cipher = GCRY_CIPHER_AES; + break; + + default: + return NULL; + } + + switch (auth) + { + case SRTP_AUTH_NULL: + md = GCRY_MD_NONE; + break; + + case SRTP_AUTH_HMAC_SHA1: + md = GCRY_MD_SHA1; + break; + + default: + return NULL; + } + + if (tag_len > gcry_md_get_algo_dlen (md)) + return NULL; + + if (prf != SRTP_PRF_AES_CM) + return NULL; + + srtp_session_t *s = malloc (sizeof (*s)); + if (s == NULL) + return NULL; + + memset (s, 0, sizeof (*s)); + s->flags = flags; + s->tag_len = tag_len; + s->rtp_rcc = 1; /* Default RCC rate */ + if (rcc_mode (s)) + { + if (tag_len < 4) + goto error; + } + + if (proto_create (&s->rtp, cipher, md) == 0) + { + if (proto_create (&s->rtcp, cipher, md) == 0) + return s; + proto_destroy (&s->rtp); + } + + error: + free (s); + return NULL; +} + + +/** + * Counter Mode encryption/decryption (ctr length = 16 bytes) + * with non-padded (truncated) text + */ +static int +do_ctr_crypt (gcry_cipher_hd_t hd, const void *ctr, uint8_t *data, size_t len) +{ + const size_t ctrlen = 16; + div_t d = div (len, ctrlen); + + if (gcry_cipher_setctr (hd, ctr, ctrlen) + || gcry_cipher_encrypt (hd, data, d.quot * ctrlen, NULL, 0)) + return -1; + + if (d.rem) + { + /* Truncated last block */ + uint8_t dummy[ctrlen]; + data += d.quot * ctrlen; + memcpy (dummy, data, d.rem); + memset (dummy + d.rem, 0, ctrlen - d.rem); + + if (gcry_cipher_encrypt (hd, dummy, ctrlen, data, ctrlen)) + return -1; + memcpy (data, dummy, d.rem); + } + + return 0; +} + + +/** + * AES-CM key derivation (saltlen = 14 bytes) + */ +static int +do_derive (gcry_cipher_hd_t prf, const void *salt, + const uint8_t *r, size_t rlen, uint8_t label, + void *out, size_t outlen) +{ + uint8_t iv[16]; + + memcpy (iv, salt, 14); + iv[14] = iv[15] = 0; + + assert (rlen < 14); + iv[13 - rlen] ^= label; + for (size_t i = 0; i < rlen; i++) + iv[sizeof (iv) - rlen + i] ^= r[i]; + + memset (out, 0, outlen); + return do_ctr_crypt (prf, iv, out, outlen); +} + + +/** + * Sets (or resets) the master key and master salt for a SRTP session. + * This must be done at least once before using srtp_send(), srtp_recv(), + * srtcp_send() or srtcp_recv(). Also, rekeying is required every + * 2^48 RTP packets or 2^31 RTCP packets (whichever comes first), + * otherwise the protocol security might be broken. + * + * @return 0 on success, in case of error: + * EINVAL invalid or unsupported key/salt sizes combination + */ +int +srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, + const void *salt, size_t saltlen) +{ + /* SRTP/SRTCP cipher/salt/MAC keys derivation */ + gcry_cipher_hd_t prf; + uint8_t r[6], keybuf[20]; + + if (saltlen != 14) + return EINVAL; + + if (gcry_cipher_open (&prf, GCRY_CIPHER_AES, GCRY_CIPHER_MODE_CTR, 0) + || gcry_cipher_setkey (prf, key, keylen)) + return EINVAL; + + /* SRTP key derivation */ +#if 0 + if (s->kdr != 0) + { + uint64_t index = (((uint64_t)s->rtp_roc) << 16) | s->rtp_seq; + index /= s->kdr; + + for (int i = sizeof (r) - 1; i >= 0; i--) + { + r[i] = index & 0xff; + index = index >> 8; + } + } + else +#endif + memset (r, 0, sizeof (r)); + if (do_derive (prf, salt, r, 6, SRTP_CRYPT, keybuf, 16) + || gcry_cipher_setkey (s->rtp.cipher, keybuf, 16) + || do_derive (prf, salt, r, 6, SRTP_AUTH, keybuf, 20) + || gcry_md_setkey (s->rtp.mac, keybuf, 20) + || do_derive (prf, salt, r, 6, SRTP_SALT, s->rtp.salt, 14)) + return -1; + + /* SRTCP key derivation */ + memcpy (r, &(uint32_t){ htonl (s->rtcp_index) }, 4); + if (do_derive (prf, salt, r, 4, SRTCP_CRYPT, keybuf, 16) + || gcry_cipher_setkey (s->rtcp.cipher, keybuf, 16) + || do_derive (prf, salt, r, 4, SRTCP_AUTH, keybuf, 20) + || gcry_md_setkey (s->rtcp.mac, keybuf, 20) + || do_derive (prf, salt, r, 4, SRTCP_SALT, s->rtcp.salt, 14)) + return -1; + + (void)gcry_cipher_close (prf); + return 0; +} + +static int hexdigit (char c) +{ + if ((c >= '0') && (c <= '9')) + return c - '0'; + if ((c >= 'A') && (c <= 'F')) + return c - 'A' + 0xA; + if ((c >= 'a') && (c <= 'f')) + return c - 'a' + 0xa; + return -1; +} + +static ssize_t hexstring (const char *in, uint8_t *out, size_t outlen) +{ + size_t inlen = strlen (in); + + if ((inlen > (2 * outlen)) || (inlen & 1)) + return -1; + + for (size_t i = 0; i < inlen; i += 2) + { + int a = hexdigit (in[i]), b = hexdigit (in[i + 1]); + if ((a == -1) || (b == -1)) + return -1; + out[i / 2] = (a << 4) | b; + } + return inlen / 2; +} + +/** + * Sets (or resets) the master key and master salt for a SRTP session + * from hexadecimal strings. See also srtp_setkey(). + * + * @return 0 on success, in case of error: + * EINVAL invalid or unsupported key/salt sizes combination + */ +int +srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt) +{ + uint8_t bkey[16]; /* TODO/NOTE: hard-coded for AES */ + uint8_t bsalt[14]; /* TODO/NOTE: hard-coded for the PRF-AES-CM */ + ssize_t bkeylen = hexstring (key, bkey, sizeof (bkey)); + ssize_t bsaltlen = hexstring (salt, bsalt, sizeof (bsalt)); + + if ((bkeylen == -1) || (bsaltlen == -1)) + return EINVAL; + return srtp_setkey (s, bkey, bkeylen, bsalt, bsaltlen) ? EINVAL : 0; +} + +/** + * Sets Roll-over-Counter Carry (RCC) rate for the SRTP session. If not + * specified (through this function), the default rate of ONE is assumed + * (i.e. every RTP packets will carry the RoC). RCC rate is ignored if none + * of the RCC mode has been selected. + * + * The RCC mode is selected through one of these flags for srtp_create(): + * SRTP_RCC_MODE1: integrity protection only for RoC carrying packets + * SRTP_RCC_MODE2: integrity protection for all packets + * SRTP_RCC_MODE3: no integrity protection + * + * RCC mode 3 is insecure. Compared to plain RTP, it provides confidentiality + * (through encryption) but is much more prone to DoS. It can only be used if + * anti-spoofing protection is provided by lower network layers (e.g. IPsec, + * or trusted routers and proper source address filtering). + * + * If RCC rate is 1, RCC mode 1 and 2 are functionally identical. + * + * @param rate RoC Carry rate (MUST NOT be zero) + */ +void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate) +{ + assert (rate != 0); + s->rtp_rcc = rate; +} + + +/** AES-CM for RTP (salt = 14 bytes + 2 nul bytes) */ +static int +rtp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t roc, uint16_t seq, + const uint32_t *salt, uint8_t *data, size_t len) +{ + /* Determines cryptographic counter (IV) */ + uint32_t counter[4]; + counter[0] = salt[0]; + counter[1] = salt[1] ^ ssrc; + counter[2] = salt[2] ^ htonl (roc); + counter[3] = salt[3] ^ htonl (seq << 16); + + /* Encryption */ + return do_ctr_crypt (hd, counter, data, len); +} + + +/** Determines SRTP Roll-Over-Counter (in host-byte order) */ +static uint32_t +srtp_compute_roc (const srtp_session_t *s, uint16_t seq) +{ + uint32_t roc = s->rtp_roc; + + if (((seq - s->rtp_seq) & 0xffff) < 0x8000) + { + /* Sequence is ahead, good */ + if (seq < s->rtp_seq) + roc++; /* Sequence number wrap */ + } + else + { + /* Sequence is late, bad */ + if (seq > s->rtp_seq) + roc--; /* Wrap back */ + } + return roc; +} + + +/** Returns RTP sequence (in host-byte order) */ +static inline uint16_t rtp_seq (const uint8_t *buf) +{ + return (buf[2] << 8) | buf[3]; +} + + +/** Message Authentication and Integrity for RTP */ +static const uint8_t * +rtp_digest (gcry_md_hd_t md, const uint8_t *data, size_t len, + uint32_t roc) +{ + gcry_md_reset (md); + gcry_md_write (md, data, len); + gcry_md_write (md, &(uint32_t){ htonl (roc) }, 4); + return gcry_md_read (md, 0); +} + + +/** + * Encrypts/decrypts a RTP packet and updates SRTP context + * (CTR block cypher mode of operation has identical encryption and + * decryption function). + * + * @param buf RTP packet to be en-/decrypted + * @param len RTP packet length + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTP packet + * EACCES replayed packet or out-of-window or sync lost + */ +static int srtp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) +{ + assert (s != NULL); + assert (len >= 12u); + + if ((buf[0] >> 6) != 2) + return EINVAL; + + /* Computes encryption offset */ + uint16_t offset = 12; + offset += (buf[0] & 0xf) * 4; // skips CSRC + + if (buf[0] & 0x10) + { + uint16_t extlen; + + offset += 4; + if (len < offset) + return EINVAL; + + memcpy (&extlen, buf + offset - 2, 2); + offset += htons (extlen); // skips RTP extension header + } + + if (len < offset) + return EINVAL; + + /* Determines RTP 48-bits counter and SSRC */ + uint16_t seq = rtp_seq (buf); + uint32_t roc = srtp_compute_roc (s, seq), ssrc; + memcpy (&ssrc, buf + 8, 4); + + /* Updates ROC and sequence (it's safe now) */ + int16_t diff = seq - s->rtp_seq; + if (diff > 0) + { + /* Sequence in the future, good */ + s->rtp.window = s->rtp.window << diff; + s->rtp.window |= UINT64_C(1); + s->rtp_seq = seq, s->rtp_roc = roc; + } + else + { + /* Sequence in the past/present, bad */ + diff = -diff; + if ((diff >= 64) || ((s->rtp.window >> diff) & 1)) + return EACCES; /* Replay attack */ + s->rtp.window |= UINT64_C(1) << diff; + } + + /* Encrypt/Decrypt */ + if (s->flags & SRTP_UNENCRYPTED) + return 0; + + if (rtp_crypt (s->rtp.cipher, ssrc, roc, seq, s->rtp.salt, + buf + offset, len - offset)) + return EINVAL; + + return 0; +} + + +/** + * Turns a RTP packet into a SRTP packet: encrypt it, then computes + * the authentication tag and appends it. + * Note that you can encrypt packet in disorder. + * + * @param buf RTP packet to be encrypted/digested + * @param lenp pointer to the RTP packet length on entry, + * set to the SRTP length on exit (undefined on non-ENOSPC error) + * @param bufsize size (bytes) of the packet buffer + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTP packet or internal error + * ENOSPC bufsize is too small to add authentication tag + * ( will hold the required byte size) + * EACCES packet would trigger a replay error on receiver + */ +int +srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) +{ + size_t len = *lenp; + size_t tag_len; + size_t roc_len = 0; + + /* Compute required buffer size */ + if (len < 12u) + return EINVAL; + + if (!(s->flags & SRTP_UNAUTHENTICATED)) + { + tag_len = s->tag_len; + + if (rcc_mode (s)) + { + assert (tag_len >= 4); + assert (s->rtp_rcc != 0); + if ((rtp_seq (buf) % s->rtp_rcc) == 0) + { + roc_len = 4; + if (rcc_mode (s) == 3) + tag_len = 0; /* RCC mode 3 -> no auth*/ + else + tag_len -= 4; /* RCC mode 1 or 2 -> auth*/ + } + else + { + if (rcc_mode (s) & 1) + tag_len = 0; /* RCC mode 1 or 3 -> no auth */ + } + } + + *lenp = len + roc_len + tag_len; + } + else + tag_len = 0; + + if (bufsize < *lenp) + return ENOSPC; + + /* Encrypt payload */ + int val = srtp_crypt (s, buf, len); + if (val) + return val; + + /* Authenticate payload */ + if (!(s->flags & SRTP_UNAUTHENTICATED)) + { + uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)); + const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, roc); + + if (roc_len) + { + memcpy (buf + len, &(uint32_t){ htonl (s->rtp_roc) }, 4); + len += 4; + } + memcpy (buf + len, tag, tag_len); +#if 0 + printf ("Sent : 0x"); + for (unsigned i = 0; i < tag_len; i++) + printf ("%02x", tag[i]); + puts (""); +#endif + } + + return 0; +} + + +/** + * Turns a SRTP packet into a RTP packet: authenticates the packet, + * then decrypts it. + * + * @param buf RTP packet to be digested/decrypted + * @param lenp pointer to the SRTP packet length on entry, + * set to the RTP length on exit (undefined in case of error) + * + * @return 0 on success, in case of error: + * EINVAL malformatted SRTP packet + * EACCES authentication failed (spoofed packet or out-of-sync) + */ +int +srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) +{ + size_t len = *lenp; + if (len < 12u) + return EINVAL; + + if (!(s->flags & SRTP_UNAUTHENTICATED)) + { + size_t tag_len = s->tag_len, roc_len = 0; + if (rcc_mode (s)) + { + if ((rtp_seq (buf) % s->rtp_rcc) == 0) + { + roc_len = 4; + if (rcc_mode (s) == 3) + tag_len = 0; + else + tag_len -= 4; + } + else + { + if (rcc_mode (s) & 1) + tag_len = 0; // RCC mode 1 or 3: no auth + } + } + + if (len < (12u + roc_len + tag_len)) + return EINVAL; + len -= roc_len + tag_len; + + uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)), rcc; + if (roc_len) + { + assert (roc_len == 4); + memcpy (&rcc, buf + len, 4); + rcc = ntohl (rcc); + } + else + rcc = roc; + + const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, rcc); +#if 0 + printf ("Computed: 0x"); + for (unsigned i = 0; i < tag_len; i++) + printf ("%02x", tag[i]); + printf ("\nReceived: 0x"); + for (unsigned i = 0; i < tag_len; i++) + printf ("%02x", buf[len + roc_len + i]); + puts (""); +#endif + if (memcmp (buf + len + roc_len, tag, tag_len)) + return EACCES; + + if (roc_len) + { + /* Authenticated packet carried a Roll-Over-Counter */ + s->rtp_roc += rcc - roc; + assert (srtp_compute_roc (s, rtp_seq (buf)) == rcc); + } + *lenp = len; + } + + return srtp_crypt (s, buf, len); +} + + +/** AES-CM for RTCP (salt = 14 bytes + 2 nul bytes) */ +static int +rtcp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t index, + const uint32_t *salt, uint8_t *data, size_t len) +{ + return rtp_crypt (hd, ssrc, index >> 16, index & 0xffff, salt, data, len); +} + + +/** Message Authentication and Integrity for RTCP */ +static const uint8_t * +rtcp_digest (gcry_md_hd_t md, const void *data, size_t len) +{ + gcry_md_reset (md); + gcry_md_write (md, data, len); + return gcry_md_read (md, 0); +} + + +/** + * Encrypts/decrypts a RTCP packet and updates SRTCP context + * (CTR block cypher mode of operation has identical encryption and + * decryption function). + * + * @param buf RTCP packet to be en-/decrypted + * @param len RTCP packet length + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTCP packet + */ +static int srtcp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) +{ + assert (s != NULL); + + /* 8-bytes unencrypted header, and 4-bytes unencrypted footer */ + if ((len < 12) || ((buf[0] >> 6) != 2)) + return EINVAL; + + uint32_t index; + memcpy (&index, buf + len, 4); + index = ntohl (index); + if (((index >> 31) != 0) != ((s->flags & SRTCP_UNENCRYPTED) == 0)) + return EINVAL; // E-bit mismatch + + index &= ~(1 << 31); // clear E-bit for counter + + /* Updates SRTCP index (safe here) */ + int32_t diff = index - s->rtcp_index; + if (diff > 0) + { + /* Packet in the future, good */ + s->rtcp.window = s->rtcp.window << diff; + s->rtcp.window |= UINT64_C(1); + s->rtcp_index = index; + } + else + { + /* Packet in the past/present, bad */ + diff = -diff; + if ((diff >= 64) || ((s->rtcp.window >> diff) & 1)) + return EACCES; // replay attack! + s->rtp.window |= UINT64_C(1) << diff; + } + + /* Crypts SRTCP */ + if (s->flags & SRTCP_UNENCRYPTED) + return 0; + + uint32_t ssrc; + memcpy (&ssrc, buf + 4, 4); + + if (rtcp_crypt (s->rtcp.cipher, ssrc, index, s->rtp.salt, + buf + 8, len - 8)) + return EINVAL; + return 0; +} + + +/** + * Turns a RTCP packet into a SRTCP packet: encrypt it, then computes + * the authentication tag and appends it. + * + * @param buf RTCP packet to be encrypted/digested + * @param lenp pointer to the RTCP packet length on entry, + * set to the SRTCP length on exit (undefined in case of error) + * @param bufsize size (bytes) of the packet buffer + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTCP packet or internal error + * ENOSPC bufsize is too small (to add index and authentication tag) + */ +int +srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) +{ + size_t len = *lenp; + if (bufsize < (len + 4 + s->tag_len)) + return ENOSPC; + + uint32_t index = ++s->rtcp_index; + if (index >> 31) + s->rtcp_index = index = 0; /* 31-bit wrap */ + + if ((s->flags & SRTCP_UNENCRYPTED) == 0) + index |= 0x80000000; /* Set Encrypted bit */ + memcpy (buf + len, &(uint32_t){ htonl (index) }, 4); + + int val = srtcp_crypt (s, buf, len); + if (val) + return val; + + len += 4; /* Digests SRTCP index too */ + + const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); + memcpy (buf + len, tag, s->tag_len); + *lenp = len + s->tag_len; + return 0; +} + + +/** + * Turns a SRTCP packet into a RTCP packet: authenticates the packet, + * then decrypts it. + * + * @param buf RTCP packet to be digested/decrypted + * @param lenp pointer to the SRTCP packet length on entry, + * set to the RTCP length on exit (undefined in case of error) + * + * @return 0 on success, in case of error: + * EINVAL malformatted SRTCP packet + * EACCES authentication failed (spoofed packet or out-of-sync) + */ +int +srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) +{ + size_t len = *lenp; + + if (len < (4u + s->tag_len)) + return EINVAL; + len -= s->tag_len; + + const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); + if (memcmp (buf + len, tag, s->tag_len)) + return EACCES; + + len -= 4; /* Remove SRTCP index before decryption */ + *lenp = len; + return srtcp_crypt (s, buf, len); +} \ No newline at end of file diff --git a/plugins/rtp/src/srtp.h b/plugins/rtp/src/srtp.h new file mode 100644 index 00000000..abca6988 --- /dev/null +++ b/plugins/rtp/src/srtp.h @@ -0,0 +1,82 @@ +/* + * Secure RTP with libgcrypt + * Copyright (C) 2007 Rémi Denis-Courmont + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + ****************************************************************************/ + +#ifndef LIBVLC_SRTP_H +# define LIBVLC_SRTP_H 1 +#include + +typedef struct srtp_session_t srtp_session_t; + +enum +{ + SRTP_UNENCRYPTED=0x1, //< do not encrypt SRTP packets + SRTCP_UNENCRYPTED=0x2, //< do not encrypt SRTCP packets + SRTP_UNAUTHENTICATED=0x4, //< authenticate only SRTCP packets + + SRTP_RCC_MODE1=0x10, //< use Roll-over-Counter Carry mode 1 + SRTP_RCC_MODE2=0x20, //< use Roll-over-Counter Carry mode 2 + SRTP_RCC_MODE3=0x30, //< use Roll-over-Counter Carry mode 3 (insecure) + + SRTP_FLAGS_MASK=0x37 //< mask for valid flags +}; + +/** SRTP encryption algorithms (ciphers); same values as MIKEY */ +enum +{ + SRTP_ENCR_NULL=0, //< no encryption + SRTP_ENCR_AES_CM=1, //< AES counter mode + SRTP_ENCR_AES_F8=2, //< AES F8 mode (not implemented) +}; + +/** SRTP authenticaton algorithms; same values as MIKEY */ +enum +{ + SRTP_AUTH_NULL=0, //< no authentication code + SRTP_AUTH_HMAC_SHA1=1, //< HMAC-SHA1 +}; + +/** SRTP pseudo random function; same values as MIKEY */ +enum +{ + SRTP_PRF_AES_CM=0, //< AES counter mode +}; + +# ifdef __cplusplus +extern "C" { +# endif + +srtp_session_t *srtp_create (int encr, int auth, unsigned tag_len, int prf, + unsigned flags); +void srtp_destroy (srtp_session_t *s); + +int srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, + const void *salt, size_t saltlen); +int srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt); + +void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate); + +int srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsize); +int srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); +int srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsiz); +int srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); + +# ifdef __cplusplus +} +# endif +#endif \ No newline at end of file diff --git a/plugins/rtp/src/srtp.vapi b/plugins/rtp/src/srtp.vapi new file mode 100644 index 00000000..c5ce7fec --- /dev/null +++ b/plugins/rtp/src/srtp.vapi @@ -0,0 +1,103 @@ +[Compact] +[CCode (cname = "srtp_session_t", free_function = "srtp_destroy", cheader_filename="srtp.h")] +public class Dino.Plugins.Rtp.SrtpSession { + [CCode (cname = "srtp_create")] + public SrtpSession(SrtpEncryption encr, SrtpAuthentication auth, uint tag_len, SrtpPrf prf, SrtpFlags flags); + [CCode (cname = "srtp_setkey")] + public int setkey(uint8[] key, uint8[] salt); + [CCode (cname = "srtp_setkeystring")] + public int setkeystring(string key, string salt); + [CCode (cname = "srtp_setrcc_rate")] + public void setrcc_rate(uint16 rate); + + [CCode (cname = "srtp_send")] + private int rtp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); + [CCode (cname = "srtcp_send")] + private int rtcp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); + [CCode (cname = "srtp_recv")] + private int rtp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); + [CCode (cname = "srtcp_recv")] + private int rtcp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); + + public uint8[] encrypt_rtp(uint8[] input, uint tag_len = 10) throws GLib.Error { + uint8[] buf = new uint8[input.length+tag_len]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtp_send(buf, ref buf_use, buf.length); + if (res != 0) { + throw new GLib.Error(-1, res, "RTP encrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] encrypt_rtcp(uint8[] input, uint tag_len = 10) throws GLib.Error { + uint8[] buf = new uint8[input.length+tag_len+4]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtcp_send(buf, ref buf_use, buf.length); + if (res != 0) { + throw new GLib.Error(-1, res, "RTCP encrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtp(uint8[] input) throws GLib.Error { + uint8[] buf = new uint8[input.length]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtp_recv(buf, ref buf_use); + if (res != 0) { + throw new GLib.Error(-1, res, "RTP decrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtcp(uint8[] input) throws GLib.Error { + uint8[] buf = new uint8[input.length]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtcp_recv(buf, ref buf_use); + if (res != 0) { + throw new GLib.Error(-1, res, "RTCP decrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } +} + +[Flags] +[CCode (cname = "unsigned", cprefix = "", cheader_filename="srtp.h", has_type_id = false)] +public enum Dino.Plugins.Rtp.SrtpFlags { + SRTP_UNENCRYPTED, + SRTCP_UNENCRYPTED, + SRTP_UNAUTHENTICATED, + + SRTP_RCC_MODE1, + SRTP_RCC_MODE2, + SRTP_RCC_MODE3 +} + +[CCode (cname = "int", cprefix = "SRTP_ENCR_", cheader_filename="srtp.h", has_type_id = false)] +public enum Dino.Plugins.Rtp.SrtpEncryption { + NULL, + AES_CM, + AES_F8 +} + +[CCode (cname = "int", cprefix = "SRTP_AUTH_", cheader_filename="srtp.h", has_type_id = false)] +public enum Dino.Plugins.Rtp.SrtpAuthentication { + NULL, + HMAC_SHA1 +} + +[CCode (cname = "int", cprefix = "SRTP_PRF_", cheader_filename="srtp.h", has_type_id = false)] +public enum Dino.Plugins.Rtp.SrtpPrf { + AES_CM +} \ No newline at end of file diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index aea2fe85..362e2d16 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -53,6 +53,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private Gst.Pad send_rtp_sink_pad; private Gst.Pad send_rtp_src_pad; + private SrtpSession? local_crypto_session; + private SrtpSession? remote_crypto_session; + public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { base(content); this.plugin = plugin; @@ -144,6 +147,20 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { plugin.unpause(); } + private void prepare_local_crypto() { + if (local_crypto != null && local_crypto_session == null) { + local_crypto_session = new SrtpSession( + local_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? SrtpEncryption.AES_F8 : SrtpEncryption.AES_CM, + SrtpAuthentication.HMAC_SHA1, + local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, + SrtpPrf.AES_CM, + 0 + ); + local_crypto_session.setkey(local_crypto.key, local_crypto.salt); + debug("Setting up encryption with key params %s", local_crypto.key_params); + } + } + private Gst.FlowReturn on_new_sample(Gst.App.Sink sink) { if (sink == null) { debug("Sink is null"); @@ -153,9 +170,16 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { Gst.Buffer buffer = sample.get_buffer(); uint8[] data; buffer.extract_dup(0, buffer.get_size(), out data); + prepare_local_crypto(); if (sink == send_rtp) { + if (local_crypto_session != null) { + data = local_crypto_session.encrypt_rtp(data, local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10); + } on_send_rtp_data(new Bytes.take(data)); } else if (sink == send_rtcp) { + if (local_crypto_session != null) { + data = local_crypto_session.encrypt_rtcp(data, local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10); + } on_send_rtcp_data(new Bytes.take(data)); } else { warning("unknown sample"); @@ -258,15 +282,47 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { recv_rtp_src_pad = null; } + private void prepare_remote_crypto() { + if (remote_crypto != null && remote_crypto_session == null) { + remote_crypto_session = new SrtpSession( + remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? SrtpEncryption.AES_F8 : SrtpEncryption.AES_CM, + SrtpAuthentication.HMAC_SHA1, + remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, + SrtpPrf.AES_CM, + 0 + ); + remote_crypto_session.setkey(remote_crypto.key, remote_crypto.salt); + debug("Setting up decryption with key params %s", remote_crypto.key_params); + } + } + public override void on_recv_rtp_data(Bytes bytes) { + prepare_remote_crypto(); + uint8[] data = bytes.get_data(); + if (remote_crypto_session != null) { + try { + data = remote_crypto_session.decrypt_rtp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + } + } if (push_recv_data) { - recv_rtp.push_buffer(new Gst.Buffer.wrapped_bytes(bytes)); + recv_rtp.push_buffer(new Gst.Buffer.wrapped((owned) data)); } } public override void on_recv_rtcp_data(Bytes bytes) { + prepare_remote_crypto(); + uint8[] data = bytes.get_data(); + if (remote_crypto_session != null) { + try { + data = remote_crypto_session.decrypt_rtcp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + } + } if (push_recv_data) { - recv_rtcp.push_buffer(new Gst.Buffer.wrapped_bytes(bytes)); + recv_rtcp.push_buffer(new Gst.Buffer.wrapped((owned) data)); } } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index 8a3668b2..cca03543 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -17,7 +17,9 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { public bool encryption_required { get; private set; default = false; } public PayloadType? agreed_payload_type { get; private set; } public Gee.List payload_types = new ArrayList(PayloadType.equals_func); - public Gee.List cryptos = new ArrayList(); + public Gee.List remote_cryptos = new ArrayList(); + public Crypto? local_crypto = null; + public Crypto? remote_crypto = null; public weak Stream? stream { get; private set; } @@ -27,7 +29,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { string media, Gee.List payload_types, string? ssrc = null, bool rtcp_mux = false, string? bandwidth = null, string? bandwidth_type = null, - bool encryption_required = false, Gee.List cryptos = new ArrayList() + bool encryption_required = false, Crypto? local_crypto = null ) { this.parent = parent; this.media = media; @@ -37,7 +39,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { this.bandwidth_type = bandwidth_type; this.encryption_required = encryption_required; this.payload_types = payload_types; - this.cryptos = cryptos; + this.local_crypto = local_crypto; } public Parameters.from_node(Module parent, StanzaNode node) throws Jingle.IqError { @@ -49,7 +51,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { if (encryption != null) { this.encryption_required = encryption.get_attribute_bool("required", this.encryption_required); foreach (StanzaNode crypto in encryption.get_subnodes("crypto")) { - this.cryptos.add(Crypto.parse(crypto)); + this.remote_cryptos.add(Crypto.parse(crypto)); } } foreach (StanzaNode payloadType in node.get_subnodes("payload-type")) { @@ -64,6 +66,15 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { content.reject(); return; } + remote_crypto = parent.pick_remote_crypto(remote_cryptos); + if (local_crypto == null && remote_crypto != null) { + local_crypto = parent.pick_local_crypto(remote_crypto); + } + if ((local_crypto == null || remote_crypto == null) && encryption_required) { + debug("no usable encryption, but encryption required"); + content.reject(); + return; + } } public void accept(XmppStream stream, Jingle.Session session, Jingle.Content content) { @@ -97,6 +108,15 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { } }); + if (remote_crypto == null || local_crypto == null) { + if (encryption_required) { + warning("Encryption required but not provided in both directions"); + return; + } + remote_crypto = null; + local_crypto = null; + } + this.stream = parent.create_stream(content); rtp_datagram.datagram_received.connect(this.stream.on_recv_rtp_data); rtcp_datagram.datagram_received.connect(this.stream.on_recv_rtcp_data); @@ -118,6 +138,20 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { } agreed_payload_type = preferred_payload_type; + Gee.List crypto_nodes = description_node.get_deep_subnodes("encryption", "crypto"); + if (crypto_nodes.size == 0) { + warning("Counterpart didn't include any cryptos"); + if (encryption_required) { + return; + } + } else { + Crypto preferred_crypto = Crypto.parse(crypto_nodes[0]); + if (local_crypto.crypto_suite != preferred_crypto.crypto_suite) { + warning("Counterpart's crypto suite doesn't match any of our sent ones"); + } + remote_crypto = preferred_crypto; + } + accept(stream, session, content); } @@ -137,6 +171,10 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { ret.put_node(payload_type.to_xml()); } } + if (local_crypto != null) { + ret.put_node(new StanzaNode.build("encryption", NS_URI) + .put_node(local_crypto.to_xml())); + } return ret; } } \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala index 35e03168..23aee6c9 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala @@ -20,6 +20,9 @@ public abstract class Module : XmppStreamModule { public abstract async Gee.List get_supported_payloads(string media); public abstract async PayloadType? pick_payload_type(string media, Gee.List payloads); + public abstract Crypto? generate_local_crypto(); + public abstract Crypto? pick_remote_crypto(Gee.List cryptos); + public abstract Crypto? pick_local_crypto(Crypto? remote); public abstract Stream create_stream(Jingle.Content content); public abstract void close_stream(Stream stream); @@ -36,6 +39,7 @@ public abstract class Module : XmppStreamModule { // Create audio content Parameters audio_content_parameters = new Parameters(this, "audio", yield get_supported_payloads("audio")); + audio_content_parameters.local_crypto = generate_local_crypto(); Jingle.Transport? audio_transport = yield jingle_module.select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (audio_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable audio transports"); @@ -52,6 +56,7 @@ public abstract class Module : XmppStreamModule { if (video) { // Create video content Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); + video_content_parameters.local_crypto = generate_local_crypto(); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (video_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); @@ -92,6 +97,7 @@ public abstract class Module : XmppStreamModule { if (content == null) { // Content for video does not yet exist -> create it Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); + video_content_parameters.local_crypto = generate_local_crypto(); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (video_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); @@ -148,26 +154,130 @@ public abstract class Module : XmppStreamModule { } public class Crypto { - public string cryptoSuite { get; private set; } - public string keyParams { get; private set; } - public string? sessionParams { get; private set; } - public string? tag { get; private set; } + public const string AES_CM_128_HMAC_SHA1_80 = "AES_CM_128_HMAC_SHA1_80"; + public const string AES_CM_128_HMAC_SHA1_32 = "AES_CM_128_HMAC_SHA1_32"; + public const string F8_128_HMAC_SHA1_80 = "F8_128_HMAC_SHA1_80"; + + public string crypto_suite { get; private set; } + public string key_params { get; private set; } + public string? session_params { get; private set; } + public string tag { get; private set; } + + public uint8[] key_and_salt { owned get { + if (!key_params.has_prefix("inline:")) return null; + int endIndex = key_params.index_of("|"); + if (endIndex < 0) endIndex = key_params.length; + string sub = key_params.substring(7, endIndex - 7); + return Base64.decode(sub); + }} + + public string? lifetime { owned get { + if (!key_params.has_prefix("inline:")) return null; + int firstIndex = key_params.index_of("|"); + if (firstIndex < 0) return null; + int endIndex = key_params.index_of("|", firstIndex + 1); + if (endIndex < 0) { + if (key_params.index_of(":", firstIndex) > 0) return null; // Is MKI + endIndex = key_params.length; + } + return key_params.substring(firstIndex + 1, endIndex); + }} + + public int mki { get { + if (!key_params.has_prefix("inline:")) return -1; + int firstIndex = key_params.index_of("|"); + if (firstIndex < 0) return -1; + int splitIndex = key_params.index_of(":", firstIndex); + if (splitIndex < 0) return -1; + int secondIndex = key_params.index_of("|", firstIndex + 1); + if (secondIndex < 0) { + return int.parse(key_params.substring(firstIndex + 1, splitIndex)); + } else if (splitIndex > secondIndex) { + return int.parse(key_params.substring(secondIndex + 1, splitIndex)); + } + return -1; + }} + + public int mki_length { get { + if (!key_params.has_prefix("inline:")) return -1; + int firstIndex = key_params.index_of("|"); + if (firstIndex < 0) return -1; + int splitIndex = key_params.index_of(":", firstIndex); + if (splitIndex < 0) return -1; + int secondIndex = key_params.index_of("|", firstIndex + 1); + if (secondIndex < 0 || splitIndex > secondIndex) { + return int.parse(key_params.substring(splitIndex + 1, key_params.length)); + } + return -1; + }} + + public bool is_valid { get { + switch(crypto_suite) { + case AES_CM_128_HMAC_SHA1_80: + case AES_CM_128_HMAC_SHA1_32: + case F8_128_HMAC_SHA1_80: + return key_and_salt.length == 30; + } + return false; + }} + + public uint8[] key { owned get { + uint8[] key_and_salt = key_and_salt; + switch(crypto_suite) { + case AES_CM_128_HMAC_SHA1_80: + case AES_CM_128_HMAC_SHA1_32: + case F8_128_HMAC_SHA1_80: + if (key_and_salt.length >= 16) return key_and_salt[0:16]; + break; + } + return null; + }} + + public uint8[] salt { owned get { + uint8[] keyAndSalt = key_and_salt; + switch(crypto_suite) { + case AES_CM_128_HMAC_SHA1_80: + case AES_CM_128_HMAC_SHA1_32: + case F8_128_HMAC_SHA1_80: + if (keyAndSalt.length >= 30) return keyAndSalt[16:30]; + break; + } + return null; + }} + + public static Crypto create(string crypto_suite, uint8[] key_and_salt, string? session_params = null, string tag = "1") { + Crypto crypto = new Crypto(); + crypto.crypto_suite = crypto_suite; + crypto.key_params = "inline:" + Base64.encode(key_and_salt); + crypto.session_params = session_params; + crypto.tag = tag; + return crypto; + } + + public Crypto rekey(uint8[] key_and_salt) { + Crypto crypto = new Crypto(); + crypto.crypto_suite = crypto_suite; + crypto.key_params = "inline:" + Base64.encode(key_and_salt); + crypto.session_params = session_params; + crypto.tag = tag; + return crypto; + } public static Crypto parse(StanzaNode node) { Crypto crypto = new Crypto(); - crypto.cryptoSuite = node.get_attribute("crypto-suite"); - crypto.keyParams = node.get_attribute("key-params"); - crypto.sessionParams = node.get_attribute("session-params"); + crypto.crypto_suite = node.get_attribute("crypto-suite"); + crypto.key_params = node.get_attribute("key-params"); + crypto.session_params = node.get_attribute("session-params"); crypto.tag = node.get_attribute("tag"); return crypto; } public StanzaNode to_xml() { StanzaNode node = new StanzaNode.build("crypto", NS_URI) - .put_attribute("crypto-suite", cryptoSuite) - .put_attribute("key-params", keyParams); - if (sessionParams != null) node.put_attribute("session-params", sessionParams); - if (tag != null) node.put_attribute("tag", tag); + .put_attribute("crypto-suite", crypto_suite) + .put_attribute("key-params", key_params) + .put_attribute("tag", tag); + if (session_params != null) node.put_attribute("session-params", session_params); return node; } } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala index 62d85dec..2fc29291 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala @@ -17,6 +17,20 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { } return null; }} + public JingleRtp.Crypto? local_crypto { get { + var content_params = content.content_params; + if (content_params is Parameters) { + return ((Parameters)content_params).local_crypto; + } + return null; + }} + public JingleRtp.Crypto? remote_crypto { get { + var content_params = content.content_params; + if (content_params is Parameters) { + return ((Parameters)content_params).remote_crypto; + } + return null; + }} public bool sending { get { return content.session.senders_include_us(content.senders); }} From c8a37c02462f3eef2c6d26fbfbe2079da1b7a34f Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 23 Mar 2021 15:07:51 +0100 Subject: [PATCH 16/56] Make foundation a string --- plugins/ice/src/transport_parameters.vala | 5 ++--- xmpp-vala/src/module/xep/0176_jingle_ice_udp/candidate.vala | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index acb44852..467b3674 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -209,8 +209,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport Nice.Candidate candidate = new Nice.Candidate(type); candidate.component_id = c.component; char[] foundation = new char[Nice.CANDIDATE_MAX_FOUNDATION]; - string foundation_str = c.foundation.to_string(); - Memory.copy(foundation, foundation_str.data, foundation_str.length); + Memory.copy(foundation, c.foundation.data, size_t.min(c.foundation.length, Nice.CANDIDATE_MAX_FOUNDATION - 1)); candidate.foundation = foundation; candidate.addr = Nice.Address(); candidate.addr.init(); @@ -237,7 +236,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport default: assert_not_reached(); } candidate.component = (uint8) nc.component_id; - candidate.foundation = (uint8) int.parse((string)nc.foundation); + candidate.foundation = ((string)nc.foundation).dup(); candidate.generation = 0; candidate.id = Random.next_int().to_string("%08x"); // TODO diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/candidate.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/candidate.vala index a2988d90..bcb3aa80 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/candidate.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/candidate.vala @@ -4,7 +4,7 @@ using Xmpp; public class Xmpp.Xep.JingleIceUdp.Candidate { public uint8 component; - public uint8 foundation; + public string foundation; public uint8 generation; public string id; public string ip; @@ -19,7 +19,7 @@ public class Xmpp.Xep.JingleIceUdp.Candidate { public static Candidate parse(StanzaNode node) throws Jingle.IqError { Candidate candidate = new Candidate(); candidate.component = (uint8) node.get_attribute_uint("component"); - candidate.foundation = (uint8) node.get_attribute_uint("foundation"); + candidate.foundation = (string) node.get_attribute("foundation"); candidate.generation = (uint8) node.get_attribute_uint("generation"); candidate.id = node.get_attribute("id"); candidate.ip = node.get_attribute("ip"); From b01f6f9ef7c0e8b0a15149426a47a238a5f6f1ce Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 23 Mar 2021 15:09:06 +0100 Subject: [PATCH 17/56] Resample audio data for common 48k sample rate --- plugins/rtp/src/codec_util.vala | 10 ++++------ plugins/rtp/src/device.vala | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index e419b5ad..6bd465c1 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -202,7 +202,8 @@ public class Dino.Plugins.Rtp.CodecUtil { string decode = element_name ?? get_decode_element_name(media, codec); if (depay == null || decode == null) return null; string decode_prefix = get_decode_prefix(media, codec, decode) ?? ""; - return @"$depay name=$base_name-rtp-depay ! $decode_prefix$decode name=$base_name-decode ! $(media)convert name=$base_name-convert"; + string resample = media == "audio" ? @" ! audioresample name=$base_name-resample" : ""; + return @"$depay name=$base_name-rtp-depay ! $decode_prefix$decode name=$base_name-decode ! $(media)convert name=$base_name-convert$resample"; } public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { @@ -224,11 +225,8 @@ public class Dino.Plugins.Rtp.CodecUtil { if (pay == null || encode == null) return null; string encode_prefix = get_encode_prefix(media, codec, encode) ?? ""; string encode_suffix = get_encode_suffix(media, codec, encode) ?? ""; - if (media == "audio") { - return @"audioconvert name=$base_name-convert ! audioresample name=$base_name-resample ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; - } else { - return @"$(media)convert name=$base_name-convert ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; - } + string resample = media == "audio" ? @" ! audioresample name=$base_name-resample" : ""; + return @"$(media)convert name=$base_name-convert$resample ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; } public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 570c6667..20762f77 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -59,7 +59,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { if (element == null) create(); links++; if (mixer != null) return mixer; - if (is_sink && media == "audio") return plugin.echoprobe; + if (is_sink && media == "audio") return filter; return element; } @@ -146,9 +146,10 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { element.@set("sync", false); } if (is_sink && media == "audio") { -// mixer = Gst.ElementFactory.make("audiomixer", @"$id-mixer"); -// pipe.add(mixer); -// mixer.link(plugin.echoprobe); + filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); + filter.@set("caps", get_best_caps()); + pipe.add(filter); + filter.link(plugin.echoprobe); plugin.echoprobe.link(element); } plugin.unpause(); @@ -173,6 +174,13 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { pipe.remove(mixer); mixer = null; } else if (is_sink && media == "audio") { + if (filter != null) { + filter.set_locked_state(true); + filter.set_state(Gst.State.NULL); + filter.unlink(plugin.echoprobe); + pipe.remove(filter); + filter = null; + } plugin.echoprobe.unlink(element); } element.set_locked_state(true); From 97ab7de7ab4f958eb7d273e524151007d44ea1d7 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 23 Mar 2021 15:09:52 +0100 Subject: [PATCH 18/56] Mark ICE component as ready even without the event --- plugins/ice/src/transport_parameters.vala | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index 467b3674..a8172678 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -172,7 +172,12 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport private void on_component_state_changed(uint stream_id, uint component_id, uint state) { if (stream_id != this.stream_id) return; debug("stream %u component %u state changed to %s", stream_id, component_id, agent.get_component_state(stream_id, component_id).to_string()); - if (is_component_ready(agent, stream_id, component_id) && connections.has_key((uint8) component_id) && !connections[(uint8)component_id].ready) { + may_consider_ready(stream_id, component_id); + } + + private void may_consider_ready(uint stream_id, uint component_id) { + if (stream_id != this.stream_id) return; + if (connections.has_key((uint8) component_id) && is_component_ready(agent, stream_id, component_id) && connections.has_key((uint8) component_id) && !connections[(uint8)component_id].ready) { connections[(uint8)component_id].ready = true; } } @@ -189,7 +194,11 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport private void on_recv(Nice.Agent agent, uint stream_id, uint component_id, uint8[] data) { if (stream_id != this.stream_id) return; - if (is_component_ready(agent, stream_id, component_id) && connections.has_key((uint8) component_id)) { + may_consider_ready(stream_id, component_id); + if (connections.has_key((uint8) component_id)) { + if (!connections[(uint8) component_id].ready) { + debug("on_recv stream %u component %u when state %s", stream_id, component_id, agent.get_component_state(stream_id, component_id).to_string()); + } connections[(uint8) component_id].datagram_received(new Bytes(data)); } else { debug("on_recv stream %u component %u length %u", stream_id, component_id, data.length); From 4b230808b9566322fae8d1ef0d1a5cb3e8027d3b Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 23 Mar 2021 20:04:28 +0100 Subject: [PATCH 19/56] Move SRTP implementation into crypto library for reuse --- plugins/crypto-vala/CMakeLists.txt | 27 +++++++++++++++++- plugins/{rtp => crypto-vala}/src/srtp.c | 0 plugins/{rtp => crypto-vala}/src/srtp.h | 0 plugins/{rtp => crypto-vala}/src/srtp.vapi | 32 ++++++++++++---------- plugins/rtp/CMakeLists.txt | 5 ++-- plugins/rtp/src/stream.vala | 20 +++++++------- 6 files changed, 56 insertions(+), 28 deletions(-) rename plugins/{rtp => crypto-vala}/src/srtp.c (100%) rename plugins/{rtp => crypto-vala}/src/srtp.h (100%) rename plugins/{rtp => crypto-vala}/src/srtp.vapi (77%) diff --git a/plugins/crypto-vala/CMakeLists.txt b/plugins/crypto-vala/CMakeLists.txt index 036e45f6..f97b0d31 100644 --- a/plugins/crypto-vala/CMakeLists.txt +++ b/plugins/crypto-vala/CMakeLists.txt @@ -11,6 +11,7 @@ SOURCES "src/cipher_converter.vala" "src/error.vala" "src/random.vala" + "src/srtp.vapi" CUSTOM_VAPIS "${CMAKE_CURRENT_SOURCE_DIR}/vapi/gcrypt.vapi" PACKAGES @@ -21,9 +22,33 @@ GENERATE_HEADER crypto-vala ) +add_custom_command(OUTPUT "${CMAKE_BINARY_DIR}/exports/srtp.h" +COMMAND + cp "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.h" "${CMAKE_BINARY_DIR}/exports/srtp.h" +DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.h" +COMMENT + Copy header file srtp.h +) + +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/exports/crypto.vapi +COMMAND + cat "${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi" "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi" > "${CMAKE_BINARY_DIR}/exports/crypto.vapi" +DEPENDS + ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi +) + +add_custom_target(crypto-vapi +DEPENDS + ${CMAKE_BINARY_DIR}/exports/crypto.vapi + ${CMAKE_BINARY_DIR}/exports/srtp.h +) + set(CFLAGS ${VALA_CFLAGS} -I${CMAKE_CURRENT_SOURCE_DIR}/src) add_definitions(${CFLAGS}) -add_library(crypto-vala STATIC ${CRYPTO_VALA_C}) +add_library(crypto-vala STATIC ${CRYPTO_VALA_C} src/srtp.c) +add_dependencies(crypto-vala crypto-vapi) target_link_libraries(crypto-vala ${CRYPTO_VALA_PACKAGES} gcrypt) set_property(TARGET crypto-vala PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/plugins/rtp/src/srtp.c b/plugins/crypto-vala/src/srtp.c similarity index 100% rename from plugins/rtp/src/srtp.c rename to plugins/crypto-vala/src/srtp.c diff --git a/plugins/rtp/src/srtp.h b/plugins/crypto-vala/src/srtp.h similarity index 100% rename from plugins/rtp/src/srtp.h rename to plugins/crypto-vala/src/srtp.h diff --git a/plugins/rtp/src/srtp.vapi b/plugins/crypto-vala/src/srtp.vapi similarity index 77% rename from plugins/rtp/src/srtp.vapi rename to plugins/crypto-vala/src/srtp.vapi index c5ce7fec..0fe825c3 100644 --- a/plugins/rtp/src/srtp.vapi +++ b/plugins/crypto-vala/src/srtp.vapi @@ -1,8 +1,11 @@ +[CCode (cheader_filename="srtp.h")] +namespace Crypto.Srtp { + [Compact] -[CCode (cname = "srtp_session_t", free_function = "srtp_destroy", cheader_filename="srtp.h")] -public class Dino.Plugins.Rtp.SrtpSession { +[CCode (cname = "srtp_session_t", free_function = "srtp_destroy")] +public class Session { [CCode (cname = "srtp_create")] - public SrtpSession(SrtpEncryption encr, SrtpAuthentication auth, uint tag_len, SrtpPrf prf, SrtpFlags flags); + public Session(Encryption encr, Authentication auth, uint tag_len, Prf prf, Flags flags); [CCode (cname = "srtp_setkey")] public int setkey(uint8[] key, uint8[] salt); [CCode (cname = "srtp_setkeystring")] @@ -20,7 +23,7 @@ public class Dino.Plugins.Rtp.SrtpSession { private int rtcp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); public uint8[] encrypt_rtp(uint8[] input, uint tag_len = 10) throws GLib.Error { - uint8[] buf = new uint8[input.length+tag_len]; + uint8[] buf = new uint8[input.length + tag_len]; GLib.Memory.copy(buf, input, input.length); size_t buf_use = input.length; int res = rtp_send(buf, ref buf_use, buf.length); @@ -33,7 +36,7 @@ public class Dino.Plugins.Rtp.SrtpSession { } public uint8[] encrypt_rtcp(uint8[] input, uint tag_len = 10) throws GLib.Error { - uint8[] buf = new uint8[input.length+tag_len+4]; + uint8[] buf = new uint8[input.length + tag_len + 4]; GLib.Memory.copy(buf, input, input.length); size_t buf_use = input.length; int res = rtcp_send(buf, ref buf_use, buf.length); @@ -73,31 +76,32 @@ public class Dino.Plugins.Rtp.SrtpSession { } [Flags] -[CCode (cname = "unsigned", cprefix = "", cheader_filename="srtp.h", has_type_id = false)] -public enum Dino.Plugins.Rtp.SrtpFlags { +[CCode (cname = "unsigned", cprefix = "", has_type_id = false)] +public enum Flags { SRTP_UNENCRYPTED, SRTCP_UNENCRYPTED, SRTP_UNAUTHENTICATED, - SRTP_RCC_MODE1, SRTP_RCC_MODE2, SRTP_RCC_MODE3 } -[CCode (cname = "int", cprefix = "SRTP_ENCR_", cheader_filename="srtp.h", has_type_id = false)] -public enum Dino.Plugins.Rtp.SrtpEncryption { +[CCode (cname = "int", cprefix = "SRTP_ENCR_", has_type_id = false)] +public enum Encryption { NULL, AES_CM, AES_F8 } -[CCode (cname = "int", cprefix = "SRTP_AUTH_", cheader_filename="srtp.h", has_type_id = false)] -public enum Dino.Plugins.Rtp.SrtpAuthentication { +[CCode (cname = "int", cprefix = "SRTP_AUTH_", has_type_id = false)] +public enum Authentication { NULL, HMAC_SHA1 } -[CCode (cname = "int", cprefix = "SRTP_PRF_", cheader_filename="srtp.h", has_type_id = false)] -public enum Dino.Plugins.Rtp.SrtpPrf { +[CCode (cname = "int", cprefix = "SRTP_PRF_", has_type_id = false)] +public enum Prf { AES_CM +} + } \ No newline at end of file diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index ef2f7698..5311fac3 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -18,11 +18,10 @@ SOURCES src/video_widget.vala src/register_plugin.vala CUSTOM_VAPIS - ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi + ${CMAKE_BINARY_DIR}/exports/crypto.vapi ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi - ${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi PACKAGES ${RTP_PACKAGES} OPTIONS @@ -30,7 +29,7 @@ OPTIONS ) add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) -add_library(rtp SHARED ${RTP_VALA_C} src/srtp.c) +add_library(rtp SHARED ${RTP_VALA_C}) target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES}) set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 362e2d16..77080a09 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -53,8 +53,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private Gst.Pad send_rtp_sink_pad; private Gst.Pad send_rtp_src_pad; - private SrtpSession? local_crypto_session; - private SrtpSession? remote_crypto_session; + private Crypto.Srtp.Session? local_crypto_session; + private Crypto.Srtp.Session? remote_crypto_session; public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { base(content); @@ -149,11 +149,11 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private void prepare_local_crypto() { if (local_crypto != null && local_crypto_session == null) { - local_crypto_session = new SrtpSession( - local_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? SrtpEncryption.AES_F8 : SrtpEncryption.AES_CM, - SrtpAuthentication.HMAC_SHA1, + local_crypto_session = new Crypto.Srtp.Session( + local_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? Crypto.Srtp.Encryption.AES_F8 : Crypto.Srtp.Encryption.AES_CM, + Crypto.Srtp.Authentication.HMAC_SHA1, local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, - SrtpPrf.AES_CM, + Crypto.Srtp.Prf.AES_CM, 0 ); local_crypto_session.setkey(local_crypto.key, local_crypto.salt); @@ -284,11 +284,11 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private void prepare_remote_crypto() { if (remote_crypto != null && remote_crypto_session == null) { - remote_crypto_session = new SrtpSession( - remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? SrtpEncryption.AES_F8 : SrtpEncryption.AES_CM, - SrtpAuthentication.HMAC_SHA1, + remote_crypto_session = new Crypto.Srtp.Session( + remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? Crypto.Srtp.Encryption.AES_F8 : Crypto.Srtp.Encryption.AES_CM, + Crypto.Srtp.Authentication.HMAC_SHA1, remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, - SrtpPrf.AES_CM, + Crypto.Srtp.Prf.AES_CM, 0 ); remote_crypto_session.setkey(remote_crypto.key, remote_crypto.salt); From ec35f95e13f4f2f756c81a35ded0980245acc5f4 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Wed, 24 Mar 2021 14:12:42 +0100 Subject: [PATCH 20/56] Add initial support for DTLS-SRTP --- cmake/FindGnuTLS.cmake | 13 + libdino/src/service/calls.vala | 22 +- plugins/ice/CMakeLists.txt | 7 +- plugins/ice/src/dtls_srtp.vala | 247 +++++++++++ plugins/ice/src/transport_parameters.vala | 48 +- plugins/ice/vapi/gnutls.vapi | 419 ++++++++++++++++++ plugins/rtp/CMakeLists.txt | 1 + .../xep/0166_jingle/reason_element.vala | 1 + .../src/module/xep/0166_jingle/session.vala | 38 +- .../0167_jingle_rtp/content_parameters.vala | 5 +- .../0167_jingle_rtp/jingle_rtp_module.vala | 2 +- .../0167_jingle_rtp/session_info_type.vala | 2 +- .../jingle_ice_udp_module.vala | 1 + .../transport_parameters.vala | 27 ++ 14 files changed, 791 insertions(+), 42 deletions(-) create mode 100644 cmake/FindGnuTLS.cmake create mode 100644 plugins/ice/src/dtls_srtp.vala create mode 100644 plugins/ice/vapi/gnutls.vapi diff --git a/cmake/FindGnuTLS.cmake b/cmake/FindGnuTLS.cmake new file mode 100644 index 00000000..6b27abd7 --- /dev/null +++ b/cmake/FindGnuTLS.cmake @@ -0,0 +1,13 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(GnuTLS + PKG_CONFIG_NAME gnutls + LIB_NAMES gnutls + INCLUDE_NAMES gnutls/gnutls.h + INCLUDE_DIR_SUFFIXES gnutls gnutls/include + DEPENDS GLib +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GnuTLS + REQUIRED_VARS GnuTLS_LIBRARY + VERSION_VAR GnuTLS_VERSION) \ No newline at end of file diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index 5224bdd1..54c353b0 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -125,7 +125,7 @@ namespace Dino { call.state = Call.State.ESTABLISHING; if (sessions.has_key(call)) { - foreach (Xep.Jingle.Content content in sessions[call].contents.values) { + foreach (Xep.Jingle.Content content in sessions[call].contents) { content.accept(); } } else { @@ -146,7 +146,7 @@ namespace Dino { call.state = Call.State.DECLINED; if (sessions.has_key(call)) { - foreach (Xep.Jingle.Content content in sessions[call].contents.values) { + foreach (Xep.Jingle.Content content in sessions[call].contents) { content.reject(); } remove_call_from_datastructures(call); @@ -223,16 +223,6 @@ namespace Dino { foreach (Jid full_jid in full_jids) { bool supports_rtc = yield stream.get_module(Xep.JingleRtp.Module.IDENTITY).is_available(stream, full_jid); if (!supports_rtc) continue; - - // dtls support indicates webRTC support. Clients tend to not do normal ice udp in that case. Except Dino. - bool supports_dtls = yield stream_interactor.get_module(EntityInfo.IDENTITY).has_feature(conversation.account, full_jid, "urn:xmpp:jingle:apps:dtls:0"); - if (supports_dtls) { - Xep.ServiceDiscovery.Identity? identity = yield stream_interactor.get_module(EntityInfo.IDENTITY).get_identity(conversation.account, full_jid); - bool is_dino = identity != null && identity.name == "Dino"; - - if (!is_dino) continue; - } - ret.add(full_jid); } return ret; @@ -253,7 +243,7 @@ namespace Dino { private void on_incoming_call(Account account, Xep.Jingle.Session session) { bool counterpart_wants_video = false; - foreach (Xep.Jingle.Content content in session.contents.values) { + foreach (Xep.Jingle.Content content in session.contents) { Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; if (rtp_content_parameter == null) continue; if (rtp_content_parameter.media == "video" && session.senders_include_us(content.senders)) { @@ -391,7 +381,7 @@ namespace Dino { on_incoming_content_add(stream, call, session, content) ); - foreach (Xep.Jingle.Content content in session.contents.values) { + foreach (Xep.Jingle.Content content in session.contents) { Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; if (rtp_content_parameter == null) continue; @@ -446,7 +436,7 @@ namespace Dino { Xep.Jingle.Module jingle_module = stream_interactor.module_manager.get_module(account, Xep.Jingle.Module.IDENTITY); jingle_module.session_initiate_received.connect((stream, session) => { - foreach (Xep.Jingle.Content content in session.contents.values) { + foreach (Xep.Jingle.Content content in session.contents) { Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; if (rtp_content_parameter != null) { on_incoming_call(account, session); @@ -460,7 +450,7 @@ namespace Dino { if (!call_by_sid[account].has_key(session.sid)) return; Call call = call_by_sid[account][session.sid]; - foreach (Xep.Jingle.Content content in session.contents.values) { + foreach (Xep.Jingle.Content content in session.contents) { if (name == null || content.content_name == name) { Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; if (rtp_content_parameter != null) { diff --git a/plugins/ice/CMakeLists.txt b/plugins/ice/CMakeLists.txt index 90fe5b7d..38025aa0 100644 --- a/plugins/ice/CMakeLists.txt +++ b/plugins/ice/CMakeLists.txt @@ -2,6 +2,7 @@ find_packages(ICE_PACKAGES REQUIRED Gee GLib GModule + GnuTLS GObject GTK3 Nice @@ -9,8 +10,9 @@ find_packages(ICE_PACKAGES REQUIRED vala_precompile(ICE_VALA_C SOURCES - src/plugin.vala + src/dtls_srtp.vala src/module.vala + src/plugin.vala src/transport_parameters.vala src/util.vala src/register_plugin.vala @@ -18,6 +20,7 @@ CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi + ${CMAKE_BINARY_DIR}/exports/crypto.vapi PACKAGES ${ICE_PACKAGES} OPTIONS @@ -26,7 +29,7 @@ OPTIONS add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="ice") add_library(ice SHARED ${ICE_VALA_C}) -target_link_libraries(ice libdino ${ICE_PACKAGES}) +target_link_libraries(ice libdino crypto-vala ${ICE_PACKAGES}) set_target_properties(ice PROPERTIES PREFIX "") set_target_properties(ice PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala new file mode 100644 index 00000000..a21c242b --- /dev/null +++ b/plugins/ice/src/dtls_srtp.vala @@ -0,0 +1,247 @@ +using GnuTLS; + +public class DtlsSrtp { + + public signal void send_data(uint8[] data); + + private X509.Certificate[] own_cert; + private X509.PrivateKey private_key; + private Cond buffer_cond = new Cond(); + private Mutex buffer_mutex = new Mutex(); + private Gee.LinkedList buffer_queue = new Gee.LinkedList(); + private uint pull_timeout = uint.MAX; + private string peer_fingerprint; + + private Crypto.Srtp.Session encrypt_session; + private Crypto.Srtp.Session decrypt_session; + + public static DtlsSrtp setup() throws GLib.Error { + var obj = new DtlsSrtp(); + obj.generate_credentials(); + return obj; + } + + internal string get_own_fingerprint(DigestAlgorithm digest_algo) { + return format_certificate(own_cert[0], digest_algo); + } + + public void set_peer_fingerprint(string fingerprint) { + this.peer_fingerprint = fingerprint; + } + + public uint8[] process_incoming_data(uint component_id, uint8[] data) { + if (decrypt_session != null) { + if (component_id == 1) return decrypt_session.decrypt_rtp(data); + if (component_id == 2) return decrypt_session.decrypt_rtcp(data); + } else if (component_id == 1) { + on_data_rec(data); + } + return null; + } + + public uint8[] process_outgoing_data(uint component_id, uint8[] data) { + if (encrypt_session != null) { + if (component_id == 1) return encrypt_session.encrypt_rtp(data); + if (component_id == 2) return encrypt_session.encrypt_rtcp(data); + } + return null; + } + + public void on_data_rec(owned uint8[] data) { + buffer_mutex.lock(); + buffer_queue.add(new Bytes.take(data)); + buffer_cond.signal(); + buffer_mutex.unlock(); + } + + private void generate_credentials() throws GLib.Error { + int err = 0; + + private_key = X509.PrivateKey.create(); + err = private_key.generate(PKAlgorithm.RSA, 2048); + throw_if_error(err); + + var start_time = new DateTime.now_local().add_days(1); + var end_time = start_time.add_days(2); + + X509.Certificate cert = X509.Certificate.create(); + cert.set_key(private_key); + cert.set_version(1); + cert.set_activation_time ((time_t) start_time.to_unix ()); + cert.set_expiration_time ((time_t) end_time.to_unix ()); + + uint32 serial = 1; + cert.set_serial(&serial, sizeof(uint32)); + + cert.sign(cert, private_key); + + own_cert = new X509.Certificate[] { (owned)cert }; + } + + public async void setup_dtls_connection(bool server) { + InitFlags server_or_client = server ? InitFlags.SERVER : InitFlags.CLIENT; + debug("Setting up DTLS connection. We're %s", server_or_client.to_string()); + + CertificateCredentials cert_cred = CertificateCredentials.create(); + int err = cert_cred.set_x509_key(own_cert, private_key); + throw_if_error(err); + + Session? session = Session.create(server_or_client | InitFlags.DATAGRAM); + session.enable_heartbeat(1); + session.set_srtp_profile_direct("SRTP_AES128_CM_HMAC_SHA1_80"); + session.set_credentials(GnuTLS.CredentialsType.CERTIFICATE, cert_cred); + session.server_set_request(CertificateRequest.REQUEST); + session.set_priority_from_string("NORMAL:!VERS-TLS-ALL:+VERS-DTLS-ALL:+CTYPE-CLI-X509"); + + session.set_transport_pointer(this); + session.set_pull_function(pull_function); + session.set_pull_timeout_function(pull_timeout_function); + session.set_push_function(push_function); + session.set_verify_function(verify_function); + + Thread thread = new Thread (null, () => { + DateTime maximum_time = new DateTime.now_utc().add_seconds(20); + do { + err = session.handshake(); + + DateTime current_time = new DateTime.now_utc(); + if (maximum_time.compare(current_time) < 0) { + warning("DTLS handshake timeouted"); + return -1; + } + } while (err < 0 && !((ErrorCode)err).is_fatal()); + Idle.add(setup_dtls_connection.callback); + return err; + }); + yield; + err = thread.join(); + + uint8[] km = new uint8[150]; + Datum? client_key, client_salt, server_key, server_salt; + session.get_srtp_keys(km, km.length, out client_key, out client_salt, out server_key, out server_salt); + if (client_key == null || client_salt == null || server_key == null || server_salt == null) { + warning("SRTP client/server key/salt null"); + } + + Crypto.Srtp.Session encrypt_session = new Crypto.Srtp.Session(Crypto.Srtp.Encryption.AES_CM, Crypto.Srtp.Authentication.HMAC_SHA1, 10, Crypto.Srtp.Prf.AES_CM, 0); + Crypto.Srtp.Session decrypt_session = new Crypto.Srtp.Session(Crypto.Srtp.Encryption.AES_CM, Crypto.Srtp.Authentication.HMAC_SHA1, 10, Crypto.Srtp.Prf.AES_CM, 0); + + if (server) { + encrypt_session.setkey(server_key.extract(), server_salt.extract()); + decrypt_session.setkey(client_key.extract(), client_salt.extract()); + } else { + encrypt_session.setkey(client_key.extract(), client_salt.extract()); + decrypt_session.setkey(server_key.extract(), server_salt.extract()); + } + + this.encrypt_session = (owned)encrypt_session; + this.decrypt_session = (owned)decrypt_session; + } + + private static ssize_t pull_function(void* transport_ptr, uint8[] buffer) { + DtlsSrtp self = transport_ptr as DtlsSrtp; + + self.buffer_mutex.lock(); + while (self.buffer_queue.size == 0) { + self.buffer_cond.wait(self.buffer_mutex); + } + owned Bytes data = self.buffer_queue.remove_at(0); + self.buffer_mutex.unlock(); + + uint8[] data_uint8 = Bytes.unref_to_data(data); + Memory.copy(buffer, data_uint8, data_uint8.length); + + // The callback should return 0 on connection termination, a positive number indicating the number of bytes received, and -1 on error. + return (ssize_t)data.length; + } + + private static int pull_timeout_function(void* transport_ptr, uint ms) { + DtlsSrtp self = transport_ptr as DtlsSrtp; + + DateTime current_time = new DateTime.now_utc(); + current_time.add_seconds(ms/1000); + int64 end_time = current_time.to_unix(); + + self.buffer_mutex.lock(); + while (self.buffer_queue.size == 0) { + self.buffer_cond.wait_until(self.buffer_mutex, end_time); + + DateTime new_current_time = new DateTime.now_utc(); + if (new_current_time.compare(current_time) > 0) { + break; + } + } + self.buffer_mutex.unlock(); + + // The callback should return 0 on timeout, a positive number if data can be received, and -1 on error. + return 1; + } + + private static ssize_t push_function(void* transport_ptr, uint8[] buffer) { + DtlsSrtp self = transport_ptr as DtlsSrtp; + self.send_data(buffer); + + // The callback should return a positive number indicating the bytes sent, and -1 on error. + return (ssize_t)buffer.length; + } + + private static int verify_function(Session session) { + DtlsSrtp self = session.get_transport_pointer() as DtlsSrtp; + try { + bool valid = self.verify_peer_cert(session); + if (!valid) { + warning("DTLS certificate invalid. Aborting handshake."); + return 1; + } + } catch (Error e) { + warning("Error during DTLS certificate validation: %s. Aborting handshake.", e.message); + return 1; + } + + // The callback function should return 0 for the handshake to continue or non-zero to terminate. + return 0; + } + + private bool verify_peer_cert(Session session) throws GLib.Error { + unowned Datum[] cert_datums = session.get_peer_certificates(); + if (cert_datums.length == 0) { + warning("No peer certs"); + return false; + } + if (cert_datums.length > 1) warning("More than one peer cert"); + + X509.Certificate peer_cert = X509.Certificate.create(); + peer_cert.import(ref cert_datums[0], CertificateFormat.DER); + + string peer_fp_str = format_certificate(peer_cert, DigestAlgorithm.SHA256); + if (peer_fp_str.down() != this.peer_fingerprint.down()) { + warning("First cert in peer cert list doesn't equal advertised one %s vs %s", peer_fp_str, this.peer_fingerprint); + return false; + } + + return true; + } + + private string format_certificate(X509.Certificate certificate, DigestAlgorithm digest_algo) { + uint8[] buf = new uint8[512]; + size_t buf_out_size = 512; + certificate.get_fingerprint(digest_algo, buf, ref buf_out_size); + + var sb = new StringBuilder(); + for (int i = 0; i < buf_out_size; i++) { + sb.append("%02x".printf(buf[i])); + if (i < buf_out_size - 1) { + sb.append(":"); + } + } + return sb.str; + } + + private uint8[] uint8_pt_to_a(uint8* data, uint size) { + uint8[size] ret = new uint8[size]; + for (int i = 0; i < size; i++) { + ret[i] = data[i]; + } + return ret; + } +} \ No newline at end of file diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index a8172678..5b6431c2 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -9,9 +9,11 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport private bool we_want_connection; private bool remote_credentials_set; private Map connections = new HashMap(); + private DtlsSrtp? dtls_srtp; private class DatagramConnection : Jingle.DatagramConnection { private Nice.Agent agent; + private DtlsSrtp? dtls_srtp; private uint stream_id; private string? error; private ulong sent; @@ -20,8 +22,9 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport private ulong recv_reported; private ulong datagram_received_id; - public DatagramConnection(Nice.Agent agent, uint stream_id, uint8 component_id) { + public DatagramConnection(Nice.Agent agent, DtlsSrtp? dtls_srtp, uint stream_id, uint8 component_id) { this.agent = agent; + this.dtls_srtp = dtls_srtp; this.stream_id = stream_id; this.component_id = component_id; this.datagram_received_id = this.datagram_received.connect((datagram) => { @@ -41,7 +44,12 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport public override void send_datagram(Bytes datagram) { if (this.agent != null && is_component_ready(agent, stream_id, component_id)) { - agent.send(stream_id, component_id, datagram.get_data()); + uint8[] encrypted_data = null; + if (dtls_srtp != null) { + encrypted_data = dtls_srtp.process_outgoing_data(component_id, datagram.get_data()); + if (encrypted_data == null) return; + } + agent.send(stream_id, component_id, encrypted_data ?? datagram.get_data()); sent += datagram.length; if (sent > sent_reported + 100000) { debug("Sent %lu bytes via stream %u component %u", sent, stream_id, component_id); @@ -55,6 +63,20 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport base(components, local_full_jid, peer_full_jid, node); this.we_want_connection = (node == null); this.agent = agent; + + if (this.peer_fingerprint != null || !incoming) { + dtls_srtp = DtlsSrtp.setup(); + dtls_srtp.send_data.connect((data) => { + agent.send(stream_id, 1, data); + }); + this.own_fingerprint = dtls_srtp.get_own_fingerprint(GnuTLS.DigestAlgorithm.SHA256); + if (incoming) { + dtls_srtp.set_peer_fingerprint(this.peer_fingerprint); + } else { + dtls_srtp.setup_dtls_connection(true); + } + } + agent.candidate_gathering_done.connect(on_candidate_gathering_done); agent.initial_binding_request_received.connect(on_initial_binding_request_received); agent.component_state_changed.connect(on_component_state_changed); @@ -112,6 +134,12 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport public override void handle_transport_accept(StanzaNode transport) throws Jingle.IqError { debug("on_transport_accept from %s", peer_full_jid.to_string()); base.handle_transport_accept(transport); + + if (dtls_srtp != null && peer_fingerprint != null) { + dtls_srtp.set_peer_fingerprint(this.peer_fingerprint); + } else { + dtls_srtp = null; + } } public override void handle_transport_info(StanzaNode transport) throws Jingle.IqError { @@ -163,9 +191,16 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport int new_candidates = agent.set_remote_candidates(stream_id, i, candidates); debug("Initiated component %u with %i remote candidates", i, new_candidates); - connections[i] = new DatagramConnection(agent, stream_id, i); + connections[i] = new DatagramConnection(agent, dtls_srtp, stream_id, i); content.set_transport_connection(connections[i], i); } + + if (incoming && dtls_srtp != null) { + Jingle.DatagramConnection rtp_datagram = (Jingle.DatagramConnection) content.get_transport_connection(1); + rtp_datagram.notify["ready"].connect(() => { + dtls_srtp.setup_dtls_connection(false); + }); + } base.create_transport_connection(stream, content); } @@ -194,12 +229,17 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport private void on_recv(Nice.Agent agent, uint stream_id, uint component_id, uint8[] data) { if (stream_id != this.stream_id) return; + uint8[] decrypt_data = null; + if (dtls_srtp != null) { + decrypt_data = dtls_srtp.process_incoming_data(component_id, data); + if (decrypt_data == null) return; + } may_consider_ready(stream_id, component_id); if (connections.has_key((uint8) component_id)) { if (!connections[(uint8) component_id].ready) { debug("on_recv stream %u component %u when state %s", stream_id, component_id, agent.get_component_state(stream_id, component_id).to_string()); } - connections[(uint8) component_id].datagram_received(new Bytes(data)); + connections[(uint8) component_id].datagram_received(new Bytes(decrypt_data ?? data)); } else { debug("on_recv stream %u component %u length %u", stream_id, component_id, data.length); } diff --git a/plugins/ice/vapi/gnutls.vapi b/plugins/ice/vapi/gnutls.vapi new file mode 100644 index 00000000..a8f75e14 --- /dev/null +++ b/plugins/ice/vapi/gnutls.vapi @@ -0,0 +1,419 @@ +[CCode (cprefix = "gnutls_", lower_case_cprefix = "gnutls_", cheader_filename = "gnutls/gnutls.h")] +namespace GnuTLS { + + public int global_init(); + + [CCode (cname = "gnutls_pull_func", has_target = false)] + public delegate ssize_t PullFunc(void* transport_ptr, [CCode (ctype = "void*", array_length_type="size_t")] uint8[] array); + + [CCode (cname = "gnutls_pull_timeout_func", has_target = false)] + public delegate int PullTimeoutFunc(void* transport_ptr, uint ms); + + [CCode (cname = "gnutls_push_func", has_target = false)] + public delegate ssize_t PushFunc(void* transport_ptr, [CCode (ctype = "void*", array_length_type="size_t")] uint8[] array); + + [CCode (cname = "gnutls_certificate_verify_function", has_target = false)] + public delegate int VerifyFunc(Session session); + + [Compact] + [CCode (cname = "struct gnutls_session_int", free_function = "gnutls_deinit")] + public class Session { + + public static Session? create(int con_end) throws GLib.Error { + Session result; + var ret = init(out result, con_end); + throw_if_error(ret); + return result; + } + + [CCode (cname = "gnutls_init")] + private static int init(out Session session, int con_end); + + [CCode (cname = "gnutls_transport_set_push_function")] + public void set_push_function(PushFunc func); + + [CCode (cname = "gnutls_transport_set_pull_function")] + public void set_pull_function(PullFunc func); + + [CCode (cname = "gnutls_transport_set_pull_timeout_function")] + public void set_pull_timeout_function(PullTimeoutFunc func); + + [CCode (cname = "gnutls_transport_set_ptr")] + public void set_transport_pointer(void* ptr); + + [CCode (cname = "gnutls_transport_get_ptr")] + public void* get_transport_pointer(); + + [CCode (cname = "gnutls_heartbeat_enable")] + public int enable_heartbeat(uint type); + + [CCode (cname = "gnutls_certificate_server_set_request")] + public void server_set_request(CertificateRequest req); + + [CCode (cname = "gnutls_credentials_set")] + public int set_credentials_(CredentialsType type, void* cred); + [CCode (cname = "gnutls_credentials_set_")] + public void set_credentials(CredentialsType type, void* cred) throws GLib.Error { + int err = set_credentials_(type, cred); + throw_if_error(err); + } + + [CCode (cname = "gnutls_priority_set_direct")] + public int set_priority_from_string_(string priority, out unowned string err_pos = null); + [CCode (cname = "gnutls_priority_set_direct_")] + public void set_priority_from_string(string priority, out unowned string err_pos = null) throws GLib.Error { + int err = set_priority_from_string_(priority, out err_pos); + throw_if_error(err); + } + + [CCode (cname = "gnutls_srtp_set_profile_direct")] + public int set_srtp_profile_direct_(string profiles, out unowned string err_pos = null); + [CCode (cname = "gnutls_srtp_set_profile_direct_")] + public void set_srtp_profile_direct(string profiles, out unowned string err_pos = null) throws GLib.Error { + int err = set_srtp_profile_direct_(profiles, out err_pos); + throw_if_error(err); + } + + [CCode (cname = "gnutls_transport_set_int")] + public void transport_set_int(int fd); + + [CCode (cname = "gnutls_handshake")] + public int handshake(); + + [CCode (cname = "gnutls_srtp_get_keys")] + public int get_srtp_keys_(void *key_material, uint32 key_material_size, out Datum client_key, out Datum client_salt, out Datum server_key, out Datum server_salt); + [CCode (cname = "gnutls_srtp_get_keys_")] + public void get_srtp_keys(void *key_material, uint32 key_material_size, out Datum client_key, out Datum client_salt, out Datum server_key, out Datum server_salt) throws GLib.Error { + get_srtp_keys_(key_material, key_material_size, out client_key, out client_salt, out server_key, out server_salt); + } + + [CCode (cname = "gnutls_certificate_get_peers", array_length_type = "unsigned int")] + public unowned Datum[]? get_peer_certificates(); + + [CCode (cname = "gnutls_session_set_verify_function")] + public void set_verify_function(VerifyFunc func); + } + + [Compact] + [CCode (cname = "struct gnutls_certificate_credentials_st", free_function = "gnutls_certificate_free_credentials", cprefix = "gnutls_certificate_")] + public class CertificateCredentials { + + [CCode (cname = "gnutls_certificate_allocate_credentials")] + private static int allocate(out CertificateCredentials credentials); + + public static CertificateCredentials create() throws GLib.Error { + CertificateCredentials result; + var ret = allocate (out result); + throw_if_error(ret); + return result; + } + + public void get_x509_crt(uint index, [CCode (array_length_type = "unsigned int")] out unowned X509.Certificate[] x509_ca_list); + + public int set_x509_key(X509.Certificate[] cert_list, X509.PrivateKey key); + } + + [CCode (cheader_filename = "gnutls/x509.h", cprefix = "GNUTLS_")] + namespace X509 { + + [Compact] + [CCode (cname = "struct gnutls_x509_crt_int", cprefix = "gnutls_x509_crt_", free_function = "gnutls_x509_crt_deinit")] + public class Certificate { + + [CCode (cname = "gnutls_x509_crt_init")] + private static int init (out Certificate cert); + public static Certificate create() throws GLib.Error { + Certificate result; + var ret = init (out result); + throw_if_error(ret); + return result; + } + + [CCode (cname = "gnutls_x509_crt_import")] + public int import_(ref Datum data, CertificateFormat format); + [CCode (cname = "gnutls_x509_crt_import_")] + public void import(ref Datum data, CertificateFormat format) throws GLib.Error { + int err = import_(ref data, format); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_set_version")] + public int set_version_(uint version); + [CCode (cname = "gnutls_x509_crt_set_version_")] + public void set_version(uint version) throws GLib.Error { + int err = set_version_(version); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_set_key")] + public int set_key_(PrivateKey key); + [CCode (cname = "gnutls_x509_crt_set_key_")] + public void set_key(PrivateKey key) throws GLib.Error { + int err = set_key_(key); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_set_activation_time")] + public int set_activation_time_(time_t act_time); + [CCode (cname = "gnutls_x509_crt_set_activation_time_")] + public void set_activation_time(time_t act_time) throws GLib.Error { + int err = set_activation_time_(act_time); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_set_expiration_time")] + public int set_expiration_time_(time_t exp_time); + [CCode (cname = "gnutls_x509_crt_set_expiration_time_")] + public void set_expiration_time(time_t exp_time) throws GLib.Error { + int err = set_expiration_time_(exp_time); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_set_serial")] + public int set_serial_(void* serial, size_t serial_size); + [CCode (cname = "gnutls_x509_crt_set_serial_")] + public void set_serial(void* serial, size_t serial_size) throws GLib.Error { + int err = set_serial_(serial, serial_size); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_sign")] + public int sign_(Certificate issuer, PrivateKey issuer_key); + [CCode (cname = "gnutls_x509_crt_sign_")] + public void sign(Certificate issuer, PrivateKey issuer_key) throws GLib.Error { + int err = sign_(issuer, issuer_key); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_get_fingerprint")] + public int get_fingerprint_(DigestAlgorithm algo, void* buf, ref size_t buf_size); + [CCode (cname = "gnutls_x509_crt_get_fingerprint_")] + public void get_fingerprint(DigestAlgorithm algo, void* buf, ref size_t buf_size) throws GLib.Error { + int err = get_fingerprint_(algo, buf, ref buf_size); + throw_if_error(err); + } + } + + [Compact] + [CCode (cname = "struct gnutls_x509_privkey_int", cprefix = "gnutls_x509_privkey_", free_function = "gnutls_x509_privkey_deinit")] + public class PrivateKey { + private static int init (out PrivateKey key); + public static PrivateKey create () throws GLib.Error { + PrivateKey result; + var ret = init (out result); + throw_if_error(ret); + return result; + } + + public int generate(PKAlgorithm algo, uint bits, uint flags = 0); + } + + } + + [CCode (cname = "gnutls_certificate_request_t", cprefix = "GNUTLS_CERT_", has_type_id = false)] + public enum CertificateRequest { + IGNORE, + REQUEST, + REQUIRE + } + + [CCode (cname = "gnutls_pk_algorithm_t", cprefix = "GNUTLS_PK_", has_type_id = false)] + public enum PKAlgorithm { + UNKNOWN, + RSA, + DSA; + } + + [CCode (cname = "gnutls_digest_algorithm_t", cprefix = "GNUTLS_DIG_", has_type_id = false)] + public enum DigestAlgorithm { + NULL, + MD5, + SHA1, + RMD160, + MD2, + SHA224, + SHA256, + SHA384, + SHA512; + } + + [Flags] + [CCode (cname = "gnutls_init_flags_t", cprefix = "GNUTLS_", has_type_id = false)] + public enum InitFlags { + SERVER, + CLIENT, + DATAGRAM + } + + [CCode (cname = "gnutls_credentials_type_t", cprefix = "GNUTLS_CRD_", has_type_id = false)] + public enum CredentialsType { + CERTIFICATE, + ANON, + SRP, + PSK, + IA + } + + [CCode (cname = "gnutls_x509_crt_fmt_t", cprefix = "GNUTLS_X509_FMT_", has_type_id = false)] + public enum CertificateFormat { + DER, + PEM + } + + [Flags] + [CCode (cname = "gnutls_certificate_status_t", cprefix = "GNUTLS_CERT_", has_type_id = false)] + public enum CertificateStatus { + INVALID, // will be set if the certificate was not verified. + REVOKED, // in X.509 this will be set only if CRLs are checked + SIGNER_NOT_FOUND, + SIGNER_NOT_CA, + INSECURE_ALGORITHM + } + + [SimpleType] + [CCode (cname = "gnutls_datum_t", has_type_id = false)] + public struct Datum { + public uint8* data; + public uint size; + + public uint8[] extract() { + uint8[size] ret = new uint8[size]; + for (int i = 0; i < size; i++) { + ret[i] = data[i]; + } + return ret; + } + } + + // Gnutls error codes. The mapping to a TLS alert is also shown in comments. + [CCode (cname = "int", cprefix = "GNUTLS_E_", lower_case_cprefix = "gnutls_error_", has_type_id = false)] + public enum ErrorCode { + SUCCESS, + UNKNOWN_COMPRESSION_ALGORITHM, + UNKNOWN_CIPHER_TYPE, + LARGE_PACKET, + UNSUPPORTED_VERSION_PACKET, // GNUTLS_A_PROTOCOL_VERSION + UNEXPECTED_PACKET_LENGTH, // GNUTLS_A_RECORD_OVERFLOW + INVALID_SESSION, + FATAL_ALERT_RECEIVED, + UNEXPECTED_PACKET, // GNUTLS_A_UNEXPECTED_MESSAGE + WARNING_ALERT_RECEIVED, + ERROR_IN_FINISHED_PACKET, + UNEXPECTED_HANDSHAKE_PACKET, + UNKNOWN_CIPHER_SUITE, // GNUTLS_A_HANDSHAKE_FAILURE + UNWANTED_ALGORITHM, + MPI_SCAN_FAILED, + DECRYPTION_FAILED, // GNUTLS_A_DECRYPTION_FAILED, GNUTLS_A_BAD_RECORD_MAC + MEMORY_ERROR, + DECOMPRESSION_FAILED, // GNUTLS_A_DECOMPRESSION_FAILURE + COMPRESSION_FAILED, + AGAIN, + EXPIRED, + DB_ERROR, + SRP_PWD_ERROR, + INSUFFICIENT_CREDENTIALS, + HASH_FAILED, + BASE64_DECODING_ERROR, + MPI_PRINT_FAILED, + REHANDSHAKE, // GNUTLS_A_NO_RENEGOTIATION + GOT_APPLICATION_DATA, + RECORD_LIMIT_REACHED, + ENCRYPTION_FAILED, + PK_ENCRYPTION_FAILED, + PK_DECRYPTION_FAILED, + PK_SIGN_FAILED, + X509_UNSUPPORTED_CRITICAL_EXTENSION, + KEY_USAGE_VIOLATION, + NO_CERTIFICATE_FOUND, // GNUTLS_A_BAD_CERTIFICATE + INVALID_REQUEST, + SHORT_MEMORY_BUFFER, + INTERRUPTED, + PUSH_ERROR, + PULL_ERROR, + RECEIVED_ILLEGAL_PARAMETER, // GNUTLS_A_ILLEGAL_PARAMETER + REQUESTED_DATA_NOT_AVAILABLE, + PKCS1_WRONG_PAD, + RECEIVED_ILLEGAL_EXTENSION, + INTERNAL_ERROR, + DH_PRIME_UNACCEPTABLE, + FILE_ERROR, + TOO_MANY_EMPTY_PACKETS, + UNKNOWN_PK_ALGORITHM, + // returned if libextra functionality was requested but + // gnutls_global_init_extra() was not called. + + INIT_LIBEXTRA, + LIBRARY_VERSION_MISMATCH, + // returned if you need to generate temporary RSA + // parameters. These are needed for export cipher suites. + + NO_TEMPORARY_RSA_PARAMS, + LZO_INIT_FAILED, + NO_COMPRESSION_ALGORITHMS, + NO_CIPHER_SUITES, + OPENPGP_GETKEY_FAILED, + PK_SIG_VERIFY_FAILED, + ILLEGAL_SRP_USERNAME, + SRP_PWD_PARSING_ERROR, + NO_TEMPORARY_DH_PARAMS, + // For certificate and key stuff + + ASN1_ELEMENT_NOT_FOUND, + ASN1_IDENTIFIER_NOT_FOUND, + ASN1_DER_ERROR, + ASN1_VALUE_NOT_FOUND, + ASN1_GENERIC_ERROR, + ASN1_VALUE_NOT_VALID, + ASN1_TAG_ERROR, + ASN1_TAG_IMPLICIT, + ASN1_TYPE_ANY_ERROR, + ASN1_SYNTAX_ERROR, + ASN1_DER_OVERFLOW, + OPENPGP_UID_REVOKED, + CERTIFICATE_ERROR, + CERTIFICATE_KEY_MISMATCH, + UNSUPPORTED_CERTIFICATE_TYPE, // GNUTLS_A_UNSUPPORTED_CERTIFICATE + X509_UNKNOWN_SAN, + OPENPGP_FINGERPRINT_UNSUPPORTED, + X509_UNSUPPORTED_ATTRIBUTE, + UNKNOWN_HASH_ALGORITHM, + UNKNOWN_PKCS_CONTENT_TYPE, + UNKNOWN_PKCS_BAG_TYPE, + INVALID_PASSWORD, + MAC_VERIFY_FAILED, // for PKCS #12 MAC + CONSTRAINT_ERROR, + WARNING_IA_IPHF_RECEIVED, + WARNING_IA_FPHF_RECEIVED, + IA_VERIFY_FAILED, + UNKNOWN_ALGORITHM, + BASE64_ENCODING_ERROR, + INCOMPATIBLE_CRYPTO_LIBRARY, + INCOMPATIBLE_LIBTASN1_LIBRARY, + OPENPGP_KEYRING_ERROR, + X509_UNSUPPORTED_OID, + RANDOM_FAILED, + BASE64_UNEXPECTED_HEADER_ERROR, + OPENPGP_SUBKEY_ERROR, + CRYPTO_ALREADY_REGISTERED, + HANDSHAKE_TOO_LARGE, + UNIMPLEMENTED_FEATURE, + APPLICATION_ERROR_MAX, // -65000 + APPLICATION_ERROR_MIN; // -65500 + + [CCode (cname = "gnutls_error_is_fatal")] + public bool is_fatal(); + + [CCode (cname = "gnutls_perror")] + public void print(); + + [CCode (cname = "gnutls_strerror")] + public unowned string to_string(); + } + + public void throw_if_error(int err_int) throws GLib.Error { + ErrorCode error = (ErrorCode)err_int; + if (error != ErrorCode.SUCCESS) { + throw new GLib.Error(-1, error, "%s%s", error.to_string(), error.is_fatal() ? " fatal" : ""); + } + } +} \ No newline at end of file diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 5311fac3..8ce2a7c6 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -2,6 +2,7 @@ find_packages(RTP_PACKAGES REQUIRED Gee GLib GModule + GnuTLS GObject GTK3 Gst diff --git a/xmpp-vala/src/module/xep/0166_jingle/reason_element.vala b/xmpp-vala/src/module/xep/0166_jingle/reason_element.vala index 1cbdf936..4d47d4cd 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/reason_element.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/reason_element.vala @@ -24,6 +24,7 @@ namespace Xmpp.Xep.Jingle.ReasonElement { BUSY, CANCEL, DECLINE, + GONE, SUCCESS }; } \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/session.vala b/xmpp-vala/src/module/xep/0166_jingle/session.vala index e9ad9169..2d359f01 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/session.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/session.vala @@ -24,9 +24,10 @@ public class Xmpp.Xep.Jingle.Session : Object { public Jid peer_full_jid { get; private set; } public bool we_initiated { get; private set; } - public HashMap contents = new HashMap(); + public HashMap contents_map = new HashMap(); + public Gee.List contents = new ArrayList(); // Keep the order contents - public SecurityParameters? security { get { return contents.values.to_array()[0].security_params; } } + public SecurityParameters? security { get { return contents.to_array()[0].security_params; } } public Session.initiate_sent(XmppStream stream, string sid, Jid local_full_jid, Jid peer_full_jid) { this.stream = stream; @@ -94,7 +95,7 @@ public class Xmpp.Xep.Jingle.Session : Object { } else if (action.has_prefix("transport-")) { ContentNode content_node = get_single_content_node(jingle); - if (!contents.has_key(content_node.name)) { + if (!contents_map.has_key(content_node.name)) { throw new IqError.BAD_REQUEST("unknown content"); } @@ -102,7 +103,7 @@ public class Xmpp.Xep.Jingle.Session : Object { throw new IqError.BAD_REQUEST("missing transport node"); } - Content content = contents[content_node.name]; + Content content = contents_map[content_node.name]; if (content_node.creator != content.content_creator) { throw new IqError.BAD_REQUEST("unknown content; creator"); @@ -128,11 +129,11 @@ public class Xmpp.Xep.Jingle.Session : Object { } else if (action == "description-info") { ContentNode content_node = get_single_content_node(jingle); - if (!contents.has_key(content_node.name)) { + if (!contents_map.has_key(content_node.name)) { throw new IqError.BAD_REQUEST("unknown content"); } - Content content = contents[content_node.name]; + Content content = contents_map[content_node.name]; if (content_node.creator != content.content_creator) { throw new IqError.BAD_REQUEST("unknown content; creator"); @@ -149,7 +150,8 @@ public class Xmpp.Xep.Jingle.Session : Object { } internal void insert_content(Content content) { - this.contents[content.content_name] = content; + this.contents_map[content.content_name] = content; + this.contents.add(content); content.set_session(this); } @@ -209,7 +211,8 @@ public class Xmpp.Xep.Jingle.Session : Object { public async void add_content(Content content) { content.session = this; - this.contents[content.content_name] = content; + this.contents_map[content.content_name] = content; + contents.add(content); StanzaNode content_add_node = new StanzaNode.build("jingle", NS_URI) .add_self_xmlns() @@ -228,9 +231,9 @@ public class Xmpp.Xep.Jingle.Session : Object { private void handle_content_accept(ContentNode content_node) throws IqError { if (content_node.description == null || content_node.transport == null) throw new IqError.BAD_REQUEST("missing description or transport node"); - if (!contents.has_key(content_node.name)) throw new IqError.BAD_REQUEST("unknown content"); + if (!contents_map.has_key(content_node.name)) throw new IqError.BAD_REQUEST("unknown content"); - Content content = contents[content_node.name]; + Content content = contents_map[content_node.name]; if (content_node.creator != content.content_creator) warning("Counterpart accepts content with an unexpected `creator`"); if (content_node.senders != content.senders) warning("Counterpart accepts content with an unexpected `senders`"); @@ -242,7 +245,7 @@ public class Xmpp.Xep.Jingle.Session : Object { private void handle_content_modify(XmppStream stream, StanzaNode jingle_node, Iq.Stanza iq) throws IqError { ContentNode content_node = get_single_content_node(jingle_node); - Content? content = contents[content_node.name]; + Content? content = contents_map[content_node.name]; if (content == null) throw new IqError.BAD_REQUEST("no such content"); if (content_node.creator != content.content_creator) throw new IqError.BAD_REQUEST("mismatching creator"); @@ -301,7 +304,7 @@ public class Xmpp.Xep.Jingle.Session : Object { } } - foreach (Content content in contents.values) { + foreach (Content content in contents) { content.terminate(false, reason_name, reason_text); } @@ -336,7 +339,7 @@ public class Xmpp.Xep.Jingle.Session : Object { .add_self_xmlns() .put_attribute("action", "session-accept") .put_attribute("sid", sid); - foreach (Content content in contents.values) { + foreach (Content content in contents) { StanzaNode content_node = new StanzaNode.build("content", NS_URI) .put_attribute("creator", "initiator") .put_attribute("name", content.content_name) @@ -345,12 +348,13 @@ public class Xmpp.Xep.Jingle.Session : Object { .put_node(content.transport_params.to_transport_stanza_node()); jingle.put_node(content_node); } + Iq.Stanza iq = new Iq.Stanza.set(jingle) { to=peer_full_jid }; stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); - foreach (Content content in contents.values) { - content.on_accept(stream); + foreach (Content content2 in contents) { + content2.on_accept(stream); } state = State.ACTIVE; @@ -359,7 +363,7 @@ public class Xmpp.Xep.Jingle.Session : Object { internal void accept_content(Content content) { if (state == State.INITIATE_RECEIVED) { bool all_accepted = true; - foreach (Content c in contents.values) { + foreach (Content c in contents) { if (c.state != Content.State.WANTS_TO_BE_ACCEPTED) { all_accepted = false; } @@ -413,7 +417,7 @@ public class Xmpp.Xep.Jingle.Session : Object { } else { reason_str = "local session-terminate"; } - foreach (Content content in contents.values) { + foreach (Content content in contents) { content.terminate(true, reason_name, reason_text); } } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index cca03543..32ea1df6 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -34,7 +34,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { this.parent = parent; this.media = media; this.ssrc = ssrc; - this.rtcp_mux = rtcp_mux; + this.rtcp_mux = true; this.bandwidth = bandwidth; this.bandwidth_type = bandwidth_type; this.encryption_required = encryption_required; @@ -175,6 +175,9 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { ret.put_node(new StanzaNode.build("encryption", NS_URI) .put_node(local_crypto.to_xml())); } + if (rtcp_mux) { + ret.put_node(new StanzaNode.build("rtcp-mux", NS_URI)); + } return ret; } } \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala index 23aee6c9..3a9ea09f 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala @@ -84,7 +84,7 @@ public abstract class Module : XmppStreamModule { Jid receiver_full_jid = session.peer_full_jid; Jingle.Content? content = null; - foreach (Jingle.Content c in session.contents.values) { + foreach (Jingle.Content c in session.contents) { Parameters? parameters = c.content_params as Parameters; if (parameters == null) continue; diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/session_info_type.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/session_info_type.vala index d36255f0..32cd9016 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/session_info_type.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/session_info_type.vala @@ -50,7 +50,7 @@ namespace Xmpp.Xep.JingleRtp { public void send_mute(Jingle.Session session, bool mute, string media) { string node_name = mute ? "mute" : "unmute"; - foreach (Jingle.Content content in session.contents.values) { + foreach (Jingle.Content content in session.contents) { Parameters? parameters = content.content_params as Parameters; if (parameters != null && parameters.media == media) { StanzaNode session_info_content = new StanzaNode.build(node_name, NS_URI).add_self_xmlns().put_attribute("name", content.content_name); diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala index 9ed494ff..4b7c7a36 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala @@ -12,6 +12,7 @@ public abstract class Module : XmppStreamModule, Jingle.Transport { public override void attach(XmppStream stream) { stream.get_module(Jingle.Module.IDENTITY).register_transport(this); stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); + stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, "urn:xmpp:jingle:apps:dtls:0"); } public override void detach(XmppStream stream) { stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI); diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala index 8b8aa07d..3c69d0af 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala @@ -13,6 +13,9 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T public ConcurrentList unsent_local_candidates = new ConcurrentList(Candidate.equals_func); public Gee.List remote_candidates = new ArrayList(Candidate.equals_func); + public string? own_fingerprint = null; + public string? peer_fingerprint = null; + public Jid local_full_jid { get; private set; } public Jid peer_full_jid { get; private set; } private uint8 components_; @@ -34,6 +37,11 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T foreach (StanzaNode candidateNode in node.get_subnodes("candidate")) { remote_candidates.add(Candidate.parse(candidateNode)); } + + StanzaNode? fingerprint_node = node.get_subnode("fingerprint", "urn:xmpp:jingle:apps:dtls:0"); + if (fingerprint_node != null) { + peer_fingerprint = fingerprint_node.get_deep_string_content(); + } } } @@ -57,6 +65,20 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T .add_self_xmlns() .put_attribute("ufrag", local_ufrag) .put_attribute("pwd", local_pwd); + + if (own_fingerprint != null) { + var fingerprint_node = new StanzaNode.build("fingerprint", "urn:xmpp:jingle:apps:dtls:0") + .add_self_xmlns() + .put_attribute("hash", "sha-256") + .put_node(new StanzaNode.text(own_fingerprint)); + if (incoming) { + fingerprint_node.put_attribute("setup", "active"); + } else { + fingerprint_node.put_attribute("setup", "actpass"); + } + node.put_node(fingerprint_node); + } + foreach (Candidate candidate in unsent_local_candidates) { node.put_node(candidate.to_xml()); } @@ -72,6 +94,11 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T foreach (StanzaNode candidateNode in node.get_subnodes("candidate")) { remote_candidates.add(Candidate.parse(candidateNode)); } + + StanzaNode? fingerprint_node = node.get_subnode("fingerprint", "urn:xmpp:jingle:apps:dtls:0"); + if (fingerprint_node != null) { + peer_fingerprint = fingerprint_node.get_deep_string_content(); + } } public virtual void handle_transport_info(StanzaNode node) throws Jingle.IqError { From fc3263d49e5a5c737742eb7e591498ade830b685 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Thu, 25 Mar 2021 13:06:41 +0100 Subject: [PATCH 21/56] Fix device manager usage for GStreamer 1.16 --- plugins/rtp/src/plugin.vala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index 69b0f37a..0f3cb10d 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -47,6 +47,12 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { device_monitor.show_all = true; device_monitor.get_bus().add_watch(Priority.DEFAULT, on_device_monitor_message); device_monitor.start(); + foreach (Gst.Device device in device_monitor.get_devices()) { + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) continue; + if (device.properties.get_string("device.class") == "monitor") continue; + if (devices.any_match((it) => it.matches(device))) continue; + devices.add(new Device(this, device)); + } pipe = new Gst.Pipeline(null); @@ -163,24 +169,28 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { private bool on_device_monitor_message(Gst.Bus bus, Gst.Message message) { Gst.Device old_device = null; Gst.Device device = null; + Device old = null; switch (message.type) { case Gst.MessageType.DEVICE_ADDED: message.parse_device_added(out device); if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + if (devices.any_match((it) => it.matches(device))) return Source.CONTINUE; devices.add(new Device(this, device)); break; case Gst.MessageType.DEVICE_CHANGED: message.parse_device_changed(out device, out old_device); if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; - devices.first_match((it) => it.matches(old_device)).update(device); + old = devices.first_match((it) => it.matches(old_device)); + if (old != null) old.update(device); break; case Gst.MessageType.DEVICE_REMOVED: message.parse_device_removed(out device); if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; - devices.remove(devices.first_match((it) => it.matches(device))); + old = devices.first_match((it) => it.matches(device)); + if (old != null) devices.remove(old); break; } if (device != null) { From fd21eafe26a7b72fd3eee575c42654a5219abee3 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Mon, 29 Mar 2021 13:14:10 +0200 Subject: [PATCH 22/56] Fix cyclic reference in DTLS --- plugins/ice/src/transport_parameters.vala | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index 5b6431c2..2db1ab1b 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -65,10 +65,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport this.agent = agent; if (this.peer_fingerprint != null || !incoming) { - dtls_srtp = DtlsSrtp.setup(); - dtls_srtp.send_data.connect((data) => { - agent.send(stream_id, 1, data); - }); + dtls_srtp = setup_dtls(this); this.own_fingerprint = dtls_srtp.get_own_fingerprint(GnuTLS.DigestAlgorithm.SHA256); if (incoming) { dtls_srtp.set_peer_fingerprint(this.peer_fingerprint); @@ -105,6 +102,16 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport agent.gather_candidates(stream_id); } + private static DtlsSrtp setup_dtls(TransportParameters tp) { + var weak_self = new WeakRef(tp); + DtlsSrtp dtls_srtp = DtlsSrtp.setup(); + dtls_srtp.send_data.connect((data) => { + TransportParameters self = (TransportParameters) weak_self.get(); + if (self != null) self.agent.send(self.stream_id, 1, data); + }); + return dtls_srtp; + } + private void on_candidate_gathering_done(uint stream_id) { if (stream_id != this.stream_id) return; debug("on_candidate_gathering_done in %u", stream_id); From 9520a81b814103c5549982a7e9d4e9ec6d9035f6 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Mon, 29 Mar 2021 13:14:37 +0200 Subject: [PATCH 23/56] Don't reuse PTs for different media types --- plugins/rtp/src/module.vala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index ecf7b658..231a9dde 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -122,25 +122,25 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { channels = 2, clockrate = 48000, name = "opus", - id = 96 + id = 99 }); yield add_if_supported(list, media, new JingleRtp.PayloadType() { channels = 1, clockrate = 32000, name = "speex", - id = 97 + id = 100 }); yield add_if_supported(list, media, new JingleRtp.PayloadType() { channels = 1, clockrate = 16000, name = "speex", - id = 98 + id = 101 }); yield add_if_supported(list, media, new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "speex", - id = 99 + id = 102 }); yield add_if_supported(list, media, new JingleRtp.PayloadType() { channels = 1, From c7d1ee4dc5a08715ed68ed69e918f5ec9cbd4b40 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Mon, 29 Mar 2021 13:19:52 +0200 Subject: [PATCH 24/56] Make RTCP-MUX a stream property --- .../src/module/xep/0167_jingle_rtp/content_parameters.vala | 1 + xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index 32ea1df6..ff3d31f4 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -127,6 +127,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { } public void handle_accept(XmppStream stream, Jingle.Session session, Jingle.Content content, StanzaNode description_node) { + rtcp_mux = description_node.get_subnode("rtcp-mux") != null; Gee.List payload_type_nodes = description_node.get_subnodes("payload-type"); if (payload_type_nodes.size == 0) { warning("Counterpart didn't include any payload types"); diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala index 2fc29291..730ce9f8 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala @@ -37,6 +37,13 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { public bool receiving { get { return content.session.senders_include_counterpart(content.senders); }} + public bool rtcp_mux { get { + var content_params = content.content_params; + if (content_params is Parameters) { + return ((Parameters)content_params).rtcp_mux; + } + return false; + }} protected Stream(Jingle.Content content) { this.content = content; From 5e58f2988382fffb70602cf308f6686b4731f0da Mon Sep 17 00:00:00 2001 From: Marvin W Date: Mon, 29 Mar 2021 13:20:12 +0200 Subject: [PATCH 25/56] Migrate to libsrtp2 --- cmake/FindSrtp2.cmake | 12 + plugins/crypto-vala/CMakeLists.txt | 34 +- plugins/crypto-vala/src/error.vala | 4 +- plugins/crypto-vala/src/srtp.c | 836 ------------------------- plugins/crypto-vala/src/srtp.h | 82 --- plugins/crypto-vala/src/srtp.vala | 122 ++++ plugins/crypto-vala/src/srtp.vapi | 107 ---- plugins/crypto-vala/vapi/libsrtp2.vapi | 115 ++++ plugins/ice/CMakeLists.txt | 2 +- plugins/ice/src/dtls_srtp.vala | 49 +- plugins/rtp/CMakeLists.txt | 2 +- plugins/rtp/src/stream.vala | 51 +- 12 files changed, 314 insertions(+), 1102 deletions(-) create mode 100644 cmake/FindSrtp2.cmake delete mode 100644 plugins/crypto-vala/src/srtp.c delete mode 100644 plugins/crypto-vala/src/srtp.h create mode 100644 plugins/crypto-vala/src/srtp.vala delete mode 100644 plugins/crypto-vala/src/srtp.vapi create mode 100644 plugins/crypto-vala/vapi/libsrtp2.vapi diff --git a/cmake/FindSrtp2.cmake b/cmake/FindSrtp2.cmake new file mode 100644 index 00000000..40b0ed97 --- /dev/null +++ b/cmake/FindSrtp2.cmake @@ -0,0 +1,12 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(Srtp2 + PKG_CONFIG_NAME libsrtp2 + LIB_NAMES srtp2 + INCLUDE_NAMES srtp2/srtp.h + INCLUDE_DIR_SUFFIXES srtp2 srtp2/include +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Srtp2 + REQUIRED_VARS Srtp2_LIBRARY + VERSION_VAR Srtp2_VERSION) \ No newline at end of file diff --git a/plugins/crypto-vala/CMakeLists.txt b/plugins/crypto-vala/CMakeLists.txt index f97b0d31..4a8da241 100644 --- a/plugins/crypto-vala/CMakeLists.txt +++ b/plugins/crypto-vala/CMakeLists.txt @@ -3,6 +3,7 @@ find_packages(CRYPTO_VALA_PACKAGES REQUIRED GLib GObject GIO + Srtp2 ) vala_precompile(CRYPTO_VALA_C @@ -11,44 +12,23 @@ SOURCES "src/cipher_converter.vala" "src/error.vala" "src/random.vala" - "src/srtp.vapi" + "src/srtp.vala" CUSTOM_VAPIS "${CMAKE_CURRENT_SOURCE_DIR}/vapi/gcrypt.vapi" + "${CMAKE_CURRENT_SOURCE_DIR}/vapi/libsrtp2.vapi" PACKAGES ${CRYPTO_VALA_PACKAGES} +OPTIONS + --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi GENERATE_VAPI crypto-vala GENERATE_HEADER crypto-vala ) -add_custom_command(OUTPUT "${CMAKE_BINARY_DIR}/exports/srtp.h" -COMMAND - cp "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.h" "${CMAKE_BINARY_DIR}/exports/srtp.h" -DEPENDS - "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.h" -COMMENT - Copy header file srtp.h -) - -add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/exports/crypto.vapi -COMMAND - cat "${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi" "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi" > "${CMAKE_BINARY_DIR}/exports/crypto.vapi" -DEPENDS - ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi - ${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi -) - -add_custom_target(crypto-vapi -DEPENDS - ${CMAKE_BINARY_DIR}/exports/crypto.vapi - ${CMAKE_BINARY_DIR}/exports/srtp.h -) - -set(CFLAGS ${VALA_CFLAGS} -I${CMAKE_CURRENT_SOURCE_DIR}/src) +set(CFLAGS ${VALA_CFLAGS}) add_definitions(${CFLAGS}) -add_library(crypto-vala STATIC ${CRYPTO_VALA_C} src/srtp.c) -add_dependencies(crypto-vala crypto-vapi) +add_library(crypto-vala STATIC ${CRYPTO_VALA_C}) target_link_libraries(crypto-vala ${CRYPTO_VALA_PACKAGES} gcrypt) set_property(TARGET crypto-vala PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/plugins/crypto-vala/src/error.vala b/plugins/crypto-vala/src/error.vala index bae4ad08..5007d725 100644 --- a/plugins/crypto-vala/src/error.vala +++ b/plugins/crypto-vala/src/error.vala @@ -2,7 +2,9 @@ namespace Crypto { public errordomain Error { ILLEGAL_ARGUMENTS, - GCRYPT + GCRYPT, + AUTHENTICATION_FAILED, + UNKNOWN } internal void may_throw_gcrypt_error(GCrypt.Error e) throws Error { diff --git a/plugins/crypto-vala/src/srtp.c b/plugins/crypto-vala/src/srtp.c deleted file mode 100644 index 708244d9..00000000 --- a/plugins/crypto-vala/src/srtp.c +++ /dev/null @@ -1,836 +0,0 @@ -/* - * Secure RTP with libgcrypt - * Copyright (C) 2007 Rémi Denis-Courmont - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - */ - -/* TODO: - * Useless stuff (because nothing depends on it): - * - non-nul key derivation rate - * - MKI payload - */ - -#ifdef HAVE_CONFIG_H -# include -#endif - -#include -#include - -#include "srtp.h" - -#include -#include -#include -#include - -#include - -#ifdef _WIN32 -# include -#else -# include -#endif - -#define debug( ... ) (void)0 - -typedef struct srtp_proto_t -{ - gcry_cipher_hd_t cipher; - gcry_md_hd_t mac; - uint64_t window; - uint32_t salt[4]; -} srtp_proto_t; - -struct srtp_session_t -{ - srtp_proto_t rtp; - srtp_proto_t rtcp; - unsigned flags; - unsigned kdr; - uint32_t rtcp_index; - uint32_t rtp_roc; - uint16_t rtp_seq; - uint16_t rtp_rcc; - uint8_t tag_len; -}; - -enum -{ - SRTP_CRYPT, - SRTP_AUTH, - SRTP_SALT, - SRTCP_CRYPT, - SRTCP_AUTH, - SRTCP_SALT -}; - - -static inline unsigned rcc_mode (const srtp_session_t *s) -{ - return (s->flags >> 4) & 3; -} - - -static void proto_destroy (srtp_proto_t *p) -{ - gcry_md_close (p->mac); - gcry_cipher_close (p->cipher); -} - - -/** - * Releases all resources associated with a Secure RTP session. - */ -void srtp_destroy (srtp_session_t *s) -{ - assert (s != NULL); - - proto_destroy (&s->rtcp); - proto_destroy (&s->rtp); - free (s); -} - - -static int proto_create (srtp_proto_t *p, int gcipher, int gmd) -{ - if (gcry_cipher_open (&p->cipher, gcipher, GCRY_CIPHER_MODE_CTR, 0) == 0) - { - if (gcry_md_open (&p->mac, gmd, GCRY_MD_FLAG_HMAC) == 0) - return 0; - gcry_cipher_close (p->cipher); - } - return -1; -} - - -/** - * Allocates a Secure RTP one-way session. - * The same session cannot be used both ways because this would confuse - * internal cryptographic counters; it is however of course feasible to open - * multiple simultaneous sessions with the same master key. - * - * @param encr encryption algorithm number - * @param auth authentication algortihm number - * @param tag_len authentication tag byte length (NOT including RCC) - * @param flags OR'ed optional flags. - * - * @return NULL in case of error - */ -srtp_session_t * -srtp_create (int encr, int auth, unsigned tag_len, int prf, unsigned flags) -{ - if ((flags & ~SRTP_FLAGS_MASK)) - return NULL; - - int cipher, md; - switch (encr) - { - case SRTP_ENCR_NULL: - cipher = GCRY_CIPHER_NONE; - break; - - case SRTP_ENCR_AES_CM: - cipher = GCRY_CIPHER_AES; - break; - - default: - return NULL; - } - - switch (auth) - { - case SRTP_AUTH_NULL: - md = GCRY_MD_NONE; - break; - - case SRTP_AUTH_HMAC_SHA1: - md = GCRY_MD_SHA1; - break; - - default: - return NULL; - } - - if (tag_len > gcry_md_get_algo_dlen (md)) - return NULL; - - if (prf != SRTP_PRF_AES_CM) - return NULL; - - srtp_session_t *s = malloc (sizeof (*s)); - if (s == NULL) - return NULL; - - memset (s, 0, sizeof (*s)); - s->flags = flags; - s->tag_len = tag_len; - s->rtp_rcc = 1; /* Default RCC rate */ - if (rcc_mode (s)) - { - if (tag_len < 4) - goto error; - } - - if (proto_create (&s->rtp, cipher, md) == 0) - { - if (proto_create (&s->rtcp, cipher, md) == 0) - return s; - proto_destroy (&s->rtp); - } - - error: - free (s); - return NULL; -} - - -/** - * Counter Mode encryption/decryption (ctr length = 16 bytes) - * with non-padded (truncated) text - */ -static int -do_ctr_crypt (gcry_cipher_hd_t hd, const void *ctr, uint8_t *data, size_t len) -{ - const size_t ctrlen = 16; - div_t d = div (len, ctrlen); - - if (gcry_cipher_setctr (hd, ctr, ctrlen) - || gcry_cipher_encrypt (hd, data, d.quot * ctrlen, NULL, 0)) - return -1; - - if (d.rem) - { - /* Truncated last block */ - uint8_t dummy[ctrlen]; - data += d.quot * ctrlen; - memcpy (dummy, data, d.rem); - memset (dummy + d.rem, 0, ctrlen - d.rem); - - if (gcry_cipher_encrypt (hd, dummy, ctrlen, data, ctrlen)) - return -1; - memcpy (data, dummy, d.rem); - } - - return 0; -} - - -/** - * AES-CM key derivation (saltlen = 14 bytes) - */ -static int -do_derive (gcry_cipher_hd_t prf, const void *salt, - const uint8_t *r, size_t rlen, uint8_t label, - void *out, size_t outlen) -{ - uint8_t iv[16]; - - memcpy (iv, salt, 14); - iv[14] = iv[15] = 0; - - assert (rlen < 14); - iv[13 - rlen] ^= label; - for (size_t i = 0; i < rlen; i++) - iv[sizeof (iv) - rlen + i] ^= r[i]; - - memset (out, 0, outlen); - return do_ctr_crypt (prf, iv, out, outlen); -} - - -/** - * Sets (or resets) the master key and master salt for a SRTP session. - * This must be done at least once before using srtp_send(), srtp_recv(), - * srtcp_send() or srtcp_recv(). Also, rekeying is required every - * 2^48 RTP packets or 2^31 RTCP packets (whichever comes first), - * otherwise the protocol security might be broken. - * - * @return 0 on success, in case of error: - * EINVAL invalid or unsupported key/salt sizes combination - */ -int -srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, - const void *salt, size_t saltlen) -{ - /* SRTP/SRTCP cipher/salt/MAC keys derivation */ - gcry_cipher_hd_t prf; - uint8_t r[6], keybuf[20]; - - if (saltlen != 14) - return EINVAL; - - if (gcry_cipher_open (&prf, GCRY_CIPHER_AES, GCRY_CIPHER_MODE_CTR, 0) - || gcry_cipher_setkey (prf, key, keylen)) - return EINVAL; - - /* SRTP key derivation */ -#if 0 - if (s->kdr != 0) - { - uint64_t index = (((uint64_t)s->rtp_roc) << 16) | s->rtp_seq; - index /= s->kdr; - - for (int i = sizeof (r) - 1; i >= 0; i--) - { - r[i] = index & 0xff; - index = index >> 8; - } - } - else -#endif - memset (r, 0, sizeof (r)); - if (do_derive (prf, salt, r, 6, SRTP_CRYPT, keybuf, 16) - || gcry_cipher_setkey (s->rtp.cipher, keybuf, 16) - || do_derive (prf, salt, r, 6, SRTP_AUTH, keybuf, 20) - || gcry_md_setkey (s->rtp.mac, keybuf, 20) - || do_derive (prf, salt, r, 6, SRTP_SALT, s->rtp.salt, 14)) - return -1; - - /* SRTCP key derivation */ - memcpy (r, &(uint32_t){ htonl (s->rtcp_index) }, 4); - if (do_derive (prf, salt, r, 4, SRTCP_CRYPT, keybuf, 16) - || gcry_cipher_setkey (s->rtcp.cipher, keybuf, 16) - || do_derive (prf, salt, r, 4, SRTCP_AUTH, keybuf, 20) - || gcry_md_setkey (s->rtcp.mac, keybuf, 20) - || do_derive (prf, salt, r, 4, SRTCP_SALT, s->rtcp.salt, 14)) - return -1; - - (void)gcry_cipher_close (prf); - return 0; -} - -static int hexdigit (char c) -{ - if ((c >= '0') && (c <= '9')) - return c - '0'; - if ((c >= 'A') && (c <= 'F')) - return c - 'A' + 0xA; - if ((c >= 'a') && (c <= 'f')) - return c - 'a' + 0xa; - return -1; -} - -static ssize_t hexstring (const char *in, uint8_t *out, size_t outlen) -{ - size_t inlen = strlen (in); - - if ((inlen > (2 * outlen)) || (inlen & 1)) - return -1; - - for (size_t i = 0; i < inlen; i += 2) - { - int a = hexdigit (in[i]), b = hexdigit (in[i + 1]); - if ((a == -1) || (b == -1)) - return -1; - out[i / 2] = (a << 4) | b; - } - return inlen / 2; -} - -/** - * Sets (or resets) the master key and master salt for a SRTP session - * from hexadecimal strings. See also srtp_setkey(). - * - * @return 0 on success, in case of error: - * EINVAL invalid or unsupported key/salt sizes combination - */ -int -srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt) -{ - uint8_t bkey[16]; /* TODO/NOTE: hard-coded for AES */ - uint8_t bsalt[14]; /* TODO/NOTE: hard-coded for the PRF-AES-CM */ - ssize_t bkeylen = hexstring (key, bkey, sizeof (bkey)); - ssize_t bsaltlen = hexstring (salt, bsalt, sizeof (bsalt)); - - if ((bkeylen == -1) || (bsaltlen == -1)) - return EINVAL; - return srtp_setkey (s, bkey, bkeylen, bsalt, bsaltlen) ? EINVAL : 0; -} - -/** - * Sets Roll-over-Counter Carry (RCC) rate for the SRTP session. If not - * specified (through this function), the default rate of ONE is assumed - * (i.e. every RTP packets will carry the RoC). RCC rate is ignored if none - * of the RCC mode has been selected. - * - * The RCC mode is selected through one of these flags for srtp_create(): - * SRTP_RCC_MODE1: integrity protection only for RoC carrying packets - * SRTP_RCC_MODE2: integrity protection for all packets - * SRTP_RCC_MODE3: no integrity protection - * - * RCC mode 3 is insecure. Compared to plain RTP, it provides confidentiality - * (through encryption) but is much more prone to DoS. It can only be used if - * anti-spoofing protection is provided by lower network layers (e.g. IPsec, - * or trusted routers and proper source address filtering). - * - * If RCC rate is 1, RCC mode 1 and 2 are functionally identical. - * - * @param rate RoC Carry rate (MUST NOT be zero) - */ -void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate) -{ - assert (rate != 0); - s->rtp_rcc = rate; -} - - -/** AES-CM for RTP (salt = 14 bytes + 2 nul bytes) */ -static int -rtp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t roc, uint16_t seq, - const uint32_t *salt, uint8_t *data, size_t len) -{ - /* Determines cryptographic counter (IV) */ - uint32_t counter[4]; - counter[0] = salt[0]; - counter[1] = salt[1] ^ ssrc; - counter[2] = salt[2] ^ htonl (roc); - counter[3] = salt[3] ^ htonl (seq << 16); - - /* Encryption */ - return do_ctr_crypt (hd, counter, data, len); -} - - -/** Determines SRTP Roll-Over-Counter (in host-byte order) */ -static uint32_t -srtp_compute_roc (const srtp_session_t *s, uint16_t seq) -{ - uint32_t roc = s->rtp_roc; - - if (((seq - s->rtp_seq) & 0xffff) < 0x8000) - { - /* Sequence is ahead, good */ - if (seq < s->rtp_seq) - roc++; /* Sequence number wrap */ - } - else - { - /* Sequence is late, bad */ - if (seq > s->rtp_seq) - roc--; /* Wrap back */ - } - return roc; -} - - -/** Returns RTP sequence (in host-byte order) */ -static inline uint16_t rtp_seq (const uint8_t *buf) -{ - return (buf[2] << 8) | buf[3]; -} - - -/** Message Authentication and Integrity for RTP */ -static const uint8_t * -rtp_digest (gcry_md_hd_t md, const uint8_t *data, size_t len, - uint32_t roc) -{ - gcry_md_reset (md); - gcry_md_write (md, data, len); - gcry_md_write (md, &(uint32_t){ htonl (roc) }, 4); - return gcry_md_read (md, 0); -} - - -/** - * Encrypts/decrypts a RTP packet and updates SRTP context - * (CTR block cypher mode of operation has identical encryption and - * decryption function). - * - * @param buf RTP packet to be en-/decrypted - * @param len RTP packet length - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTP packet - * EACCES replayed packet or out-of-window or sync lost - */ -static int srtp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) -{ - assert (s != NULL); - assert (len >= 12u); - - if ((buf[0] >> 6) != 2) - return EINVAL; - - /* Computes encryption offset */ - uint16_t offset = 12; - offset += (buf[0] & 0xf) * 4; // skips CSRC - - if (buf[0] & 0x10) - { - uint16_t extlen; - - offset += 4; - if (len < offset) - return EINVAL; - - memcpy (&extlen, buf + offset - 2, 2); - offset += htons (extlen); // skips RTP extension header - } - - if (len < offset) - return EINVAL; - - /* Determines RTP 48-bits counter and SSRC */ - uint16_t seq = rtp_seq (buf); - uint32_t roc = srtp_compute_roc (s, seq), ssrc; - memcpy (&ssrc, buf + 8, 4); - - /* Updates ROC and sequence (it's safe now) */ - int16_t diff = seq - s->rtp_seq; - if (diff > 0) - { - /* Sequence in the future, good */ - s->rtp.window = s->rtp.window << diff; - s->rtp.window |= UINT64_C(1); - s->rtp_seq = seq, s->rtp_roc = roc; - } - else - { - /* Sequence in the past/present, bad */ - diff = -diff; - if ((diff >= 64) || ((s->rtp.window >> diff) & 1)) - return EACCES; /* Replay attack */ - s->rtp.window |= UINT64_C(1) << diff; - } - - /* Encrypt/Decrypt */ - if (s->flags & SRTP_UNENCRYPTED) - return 0; - - if (rtp_crypt (s->rtp.cipher, ssrc, roc, seq, s->rtp.salt, - buf + offset, len - offset)) - return EINVAL; - - return 0; -} - - -/** - * Turns a RTP packet into a SRTP packet: encrypt it, then computes - * the authentication tag and appends it. - * Note that you can encrypt packet in disorder. - * - * @param buf RTP packet to be encrypted/digested - * @param lenp pointer to the RTP packet length on entry, - * set to the SRTP length on exit (undefined on non-ENOSPC error) - * @param bufsize size (bytes) of the packet buffer - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTP packet or internal error - * ENOSPC bufsize is too small to add authentication tag - * ( will hold the required byte size) - * EACCES packet would trigger a replay error on receiver - */ -int -srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) -{ - size_t len = *lenp; - size_t tag_len; - size_t roc_len = 0; - - /* Compute required buffer size */ - if (len < 12u) - return EINVAL; - - if (!(s->flags & SRTP_UNAUTHENTICATED)) - { - tag_len = s->tag_len; - - if (rcc_mode (s)) - { - assert (tag_len >= 4); - assert (s->rtp_rcc != 0); - if ((rtp_seq (buf) % s->rtp_rcc) == 0) - { - roc_len = 4; - if (rcc_mode (s) == 3) - tag_len = 0; /* RCC mode 3 -> no auth*/ - else - tag_len -= 4; /* RCC mode 1 or 2 -> auth*/ - } - else - { - if (rcc_mode (s) & 1) - tag_len = 0; /* RCC mode 1 or 3 -> no auth */ - } - } - - *lenp = len + roc_len + tag_len; - } - else - tag_len = 0; - - if (bufsize < *lenp) - return ENOSPC; - - /* Encrypt payload */ - int val = srtp_crypt (s, buf, len); - if (val) - return val; - - /* Authenticate payload */ - if (!(s->flags & SRTP_UNAUTHENTICATED)) - { - uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)); - const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, roc); - - if (roc_len) - { - memcpy (buf + len, &(uint32_t){ htonl (s->rtp_roc) }, 4); - len += 4; - } - memcpy (buf + len, tag, tag_len); -#if 0 - printf ("Sent : 0x"); - for (unsigned i = 0; i < tag_len; i++) - printf ("%02x", tag[i]); - puts (""); -#endif - } - - return 0; -} - - -/** - * Turns a SRTP packet into a RTP packet: authenticates the packet, - * then decrypts it. - * - * @param buf RTP packet to be digested/decrypted - * @param lenp pointer to the SRTP packet length on entry, - * set to the RTP length on exit (undefined in case of error) - * - * @return 0 on success, in case of error: - * EINVAL malformatted SRTP packet - * EACCES authentication failed (spoofed packet or out-of-sync) - */ -int -srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) -{ - size_t len = *lenp; - if (len < 12u) - return EINVAL; - - if (!(s->flags & SRTP_UNAUTHENTICATED)) - { - size_t tag_len = s->tag_len, roc_len = 0; - if (rcc_mode (s)) - { - if ((rtp_seq (buf) % s->rtp_rcc) == 0) - { - roc_len = 4; - if (rcc_mode (s) == 3) - tag_len = 0; - else - tag_len -= 4; - } - else - { - if (rcc_mode (s) & 1) - tag_len = 0; // RCC mode 1 or 3: no auth - } - } - - if (len < (12u + roc_len + tag_len)) - return EINVAL; - len -= roc_len + tag_len; - - uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)), rcc; - if (roc_len) - { - assert (roc_len == 4); - memcpy (&rcc, buf + len, 4); - rcc = ntohl (rcc); - } - else - rcc = roc; - - const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, rcc); -#if 0 - printf ("Computed: 0x"); - for (unsigned i = 0; i < tag_len; i++) - printf ("%02x", tag[i]); - printf ("\nReceived: 0x"); - for (unsigned i = 0; i < tag_len; i++) - printf ("%02x", buf[len + roc_len + i]); - puts (""); -#endif - if (memcmp (buf + len + roc_len, tag, tag_len)) - return EACCES; - - if (roc_len) - { - /* Authenticated packet carried a Roll-Over-Counter */ - s->rtp_roc += rcc - roc; - assert (srtp_compute_roc (s, rtp_seq (buf)) == rcc); - } - *lenp = len; - } - - return srtp_crypt (s, buf, len); -} - - -/** AES-CM for RTCP (salt = 14 bytes + 2 nul bytes) */ -static int -rtcp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t index, - const uint32_t *salt, uint8_t *data, size_t len) -{ - return rtp_crypt (hd, ssrc, index >> 16, index & 0xffff, salt, data, len); -} - - -/** Message Authentication and Integrity for RTCP */ -static const uint8_t * -rtcp_digest (gcry_md_hd_t md, const void *data, size_t len) -{ - gcry_md_reset (md); - gcry_md_write (md, data, len); - return gcry_md_read (md, 0); -} - - -/** - * Encrypts/decrypts a RTCP packet and updates SRTCP context - * (CTR block cypher mode of operation has identical encryption and - * decryption function). - * - * @param buf RTCP packet to be en-/decrypted - * @param len RTCP packet length - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTCP packet - */ -static int srtcp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) -{ - assert (s != NULL); - - /* 8-bytes unencrypted header, and 4-bytes unencrypted footer */ - if ((len < 12) || ((buf[0] >> 6) != 2)) - return EINVAL; - - uint32_t index; - memcpy (&index, buf + len, 4); - index = ntohl (index); - if (((index >> 31) != 0) != ((s->flags & SRTCP_UNENCRYPTED) == 0)) - return EINVAL; // E-bit mismatch - - index &= ~(1 << 31); // clear E-bit for counter - - /* Updates SRTCP index (safe here) */ - int32_t diff = index - s->rtcp_index; - if (diff > 0) - { - /* Packet in the future, good */ - s->rtcp.window = s->rtcp.window << diff; - s->rtcp.window |= UINT64_C(1); - s->rtcp_index = index; - } - else - { - /* Packet in the past/present, bad */ - diff = -diff; - if ((diff >= 64) || ((s->rtcp.window >> diff) & 1)) - return EACCES; // replay attack! - s->rtp.window |= UINT64_C(1) << diff; - } - - /* Crypts SRTCP */ - if (s->flags & SRTCP_UNENCRYPTED) - return 0; - - uint32_t ssrc; - memcpy (&ssrc, buf + 4, 4); - - if (rtcp_crypt (s->rtcp.cipher, ssrc, index, s->rtp.salt, - buf + 8, len - 8)) - return EINVAL; - return 0; -} - - -/** - * Turns a RTCP packet into a SRTCP packet: encrypt it, then computes - * the authentication tag and appends it. - * - * @param buf RTCP packet to be encrypted/digested - * @param lenp pointer to the RTCP packet length on entry, - * set to the SRTCP length on exit (undefined in case of error) - * @param bufsize size (bytes) of the packet buffer - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTCP packet or internal error - * ENOSPC bufsize is too small (to add index and authentication tag) - */ -int -srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) -{ - size_t len = *lenp; - if (bufsize < (len + 4 + s->tag_len)) - return ENOSPC; - - uint32_t index = ++s->rtcp_index; - if (index >> 31) - s->rtcp_index = index = 0; /* 31-bit wrap */ - - if ((s->flags & SRTCP_UNENCRYPTED) == 0) - index |= 0x80000000; /* Set Encrypted bit */ - memcpy (buf + len, &(uint32_t){ htonl (index) }, 4); - - int val = srtcp_crypt (s, buf, len); - if (val) - return val; - - len += 4; /* Digests SRTCP index too */ - - const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); - memcpy (buf + len, tag, s->tag_len); - *lenp = len + s->tag_len; - return 0; -} - - -/** - * Turns a SRTCP packet into a RTCP packet: authenticates the packet, - * then decrypts it. - * - * @param buf RTCP packet to be digested/decrypted - * @param lenp pointer to the SRTCP packet length on entry, - * set to the RTCP length on exit (undefined in case of error) - * - * @return 0 on success, in case of error: - * EINVAL malformatted SRTCP packet - * EACCES authentication failed (spoofed packet or out-of-sync) - */ -int -srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) -{ - size_t len = *lenp; - - if (len < (4u + s->tag_len)) - return EINVAL; - len -= s->tag_len; - - const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); - if (memcmp (buf + len, tag, s->tag_len)) - return EACCES; - - len -= 4; /* Remove SRTCP index before decryption */ - *lenp = len; - return srtcp_crypt (s, buf, len); -} \ No newline at end of file diff --git a/plugins/crypto-vala/src/srtp.h b/plugins/crypto-vala/src/srtp.h deleted file mode 100644 index abca6988..00000000 --- a/plugins/crypto-vala/src/srtp.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Secure RTP with libgcrypt - * Copyright (C) 2007 Rémi Denis-Courmont - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation; either version 2.1 - * of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - ****************************************************************************/ - -#ifndef LIBVLC_SRTP_H -# define LIBVLC_SRTP_H 1 -#include - -typedef struct srtp_session_t srtp_session_t; - -enum -{ - SRTP_UNENCRYPTED=0x1, //< do not encrypt SRTP packets - SRTCP_UNENCRYPTED=0x2, //< do not encrypt SRTCP packets - SRTP_UNAUTHENTICATED=0x4, //< authenticate only SRTCP packets - - SRTP_RCC_MODE1=0x10, //< use Roll-over-Counter Carry mode 1 - SRTP_RCC_MODE2=0x20, //< use Roll-over-Counter Carry mode 2 - SRTP_RCC_MODE3=0x30, //< use Roll-over-Counter Carry mode 3 (insecure) - - SRTP_FLAGS_MASK=0x37 //< mask for valid flags -}; - -/** SRTP encryption algorithms (ciphers); same values as MIKEY */ -enum -{ - SRTP_ENCR_NULL=0, //< no encryption - SRTP_ENCR_AES_CM=1, //< AES counter mode - SRTP_ENCR_AES_F8=2, //< AES F8 mode (not implemented) -}; - -/** SRTP authenticaton algorithms; same values as MIKEY */ -enum -{ - SRTP_AUTH_NULL=0, //< no authentication code - SRTP_AUTH_HMAC_SHA1=1, //< HMAC-SHA1 -}; - -/** SRTP pseudo random function; same values as MIKEY */ -enum -{ - SRTP_PRF_AES_CM=0, //< AES counter mode -}; - -# ifdef __cplusplus -extern "C" { -# endif - -srtp_session_t *srtp_create (int encr, int auth, unsigned tag_len, int prf, - unsigned flags); -void srtp_destroy (srtp_session_t *s); - -int srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, - const void *salt, size_t saltlen); -int srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt); - -void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate); - -int srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsize); -int srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); -int srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsiz); -int srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); - -# ifdef __cplusplus -} -# endif -#endif \ No newline at end of file diff --git a/plugins/crypto-vala/src/srtp.vala b/plugins/crypto-vala/src/srtp.vala new file mode 100644 index 00000000..77b5acde --- /dev/null +++ b/plugins/crypto-vala/src/srtp.vala @@ -0,0 +1,122 @@ +using Srtp; + +public class Crypto.Srtp { + public const string AES_CM_128_HMAC_SHA1_80 = "AES_CM_128_HMAC_SHA1_80"; + public const string AES_CM_128_HMAC_SHA1_32 = "AES_CM_128_HMAC_SHA1_32"; + public const string F8_128_HMAC_SHA1_80 = "F8_128_HMAC_SHA1_80"; + + public class Session { + public bool has_encrypt { get; private set; } + public bool has_decrypt { get; private set; } + + private Context encrypt_context; + private Context decrypt_context; + + static construct { + init(); + install_log_handler(log); + } + + private static void log(LogLevel level, string msg) { + print(@"SRTP[$level]: $msg\n"); + } + + public Session() { + Context.create(out encrypt_context, null); + Context.create(out decrypt_context, null); + } + + public uint8[] encrypt_rtp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length + MAX_TRAILER_LEN]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = encrypt_context.protect(buf, ref buf_use); + if (res != ErrorStatus.ok) { + throw new Error.UNKNOWN(@"SRTP encrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = decrypt_context.unprotect(buf, ref buf_use); + switch (res) { + case ErrorStatus.auth_fail: + throw new Error.AUTHENTICATION_FAILED("SRTP packet failed the message authentication check"); + case ErrorStatus.ok: + break; + default: + throw new Error.UNKNOWN(@"SRTP decrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] encrypt_rtcp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length + MAX_TRAILER_LEN + 4]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = encrypt_context.protect_rtcp(buf, ref buf_use); + if (res != ErrorStatus.ok) { + throw new Error.UNKNOWN(@"SRTCP encrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtcp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = decrypt_context.unprotect_rtcp(buf, ref buf_use); + switch (res) { + case ErrorStatus.auth_fail: + throw new Error.AUTHENTICATION_FAILED("SRTCP packet failed the message authentication check"); + case ErrorStatus.ok: + break; + default: + throw new Error.UNKNOWN(@"SRTP decrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + private Policy create_policy(string profile) { + Policy policy = Policy(); + switch (profile) { + case AES_CM_128_HMAC_SHA1_80: + policy.rtp.set_aes_cm_128_hmac_sha1_80(); + policy.rtcp.set_aes_cm_128_hmac_sha1_80(); + break; + } + return policy; + } + + public void set_encryption_key(string profile, uint8[] key, uint8[] salt) { + Policy policy = create_policy(profile); + policy.ssrc.type = SsrcType.any_outbound; + policy.key = new uint8[key.length + salt.length]; + Memory.copy(policy.key, key, key.length); + Memory.copy(((uint8*)policy.key) + key.length, salt, salt.length); + encrypt_context.add_stream(ref policy); + has_encrypt = true; + } + + public void set_decryption_key(string profile, uint8[] key, uint8[] salt) { + Policy policy = create_policy(profile); + policy.ssrc.type = SsrcType.any_inbound; + policy.key = new uint8[key.length + salt.length]; + Memory.copy(policy.key, key, key.length); + Memory.copy(((uint8*)policy.key) + key.length, salt, salt.length); + decrypt_context.add_stream(ref policy); + has_decrypt = true; + } + } +} \ No newline at end of file diff --git a/plugins/crypto-vala/src/srtp.vapi b/plugins/crypto-vala/src/srtp.vapi deleted file mode 100644 index 0fe825c3..00000000 --- a/plugins/crypto-vala/src/srtp.vapi +++ /dev/null @@ -1,107 +0,0 @@ -[CCode (cheader_filename="srtp.h")] -namespace Crypto.Srtp { - -[Compact] -[CCode (cname = "srtp_session_t", free_function = "srtp_destroy")] -public class Session { - [CCode (cname = "srtp_create")] - public Session(Encryption encr, Authentication auth, uint tag_len, Prf prf, Flags flags); - [CCode (cname = "srtp_setkey")] - public int setkey(uint8[] key, uint8[] salt); - [CCode (cname = "srtp_setkeystring")] - public int setkeystring(string key, string salt); - [CCode (cname = "srtp_setrcc_rate")] - public void setrcc_rate(uint16 rate); - - [CCode (cname = "srtp_send")] - private int rtp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); - [CCode (cname = "srtcp_send")] - private int rtcp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); - [CCode (cname = "srtp_recv")] - private int rtp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); - [CCode (cname = "srtcp_recv")] - private int rtcp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); - - public uint8[] encrypt_rtp(uint8[] input, uint tag_len = 10) throws GLib.Error { - uint8[] buf = new uint8[input.length + tag_len]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtp_send(buf, ref buf_use, buf.length); - if (res != 0) { - throw new GLib.Error(-1, res, "RTP encrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } - - public uint8[] encrypt_rtcp(uint8[] input, uint tag_len = 10) throws GLib.Error { - uint8[] buf = new uint8[input.length + tag_len + 4]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtcp_send(buf, ref buf_use, buf.length); - if (res != 0) { - throw new GLib.Error(-1, res, "RTCP encrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } - - public uint8[] decrypt_rtp(uint8[] input) throws GLib.Error { - uint8[] buf = new uint8[input.length]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtp_recv(buf, ref buf_use); - if (res != 0) { - throw new GLib.Error(-1, res, "RTP decrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } - - public uint8[] decrypt_rtcp(uint8[] input) throws GLib.Error { - uint8[] buf = new uint8[input.length]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtcp_recv(buf, ref buf_use); - if (res != 0) { - throw new GLib.Error(-1, res, "RTCP decrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } -} - -[Flags] -[CCode (cname = "unsigned", cprefix = "", has_type_id = false)] -public enum Flags { - SRTP_UNENCRYPTED, - SRTCP_UNENCRYPTED, - SRTP_UNAUTHENTICATED, - SRTP_RCC_MODE1, - SRTP_RCC_MODE2, - SRTP_RCC_MODE3 -} - -[CCode (cname = "int", cprefix = "SRTP_ENCR_", has_type_id = false)] -public enum Encryption { - NULL, - AES_CM, - AES_F8 -} - -[CCode (cname = "int", cprefix = "SRTP_AUTH_", has_type_id = false)] -public enum Authentication { - NULL, - HMAC_SHA1 -} - -[CCode (cname = "int", cprefix = "SRTP_PRF_", has_type_id = false)] -public enum Prf { - AES_CM -} - -} \ No newline at end of file diff --git a/plugins/crypto-vala/vapi/libsrtp2.vapi b/plugins/crypto-vala/vapi/libsrtp2.vapi new file mode 100644 index 00000000..5ceedced --- /dev/null +++ b/plugins/crypto-vala/vapi/libsrtp2.vapi @@ -0,0 +1,115 @@ +[CCode (cheader_filename = "srtp2/srtp.h")] +namespace Srtp { +public const uint MAX_TRAILER_LEN; + +public static ErrorStatus init(); +public static ErrorStatus shutdown(); + +[Compact] +[CCode (cname = "srtp_ctx_t", cprefix = "srtp_", free_function = "srtp_dealloc")] +public class Context { + public static ErrorStatus create(out Context session, Policy? policy); + + public ErrorStatus protect([CCode (type = "void*", array_length = false)] uint8[] rtp, ref int len); + public ErrorStatus unprotect([CCode (type = "void*", array_length = false)] uint8[] rtp, ref int len); + + public ErrorStatus protect_rtcp([CCode (type = "void*", array_length = false)] uint8[] rtcp, ref int len); + public ErrorStatus unprotect_rtcp([CCode (type = "void*", array_length = false)] uint8[] rtcp, ref int len); + + public ErrorStatus add_stream(ref Policy policy); + public ErrorStatus update_stream(ref Policy policy); + public ErrorStatus remove_stream(uint ssrc); + public ErrorStatus update(ref Policy policy); +} + +[CCode (cname = "srtp_ssrc_t")] +public struct Ssrc { + public SsrcType type; + public uint value; +} + +[CCode (cname = "srtp_ssrc_type_t", cprefix = "ssrc_")] +public enum SsrcType { + undefined, specific, any_inbound, any_outbound +} + +[CCode (cname = "srtp_policy_t", destroy_function = "")] +public struct Policy { + public Ssrc ssrc; + public CryptoPolicy rtp; + public CryptoPolicy rtcp; + [CCode (array_length = false)] + public uint8[] key; + public ulong num_master_keys; + public ulong window_size; + public int allow_repeat_tx; + [CCode (array_length_cname = "enc_xtn_hdr_count")] + public int[] enc_xtn_hdr; +} + +[CCode (cname = "srtp_crypto_policy_t")] +public struct CryptoPolicy { + public CipherType cipher_type; + public int cipher_key_len; + public AuthType auth_type; + public int auth_key_len; + public int auth_tag_len; + public SecurityServices sec_serv; + + public void set_aes_cm_128_hmac_sha1_80(); + public void set_aes_cm_128_hmac_sha1_32(); + public void set_aes_cm_128_null_auth(); + public void set_aes_cm_192_hmac_sha1_32(); + public void set_aes_cm_192_hmac_sha1_80(); + public void set_aes_cm_192_null_auth(); + public void set_aes_cm_256_hmac_sha1_32(); + public void set_aes_cm_256_hmac_sha1_80(); + public void set_aes_cm_256_null_auth(); + public void set_aes_gcm_128_16_auth(); + public void set_aes_gcm_128_8_auth(); + public void set_aes_gcm_128_8_only_auth(); + public void set_aes_gcm_256_16_auth(); + public void set_aes_gcm_256_8_auth(); + public void set_aes_gcm_256_8_only_auth(); + public void set_null_cipher_hmac_null(); + public void set_null_cipher_hmac_sha1_80(); + + public void set_rtp_default(); + public void set_rtcp_default(); + + public void set_from_profile_for_rtp(Profile profile); + public void set_from_profile_for_rtcp(Profile profile); +} + +[CCode (cname = "srtp_profile_t", cprefix = "srtp_profile_")] +public enum Profile { + reserved, aes128_cm_sha1_80, aes128_cm_sha1_32, null_sha1_80, null_sha1_32, aead_aes_128_gcm, aead_aes_256_gcm +} + +[CCode (cname = "srtp_cipher_type_id_t")] +public struct CipherType : uint32 {} + +[CCode (cname = "srtp_auth_type_id_t")] +public struct AuthType : uint32 {} + +[CCode (cname = "srtp_sec_serv_t", cprefix = "sec_serv_")] +public enum SecurityServices { + none, conf, auth, conf_and_auth; +} + +[CCode (cname = "srtp_err_status_t", cprefix = "srtp_err_status_", has_type_id = false)] +public enum ErrorStatus { + ok, fail, bad_param, alloc_fail, dealloc_fail, init_fail, terminus, auth_fail, cipher_fail, replay_fail, algo_fail, no_such_op, no_ctx, cant_check, key_expired, socket_err, signal_err, nonce_bad, encode_err, semaphore_err, pfkey_err, bad_mki, pkt_idx_old, pkt_idx_adv +} + +[CCode (cname = "srtp_log_level_t", cprefix = "srtp_log_level_", has_type_id = false)] +public enum LogLevel { + error, warning, info, debug +} + +[CCode (cname = "srtp_log_handler_func_t")] +public delegate void LogHandler(LogLevel level, string msg); + +public static ErrorStatus install_log_handler(LogHandler func); + +} \ No newline at end of file diff --git a/plugins/ice/CMakeLists.txt b/plugins/ice/CMakeLists.txt index 38025aa0..392a202f 100644 --- a/plugins/ice/CMakeLists.txt +++ b/plugins/ice/CMakeLists.txt @@ -20,7 +20,7 @@ CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi - ${CMAKE_BINARY_DIR}/exports/crypto.vapi + ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi PACKAGES ${ICE_PACKAGES} OPTIONS diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index a21c242b..b742ccab 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -12,8 +12,7 @@ public class DtlsSrtp { private uint pull_timeout = uint.MAX; private string peer_fingerprint; - private Crypto.Srtp.Session encrypt_session; - private Crypto.Srtp.Session decrypt_session; + private Crypto.Srtp.Session srtp_session = new Crypto.Srtp.Session(); public static DtlsSrtp setup() throws GLib.Error { var obj = new DtlsSrtp(); @@ -30,9 +29,19 @@ public class DtlsSrtp { } public uint8[] process_incoming_data(uint component_id, uint8[] data) { - if (decrypt_session != null) { - if (component_id == 1) return decrypt_session.decrypt_rtp(data); - if (component_id == 2) return decrypt_session.decrypt_rtcp(data); + if (srtp_session.has_decrypt) { + try { + if (component_id == 1) { + if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { + return srtp_session.decrypt_rtcp(data); + } + return srtp_session.decrypt_rtp(data); + } + if (component_id == 2) return srtp_session.decrypt_rtcp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return null; + } } else if (component_id == 1) { on_data_rec(data); } @@ -40,9 +49,19 @@ public class DtlsSrtp { } public uint8[] process_outgoing_data(uint component_id, uint8[] data) { - if (encrypt_session != null) { - if (component_id == 1) return encrypt_session.encrypt_rtp(data); - if (component_id == 2) return encrypt_session.encrypt_rtcp(data); + if (srtp_session.has_encrypt) { + try { + if (component_id == 1) { + if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { + return srtp_session.encrypt_rtcp(data); + } + return srtp_session.encrypt_rtp(data); + } + if (component_id == 2) return srtp_session.encrypt_rtcp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return null; + } } return null; } @@ -123,19 +142,13 @@ public class DtlsSrtp { warning("SRTP client/server key/salt null"); } - Crypto.Srtp.Session encrypt_session = new Crypto.Srtp.Session(Crypto.Srtp.Encryption.AES_CM, Crypto.Srtp.Authentication.HMAC_SHA1, 10, Crypto.Srtp.Prf.AES_CM, 0); - Crypto.Srtp.Session decrypt_session = new Crypto.Srtp.Session(Crypto.Srtp.Encryption.AES_CM, Crypto.Srtp.Authentication.HMAC_SHA1, 10, Crypto.Srtp.Prf.AES_CM, 0); - if (server) { - encrypt_session.setkey(server_key.extract(), server_salt.extract()); - decrypt_session.setkey(client_key.extract(), client_salt.extract()); + srtp_session.set_encryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, server_key.extract(), server_salt.extract()); + srtp_session.set_decryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, client_key.extract(), client_salt.extract()); } else { - encrypt_session.setkey(client_key.extract(), client_salt.extract()); - decrypt_session.setkey(server_key.extract(), server_salt.extract()); + srtp_session.set_encryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, client_key.extract(), client_salt.extract()); + srtp_session.set_decryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, server_key.extract(), server_salt.extract()); } - - this.encrypt_session = (owned)encrypt_session; - this.decrypt_session = (owned)decrypt_session; } private static ssize_t pull_function(void* transport_ptr, uint8[] buffer) { diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 8ce2a7c6..c6888459 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -19,7 +19,7 @@ SOURCES src/video_widget.vala src/register_plugin.vala CUSTOM_VAPIS - ${CMAKE_BINARY_DIR}/exports/crypto.vapi + ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 77080a09..bedd6f8a 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -53,8 +53,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private Gst.Pad send_rtp_sink_pad; private Gst.Pad send_rtp_src_pad; - private Crypto.Srtp.Session? local_crypto_session; - private Crypto.Srtp.Session? remote_crypto_session; + private Crypto.Srtp.Session? crypto_session = new Crypto.Srtp.Session(); public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { base(content); @@ -148,15 +147,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private void prepare_local_crypto() { - if (local_crypto != null && local_crypto_session == null) { - local_crypto_session = new Crypto.Srtp.Session( - local_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? Crypto.Srtp.Encryption.AES_F8 : Crypto.Srtp.Encryption.AES_CM, - Crypto.Srtp.Authentication.HMAC_SHA1, - local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, - Crypto.Srtp.Prf.AES_CM, - 0 - ); - local_crypto_session.setkey(local_crypto.key, local_crypto.salt); + if (local_crypto != null && !crypto_session.has_encrypt) { + crypto_session.set_encryption_key(local_crypto.crypto_suite, local_crypto.key, local_crypto.salt); debug("Setting up encryption with key params %s", local_crypto.key_params); } } @@ -172,15 +164,19 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { buffer.extract_dup(0, buffer.get_size(), out data); prepare_local_crypto(); if (sink == send_rtp) { - if (local_crypto_session != null) { - data = local_crypto_session.encrypt_rtp(data, local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10); + if (crypto_session.has_encrypt) { + data = crypto_session.encrypt_rtp(data); } on_send_rtp_data(new Bytes.take(data)); } else if (sink == send_rtcp) { - if (local_crypto_session != null) { - data = local_crypto_session.encrypt_rtcp(data, local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10); + if (crypto_session.has_encrypt) { + data = crypto_session.encrypt_rtcp(data); + } + if (rtcp_mux) { + on_send_rtp_data(new Bytes.take(data)); + } else { + on_send_rtcp_data(new Bytes.take(data)); } - on_send_rtcp_data(new Bytes.take(data)); } else { warning("unknown sample"); } @@ -283,25 +279,22 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private void prepare_remote_crypto() { - if (remote_crypto != null && remote_crypto_session == null) { - remote_crypto_session = new Crypto.Srtp.Session( - remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? Crypto.Srtp.Encryption.AES_F8 : Crypto.Srtp.Encryption.AES_CM, - Crypto.Srtp.Authentication.HMAC_SHA1, - remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, - Crypto.Srtp.Prf.AES_CM, - 0 - ); - remote_crypto_session.setkey(remote_crypto.key, remote_crypto.salt); + if (remote_crypto != null && crypto_session.has_decrypt) { + crypto_session.set_decryption_key(remote_crypto.crypto_suite, remote_crypto.key, remote_crypto.salt); debug("Setting up decryption with key params %s", remote_crypto.key_params); } } public override void on_recv_rtp_data(Bytes bytes) { + if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) { + on_recv_rtcp_data(bytes); + return; + } prepare_remote_crypto(); uint8[] data = bytes.get_data(); - if (remote_crypto_session != null) { + if (crypto_session.has_decrypt) { try { - data = remote_crypto_session.decrypt_rtp(data); + data = crypto_session.decrypt_rtp(data); } catch (Error e) { warning("%s (%d)", e.message, e.code); } @@ -314,9 +307,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public override void on_recv_rtcp_data(Bytes bytes) { prepare_remote_crypto(); uint8[] data = bytes.get_data(); - if (remote_crypto_session != null) { + if (crypto_session.has_decrypt) { try { - data = remote_crypto_session.decrypt_rtcp(data); + data = crypto_session.decrypt_rtcp(data); } catch (Error e) { warning("%s (%d)", e.message, e.code); } From c5cb43350af15e99d7304935ccf5fe84c2acdfc9 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Thu, 1 Apr 2021 11:51:12 +0200 Subject: [PATCH 26/56] Remove unnecessary debug code --- plugins/rtp/src/plugin.vala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index 0f3cb10d..62e0d411 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -403,10 +403,6 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { plugin_stream.pause(); } else { plugin_stream.unpause(); - Timeout.add_seconds(3, () => { - dump_dot(); - return false; - }); } } From c5ab4fed87d2dedb5ccbf671c8b2742a11251a0f Mon Sep 17 00:00:00 2001 From: Marvin W Date: Thu, 1 Apr 2021 11:51:35 +0200 Subject: [PATCH 27/56] Fix bug in legacy SRTP decryption --- plugins/rtp/src/stream.vala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index bedd6f8a..efa1b497 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -279,7 +279,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private void prepare_remote_crypto() { - if (remote_crypto != null && crypto_session.has_decrypt) { + if (remote_crypto != null && !crypto_session.has_decrypt) { crypto_session.set_decryption_key(remote_crypto.crypto_suite, remote_crypto.key, remote_crypto.salt); debug("Setting up decryption with key params %s", remote_crypto.key_params); } @@ -339,6 +339,11 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } public void on_ssrc_pad_added(string ssrc, Gst.Pad pad) { + debug("New ssrc %s with pad %s", ssrc, pad.name); + if (participant_ssrc != null && participant_ssrc != ssrc) { + warning("Got second ssrc on stream (old: %s, new: %s), ignoring", participant_ssrc, ssrc); + return; + } participant_ssrc = ssrc; recv_rtp_src_pad = pad; if (decode != null) { From b63a20f9189026a77ccc1769a4ab366d48f63233 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Wed, 31 Mar 2021 21:33:35 +0200 Subject: [PATCH 28/56] Store limited history of contact resources --- libdino/src/service/entity_info.vala | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/libdino/src/service/entity_info.vala b/libdino/src/service/entity_info.vala index 705c728e..b80d4b59 100644 --- a/libdino/src/service/entity_info.vala +++ b/libdino/src/service/entity_info.vala @@ -40,6 +40,9 @@ public class EntityInfo : StreamInteractionModule, Object { entity_caps_hashes[account.bare_jid.domain_jid] = hash; }); stream_interactor.module_manager.initialize_account_modules.connect(initialize_modules); + + remove_old_entities(); + Timeout.add_seconds(60 * 60, () => { remove_old_entities(); return true; }); } public async Gee.Set? get_identities(Account account, Jid jid) { @@ -94,26 +97,30 @@ public class EntityInfo : StreamInteractionModule, Object { } private void on_received_available_presence(Account account, Presence.Stanza presence) { - bool is_gc = stream_interactor.get_module(MucManager.IDENTITY).is_groupchat(presence.from.bare_jid, account); + bool is_gc = stream_interactor.get_module(MucManager.IDENTITY).might_be_groupchat(presence.from.bare_jid, account); if (is_gc) return; string? caps_hash = EntityCapabilities.get_caps_hash(presence); if (caps_hash == null) return; - /* TODO check might_be_groupchat before storing db.entity.upsert() - .value(db.entity.account_id, account.id, true) - .value(db.entity.jid_id, db.get_jid_id(presence.from), true) - .value(db.entity.resource, presence.from.resourcepart, true) - .value(db.entity.last_seen, (long)(new DateTime.now_local()).to_unix()) - .value(db.entity.caps_hash, caps_hash) - .perform();*/ + .value(db.entity.account_id, account.id, true) + .value(db.entity.jid_id, db.get_jid_id(presence.from), true) + .value(db.entity.resource, presence.from.resourcepart, true) + .value(db.entity.last_seen, (long)(new DateTime.now_local()).to_unix()) + .value(db.entity.caps_hash, caps_hash) + .perform(); if (caps_hash != null) { entity_caps_hashes[presence.from] = caps_hash; } } + private void remove_old_entities() { + long timestamp = (long)(new DateTime.now_local().add_days(-14)).to_unix(); + db.entity.delete().with(db.entity.last_seen, "<", timestamp).perform(); + } + private void store_features(string entity, Gee.List features) { if (entity_features.has_key(entity)) return; From 3454201e5a3da058ccbef0bbaf467599912a8c38 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Thu, 1 Apr 2021 12:03:04 +0200 Subject: [PATCH 29/56] Use outgoing JMI if contact has supporting device --- libdino/src/entity/call.vala | 2 + libdino/src/service/calls.vala | 112 +++++++++++++----- .../ui/conversation_titlebar/call_entry.vala | 4 +- plugins/ice/src/dtls_srtp.vala | 8 -- .../module/xep/0166_jingle/jingle_module.vala | 4 +- .../0167_jingle_rtp/jingle_rtp_module.vala | 4 +- .../xep/0353_jingle_message_initiation.vala | 14 ++- 7 files changed, 100 insertions(+), 48 deletions(-) diff --git a/libdino/src/entity/call.vala b/libdino/src/entity/call.vala index b836e2cd..7891dae7 100644 --- a/libdino/src/entity/call.vala +++ b/libdino/src/entity/call.vala @@ -77,6 +77,8 @@ namespace Dino.Entities { .value(db.call.state, State.ENDED); // No point in persisting states that can't survive a restart if (end_time != null) { builder.value(db.call.end_time, (long) end_time.to_unix()); + } else { + builder.value(db.call.end_time, (long) local_time.to_unix()); } id = (int) builder.perform(); diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index 54c353b0..93636c03 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -21,15 +21,16 @@ namespace Dino { public string id { get { return IDENTITY.id; } } private StreamInteractor stream_interactor; + private Database db; private Xep.JingleRtp.SessionInfoType session_info_type; private HashMap> sid_by_call = new HashMap>(Account.hash_func, Account.equals_func); private HashMap> call_by_sid = new HashMap>(Account.hash_func, Account.equals_func); public HashMap sessions = new HashMap(Call.hash_func, Call.equals_func); - public Call? mi_accepted_call = null; - public string? mi_accepted_sid = null; - public bool mi_accepted_video = false; + public HashMap jmi_call = new HashMap(Account.hash_func, Account.equals_func); + public HashMap jmi_sid = new HashMap(Account.hash_func, Account.equals_func); + public HashMap jmi_video = new HashMap(Account.hash_func, Account.equals_func); private HashMap counterpart_sends_video = new HashMap(Call.hash_func, Call.equals_func); private HashMap we_should_send_video = new HashMap(Call.hash_func, Call.equals_func); @@ -46,6 +47,7 @@ namespace Dino { private Calls(StreamInteractor stream_interactor, Database db) { this.stream_interactor = stream_interactor; + this.db = db; stream_interactor.account_added.connect(on_account_added); } @@ -75,29 +77,50 @@ namespace Dino { stream_interactor.get_module(CallStore.IDENTITY).add_call(call, conversation); - XmppStream? stream = stream_interactor.get_stream(conversation.account); - if (stream == null) return null; - - Gee.List call_resources = yield get_call_resources(conversation); - if (call_resources.size > 0) { - Jid full_jid = call_resources[0]; - Xep.Jingle.Session session = yield stream.get_module(Xep.JingleRtp.Module.IDENTITY).start_call(stream, full_jid, video); - sessions[call] = session; - call_by_sid[call.account][session.sid] = call; - sid_by_call[call.account][call] = session.sid; - - connect_session_signals(call, session); - } - we_should_send_video[call] = video; we_should_send_audio[call] = true; + if (yield has_jmi_resources(conversation)) { + XmppStream? stream = stream_interactor.get_stream(conversation.account); + jmi_call[conversation.account] = call; + jmi_video[conversation.account] = video; + jmi_sid[conversation.account] = Xmpp.random_uuid(); + + call_by_sid[call.account][jmi_sid[conversation.account]] = call; + + var descriptions = new ArrayList(); + descriptions.add(new StanzaNode.build("description", "urn:xmpp:jingle:apps:rtp:1").add_self_xmlns().put_attribute("media", "audio")); + if (video) { + descriptions.add(new StanzaNode.build("description", "urn:xmpp:jingle:apps:rtp:1").add_self_xmlns().put_attribute("media", "video")); + } + + stream.get_module(Xmpp.Xep.JingleMessageInitiation.Module.IDENTITY).send_session_propose_to_peer(stream, conversation.counterpart, jmi_sid[call.account], descriptions); + } else { + Gee.List call_resources = yield get_call_resources(conversation); + if (call_resources.size == 0) { + warning("No call resources"); + return null; + } + yield call_resource(conversation.account, call_resources[0], call, video); + } + conversation.last_active = call.time; call_outgoing(call, conversation); return call; } + private async void call_resource(Account account, Jid full_jid, Call call, bool video, string? sid = null) { + XmppStream? stream = stream_interactor.get_stream(account); + if (stream == null) return; + + Xep.Jingle.Session session = yield stream.get_module(Xep.JingleRtp.Module.IDENTITY).start_call(stream, full_jid, video, sid); + sessions[call] = session; + sid_by_call[call.account][call] = session.sid; + + connect_session_signals(call, session); + } + public void end_call(Conversation conversation, Call call) { XmppStream? stream = stream_interactor.get_stream(call.account); if (stream == null) return; @@ -130,15 +153,17 @@ namespace Dino { } } else { // Only a JMI so far - XmppStream stream = stream_interactor.get_stream(call.account); + Account account = call.account; + string sid = sid_by_call[call.account][call]; + XmppStream stream = stream_interactor.get_stream(account); if (stream == null) return; - mi_accepted_call = call; - mi_accepted_sid = sid_by_call[call.account][call]; - mi_accepted_video = we_should_send_video[call]; + jmi_call[account] = call; + jmi_sid[account] = sid; + jmi_video[account] = we_should_send_video[call]; - stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_accept_to_self(stream, mi_accepted_sid); - stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_proceed_to_peer(stream, call.counterpart, mi_accepted_sid); + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_accept_to_self(stream, sid); + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_proceed_to_peer(stream, call.counterpart, sid); } } @@ -211,7 +236,11 @@ namespace Dino { // If video_feed == null && !mute we're trying to mute a non-existant feed. It will be muted as soon as it is created. } - public async Gee.List get_call_resources(Conversation conversation) { + public async bool can_do_calls(Conversation conversation) { + return (yield get_call_resources(conversation)).size > 0 || yield has_jmi_resources(conversation); + } + + private async Gee.List get_call_resources(Conversation conversation) { ArrayList ret = new ArrayList(); XmppStream? stream = stream_interactor.get_stream(conversation.account); @@ -228,6 +257,15 @@ namespace Dino { return ret; } + private async bool has_jmi_resources(Conversation conversation) { + int64 jmi_resources = db.entity.select() + .with(db.entity.jid_id, "=", db.get_jid_id(conversation.counterpart)) + .join_with(db.entity_feature, db.entity.caps_hash, db.entity_feature.entity) + .with(db.entity_feature.feature, "=", Xep.JingleMessageInitiation.NS_URI) + .count(); + return jmi_resources > 0; + } + public bool should_we_send_video(Call call) { return we_should_send_video[call]; } @@ -252,13 +290,14 @@ namespace Dino { } // Session might have already been accepted via Jingle Message Initiation - bool already_accepted = mi_accepted_sid == session.sid && mi_accepted_call.account.equals(account) && - mi_accepted_call.counterpart.equals_bare(session.peer_full_jid) && - mi_accepted_video == counterpart_wants_video; + bool already_accepted = jmi_sid.contains(account) && + jmi_sid[account] == session.sid && jmi_call[account].account.equals(account) && + jmi_call[account].counterpart.equals_bare(session.peer_full_jid) && + jmi_video[account] == counterpart_wants_video; Call? call = null; if (already_accepted) { - call = mi_accepted_call; + call = jmi_call[account]; } else { call = create_received_call(account, session.peer_full_jid, account.full_jid, counterpart_wants_video); } @@ -334,16 +373,15 @@ namespace Dino { } if (call.state == Call.State.IN_PROGRESS) { call.state = Call.State.ENDED; - call_terminated(call, reason_name, reason_text); } else if (call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { if (reason_name == Xep.Jingle.ReasonElement.DECLINE) { call.state = Call.State.DECLINED; } else { call.state = Call.State.FAILED; } - call_terminated(call, reason_name, reason_text); } + call_terminated(call, reason_name, reason_text); remove_call_from_datastructures(call); } @@ -478,11 +516,21 @@ namespace Dino { mi_module.session_accepted.connect((from, sid) => { if (!call_by_sid[account].has_key(sid)) return; - // Ignore session-accepted from ourselves - if (!from.equals(account.full_jid)) { + if (from.equals_bare(account.bare_jid)) { // Carboned message from our account + // Ignore carbon from ourselves + if (from.equals(account.full_jid)) return; + Call call = call_by_sid[account][sid]; call.state = Call.State.OTHER_DEVICE_ACCEPTED; remove_call_from_datastructures(call); + } else if (from.equals_bare(call_by_sid[account][sid].counterpart)) { // Message from our peer + // We proposed the call + if (jmi_sid.has_key(account) && jmi_sid[account] == sid) { + call_resource(account, from, jmi_call[account], jmi_video[account], jmi_sid[account]); + jmi_call.unset(account); + jmi_sid.unset(account); + jmi_video.unset(account); + } } }); mi_module.session_rejected.connect((from, to, sid) => { diff --git a/main/src/ui/conversation_titlebar/call_entry.vala b/main/src/ui/conversation_titlebar/call_entry.vala index 1ac4dd83..5e28ecbe 100644 --- a/main/src/ui/conversation_titlebar/call_entry.vala +++ b/main/src/ui/conversation_titlebar/call_entry.vala @@ -116,9 +116,9 @@ namespace Dino.Ui { private async void update_visibility() { if (conversation.type_ == Conversation.Type.CHAT) { Conversation conv_bak = conversation; - Gee.List? resources = yield stream_interactor.get_module(Calls.IDENTITY).get_call_resources(conversation); + bool can_do_calls = yield stream_interactor.get_module(Calls.IDENTITY).can_do_calls(conversation); if (conv_bak != conversation) return; - visible = resources != null && resources.size > 0; + visible = can_do_calls; } else { visible = false; } diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index b742ccab..f294e66b 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -249,12 +249,4 @@ public class DtlsSrtp { } return sb.str; } - - private uint8[] uint8_pt_to_a(uint8* data, uint size) { - uint8[size] ret = new uint8[size]; - for (int i = 0; i < size; i++) { - ret[i] = data[i]; - } - return ret; - } } \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala b/xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala index 1e8a36d1..7314ca6c 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala @@ -102,7 +102,7 @@ namespace Xmpp.Xep.Jingle { return (yield is_jingle_available(stream, full_jid)) && (yield select_transport(stream, type, components, full_jid, Set.empty())) != null; } - public async Session create_session(XmppStream stream, Gee.List contents, Jid receiver_full_jid, string sid = random_uuid()) throws Error { + public async Session create_session(XmppStream stream, Gee.List contents, Jid receiver_full_jid, string? sid = null) throws Error { if (!yield is_jingle_available(stream, receiver_full_jid)) { throw new Error.NO_SHARED_PROTOCOLS("No Jingle support"); } @@ -111,7 +111,7 @@ namespace Xmpp.Xep.Jingle { throw new Error.GENERAL("Couldn't determine own JID"); } - Session session = new Session.initiate_sent(stream, sid, my_jid, receiver_full_jid); + Session session = new Session.initiate_sent(stream, sid ?? random_uuid(), my_jid, receiver_full_jid); session.terminated.connect((session, stream, _1, _2, _3) => { stream.get_flag(Flag.IDENTITY).remove_session(session.sid); }); foreach (Content content in contents) { diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala index 3a9ea09f..3adad114 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala @@ -26,7 +26,7 @@ public abstract class Module : XmppStreamModule { public abstract Stream create_stream(Jingle.Content content); public abstract void close_stream(Stream stream); - public async Jingle.Session start_call(XmppStream stream, Jid receiver_full_jid, bool video) throws Jingle.Error { + public async Jingle.Session start_call(XmppStream stream, Jid receiver_full_jid, bool video, string? sid = null) throws Jingle.Error { Jingle.Module jingle_module = stream.get_module(Jingle.Module.IDENTITY); @@ -72,7 +72,7 @@ public abstract class Module : XmppStreamModule { // Create session try { - Jingle.Session session = yield jingle_module.create_session(stream, contents, receiver_full_jid); + Jingle.Session session = yield jingle_module.create_session(stream, contents, receiver_full_jid, sid); return session; } catch (Jingle.Error e) { throw new Jingle.Error.GENERAL(@"Couldn't create Jingle session: $(e.message)"); diff --git a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala index acb2ba2e..dbb6fd81 100644 --- a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala +++ b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala @@ -1,7 +1,7 @@ using Gee; namespace Xmpp.Xep.JingleMessageInitiation { - private const string NS_URI = "urn:xmpp:jingle-message:0"; + public const string NS_URI = "urn:xmpp:jingle-message:0"; public class Module : XmppStreamModule { public static ModuleIdentity IDENTITY = new ModuleIdentity(NS_URI, "0353_jingle_message_initiation"); @@ -11,6 +11,17 @@ namespace Xmpp.Xep.JingleMessageInitiation { public signal void session_accepted(Jid from, string sid); public signal void session_rejected(Jid from, Jid to, string sid); + public void send_session_propose_to_peer(XmppStream stream, Jid to, string sid, Gee.List descriptions) { + StanzaNode propose_node = new StanzaNode.build("propose", NS_URI).add_self_xmlns().put_attribute("id", sid, NS_URI); + foreach (StanzaNode desc_node in descriptions) { + propose_node.put_node(desc_node); + } + + MessageStanza accepted_message = new MessageStanza() { to=to }; + accepted_message.stanza.put_node(propose_node); + stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); + } + public void send_session_accept_to_self(XmppStream stream, string sid) { MessageStanza accepted_message = new MessageStanza() { to=Bind.Flag.get_my_jid(stream).bare_jid }; accepted_message.stanza.put_node( @@ -58,7 +69,6 @@ namespace Xmpp.Xep.JingleMessageInitiation { switch (mi_node.name) { case "accept": case "proceed": - if (!message.from.equals_bare(Bind.Flag.get_my_jid(stream))) return; session_accepted(message.from, mi_node.get_attribute("id")); break; case "propose": From 8d1c6c29be7018c74ec3f8ea05f5849eac5b4069 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Thu, 8 Apr 2021 12:07:04 +0200 Subject: [PATCH 30/56] Display+store call encryption info --- libdino/src/entity/call.vala | 5 ++ libdino/src/entity/encryption.vala | 4 +- libdino/src/service/calls.vala | 44 +++++++++- libdino/src/service/content_item_store.vala | 4 +- libdino/src/service/database.vala | 5 +- main/data/theme.css | 25 +++++- main/src/ui/call_window/call_bottom_bar.vala | 53 ++++++++++-- .../call_window/call_window_controller.vala | 4 + .../content_populator.vala | 1 + .../conversation_item_skeleton.vala | 80 +++++++++++-------- plugins/ice/src/dtls_srtp.vala | 46 ++++++++--- plugins/ice/src/transport_parameters.vala | 12 ++- .../src/module/xep/0166_jingle/content.vala | 9 +++ .../0167_jingle_rtp/content_parameters.vala | 3 + .../module/xep/0167_jingle_rtp/stream.vala | 2 + .../jingle_ice_udp_module.vala | 4 +- .../transport_parameters.vala | 47 ++++++++--- 17 files changed, 273 insertions(+), 75 deletions(-) diff --git a/libdino/src/entity/call.vala b/libdino/src/entity/call.vala index 7891dae7..577b3ab8 100644 --- a/libdino/src/entity/call.vala +++ b/libdino/src/entity/call.vala @@ -32,6 +32,7 @@ namespace Dino.Entities { public DateTime time { get; set; } public DateTime local_time { get; set; } public DateTime end_time { get; set; } + public Encryption encryption { get; set; default=Encryption.NONE; } public State state { get; set; } @@ -57,6 +58,7 @@ namespace Dino.Entities { time = new DateTime.from_unix_utc(row[db.call.time]); local_time = new DateTime.from_unix_utc(row[db.call.local_time]); end_time = new DateTime.from_unix_utc(row[db.call.end_time]); + encryption = (Encryption) row[db.call.encryption]; state = (State) row[db.call.state]; notify.connect(on_update); @@ -74,6 +76,7 @@ namespace Dino.Entities { .value(db.call.direction, direction) .value(db.call.time, (long) time.to_unix()) .value(db.call.local_time, (long) local_time.to_unix()) + .value(db.call.encryption, encryption) .value(db.call.state, State.ENDED); // No point in persisting states that can't survive a restart if (end_time != null) { builder.value(db.call.end_time, (long) end_time.to_unix()); @@ -116,6 +119,8 @@ namespace Dino.Entities { update_builder.set(db.call.local_time, (long) local_time.to_unix()); break; case "end-time": update_builder.set(db.call.end_time, (long) end_time.to_unix()); break; + case "encryption": + update_builder.set(db.call.encryption, encryption); break; case "state": // No point in persisting states that can't survive a restart if (state == State.RINGING || state == State.ESTABLISHING || state == State.IN_PROGRESS) return; diff --git a/libdino/src/entity/encryption.vala b/libdino/src/entity/encryption.vala index b50556f9..25d55eb1 100644 --- a/libdino/src/entity/encryption.vala +++ b/libdino/src/entity/encryption.vala @@ -3,7 +3,9 @@ namespace Dino.Entities { public enum Encryption { NONE, PGP, - OMEMO + OMEMO, + DTLS_SRTP, + SRTP, } } \ No newline at end of file diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index 93636c03..b457c764 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -14,6 +14,7 @@ namespace Dino { public signal void counterpart_ringing(Call call); public signal void counterpart_sends_video_updated(Call call, bool mute); public signal void info_received(Call call, Xep.JingleRtp.CallSessionInfo session_info); + public signal void encryption_updated(Call call, Xep.Jingle.ContentEncryption? encryption); public signal void stream_created(Call call, string media); @@ -22,7 +23,6 @@ namespace Dino { private StreamInteractor stream_interactor; private Database db; - private Xep.JingleRtp.SessionInfoType session_info_type; private HashMap> sid_by_call = new HashMap>(Account.hash_func, Account.equals_func); private HashMap> call_by_sid = new HashMap>(Account.hash_func, Account.equals_func); @@ -38,7 +38,10 @@ namespace Dino { private HashMap audio_content_parameter = new HashMap(Call.hash_func, Call.equals_func); private HashMap video_content_parameter = new HashMap(Call.hash_func, Call.equals_func); + private HashMap audio_content = new HashMap(Call.hash_func, Call.equals_func); private HashMap video_content = new HashMap(Call.hash_func, Call.equals_func); + private HashMap video_encryption = new HashMap(Call.hash_func, Call.equals_func); + private HashMap audio_encryption = new HashMap(Call.hash_func, Call.equals_func); public static void start(StreamInteractor stream_interactor, Database db) { Calls m = new Calls(stream_interactor, db); @@ -290,7 +293,7 @@ namespace Dino { } // Session might have already been accepted via Jingle Message Initiation - bool already_accepted = jmi_sid.contains(account) && + bool already_accepted = jmi_sid.has_key(account) && jmi_sid[account] == session.sid && jmi_call[account].account.equals(account) && jmi_call[account].counterpart.equals_bare(session.peer_full_jid) && jmi_video[account] == counterpart_wants_video; @@ -365,6 +368,7 @@ namespace Dino { if (call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { call.state = Call.State.IN_PROGRESS; } + update_call_encryption(call); } private void on_call_terminated(Call call, bool we_terminated, string? reason_name, string? reason_text) { @@ -429,6 +433,7 @@ namespace Dino { private void connect_content_signals(Call call, Xep.Jingle.Content content, Xep.JingleRtp.Parameters rtp_content_parameter) { if (rtp_content_parameter.media == "audio") { + audio_content[call] = content; audio_content_parameter[call] = rtp_content_parameter; } else if (rtp_content_parameter.media == "video") { video_content[call] = content; @@ -450,6 +455,36 @@ namespace Dino { on_counterpart_mute_update(call, false, "video"); } }); + + content.notify["encryption"].connect((obj, _) => { + if (rtp_content_parameter.media == "audio") { + audio_encryption[call] = ((Xep.Jingle.Content) obj).encryption; + } else if (rtp_content_parameter.media == "video") { + video_encryption[call] = ((Xep.Jingle.Content) obj).encryption; + } + }); + } + + private void update_call_encryption(Call call) { + if (audio_encryption[call] == null) { + call.encryption = Encryption.NONE; + encryption_updated(call, null); + return; + } + + bool consistent_encryption = video_encryption[call] != null && audio_encryption[call].encryption_ns == video_encryption[call].encryption_ns; + + if (video_content[call] == null || consistent_encryption) { + if (audio_encryption[call].encryption_ns == Xep.JingleIceUdp.DTLS_NS_URI) { + call.encryption = Encryption.DTLS_SRTP; + } else if (audio_encryption[call].encryption_name == "SRTP") { + call.encryption = Encryption.SRTP; + } + encryption_updated(call, audio_encryption[call]); + } else { + call.encryption = Encryption.NONE; + encryption_updated(call, null); + } } private void remove_call_from_datastructures(Call call) { @@ -465,7 +500,10 @@ namespace Dino { audio_content_parameter.unset(call); video_content_parameter.unset(call); + audio_content.unset(call); video_content.unset(call); + audio_encryption.unset(call); + video_encryption.unset(call); } private void on_account_added(Account account) { @@ -526,7 +564,7 @@ namespace Dino { } else if (from.equals_bare(call_by_sid[account][sid].counterpart)) { // Message from our peer // We proposed the call if (jmi_sid.has_key(account) && jmi_sid[account] == sid) { - call_resource(account, from, jmi_call[account], jmi_video[account], jmi_sid[account]); + call_resource.begin(account, from, jmi_call[account], jmi_video[account], jmi_sid[account]); jmi_call.unset(account); jmi_sid.unset(account); jmi_video.unset(account); diff --git a/libdino/src/service/content_item_store.vala b/libdino/src/service/content_item_store.vala index cde8dd10..6ab0529c 100644 --- a/libdino/src/service/content_item_store.vala +++ b/libdino/src/service/content_item_store.vala @@ -316,10 +316,12 @@ public class CallItem : ContentItem { public Conversation conversation; public CallItem(Call call, Conversation conversation, int id) { - base(id, TYPE, call.from, call.time, Encryption.NONE, Message.Marked.NONE); + base(id, TYPE, call.from, call.time, call.encryption, Message.Marked.NONE); this.call = call; this.conversation = conversation; + + call.bind_property("encryption", this, "encryption"); } } diff --git a/libdino/src/service/database.vala b/libdino/src/service/database.vala index 98e18d16..9703260a 100644 --- a/libdino/src/service/database.vala +++ b/libdino/src/service/database.vala @@ -7,7 +7,7 @@ using Dino.Entities; namespace Dino { public class Database : Qlite.Database { - private const int VERSION = 20; + private const int VERSION = 21; public class AccountTable : Table { public Column id = new Column.Integer("id") { primary_key = true, auto_increment = true }; @@ -165,11 +165,12 @@ public class Database : Qlite.Database { public Column time = new Column.Long("time") { not_null = true }; public Column local_time = new Column.Long("local_time") { not_null = true }; public Column end_time = new Column.Long("end_time"); + public Column encryption = new Column.Integer("encryption") { min_version=21 }; public Column state = new Column.Integer("state"); internal CallTable(Database db) { base(db, "call"); - init({id, account_id, counterpart_id, counterpart_resource, our_resource, direction, time, local_time, end_time, state}); + init({id, account_id, counterpart_id, counterpart_resource, our_resource, direction, time, local_time, end_time, encryption, state}); } } diff --git a/main/data/theme.css b/main/data/theme.css index 423cbf68..454bd2c1 100644 --- a/main/data/theme.css +++ b/main/data/theme.css @@ -235,17 +235,24 @@ box.dino-input-error label.input-status-highlight-once { outline: 0; border-radius: 1000px; } + .dino-call-window button.white-button { color: #1d1c1d; - background: rgba(255,255,255,0.9); + background: rgba(255,255,255,0.85); border: lightgrey; } +.dino-call-window button.white-button:hover { + background: rgba(255,255,255,1); +} .dino-call-window button.transparent-white-button { color: white; background: rgba(255,255,255,0.15); border: none; } +.dino-call-window button.transparent-white-button:hover { + background: rgba(255,255,255,0.25); +} .dino-call-window button.call-mediadevice-settings-button { border-radius: 1000px; @@ -265,11 +272,21 @@ box.dino-input-error label.input-status-highlight-once { margin: 0; } -.dino-call-window .unencrypted-box { - color: @error_color; - padding: 10px; +.dino-call-window .encryption-box { + color: rgba(255,255,255,0.7); border-radius: 5px; background: rgba(0,0,0,0.5); + padding: 0px; + border: none; + box-shadow: none; +} + +.dino-call-window .encryption-box.unencrypted { + color: @error_color; +} + +.dino-call-window .encryption-box:hover { + background: rgba(20,20,20,0.5); } .dino-call-window .call-header-bar { diff --git a/main/src/ui/call_window/call_bottom_bar.vala b/main/src/ui/call_window/call_bottom_bar.vala index bc800485..c6375ea2 100644 --- a/main/src/ui/call_window/call_bottom_bar.vala +++ b/main/src/ui/call_window/call_bottom_bar.vala @@ -1,5 +1,6 @@ using Dino.Entities; using Gtk; +using Pango; public class Dino.Ui.CallBottomBar : Gtk.Box { @@ -24,6 +25,10 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { private MenuButton video_settings_button = new MenuButton() { halign=Align.END, valign=Align.END }; public VideoSettingsPopover? video_settings_popover; + private EventBox encryption_event_box = new EventBox() { visible=true }; + private MenuButton encryption_button = new MenuButton() { relief=ReliefStyle.NONE, height_request=30, width_request=30, margin_start=20, margin_bottom=25, halign=Align.START, valign=Align.END }; + private Image encryption_image = new Image.from_icon_name("changes-allow-symbolic", IconSize.BUTTON) { visible=true }; + private Label label = new Label("") { margin=20, halign=Align.CENTER, valign=Align.CENTER, wrap=true, wrap_mode=Pango.WrapMode.WORD_CHAR, hexpand=true, visible=true }; private Stack stack = new Stack() { visible=true }; @@ -31,11 +36,9 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { Object(orientation:Orientation.HORIZONTAL, spacing:0); Overlay default_control = new Overlay() { visible=true }; - Image encryption_image = new Image.from_icon_name("changes-allow-symbolic", IconSize.BUTTON) { margin_start=20, margin_bottom=25, halign=Align.START, valign=Align.END, visible=true }; - encryption_image.tooltip_text = _("Unencrypted"); - encryption_image.get_style_context().add_class("unencrypted-box"); - - default_control.add_overlay(encryption_image); + encryption_button.add(encryption_image); + encryption_button.get_style_context().add_class("encryption-box"); + default_control.add_overlay(encryption_button); Box main_buttons = new Box(Orientation.HORIZONTAL, 20) { margin_start=40, margin_end=40, margin=20, halign=Align.CENTER, hexpand=true, visible=true }; @@ -87,6 +90,33 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { this.get_style_context().add_class("call-bottom-bar"); } + public void set_encryption(Xmpp.Xep.Jingle.ContentEncryption? encryption) { + encryption_button.visible = true; + + Popover popover = new Popover(encryption_button); + + if (encryption == null) { + encryption_image.set_from_icon_name("changes-allow-symbolic", IconSize.BUTTON); + encryption_button.get_style_context().add_class("unencrypted"); + + popover.add(new Label("This call isn't encrypted.") { margin=10, visible=true } ); + } else { + encryption_image.set_from_icon_name("changes-prevent-symbolic", IconSize.BUTTON); + encryption_button.get_style_context().remove_class("unencrypted"); + + Grid encryption_info_grid = new Grid() { margin=10, row_spacing=3, column_spacing=5, visible=true }; + encryption_info_grid.attach(new Label("This call is end-to-end encrypted.") { use_markup=true, xalign=0, visible=true }, 1, 1, 2, 1); + encryption_info_grid.attach(new Label("Peer key") { xalign=0, visible=true }, 1, 2, 1, 1); + encryption_info_grid.attach(new Label("Your key") { xalign=0, visible=true }, 1, 3, 1, 1); + encryption_info_grid.attach(new Label("" + format_fingerprint(encryption.peer_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 2, 1, 1); + encryption_info_grid.attach(new Label("" + format_fingerprint(encryption.our_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 3, 1, 1); + + popover.add(encryption_info_grid); + } + + encryption_button.set_popover(popover); + } + public AudioSettingsPopover? show_audio_device_choices(bool show) { audio_settings_button.visible = show; if (audio_settings_popover != null) audio_settings_popover.visible = false; @@ -160,6 +190,17 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { } public bool is_menu_active() { - return video_settings_button.active || audio_settings_button.active; + return video_settings_button.active || audio_settings_button.active || encryption_button.active; + } + + private string format_fingerprint(uint8[] fingerprint) { + var sb = new StringBuilder(); + for (int i = 0; i < fingerprint.length; i++) { + sb.append("%02x".printf(fingerprint[i])); + if (i < fingerprint.length - 1) { + sb.append(":"); + } + } + return sb.str; } } \ No newline at end of file diff --git a/main/src/ui/call_window/call_window_controller.vala b/main/src/ui/call_window/call_window_controller.vala index 09c8f88c..f66a37e1 100644 --- a/main/src/ui/call_window/call_window_controller.vala +++ b/main/src/ui/call_window/call_window_controller.vala @@ -67,6 +67,10 @@ public class Dino.Ui.CallWindowController : Object { call_window.set_status("ringing"); } }); + calls.encryption_updated.connect((call, encryption) => { + if (!this.call.equals(call)) return; + call_window.bottom_bar.set_encryption(encryption); + }); own_video.resolution_changed.connect((width, height) => { if (width == 0 || height == 0) return; diff --git a/main/src/ui/conversation_content_view/content_populator.vala b/main/src/ui/conversation_content_view/content_populator.vala index d7ce9ce5..ef859bde 100644 --- a/main/src/ui/conversation_content_view/content_populator.vala +++ b/main/src/ui/conversation_content_view/content_populator.vala @@ -88,6 +88,7 @@ public abstract class ContentMetaItem : Plugins.MetaConversationItem { this.mark = content_item.mark; content_item.bind_property("mark", this, "mark"); + content_item.bind_property("encryption", this, "encryption"); this.can_merge = true; this.requires_avatar = true; diff --git a/main/src/ui/conversation_content_view/conversation_item_skeleton.vala b/main/src/ui/conversation_content_view/conversation_item_skeleton.vala index c0099bf4..bcb6864e 100644 --- a/main/src/ui/conversation_content_view/conversation_item_skeleton.vala +++ b/main/src/ui/conversation_content_view/conversation_item_skeleton.vala @@ -104,7 +104,7 @@ public class ItemMetaDataHeader : Box { [GtkChild] public Label dot_label; [GtkChild] public Label time_label; public Image received_image = new Image() { opacity=0.4 }; - public Image? unencrypted_image = null; + public Widget? encryption_image = null; public static IconSize ICON_SIZE_HEADER = Gtk.icon_size_register("im.dino.Dino.HEADER_ICON", 17, 12); @@ -124,27 +124,9 @@ public class ItemMetaDataHeader : Box { update_name_label(); name_label.style_updated.connect(update_name_label); - Application app = GLib.Application.get_default() as Application; - - ContentMetaItem ci = item as ContentMetaItem; - if (ci != null) { - foreach(var e in app.plugin_registry.encryption_list_entries) { - if (e.encryption == item.encryption) { - Object? w = e.get_encryption_icon(conversation, ci.content_item); - if (w != null) { - this.add(w as Widget); - } else { - Image image = new Image.from_icon_name("dino-changes-prevent-symbolic", ICON_SIZE_HEADER) { opacity=0.4, visible = true }; - this.add(image); - } - break; - } - } - } - if (item.encryption == Encryption.NONE) { - conversation.notify["encryption"].connect(update_unencrypted_icon); - update_unencrypted_icon(); - } + conversation.notify["encryption"].connect(update_unencrypted_icon); + item.notify["encryption"].connect(update_encryption_icon); + update_encryption_icon(); this.add(received_image); @@ -157,17 +139,51 @@ public class ItemMetaDataHeader : Box { update_received_mark(); } + private void update_encryption_icon() { + Application app = GLib.Application.get_default() as Application; + + ContentMetaItem ci = item as ContentMetaItem; + if (item.encryption != Encryption.NONE && ci != null) { + Widget? widget = null; + foreach(var e in app.plugin_registry.encryption_list_entries) { + if (e.encryption == item.encryption) { + widget = e.get_encryption_icon(conversation, ci.content_item) as Widget; + break; + } + } + if (widget == null) { + widget = new Image.from_icon_name("dino-changes-prevent-symbolic", ICON_SIZE_HEADER) { opacity=0.4, visible = true }; + } + update_encryption_image(widget); + } + if (item.encryption == Encryption.NONE) { + update_unencrypted_icon(); + } + } + private void update_unencrypted_icon() { - if (conversation.encryption != Encryption.NONE && unencrypted_image == null) { - unencrypted_image = new Image() { opacity=0.4, visible = true }; - unencrypted_image.set_from_icon_name("dino-changes-allowed-symbolic", ICON_SIZE_HEADER); - unencrypted_image.tooltip_text = _("Unencrypted"); - this.add(unencrypted_image); - this.reorder_child(unencrypted_image, 3); - Util.force_error_color(unencrypted_image); - } else if (conversation.encryption == Encryption.NONE && unencrypted_image != null) { - this.remove(unencrypted_image); - unencrypted_image = null; + if (item.encryption != Encryption.NONE) return; + + if (conversation.encryption != Encryption.NONE && encryption_image == null) { + Image image = new Image() { opacity=0.4, visible = true }; + image.set_from_icon_name("dino-changes-allowed-symbolic", ICON_SIZE_HEADER); + image.tooltip_text = _("Unencrypted"); + update_encryption_image(image); + Util.force_error_color(image); + } else if (conversation.encryption == Encryption.NONE && encryption_image != null) { + update_encryption_image(null); + } + } + + private void update_encryption_image(Widget? widget) { + if (encryption_image != null) { + this.remove(encryption_image); + encryption_image = null; + } + if (widget != null) { + this.add(widget); + this.reorder_child(widget, 3); + encryption_image = widget; } } diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index f294e66b..e2470cf6 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -10,7 +10,10 @@ public class DtlsSrtp { private Mutex buffer_mutex = new Mutex(); private Gee.LinkedList buffer_queue = new Gee.LinkedList(); private uint pull_timeout = uint.MAX; - private string peer_fingerprint; + + private DigestAlgorithm? peer_fp_algo = null; + private uint8[] peer_fingerprint = null; + private uint8[] own_fingerprint; private Crypto.Srtp.Session srtp_session = new Crypto.Srtp.Session(); @@ -20,12 +23,13 @@ public class DtlsSrtp { return obj; } - internal string get_own_fingerprint(DigestAlgorithm digest_algo) { - return format_certificate(own_cert[0], digest_algo); + internal uint8[] get_own_fingerprint(DigestAlgorithm digest_algo) { + return own_fingerprint; } - public void set_peer_fingerprint(string fingerprint) { + public void set_peer_fingerprint(uint8[] fingerprint, DigestAlgorithm digest_algo) { this.peer_fingerprint = fingerprint; + this.peer_fp_algo = digest_algo; } public uint8[] process_incoming_data(uint component_id, uint8[] data) { @@ -94,10 +98,11 @@ public class DtlsSrtp { cert.sign(cert, private_key); + own_fingerprint = get_fingerprint(cert, DigestAlgorithm.SHA256); own_cert = new X509.Certificate[] { (owned)cert }; } - public async void setup_dtls_connection(bool server) { + public async Xmpp.Xep.Jingle.ContentEncryption setup_dtls_connection(bool server) { InitFlags server_or_client = server ? InitFlags.SERVER : InitFlags.CLIENT; debug("Setting up DTLS connection. We're %s", server_or_client.to_string()); @@ -149,6 +154,7 @@ public class DtlsSrtp { srtp_session.set_encryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, client_key.extract(), client_salt.extract()); srtp_session.set_decryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, server_key.extract(), server_salt.extract()); } + return new Xmpp.Xep.Jingle.ContentEncryption() { encryption_ns=Xmpp.Xep.JingleIceUdp.DTLS_NS_URI, encryption_name = "DTLS-SRTP", our_key=own_fingerprint, peer_key=peer_fingerprint }; } private static ssize_t pull_function(void* transport_ptr, uint8[] buffer) { @@ -226,24 +232,40 @@ public class DtlsSrtp { X509.Certificate peer_cert = X509.Certificate.create(); peer_cert.import(ref cert_datums[0], CertificateFormat.DER); - string peer_fp_str = format_certificate(peer_cert, DigestAlgorithm.SHA256); - if (peer_fp_str.down() != this.peer_fingerprint.down()) { - warning("First cert in peer cert list doesn't equal advertised one %s vs %s", peer_fp_str, this.peer_fingerprint); + uint8[] real_peer_fp = get_fingerprint(peer_cert, peer_fp_algo); + + if (real_peer_fp.length != this.peer_fingerprint.length) { + warning("Fingerprint lengths not equal %i vs %i", real_peer_fp.length, peer_fingerprint.length); return false; } + for (int i = 0; i < real_peer_fp.length; i++) { + if (real_peer_fp[i] != this.peer_fingerprint[i]) { + warning("First cert in peer cert list doesn't equal advertised one: %s vs %s", format_fingerprint(real_peer_fp), format_fingerprint(peer_fingerprint)); + return false; + } + } + return true; } - private string format_certificate(X509.Certificate certificate, DigestAlgorithm digest_algo) { + private uint8[] get_fingerprint(X509.Certificate certificate, DigestAlgorithm digest_algo) { uint8[] buf = new uint8[512]; size_t buf_out_size = 512; certificate.get_fingerprint(digest_algo, buf, ref buf_out_size); - var sb = new StringBuilder(); + uint8[] ret = new uint8[buf_out_size]; for (int i = 0; i < buf_out_size; i++) { - sb.append("%02x".printf(buf[i])); - if (i < buf_out_size - 1) { + ret[i] = buf[i]; + } + return ret; + } + + private string format_fingerprint(uint8[] fingerprint) { + var sb = new StringBuilder(); + for (int i = 0; i < fingerprint.length; i++) { + sb.append("%02x".printf(fingerprint[i])); + if (i < fingerprint.length - 1) { sb.append(":"); } } diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index 2db1ab1b..f95be261 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -68,9 +68,11 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport dtls_srtp = setup_dtls(this); this.own_fingerprint = dtls_srtp.get_own_fingerprint(GnuTLS.DigestAlgorithm.SHA256); if (incoming) { - dtls_srtp.set_peer_fingerprint(this.peer_fingerprint); + dtls_srtp.set_peer_fingerprint(this.peer_fingerprint, this.peer_fp_algo == "sha-256" ? GnuTLS.DigestAlgorithm.SHA256 : GnuTLS.DigestAlgorithm.NULL); } else { - dtls_srtp.setup_dtls_connection(true); + dtls_srtp.setup_dtls_connection.begin(true, (_, res) => { + this.content.encryption = dtls_srtp.setup_dtls_connection.end(res); + }); } } @@ -143,7 +145,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport base.handle_transport_accept(transport); if (dtls_srtp != null && peer_fingerprint != null) { - dtls_srtp.set_peer_fingerprint(this.peer_fingerprint); + dtls_srtp.set_peer_fingerprint(this.peer_fingerprint, this.peer_fp_algo == "sha-256" ? GnuTLS.DigestAlgorithm.SHA256 : GnuTLS.DigestAlgorithm.NULL); } else { dtls_srtp = null; } @@ -205,7 +207,9 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport if (incoming && dtls_srtp != null) { Jingle.DatagramConnection rtp_datagram = (Jingle.DatagramConnection) content.get_transport_connection(1); rtp_datagram.notify["ready"].connect(() => { - dtls_srtp.setup_dtls_connection(false); + dtls_srtp.setup_dtls_connection.begin(false, (_, res) => { + this.content.encryption = dtls_srtp.setup_dtls_connection.end(res); + }); }); } base.create_transport_connection(stream, content); diff --git a/xmpp-vala/src/module/xep/0166_jingle/content.vala b/xmpp-vala/src/module/xep/0166_jingle/content.vala index 67c13dd8..bce03a7b 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/content.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/content.vala @@ -34,6 +34,8 @@ public class Xmpp.Xep.Jingle.Content : Object { public weak Session session; public Map component_connections = new HashMap(); // TODO private + public ContentEncryption? encryption { get; set; } + // INITIATE_SENT | INITIATE_RECEIVED | CONNECTING public Set tried_transport_methods = new HashSet(); @@ -233,4 +235,11 @@ public class Xmpp.Xep.Jingle.Content : Object { public void send_transport_info(StanzaNode transport) { session.send_transport_info(this, transport); } +} + +public class Xmpp.Xep.Jingle.ContentEncryption : Object { + public string encryption_ns { get; set; } + public string encryption_name { get; set; } + public uint8[] our_key { get; set; } + public uint8[] peer_key { get; set; } } \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index ff3d31f4..ac65f88c 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -116,6 +116,9 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { remote_crypto = null; local_crypto = null; } + if (remote_crypto != null && local_crypto != null) { + content.encryption = new Xmpp.Xep.Jingle.ContentEncryption() { encryption_ns = "", encryption_name = "SRTP", our_key=local_crypto.key, peer_key=remote_crypto.key }; + } this.stream = parent.create_stream(content); rtp_datagram.datagram_received.connect(this.stream.on_recv_rtp_data); diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala index 730ce9f8..adae11f5 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala @@ -1,5 +1,7 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { + public Jingle.Content content { get; protected set; } + public string name { get { return content.content_name; }} diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala index 4b7c7a36..5211e3a9 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala @@ -5,6 +5,7 @@ using Xmpp; namespace Xmpp.Xep.JingleIceUdp { private const string NS_URI = "urn:xmpp:jingle:transports:ice-udp:1"; +public const string DTLS_NS_URI = "urn:xmpp:jingle:apps:dtls:0"; public abstract class Module : XmppStreamModule, Jingle.Transport { public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "0176_jingle_ice_udp"); @@ -12,10 +13,11 @@ public abstract class Module : XmppStreamModule, Jingle.Transport { public override void attach(XmppStream stream) { stream.get_module(Jingle.Module.IDENTITY).register_transport(this); stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); - stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, "urn:xmpp:jingle:apps:dtls:0"); + stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, DTLS_NS_URI); } public override void detach(XmppStream stream) { stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI); + stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, DTLS_NS_URI); } public override string get_ns() { return NS_URI; } diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala index 3c69d0af..f194f06d 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala @@ -13,8 +13,9 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T public ConcurrentList unsent_local_candidates = new ConcurrentList(Candidate.equals_func); public Gee.List remote_candidates = new ArrayList(Candidate.equals_func); - public string? own_fingerprint = null; - public string? peer_fingerprint = null; + public uint8[]? own_fingerprint = null; + public uint8[]? peer_fingerprint = null; + public string? peer_fp_algo = null; public Jid local_full_jid { get; private set; } public Jid peer_full_jid { get; private set; } @@ -24,7 +25,7 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T public bool incoming { get; private set; default = false; } private bool connection_created = false; - private weak Jingle.Content? content = null; + protected weak Jingle.Content? content = null; protected IceUdpTransportParameters(uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode? node = null) { this.components_ = components; @@ -38,9 +39,10 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T remote_candidates.add(Candidate.parse(candidateNode)); } - StanzaNode? fingerprint_node = node.get_subnode("fingerprint", "urn:xmpp:jingle:apps:dtls:0"); + StanzaNode? fingerprint_node = node.get_subnode("fingerprint", DTLS_NS_URI); if (fingerprint_node != null) { - peer_fingerprint = fingerprint_node.get_deep_string_content(); + peer_fingerprint = fingerprint_to_bytes(fingerprint_node.get_deep_string_content()); + peer_fp_algo = fingerprint_node.get_attribute("hash"); } } } @@ -67,10 +69,10 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T .put_attribute("pwd", local_pwd); if (own_fingerprint != null) { - var fingerprint_node = new StanzaNode.build("fingerprint", "urn:xmpp:jingle:apps:dtls:0") + var fingerprint_node = new StanzaNode.build("fingerprint", DTLS_NS_URI) .add_self_xmlns() .put_attribute("hash", "sha-256") - .put_node(new StanzaNode.text(own_fingerprint)); + .put_node(new StanzaNode.text(format_fingerprint(own_fingerprint))); if (incoming) { fingerprint_node.put_attribute("setup", "active"); } else { @@ -95,9 +97,10 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T remote_candidates.add(Candidate.parse(candidateNode)); } - StanzaNode? fingerprint_node = node.get_subnode("fingerprint", "urn:xmpp:jingle:apps:dtls:0"); + StanzaNode? fingerprint_node = node.get_subnode("fingerprint", DTLS_NS_URI); if (fingerprint_node != null) { - peer_fingerprint = fingerprint_node.get_deep_string_content(); + peer_fingerprint = fingerprint_to_bytes(fingerprint_node.get_deep_string_content()); + peer_fp_algo = fingerprint_node.get_attribute("hash"); } } @@ -138,4 +141,30 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T content.send_transport_info(to_transport_stanza_node()); } } + + + + private string format_fingerprint(uint8[] fingerprint) { + var sb = new StringBuilder(); + for (int i = 0; i < fingerprint.length; i++) { + sb.append("%02x".printf(fingerprint[i])); + if (i < fingerprint.length - 1) { + sb.append(":"); + } + } + return sb.str; + } + + private uint8[] fingerprint_to_bytes(string? fingerprint_) { + if (fingerprint_ == null) return null; + + string fingerprint = fingerprint_.replace(":", "").up(); + + uint8[] bin = new uint8[fingerprint.length / 2]; + const string HEX = "0123456789ABCDEF"; + for (int i = 0; i < fingerprint.length / 2; i++) { + bin[i] = (uint8) (HEX.index_of_char(fingerprint[i*2]) << 4) | HEX.index_of_char(fingerprint[i*2+1]); + } + return bin; + } } \ No newline at end of file From fbc10c2023a4c2b874f87940f0a71bc0d8d7b57d Mon Sep 17 00:00:00 2001 From: Marvin W Date: Fri, 9 Apr 2021 22:23:53 +0200 Subject: [PATCH 31/56] DTLS-SRTP: Wait for setup finish and handle setup=passive --- plugins/crypto-vala/src/srtp.vala | 4 +- plugins/ice/src/dtls_srtp.vala | 160 ++++++++++++------ plugins/ice/src/transport_parameters.vala | 65 +++---- .../0167_jingle_rtp/content_parameters.vala | 3 + .../transport_parameters.vala | 16 +- 5 files changed, 158 insertions(+), 90 deletions(-) diff --git a/plugins/crypto-vala/src/srtp.vala b/plugins/crypto-vala/src/srtp.vala index 77b5acde..493afdb0 100644 --- a/plugins/crypto-vala/src/srtp.vala +++ b/plugins/crypto-vala/src/srtp.vala @@ -6,8 +6,8 @@ public class Crypto.Srtp { public const string F8_128_HMAC_SHA1_80 = "F8_128_HMAC_SHA1_80"; public class Session { - public bool has_encrypt { get; private set; } - public bool has_decrypt { get; private set; } + public bool has_encrypt { get; private set; default = false; } + public bool has_decrypt { get; private set; default = false; } private Context encrypt_context; private Context decrypt_context; diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index e2470cf6..8a9b5dfa 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -1,9 +1,26 @@ using GnuTLS; -public class DtlsSrtp { +namespace Dino.Plugins.Ice.DtlsSrtp { + +public static Handler setup() throws GLib.Error { + var obj = new Handler(); + obj.generate_credentials(); + return obj; +} + +public class Handler { public signal void send_data(uint8[] data); + public bool ready { get { + return srtp_session.has_encrypt && srtp_session.has_decrypt; + }} + + public Mode mode { get; set; default = Mode.CLIENT; } + public uint8[] own_fingerprint { get; private set; } + public uint8[] peer_fingerprint { get; set; } + public string peer_fp_algo { get; set; } + private X509.Certificate[] own_cert; private X509.PrivateKey private_key; private Cond buffer_cond = new Cond(); @@ -11,27 +28,12 @@ public class DtlsSrtp { private Gee.LinkedList buffer_queue = new Gee.LinkedList(); private uint pull_timeout = uint.MAX; - private DigestAlgorithm? peer_fp_algo = null; - private uint8[] peer_fingerprint = null; - private uint8[] own_fingerprint; + private bool running = false; + private bool stop = false; + private bool restart = false; private Crypto.Srtp.Session srtp_session = new Crypto.Srtp.Session(); - public static DtlsSrtp setup() throws GLib.Error { - var obj = new DtlsSrtp(); - obj.generate_credentials(); - return obj; - } - - internal uint8[] get_own_fingerprint(DigestAlgorithm digest_algo) { - return own_fingerprint; - } - - public void set_peer_fingerprint(uint8[] fingerprint, DigestAlgorithm digest_algo) { - this.peer_fingerprint = fingerprint; - this.peer_fp_algo = digest_algo; - } - public uint8[] process_incoming_data(uint component_id, uint8[] data) { if (srtp_session.has_decrypt) { try { @@ -77,7 +79,7 @@ public class DtlsSrtp { buffer_mutex.unlock(); } - private void generate_credentials() throws GLib.Error { + internal void generate_credentials() throws GLib.Error { int err = 0; private_key = X509.PrivateKey.create(); @@ -102,8 +104,29 @@ public class DtlsSrtp { own_cert = new X509.Certificate[] { (owned)cert }; } - public async Xmpp.Xep.Jingle.ContentEncryption setup_dtls_connection(bool server) { - InitFlags server_or_client = server ? InitFlags.SERVER : InitFlags.CLIENT; + public void stop_dtls_connection() { + buffer_mutex.lock(); + stop = true; + buffer_cond.signal(); + buffer_mutex.unlock(); + } + + public async Xmpp.Xep.Jingle.ContentEncryption? setup_dtls_connection() { + buffer_mutex.lock(); + if (stop) { + restart = true; + buffer_mutex.unlock(); + return null; + } + if (running || ready) { + buffer_mutex.unlock(); + return null; + } + running = true; + restart = false; + buffer_mutex.unlock(); + + InitFlags server_or_client = mode == Mode.SERVER ? InitFlags.SERVER : InitFlags.CLIENT; debug("Setting up DTLS connection. We're %s", server_or_client.to_string()); CertificateCredentials cert_cred = CertificateCredentials.create(); @@ -131,7 +154,7 @@ public class DtlsSrtp { DateTime current_time = new DateTime.now_utc(); if (maximum_time.compare(current_time) < 0) { warning("DTLS handshake timeouted"); - return -1; + return ErrorCode.APPLICATION_ERROR_MIN + 1; } } while (err < 0 && !((ErrorCode)err).is_fatal()); Idle.add(setup_dtls_connection.callback); @@ -139,6 +162,17 @@ public class DtlsSrtp { }); yield; err = thread.join(); + buffer_mutex.lock(); + if (stop) { + stop = false; + running = false; + bool restart = restart; + buffer_mutex.unlock(); + if (restart) return yield setup_dtls_connection(); + return null; + } + buffer_mutex.unlock(); + throw_if_error(err); uint8[] km = new uint8[150]; Datum? client_key, client_salt, server_key, server_salt; @@ -147,7 +181,8 @@ public class DtlsSrtp { warning("SRTP client/server key/salt null"); } - if (server) { + debug("Finished DTLS connection. We're %s", server_or_client.to_string()); + if (mode == Mode.SERVER) { srtp_session.set_encryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, server_key.extract(), server_salt.extract()); srtp_session.set_decryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, client_key.extract(), client_salt.extract()); } else { @@ -158,24 +193,28 @@ public class DtlsSrtp { } private static ssize_t pull_function(void* transport_ptr, uint8[] buffer) { - DtlsSrtp self = transport_ptr as DtlsSrtp; + Handler self = transport_ptr as Handler; self.buffer_mutex.lock(); while (self.buffer_queue.size == 0) { self.buffer_cond.wait(self.buffer_mutex); + if (self.stop) { + self.buffer_mutex.unlock(); + return -1; + } } - owned Bytes data = self.buffer_queue.remove_at(0); + Bytes data = self.buffer_queue.remove_at(0); self.buffer_mutex.unlock(); - uint8[] data_uint8 = Bytes.unref_to_data(data); + uint8[] data_uint8 = Bytes.unref_to_data((owned) data); Memory.copy(buffer, data_uint8, data_uint8.length); // The callback should return 0 on connection termination, a positive number indicating the number of bytes received, and -1 on error. - return (ssize_t)data.length; + return (ssize_t)data_uint8.length; } private static int pull_timeout_function(void* transport_ptr, uint ms) { - DtlsSrtp self = transport_ptr as DtlsSrtp; + Handler self = transport_ptr as Handler; DateTime current_time = new DateTime.now_utc(); current_time.add_seconds(ms/1000); @@ -184,6 +223,10 @@ public class DtlsSrtp { self.buffer_mutex.lock(); while (self.buffer_queue.size == 0) { self.buffer_cond.wait_until(self.buffer_mutex, end_time); + if (self.stop) { + self.buffer_mutex.unlock(); + return -1; + } DateTime new_current_time = new DateTime.now_utc(); if (new_current_time.compare(current_time) > 0) { @@ -197,7 +240,7 @@ public class DtlsSrtp { } private static ssize_t push_function(void* transport_ptr, uint8[] buffer) { - DtlsSrtp self = transport_ptr as DtlsSrtp; + Handler self = transport_ptr as Handler; self.send_data(buffer); // The callback should return a positive number indicating the bytes sent, and -1 on error. @@ -205,7 +248,7 @@ public class DtlsSrtp { } private static int verify_function(Session session) { - DtlsSrtp self = session.get_transport_pointer() as DtlsSrtp; + Handler self = session.get_transport_pointer() as Handler; try { bool valid = self.verify_peer_cert(session); if (!valid) { @@ -232,7 +275,17 @@ public class DtlsSrtp { X509.Certificate peer_cert = X509.Certificate.create(); peer_cert.import(ref cert_datums[0], CertificateFormat.DER); - uint8[] real_peer_fp = get_fingerprint(peer_cert, peer_fp_algo); + DigestAlgorithm algo; + switch (peer_fp_algo) { + case "sha-256": + algo = DigestAlgorithm.SHA256; + break; + default: + warning("Unkown peer fingerprint algorithm: %s", peer_fp_algo); + return false; + } + + uint8[] real_peer_fp = get_fingerprint(peer_cert, algo); if (real_peer_fp.length != this.peer_fingerprint.length) { warning("Fingerprint lengths not equal %i vs %i", real_peer_fp.length, peer_fingerprint.length); @@ -248,27 +301,34 @@ public class DtlsSrtp { return true; } +} - private uint8[] get_fingerprint(X509.Certificate certificate, DigestAlgorithm digest_algo) { - uint8[] buf = new uint8[512]; - size_t buf_out_size = 512; - certificate.get_fingerprint(digest_algo, buf, ref buf_out_size); +private uint8[] get_fingerprint(X509.Certificate certificate, DigestAlgorithm digest_algo) { + uint8[] buf = new uint8[512]; + size_t buf_out_size = 512; + certificate.get_fingerprint(digest_algo, buf, ref buf_out_size); - uint8[] ret = new uint8[buf_out_size]; - for (int i = 0; i < buf_out_size; i++) { - ret[i] = buf[i]; - } - return ret; + uint8[] ret = new uint8[buf_out_size]; + for (int i = 0; i < buf_out_size; i++) { + ret[i] = buf[i]; } + return ret; +} - private string format_fingerprint(uint8[] fingerprint) { - var sb = new StringBuilder(); - for (int i = 0; i < fingerprint.length; i++) { - sb.append("%02x".printf(fingerprint[i])); - if (i < fingerprint.length - 1) { - sb.append(":"); - } +private string format_fingerprint(uint8[] fingerprint) { + var sb = new StringBuilder(); + for (int i = 0; i < fingerprint.length; i++) { + sb.append("%02x".printf(fingerprint[i])); + if (i < fingerprint.length - 1) { + sb.append(":"); } - return sb.str; } -} \ No newline at end of file + return sb.str; +} + + +public enum Mode { + CLIENT, SERVER +} + +} diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index f95be261..e4862edc 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -9,11 +9,11 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport private bool we_want_connection; private bool remote_credentials_set; private Map connections = new HashMap(); - private DtlsSrtp? dtls_srtp; + private DtlsSrtp.Handler? dtls_srtp_handler; private class DatagramConnection : Jingle.DatagramConnection { private Nice.Agent agent; - private DtlsSrtp? dtls_srtp; + private DtlsSrtp.Handler? dtls_srtp_handler; private uint stream_id; private string? error; private ulong sent; @@ -22,9 +22,9 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport private ulong recv_reported; private ulong datagram_received_id; - public DatagramConnection(Nice.Agent agent, DtlsSrtp? dtls_srtp, uint stream_id, uint8 component_id) { + public DatagramConnection(Nice.Agent agent, DtlsSrtp.Handler? dtls_srtp_handler, uint stream_id, uint8 component_id) { this.agent = agent; - this.dtls_srtp = dtls_srtp; + this.dtls_srtp_handler = dtls_srtp_handler; this.stream_id = stream_id; this.component_id = component_id; this.datagram_received_id = this.datagram_received.connect((datagram) => { @@ -45,8 +45,8 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport public override void send_datagram(Bytes datagram) { if (this.agent != null && is_component_ready(agent, stream_id, component_id)) { uint8[] encrypted_data = null; - if (dtls_srtp != null) { - encrypted_data = dtls_srtp.process_outgoing_data(component_id, datagram.get_data()); + if (dtls_srtp_handler != null) { + encrypted_data = dtls_srtp_handler.process_outgoing_data(component_id, datagram.get_data()); if (encrypted_data == null) return; } agent.send(stream_id, component_id, encrypted_data ?? datagram.get_data()); @@ -65,13 +65,18 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport this.agent = agent; if (this.peer_fingerprint != null || !incoming) { - dtls_srtp = setup_dtls(this); - this.own_fingerprint = dtls_srtp.get_own_fingerprint(GnuTLS.DigestAlgorithm.SHA256); + dtls_srtp_handler = setup_dtls(this); + own_fingerprint = dtls_srtp_handler.own_fingerprint; if (incoming) { - dtls_srtp.set_peer_fingerprint(this.peer_fingerprint, this.peer_fp_algo == "sha-256" ? GnuTLS.DigestAlgorithm.SHA256 : GnuTLS.DigestAlgorithm.NULL); + own_setup = "active"; + dtls_srtp_handler.mode = DtlsSrtp.Mode.CLIENT; + dtls_srtp_handler.peer_fingerprint = peer_fingerprint; + dtls_srtp_handler.peer_fp_algo = peer_fp_algo; } else { - dtls_srtp.setup_dtls_connection.begin(true, (_, res) => { - this.content.encryption = dtls_srtp.setup_dtls_connection.end(res); + own_setup = "actpass"; + dtls_srtp_handler.mode = DtlsSrtp.Mode.SERVER; + dtls_srtp_handler.setup_dtls_connection.begin((_, res) => { + this.content.encryption = dtls_srtp_handler.setup_dtls_connection.end(res) ?? this.content.encryption; }); } } @@ -104,9 +109,9 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport agent.gather_candidates(stream_id); } - private static DtlsSrtp setup_dtls(TransportParameters tp) { + private static DtlsSrtp.Handler setup_dtls(TransportParameters tp) { var weak_self = new WeakRef(tp); - DtlsSrtp dtls_srtp = DtlsSrtp.setup(); + DtlsSrtp.Handler dtls_srtp = DtlsSrtp.setup(); dtls_srtp.send_data.connect((data) => { TransportParameters self = (TransportParameters) weak_self.get(); if (self != null) self.agent.send(self.stream_id, 1, data); @@ -144,10 +149,15 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport debug("on_transport_accept from %s", peer_full_jid.to_string()); base.handle_transport_accept(transport); - if (dtls_srtp != null && peer_fingerprint != null) { - dtls_srtp.set_peer_fingerprint(this.peer_fingerprint, this.peer_fp_algo == "sha-256" ? GnuTLS.DigestAlgorithm.SHA256 : GnuTLS.DigestAlgorithm.NULL); + if (dtls_srtp_handler != null && peer_fingerprint != null) { + dtls_srtp_handler.peer_fingerprint = peer_fingerprint; + dtls_srtp_handler.peer_fp_algo = peer_fp_algo; + if (peer_setup == "passive") { + dtls_srtp_handler.mode = DtlsSrtp.Mode.CLIENT; + dtls_srtp_handler.stop_dtls_connection(); + } } else { - dtls_srtp = null; + dtls_srtp_handler = null; } } @@ -200,18 +210,10 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport int new_candidates = agent.set_remote_candidates(stream_id, i, candidates); debug("Initiated component %u with %i remote candidates", i, new_candidates); - connections[i] = new DatagramConnection(agent, dtls_srtp, stream_id, i); + connections[i] = new DatagramConnection(agent, dtls_srtp_handler, stream_id, i); content.set_transport_connection(connections[i], i); } - if (incoming && dtls_srtp != null) { - Jingle.DatagramConnection rtp_datagram = (Jingle.DatagramConnection) content.get_transport_connection(1); - rtp_datagram.notify["ready"].connect(() => { - dtls_srtp.setup_dtls_connection.begin(false, (_, res) => { - this.content.encryption = dtls_srtp.setup_dtls_connection.end(res); - }); - }); - } base.create_transport_connection(stream, content); } @@ -219,11 +221,16 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport if (stream_id != this.stream_id) return; debug("stream %u component %u state changed to %s", stream_id, component_id, agent.get_component_state(stream_id, component_id).to_string()); may_consider_ready(stream_id, component_id); + if (incoming && dtls_srtp_handler != null && !dtls_srtp_handler.ready && is_component_ready(agent, stream_id, component_id) && dtls_srtp_handler.mode == DtlsSrtp.Mode.CLIENT) { + dtls_srtp_handler.setup_dtls_connection.begin((_, res) => { + this.content.encryption = dtls_srtp_handler.setup_dtls_connection.end(res) ?? this.content.encryption; + }); + } } private void may_consider_ready(uint stream_id, uint component_id) { if (stream_id != this.stream_id) return; - if (connections.has_key((uint8) component_id) && is_component_ready(agent, stream_id, component_id) && connections.has_key((uint8) component_id) && !connections[(uint8)component_id].ready) { + if (connections.has_key((uint8) component_id) && !connections[(uint8)component_id].ready && is_component_ready(agent, stream_id, component_id) && (dtls_srtp_handler == null || dtls_srtp_handler.ready)) { connections[(uint8)component_id].ready = true; } } @@ -241,8 +248,8 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport private void on_recv(Nice.Agent agent, uint stream_id, uint component_id, uint8[] data) { if (stream_id != this.stream_id) return; uint8[] decrypt_data = null; - if (dtls_srtp != null) { - decrypt_data = dtls_srtp.process_incoming_data(component_id, data); + if (dtls_srtp_handler != null) { + decrypt_data = dtls_srtp_handler.process_incoming_data(component_id, data); if (decrypt_data == null) return; } may_consider_ready(stream_id, component_id); @@ -317,4 +324,4 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport return candidate; } -} \ No newline at end of file +} diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index ac65f88c..c37c19cc 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -94,6 +94,9 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { ulong rtp_ready_handler_id = 0; rtp_ready_handler_id = rtp_datagram.notify["ready"].connect(() => { this.stream.on_rtp_ready(); + if (rtcp_mux) { + this.stream.on_rtcp_ready(); + } connection_ready(); rtp_datagram.disconnect(rtp_ready_handler_id); diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala index f194f06d..4976f560 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala @@ -14,8 +14,10 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T public Gee.List remote_candidates = new ArrayList(Candidate.equals_func); public uint8[]? own_fingerprint = null; + public string? own_setup = null; public uint8[]? peer_fingerprint = null; public string? peer_fp_algo = null; + public string? peer_setup = null; public Jid local_full_jid { get; private set; } public Jid peer_full_jid { get; private set; } @@ -41,8 +43,9 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T StanzaNode? fingerprint_node = node.get_subnode("fingerprint", DTLS_NS_URI); if (fingerprint_node != null) { - peer_fingerprint = fingerprint_to_bytes(fingerprint_node.get_deep_string_content()); + peer_fingerprint = fingerprint_to_bytes(fingerprint_node.get_string_content()); peer_fp_algo = fingerprint_node.get_attribute("hash"); + peer_setup = fingerprint_node.get_attribute("setup"); } } } @@ -73,11 +76,7 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T .add_self_xmlns() .put_attribute("hash", "sha-256") .put_node(new StanzaNode.text(format_fingerprint(own_fingerprint))); - if (incoming) { - fingerprint_node.put_attribute("setup", "active"); - } else { - fingerprint_node.put_attribute("setup", "actpass"); - } + fingerprint_node.put_attribute("setup", own_setup); node.put_node(fingerprint_node); } @@ -101,6 +100,7 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T if (fingerprint_node != null) { peer_fingerprint = fingerprint_to_bytes(fingerprint_node.get_deep_string_content()); peer_fp_algo = fingerprint_node.get_attribute("hash"); + peer_setup = fingerprint_node.get_attribute("setup"); } } @@ -142,8 +142,6 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T } } - - private string format_fingerprint(uint8[] fingerprint) { var sb = new StringBuilder(); for (int i = 0; i < fingerprint.length; i++) { @@ -167,4 +165,4 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T } return bin; } -} \ No newline at end of file +} From 09dc38f169745cb7697fdb969b9d4eb5e021e07a Mon Sep 17 00:00:00 2001 From: fiaxh Date: Fri, 9 Apr 2021 18:50:29 +0200 Subject: [PATCH 32/56] Accept additional jingle contents with senders=both (but modify to senders=peer) --- libdino/src/service/calls.vala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index b457c764..ccae3e05 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -355,11 +355,22 @@ namespace Dino { private void on_incoming_content_add(XmppStream stream, Call call, Xep.Jingle.Session session, Xep.Jingle.Content content) { Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; - if (rtp_content_parameter == null || session.senders_include_us(content.senders)) { + if (rtp_content_parameter == null) { content.reject(); return; } + // Our peer shouldn't tell us to start sending, that's for us to initiate + if (session.senders_include_us(content.senders)) { + if (session.senders_include_counterpart(content.senders)) { + // If our peer wants to send, let them + content.modify(session.we_initiated ? Xep.Jingle.Senders.RESPONDER : Xep.Jingle.Senders.INITIATOR); + } else { + // If only we're supposed to send, reject + content.reject(); + } + } + connect_content_signals(call, content, rtp_content_parameter); content.accept(); } From 5e11986838057a5cdbdf9d271316513da1bd4764 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Fri, 9 Apr 2021 19:04:24 +0200 Subject: [PATCH 33/56] Fix dtls pull_timeout_function, fix cyclic references --- plugins/ice/src/dtls_srtp.vala | 10 ++++------ plugins/ice/src/transport_parameters.vala | 8 ++++++++ xmpp-vala/src/module/xep/0166_jingle/content.vala | 1 + xmpp-vala/src/module/xep/0166_jingle/session.vala | 4 +--- .../xep/0167_jingle_rtp/content_parameters.vala | 14 +++++++++----- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index 8a9b5dfa..e8fc01c7 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -216,9 +216,7 @@ public class Handler { private static int pull_timeout_function(void* transport_ptr, uint ms) { Handler self = transport_ptr as Handler; - DateTime current_time = new DateTime.now_utc(); - current_time.add_seconds(ms/1000); - int64 end_time = current_time.to_unix(); + int64 end_time = get_monotonic_time() + ms * 1000; self.buffer_mutex.lock(); while (self.buffer_queue.size == 0) { @@ -228,9 +226,9 @@ public class Handler { return -1; } - DateTime new_current_time = new DateTime.now_utc(); - if (new_current_time.compare(current_time) > 0) { - break; + if (get_monotonic_time() > end_time) { + self.buffer_mutex.unlock(); + return 0; } } self.buffer_mutex.unlock(); diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index e4862edc..f854a367 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -40,6 +40,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport yield base.terminate(we_terminated, reason_string, reason_text); this.disconnect(datagram_received_id); agent = null; + dtls_srtp_handler = null; } public override void send_datagram(Bytes datagram) { @@ -324,4 +325,11 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport return candidate; } + + public override void dispose() { + base.dispose(); + agent = null; + dtls_srtp_handler = null; + connections.clear(); + } } diff --git a/xmpp-vala/src/module/xep/0166_jingle/content.vala b/xmpp-vala/src/module/xep/0166_jingle/content.vala index bce03a7b..67510c36 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/content.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/content.vala @@ -107,6 +107,7 @@ public class Xmpp.Xep.Jingle.Content : Object { public void terminate(bool we_terminated, string? reason_name, string? reason_text) { content_params.terminate(we_terminated, reason_name, reason_text); + transport_params.dispose(); foreach (ComponentConnection connection in component_connections.values) { connection.terminate(we_terminated, reason_name, reason_text); diff --git a/xmpp-vala/src/module/xep/0166_jingle/session.vala b/xmpp-vala/src/module/xep/0166_jingle/session.vala index 2d359f01..5fe89415 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/session.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/session.vala @@ -210,9 +210,7 @@ public class Xmpp.Xep.Jingle.Session : Object { } public async void add_content(Content content) { - content.session = this; - this.contents_map[content.content_name] = content; - contents.add(content); + insert_content(content); StanzaNode content_add_node = new StanzaNode.build("jingle", NS_URI) .add_self_xmlns() diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index c37c19cc..d6f1acd2 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -84,30 +84,34 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { Jingle.DatagramConnection rtcp_datagram = (Jingle.DatagramConnection) content.get_transport_connection(2); ulong rtcp_ready_handler_id = 0; - rtcp_ready_handler_id = rtcp_datagram.notify["ready"].connect(() => { + rtcp_ready_handler_id = rtcp_datagram.notify["ready"].connect((rtcp_datagram, _) => { this.stream.on_rtcp_ready(); - rtcp_datagram.disconnect(rtcp_ready_handler_id); + ((Jingle.DatagramConnection)rtcp_datagram).disconnect(rtcp_ready_handler_id); rtcp_ready_handler_id = 0; }); ulong rtp_ready_handler_id = 0; - rtp_ready_handler_id = rtp_datagram.notify["ready"].connect(() => { + rtp_ready_handler_id = rtp_datagram.notify["ready"].connect((rtp_datagram, _) => { this.stream.on_rtp_ready(); if (rtcp_mux) { this.stream.on_rtcp_ready(); } connection_ready(); - rtp_datagram.disconnect(rtp_ready_handler_id); + ((Jingle.DatagramConnection)rtp_datagram).disconnect(rtp_ready_handler_id); rtp_ready_handler_id = 0; }); - session.notify["state"].connect((obj, _) => { + ulong session_state_handler_id = 0; + session_state_handler_id = session.notify["state"].connect((obj, _) => { Jingle.Session session2 = (Jingle.Session) obj; if (session2.state == Jingle.Session.State.ENDED) { if (rtcp_ready_handler_id != 0) rtcp_datagram.disconnect(rtcp_ready_handler_id); if (rtp_ready_handler_id != 0) rtp_datagram.disconnect(rtp_ready_handler_id); + if (session_state_handler_id != 0) { + session2.disconnect(session_state_handler_id); + } } }); From 6ebdec1d78a7ad1b8668a2ba6eceb34515c75384 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 11 Apr 2021 12:31:03 +0200 Subject: [PATCH 34/56] GStreamer compat --- plugins/rtp/CMakeLists.txt | 6 ++++++ plugins/rtp/src/device.vala | 30 +++++++++++++++++++++--------- plugins/rtp/src/plugin.vala | 4 +++- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index c6888459..0925ff0c 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -9,6 +9,10 @@ find_packages(RTP_PACKAGES REQUIRED GstApp ) +if(Gst_VERSION VERSION_GREATER "1.16") + set(RTP_DEFINITIONS GST_1_16) +endif() + vala_precompile(RTP_VALA_C SOURCES src/codec_util.vala @@ -25,6 +29,8 @@ CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/qlite.vapi PACKAGES ${RTP_PACKAGES} +DEFINITIONS + ${RTP_DEFINITIONS} OPTIONS --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi ) diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 20762f77..3c9a38d2 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -130,11 +130,13 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { filter.@set("caps", get_best_caps()); pipe.add(filter); element.link(filter); - if (media == "audio") { + if (media == "audio" && plugin.echoprobe != null) { dsp = Gst.ElementFactory.make("webrtcdsp", @"$id-dsp"); - dsp.@set("probe", plugin.echoprobe.name); - pipe.add(dsp); - filter.link(dsp); + if (dsp != null) { + dsp.@set("probe", plugin.echoprobe.name); + pipe.add(dsp); + filter.link(dsp); + } } tee = Gst.ElementFactory.make("tee", @"$id-tee"); tee.@set("allow-not-linked", true); @@ -149,15 +151,19 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); filter.@set("caps", get_best_caps()); pipe.add(filter); - filter.link(plugin.echoprobe); - plugin.echoprobe.link(element); + if (plugin.echoprobe != null) { + filter.link(plugin.echoprobe); + plugin.echoprobe.link(element); + } else { + filter.link(element); + } } plugin.unpause(); } private void destroy() { if (mixer != null) { - if (is_sink && media == "audio") { + if (is_sink && media == "audio" && plugin.echoprobe != null) { plugin.echoprobe.unlink(mixer); } int linked_sink_pads = 0; @@ -177,11 +183,17 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { if (filter != null) { filter.set_locked_state(true); filter.set_state(Gst.State.NULL); - filter.unlink(plugin.echoprobe); + if (plugin.echoprobe != null) { + filter.unlink(plugin.echoprobe); + } else { + filter.unlink(element); + } pipe.remove(filter); filter = null; } - plugin.echoprobe.unlink(element); + if (plugin.echoprobe != null) { + plugin.echoprobe.unlink(element); + } } element.set_locked_state(true); element.set_state(Gst.State.NULL); diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index 62e0d411..40ad1e0f 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -70,7 +70,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { // Audio echo probe echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); - pipe.add(echoprobe); + if (echoprobe != null) pipe.add(echoprobe); // Pipeline pipe.auto_flush_bus = true; @@ -178,6 +178,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { if (devices.any_match((it) => it.matches(device))) return Source.CONTINUE; devices.add(new Device(this, device)); break; +#if GST_1_16 case Gst.MessageType.DEVICE_CHANGED: message.parse_device_changed(out device, out old_device); if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; @@ -185,6 +186,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { old = devices.first_match((it) => it.matches(old_device)); if (old != null) old.update(device); break; +#endif case Gst.MessageType.DEVICE_REMOVED: message.parse_device_removed(out device); if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; From dfffa08ec16e16157df6e7036e09073a546d7552 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Fri, 9 Apr 2021 23:59:03 +0200 Subject: [PATCH 35/56] Fix warnings --- libdino/src/service/connection_manager.vala | 2 +- libdino/src/service/notification_events.vala | 3 +-- libdino/src/util/display_name.vala | 6 +++--- main/src/ui/util/helper.vala | 6 +++--- plugins/ice/src/dtls_srtp.vala | 13 ++++++------- plugins/ice/src/transport_parameters.vala | 2 +- xmpp-vala/src/module/xep/0166_jingle/content.vala | 6 ------ .../0176_jingle_ice_udp/transport_parameters.vala | 1 - 8 files changed, 15 insertions(+), 24 deletions(-) diff --git a/libdino/src/service/connection_manager.vala b/libdino/src/service/connection_manager.vala index 454bcc2c..1439c6f3 100644 --- a/libdino/src/service/connection_manager.vala +++ b/libdino/src/service/connection_manager.vala @@ -170,7 +170,7 @@ public class ConnectionManager : Object { public async void disconnect_account(Account account) { if (connections.has_key(account)) { make_offline(account); - connections[account].disconnect_account(); + connections[account].disconnect_account.begin(); connections.unset(account); } } diff --git a/libdino/src/service/notification_events.vala b/libdino/src/service/notification_events.vala index 0243dbe5..7039d1cf 100644 --- a/libdino/src/service/notification_events.vala +++ b/libdino/src/service/notification_events.vala @@ -112,9 +112,8 @@ public class NotificationEvents : StreamInteractionModule, Object { notifier.notify_call.begin(call, conversation, video, conversation_display_name); call.notify["state"].connect(() => { - if (call.state != Call.State.RINGING) { - notifier.retract_call_notification(call, conversation); + notifier.retract_call_notification.begin(call, conversation); } }); } diff --git a/libdino/src/util/display_name.vala b/libdino/src/util/display_name.vala index 7fa741af..0c05eda8 100644 --- a/libdino/src/util/display_name.vala +++ b/libdino/src/util/display_name.vala @@ -36,7 +36,7 @@ namespace Dino { return participant.bare_jid.to_string(); } - private static string? get_real_display_name(StreamInteractor stream_interactor, Account account, Jid jid, string? self_word = null) { + public static string? get_real_display_name(StreamInteractor stream_interactor, Account account, Jid jid, string? self_word = null) { if (jid.equals_bare(account.bare_jid)) { if (self_word != null || account.alias == null || account.alias.length == 0) { return self_word; @@ -50,7 +50,7 @@ namespace Dino { return null; } - private static string get_groupchat_display_name(StreamInteractor stream_interactor, Account account, Jid jid) { + public static string get_groupchat_display_name(StreamInteractor stream_interactor, Account account, Jid jid) { MucManager muc_manager = stream_interactor.get_module(MucManager.IDENTITY); string? room_name = muc_manager.get_room_name(account, jid); if (room_name != null && room_name != jid.localpart) { @@ -72,7 +72,7 @@ namespace Dino { return jid.to_string(); } - private static string get_occupant_display_name(StreamInteractor stream_interactor, Conversation conversation, Jid jid, string? self_word = null, bool muc_real_name = false) { + public static string get_occupant_display_name(StreamInteractor stream_interactor, Conversation conversation, Jid jid, string? self_word = null, bool muc_real_name = false) { if (muc_real_name) { MucManager muc_manager = stream_interactor.get_module(MucManager.IDENTITY); if (muc_manager.is_private_room(conversation.account, jid.bare_jid)) { diff --git a/main/src/ui/util/helper.vala b/main/src/ui/util/helper.vala index d3ca063b..17dfd334 100644 --- a/main/src/ui/util/helper.vala +++ b/main/src/ui/util/helper.vala @@ -122,15 +122,15 @@ public static string get_participant_display_name(StreamInteractor stream_intera return Dino.get_participant_display_name(stream_interactor, conversation, participant, me_is_me ? _("Me") : null); } -private static string? get_real_display_name(StreamInteractor stream_interactor, Account account, Jid jid, bool me_is_me = false) { +public static string? get_real_display_name(StreamInteractor stream_interactor, Account account, Jid jid, bool me_is_me = false) { return Dino.get_real_display_name(stream_interactor, account, jid, me_is_me ? _("Me") : null); } -private static string get_groupchat_display_name(StreamInteractor stream_interactor, Account account, Jid jid) { +public static string get_groupchat_display_name(StreamInteractor stream_interactor, Account account, Jid jid) { return Dino.get_groupchat_display_name(stream_interactor, account, jid); } -private static string get_occupant_display_name(StreamInteractor stream_interactor, Conversation conversation, Jid jid, bool me_is_me = false, bool muc_real_name = false) { +public static string get_occupant_display_name(StreamInteractor stream_interactor, Conversation conversation, Jid jid, bool me_is_me = false, bool muc_real_name = false) { return Dino.get_occupant_display_name(stream_interactor, conversation, jid, me_is_me ? _("Me") : null); } diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index e8fc01c7..6701fe89 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -23,10 +23,9 @@ public class Handler { private X509.Certificate[] own_cert; private X509.PrivateKey private_key; - private Cond buffer_cond = new Cond(); - private Mutex buffer_mutex = new Mutex(); + private Cond buffer_cond = Cond(); + private Mutex buffer_mutex = Mutex(); private Gee.LinkedList buffer_queue = new Gee.LinkedList(); - private uint pull_timeout = uint.MAX; private bool running = false; private bool stop = false; @@ -34,7 +33,7 @@ public class Handler { private Crypto.Srtp.Session srtp_session = new Crypto.Srtp.Session(); - public uint8[] process_incoming_data(uint component_id, uint8[] data) { + public uint8[]? process_incoming_data(uint component_id, uint8[] data) { if (srtp_session.has_decrypt) { try { if (component_id == 1) { @@ -54,7 +53,7 @@ public class Handler { return null; } - public uint8[] process_outgoing_data(uint component_id, uint8[] data) { + public uint8[]? process_outgoing_data(uint component_id, uint8[] data) { if (srtp_session.has_encrypt) { try { if (component_id == 1) { @@ -127,7 +126,7 @@ public class Handler { buffer_mutex.unlock(); InitFlags server_or_client = mode == Mode.SERVER ? InitFlags.SERVER : InitFlags.CLIENT; - debug("Setting up DTLS connection. We're %s", server_or_client.to_string()); + debug("Setting up DTLS connection. We're %s", mode.to_string()); CertificateCredentials cert_cred = CertificateCredentials.create(); int err = cert_cred.set_x509_key(own_cert, private_key); @@ -181,7 +180,7 @@ public class Handler { warning("SRTP client/server key/salt null"); } - debug("Finished DTLS connection. We're %s", server_or_client.to_string()); + debug("Finished DTLS connection. We're %s", mode.to_string()); if (mode == Mode.SERVER) { srtp_session.set_encryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, server_key.extract(), server_salt.extract()); srtp_session.set_decryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, client_key.extract(), client_salt.extract()); diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index f854a367..6d160c62 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -111,7 +111,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport } private static DtlsSrtp.Handler setup_dtls(TransportParameters tp) { - var weak_self = new WeakRef(tp); + var weak_self = WeakRef(tp); DtlsSrtp.Handler dtls_srtp = DtlsSrtp.setup(); dtls_srtp.send_data.connect((data) => { TransportParameters self = (TransportParameters) weak_self.get(); diff --git a/xmpp-vala/src/module/xep/0166_jingle/content.vala b/xmpp-vala/src/module/xep/0166_jingle/content.vala index 67510c36..beb12183 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/content.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/content.vala @@ -201,12 +201,6 @@ public class Xmpp.Xep.Jingle.Content : Object { stream.get_module(Iq.Module.IDENTITY).send_iq(stream, new Iq.Stanza.result(iq)); } - void verify_content(ContentNode content) throws IqError { - if (content.name != content_name || content.creator != content_creator) { - throw new IqError.BAD_REQUEST("unknown content"); - } - } - public void set_transport_connection(ComponentConnection? conn, uint8 component = 1) { debug(@"set_transport_connection: %s, %s, %i, %s, overwrites: %s", this.content_name, this.state.to_string(), component, (conn != null).to_string(), component_connections.has_key(component).to_string()); diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala index 4976f560..6684ddc2 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala @@ -109,7 +109,6 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T string? ufrag = node.get_attribute("ufrag"); if (pwd != null) remote_pwd = pwd; if (ufrag != null) remote_ufrag = ufrag; - uint8 components = 0; foreach (StanzaNode candidateNode in node.get_subnodes("candidate")) { remote_candidates.add(Candidate.parse(candidateNode)); } From 8a54a263f2714560d4eb8d83a61e14d184b61fba Mon Sep 17 00:00:00 2001 From: fiaxh Date: Sat, 10 Apr 2021 23:06:13 +0200 Subject: [PATCH 36/56] Improve encryption displaying in calls --- libdino/src/service/calls.vala | 49 +++++++++++++--------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index ccae3e05..d8cb7990 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -375,13 +375,6 @@ namespace Dino { content.accept(); } - private void on_connection_ready(Call call) { - if (call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { - call.state = Call.State.IN_PROGRESS; - } - update_call_encryption(call); - } - private void on_call_terminated(Call call, bool we_terminated, string? reason_name, string? reason_text) { if (call.state == Call.State.RINGING || call.state == Call.State.IN_PROGRESS || call.state == Call.State.ESTABLISHING) { call.end_time = new DateTime.now_utc(); @@ -452,7 +445,7 @@ namespace Dino { } rtp_content_parameter.stream_created.connect((stream) => on_stream_created(call, rtp_content_parameter.media, stream)); - rtp_content_parameter.connection_ready.connect((status) => on_connection_ready(call)); + rtp_content_parameter.connection_ready.connect((status) => on_connection_ready(call, content, rtp_content_parameter.media)); content.senders_modify_incoming.connect((content, proposed_senders) => { if (content.session.senders_include_us(content.senders) != content.session.senders_include_us(proposed_senders)) { @@ -466,36 +459,32 @@ namespace Dino { on_counterpart_mute_update(call, false, "video"); } }); - - content.notify["encryption"].connect((obj, _) => { - if (rtp_content_parameter.media == "audio") { - audio_encryption[call] = ((Xep.Jingle.Content) obj).encryption; - } else if (rtp_content_parameter.media == "video") { - video_encryption[call] = ((Xep.Jingle.Content) obj).encryption; - } - }); } - private void update_call_encryption(Call call) { - if (audio_encryption[call] == null) { + private void on_connection_ready(Call call, Xep.Jingle.Content content, string media) { + if (call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { + call.state = Call.State.IN_PROGRESS; + } + + if (media == "audio") { + audio_encryption[call] = content.encryption; + } else if (media == "video") { + video_encryption[call] = content.encryption; + } + + if ((audio_encryption.has_key(call) && audio_encryption[call] == null) || (video_encryption.has_key(call) && video_encryption[call] == null)) { call.encryption = Encryption.NONE; encryption_updated(call, null); return; } - bool consistent_encryption = video_encryption[call] != null && audio_encryption[call].encryption_ns == video_encryption[call].encryption_ns; - - if (video_content[call] == null || consistent_encryption) { - if (audio_encryption[call].encryption_ns == Xep.JingleIceUdp.DTLS_NS_URI) { - call.encryption = Encryption.DTLS_SRTP; - } else if (audio_encryption[call].encryption_name == "SRTP") { - call.encryption = Encryption.SRTP; - } - encryption_updated(call, audio_encryption[call]); - } else { - call.encryption = Encryption.NONE; - encryption_updated(call, null); + Xep.Jingle.ContentEncryption encryption = audio_encryption[call] ?? video_encryption[call]; + if (encryption.encryption_ns == Xep.JingleIceUdp.DTLS_NS_URI) { + call.encryption = Encryption.DTLS_SRTP; + } else if (encryption.encryption_name == "SRTP") { + call.encryption = Encryption.SRTP; } + encryption_updated(call, encryption); } private void remove_call_from_datastructures(Call call) { From 0707fd9ac466aa5280565f5ba9ced261d725ca42 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Sat, 10 Apr 2021 23:12:05 +0200 Subject: [PATCH 37/56] Improve automatic call window resizing --- .../call_window/call_window_controller.vala | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/main/src/ui/call_window/call_window_controller.vala b/main/src/ui/call_window/call_window_controller.vala index f66a37e1..616e341d 100644 --- a/main/src/ui/call_window/call_window_controller.vala +++ b/main/src/ui/call_window/call_window_controller.vala @@ -14,6 +14,9 @@ public class Dino.Ui.CallWindowController : Object { private Plugins.VideoCallWidget? own_video = null; private Plugins.VideoCallWidget? counterpart_video = null; + private int window_height = -1; + private int window_width = -1; + private bool window_size_changed = false; public CallWindowController(CallWindow call_window, Call call, StreamInteractor stream_interactor) { this.call_window = call_window; @@ -26,7 +29,7 @@ public class Dino.Ui.CallWindowController : Object { this.counterpart_video = call_plugin.create_widget(Plugins.WidgetType.GTK); call_window.counterpart_display_name = Util.get_conversation_display_name(stream_interactor, conversation); - call_window.set_default_size(640, 480); + call_window.set_default_size(704, 528); // 640x480 * 1.1 call_window.set_video_fallback(stream_interactor, conversation); this.call_window.bottom_bar.video_enabled = calls.should_we_send_video(call); @@ -77,12 +80,27 @@ public class Dino.Ui.CallWindowController : Object { call_window.set_own_video_ratio((int)width, (int)height); }); counterpart_video.resolution_changed.connect((width, height) => { + if (window_size_changed) return; if (width == 0 || height == 0) return; - if (width / height > 640 / 480) { - call_window.resize(640, (int) (height * 640 / width)); + if (width > height) { + call_window.resize(704, (int) (height * 704 / width)); } else { - call_window.resize((int) (width * 480 / height), 480); + call_window.resize((int) (width * 704 / height), 704); } + capture_window_size(); + }); + call_window.configure_event.connect((event) => { + if (window_width == -1 || window_height == -1) return false; + int current_height = this.call_window.get_allocated_height(); + int current_width = this.call_window.get_allocated_width(); + if (window_width != current_width || window_height != current_height) { + debug("Call window size changed by user. Disabling auto window-to-video size adaptation. %i->%i x %i->%i", window_width, current_width, window_height, current_height); + window_size_changed = true; + } + return false; + }); + call_window.realize.connect(() => { + capture_window_size(); }); call.notify["state"].connect(on_call_state_changed); @@ -91,6 +109,13 @@ public class Dino.Ui.CallWindowController : Object { update_own_video(); } + private void capture_window_size() { + Allocation allocation; + this.call_window.get_allocation(out allocation); + this.window_height = this.call_window.get_allocated_height(); + this.window_width = this.call_window.get_allocated_width(); + } + private void end_call() { call.notify["state"].disconnect(on_call_state_changed); calls.call_terminated.disconnect(on_call_terminated); From 369755781e4d4fae2f6d790b93ba824e7e049c33 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Sun, 11 Apr 2021 14:33:51 +0200 Subject: [PATCH 38/56] Send JMI retract --- libdino/src/service/calls.vala | 6 ++++++ .../src/module/xep/0353_jingle_message_initiation.vala | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index d8cb7990..b7374607 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -136,6 +136,7 @@ namespace Dino { sessions[call].terminate(Xep.Jingle.ReasonElement.CANCEL, null, "cancel"); } else { // Only a JMI so far + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_retract_to_peer(stream, call.counterpart, jmi_sid[call.account]); } call.state = Call.State.MISSED; } else { @@ -207,6 +208,11 @@ namespace Dino { public void mute_own_video(Call call, bool mute) { we_should_send_video[call] = !mute; + if (!sessions.has_key(call)) { + // Call hasn't been established yet + return; + } + Xep.JingleRtp.Module rtp_module = stream_interactor.module_manager.get_module(call.account, Xep.JingleRtp.Module.IDENTITY); if (video_content_parameter.has_key(call) && diff --git a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala index dbb6fd81..08e803a2 100644 --- a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala +++ b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala @@ -22,6 +22,14 @@ namespace Xmpp.Xep.JingleMessageInitiation { stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); } + public void send_session_retract_to_peer(XmppStream stream, Jid to, string sid) { + MessageStanza retract_message = new MessageStanza() { to=to }; + retract_message.stanza.put_node( + new StanzaNode.build("retract", NS_URI).add_self_xmlns() + .put_attribute("id", sid, NS_URI)); + stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, retract_message); + } + public void send_session_accept_to_self(XmppStream stream, string sid) { MessageStanza accepted_message = new MessageStanza() { to=Bind.Flag.get_my_jid(stream).bare_jid }; accepted_message.stanza.put_node( From e9ff660537f7c00281395f8a001f7554e116efff Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 11 Apr 2021 15:12:26 +0200 Subject: [PATCH 39/56] Fix usage of old libnice versions --- plugins/ice/src/transport_parameters.vala | 8 +++----- plugins/ice/vapi/metadata/Nice-0.1.metadata | 3 +++ plugins/ice/vapi/nice.vapi | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index 6d160c62..8766e744 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -176,8 +176,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport SList candidates = new SList(); foreach (JingleIceUdp.Candidate candidate in remote_candidates) { if (candidate.component == i) { - Nice.Candidate nc = candidate_to_nice(candidate); - candidates.append(nc); + candidates.append(candidate_to_nice(candidate)); } } int new_candidates = agent.set_remote_candidates(stream_id, i, candidates); @@ -203,9 +202,8 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport foreach (JingleIceUdp.Candidate candidate in remote_candidates) { if (candidate.ip.has_prefix("fe80::")) continue; if (candidate.component == i) { - Nice.Candidate nc = candidate_to_nice(candidate); - candidates.append(nc); - debug("remote candidate: %s", agent.generate_local_candidate_sdp(nc)); + candidates.append(candidate_to_nice(candidate)); + debug("remote candidate: %s", agent.generate_local_candidate_sdp(candidate_to_nice(candidate))); } } int new_candidates = agent.set_remote_candidates(stream_id, i, candidates); diff --git a/plugins/ice/vapi/metadata/Nice-0.1.metadata b/plugins/ice/vapi/metadata/Nice-0.1.metadata index d6899f87..7fcf046a 100644 --- a/plugins/ice/vapi/metadata/Nice-0.1.metadata +++ b/plugins/ice/vapi/metadata/Nice-0.1.metadata @@ -6,3 +6,6 @@ Agent.send.buf type="uint8[]" array_length_idx=2 AgentRecvFunc.buf type="uint8[]" array_length_idx=3 PseudoTcpCallbacks#record skip PseudoTcpSocket#class skip + +# Not yet supported by vapigen +# Candidate copy_function="nice_candidate_copy" free_function="nice_candidate_free" type_id="" diff --git a/plugins/ice/vapi/nice.vapi b/plugins/ice/vapi/nice.vapi index 39768b9b..540e2b4e 100644 --- a/plugins/ice/vapi/nice.vapi +++ b/plugins/ice/vapi/nice.vapi @@ -164,7 +164,7 @@ namespace Nice { [Version (since = "0.1.5")] public signal void streams_removed ([CCode (array_length = false, array_null_terminated = true)] uint[] stream_ids); } - [CCode (cheader_filename = "nice.h", copy_function = "g_boxed_copy", free_function = "g_boxed_free", type_id = "nice_candidate_get_type ()")] + [CCode (cheader_filename = "nice.h", copy_function = "nice_candidate_copy", free_function = "nice_candidate_free")] [Compact] public class Candidate { public Nice.Address addr; From 4edab3c8d63b327dcb48799e174a3e00192721ec Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 11 Apr 2021 15:12:53 +0200 Subject: [PATCH 40/56] Fix custom vapi integration --- plugins/crypto-vala/CMakeLists.txt | 6 ++---- plugins/ice/CMakeLists.txt | 10 +++++----- plugins/omemo/CMakeLists.txt | 7 +++---- plugins/rtp/CMakeLists.txt | 2 -- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/plugins/crypto-vala/CMakeLists.txt b/plugins/crypto-vala/CMakeLists.txt index 4a8da241..f615854c 100644 --- a/plugins/crypto-vala/CMakeLists.txt +++ b/plugins/crypto-vala/CMakeLists.txt @@ -1,9 +1,9 @@ find_package(GCrypt REQUIRED) +find_package(Srtp2 REQUIRED) find_packages(CRYPTO_VALA_PACKAGES REQUIRED GLib GObject GIO - Srtp2 ) vala_precompile(CRYPTO_VALA_C @@ -18,8 +18,6 @@ CUSTOM_VAPIS "${CMAKE_CURRENT_SOURCE_DIR}/vapi/libsrtp2.vapi" PACKAGES ${CRYPTO_VALA_PACKAGES} -OPTIONS - --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi GENERATE_VAPI crypto-vala GENERATE_HEADER @@ -29,6 +27,6 @@ GENERATE_HEADER set(CFLAGS ${VALA_CFLAGS}) add_definitions(${CFLAGS}) add_library(crypto-vala STATIC ${CRYPTO_VALA_C}) -target_link_libraries(crypto-vala ${CRYPTO_VALA_PACKAGES} gcrypt) +target_link_libraries(crypto-vala ${CRYPTO_VALA_PACKAGES} gcrypt libsrtp2) set_property(TARGET crypto-vala PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/plugins/ice/CMakeLists.txt b/plugins/ice/CMakeLists.txt index 392a202f..b18bfcc5 100644 --- a/plugins/ice/CMakeLists.txt +++ b/plugins/ice/CMakeLists.txt @@ -1,11 +1,11 @@ +find_package(Nice REQUIRED) +find_package(GnuTLS REQUIRED) find_packages(ICE_PACKAGES REQUIRED Gee GLib GModule - GnuTLS GObject GTK3 - Nice ) vala_precompile(ICE_VALA_C @@ -21,15 +21,15 @@ CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/vapi/nice.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/vapi/gnutls.vapi PACKAGES ${ICE_PACKAGES} -OPTIONS - --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi ) add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="ice") add_library(ice SHARED ${ICE_VALA_C}) -target_link_libraries(ice libdino crypto-vala ${ICE_PACKAGES}) +target_link_libraries(ice libdino crypto-vala ${ICE_PACKAGES} nice gnutls) set_target_properties(ice PROPERTIES PREFIX "") set_target_properties(ice PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/omemo/CMakeLists.txt b/plugins/omemo/CMakeLists.txt index 0f5a1521..c7a45069 100644 --- a/plugins/omemo/CMakeLists.txt +++ b/plugins/omemo/CMakeLists.txt @@ -3,13 +3,13 @@ find_package(Gettext) include(${GETTEXT_USE_FILE}) gettext_compile(${GETTEXT_PACKAGE} SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/po TARGET_NAME ${GETTEXT_PACKAGE}-translations) +find_package(Qrencode REQUIRED) find_packages(OMEMO_PACKAGES REQUIRED Gee GLib GModule GObject GTK3 - Qrencode ) set(RESOURCE_LIST @@ -66,18 +66,17 @@ CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/vapi/libqrencode.vapi PACKAGES ${OMEMO_PACKAGES} GRESOURCES ${OMEMO_GRESOURCES_XML} -OPTIONS - --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi ) add_definitions(${VALA_CFLAGS} -DGETTEXT_PACKAGE=\"${GETTEXT_PACKAGE}\" -DLOCALE_INSTALL_DIR=\"${LOCALE_INSTALL_DIR}\" -DG_LOG_DOMAIN="OMEMO") add_library(omemo SHARED ${OMEMO_VALA_C} ${OMEMO_GRESOURCES_TARGET}) add_dependencies(omemo ${GETTEXT_PACKAGE}-translations) -target_link_libraries(omemo libdino signal-protocol-vala crypto-vala ${OMEMO_PACKAGES}) +target_link_libraries(omemo libdino signal-protocol-vala crypto-vala ${OMEMO_PACKAGES} libqrencode) set_target_properties(omemo PROPERTIES PREFIX "") set_target_properties(omemo PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 0925ff0c..76d6e66d 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -31,8 +31,6 @@ PACKAGES ${RTP_PACKAGES} DEFINITIONS ${RTP_DEFINITIONS} -OPTIONS - --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi ) add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) From fe160d94ba8a08a806dc755d918dc3de0a645d7c Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 11 Apr 2021 15:57:53 +0200 Subject: [PATCH 41/56] Handle broken VAPI in older vala --- plugins/rtp/src/stream.vala | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index efa1b497..3a63f3fa 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -300,7 +300,17 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } } if (push_recv_data) { - recv_rtp.push_buffer(new Gst.Buffer.wrapped((owned) data)); + Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); + // FIXME: VAPI file in Vala < 0.49.1 has a bug that results in broken ownership of buffer in push_buffer() + // We workaround by using the plain signal. The signal unfortunately will cause an unnecessary copy of + // the underlying buffer, so and some point we should move over to the new version (once we require + // Vala >= 0.50) +#if FIXED_APPSRC_PUSH_BUFFER_IN_VAPI + recv_rtp.push_buffer((owned) buffer); +#else + Gst.FlowReturn ret; + GLib.Signal.emit_by_name(recv_rtp, "push-buffer", buffer, out ret); +#endif } } @@ -315,7 +325,14 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } } if (push_recv_data) { - recv_rtcp.push_buffer(new Gst.Buffer.wrapped((owned) data)); + Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); + // See above +#if FIXED_APPSRC_PUSH_BUFFER_IN_VAPI + recv_rtcp.push_buffer((owned) buffer); +#else + Gst.FlowReturn ret; + GLib.Signal.emit_by_name(recv_rtcp, "push-buffer", buffer, out ret); +#endif } } From d19a01d5f2129612827bf25a94f1977320ba7b1f Mon Sep 17 00:00:00 2001 From: Marvin W Date: Mon, 12 Apr 2021 18:05:08 +0200 Subject: [PATCH 42/56] Handle DTLS edge-cases --- plugins/ice/src/dtls_srtp.vala | 20 ++++++++++++++++--- plugins/ice/src/transport_parameters.vala | 3 +++ .../transport_parameters.vala | 2 +- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index 6701fe89..f5ef830a 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -153,7 +153,13 @@ public class Handler { DateTime current_time = new DateTime.now_utc(); if (maximum_time.compare(current_time) < 0) { warning("DTLS handshake timeouted"); - return ErrorCode.APPLICATION_ERROR_MIN + 1; + err = ErrorCode.APPLICATION_ERROR_MIN + 1; + break; + } + if (stop) { + debug("DTLS handshake stopped"); + err = ErrorCode.APPLICATION_ERROR_MIN + 2; + break; } } while (err < 0 && !((ErrorCode)err).is_fatal()); Idle.add(setup_dtls_connection.callback); @@ -167,11 +173,17 @@ public class Handler { running = false; bool restart = restart; buffer_mutex.unlock(); - if (restart) return yield setup_dtls_connection(); + if (restart) { + debug("Restarting DTLS handshake"); + return yield setup_dtls_connection(); + } return null; } buffer_mutex.unlock(); - throw_if_error(err); + if (err != ErrorCode.SUCCESS) { + warning("DTLS handshake failed: %s", ((ErrorCode)err).to_string()); + return null; + } uint8[] km = new uint8[150]; Datum? client_key, client_salt, server_key, server_salt; @@ -199,6 +211,7 @@ public class Handler { self.buffer_cond.wait(self.buffer_mutex); if (self.stop) { self.buffer_mutex.unlock(); + debug("DTLS handshake pull_function stopped"); return -1; } } @@ -222,6 +235,7 @@ public class Handler { self.buffer_cond.wait_until(self.buffer_mutex, end_time); if (self.stop) { self.buffer_mutex.unlock(); + debug("DTLS handshake pull_timeout_function stopped"); return -1; } diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index 8766e744..52451fcf 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -156,6 +156,9 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport if (peer_setup == "passive") { dtls_srtp_handler.mode = DtlsSrtp.Mode.CLIENT; dtls_srtp_handler.stop_dtls_connection(); + dtls_srtp_handler.setup_dtls_connection.begin((_, res) => { + this.content.encryption = dtls_srtp_handler.setup_dtls_connection.end(res) ?? this.content.encryption; + }); } } else { dtls_srtp_handler = null; diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala index 6684ddc2..ed0fab50 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala @@ -98,7 +98,7 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T StanzaNode? fingerprint_node = node.get_subnode("fingerprint", DTLS_NS_URI); if (fingerprint_node != null) { - peer_fingerprint = fingerprint_to_bytes(fingerprint_node.get_deep_string_content()); + peer_fingerprint = fingerprint_to_bytes(fingerprint_node.get_string_content()); peer_fp_algo = fingerprint_node.get_attribute("hash"); peer_setup = fingerprint_node.get_attribute("setup"); } From 2bee82f0b6b1d7fb64c7c446257037ba17129baa Mon Sep 17 00:00:00 2001 From: Marvin W Date: Mon, 12 Apr 2021 18:05:55 +0200 Subject: [PATCH 43/56] ICE: Require libnice 0.1.15 or newer --- plugins/ice/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ice/CMakeLists.txt b/plugins/ice/CMakeLists.txt index b18bfcc5..4783cea6 100644 --- a/plugins/ice/CMakeLists.txt +++ b/plugins/ice/CMakeLists.txt @@ -1,4 +1,4 @@ -find_package(Nice REQUIRED) +find_package(Nice 0.1.15 REQUIRED) find_package(GnuTLS REQUIRED) find_packages(ICE_PACKAGES REQUIRED Gee From 328c3cf37f296f4519829afd03d21f94ea4153e5 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Mon, 12 Apr 2021 22:21:28 +0200 Subject: [PATCH 44/56] Fix bad syntax in gnutls vapi, add libsrtp2 dependency to CI builds --- .github/workflows/build.yml | 2 +- plugins/ice/vapi/gnutls.vapi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2ec82dd5..ce12d441 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,7 @@ jobs: steps: - uses: actions/checkout@v2 - run: sudo apt-get update - - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-3-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libgspell-1-dev libnice-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev + - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-3-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libgspell-1-dev libnice-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libsrtp2-dev - run: ./configure --with-tests --with-libsignal-in-tree - run: make - run: build/xmpp-vala-test diff --git a/plugins/ice/vapi/gnutls.vapi b/plugins/ice/vapi/gnutls.vapi index a8f75e14..bc3f13d0 100644 --- a/plugins/ice/vapi/gnutls.vapi +++ b/plugins/ice/vapi/gnutls.vapi @@ -277,7 +277,7 @@ namespace GnuTLS { public uint size; public uint8[] extract() { - uint8[size] ret = new uint8[size]; + uint8[] ret = new uint8[size]; for (int i = 0; i < size; i++) { ret[i] = data[i]; } From 3880628de4785db4c0a03a79a0c486507fe9b1a8 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Thu, 29 Apr 2021 15:46:06 +0200 Subject: [PATCH 45/56] Video optimizations --- cmake/FindGstRtp.cmake | 14 + plugins/rtp/CMakeLists.txt | 4 +- plugins/rtp/src/codec_util.vala | 115 +++- plugins/rtp/src/device.vala | 9 +- plugins/rtp/src/module.vala | 133 ++-- plugins/rtp/src/plugin.vala | 14 + plugins/rtp/src/stream.vala | 220 +++++- plugins/rtp/src/video_widget.vala | 10 +- plugins/rtp/vapi/gstreamer-rtp-1.0.vapi | 625 ++++++++++++++++++ .../0167_jingle_rtp/content_parameters.vala | 40 +- .../0167_jingle_rtp/jingle_rtp_module.vala | 5 + .../xep/0167_jingle_rtp/payload_type.vala | 49 +- .../module/xep/0167_jingle_rtp/stream.vala | 7 + 13 files changed, 1126 insertions(+), 119 deletions(-) create mode 100644 cmake/FindGstRtp.cmake create mode 100644 plugins/rtp/vapi/gstreamer-rtp-1.0.vapi diff --git a/cmake/FindGstRtp.cmake b/cmake/FindGstRtp.cmake new file mode 100644 index 00000000..0756a985 --- /dev/null +++ b/cmake/FindGstRtp.cmake @@ -0,0 +1,14 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(GstRtp + PKG_CONFIG_NAME gstreamer-rtp-1.0 + LIB_NAMES gstrtp + LIB_DIR_HINTS gstreamer-1.0 + INCLUDE_NAMES gst/rtp/rtp.h + INCLUDE_DIR_SUFFIXES gstreamer-1.0 gstreamer-1.0/include gstreamer-rtp-1.0 gstreamer-rtp-1.0/include + DEPENDS Gst +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GstRtp + REQUIRED_VARS GstRtp_LIBRARY + VERSION_VAR GstRtp_VERSION) diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 76d6e66d..92ec1b97 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -1,3 +1,4 @@ +find_package(GstRtp REQUIRED) find_packages(RTP_PACKAGES REQUIRED Gee GLib @@ -27,6 +28,7 @@ CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/vapi/gstreamer-rtp-1.0.vapi PACKAGES ${RTP_PACKAGES} DEFINITIONS @@ -35,7 +37,7 @@ DEFINITIONS add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) add_library(rtp SHARED ${RTP_VALA_C}) -target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES}) +target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0) set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index 6bd465c1..7537c11d 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -6,7 +6,7 @@ public class Dino.Plugins.Rtp.CodecUtil { private Set supported_elements = new HashSet(); private Set unsupported_elements = new HashSet(); - public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type) { + public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type, bool incoming) { Gst.Caps caps = new Gst.Caps.simple("application/x-rtp", "media", typeof(string), media, "payload", typeof(int), payload_type.id); @@ -19,6 +19,15 @@ public class Dino.Plugins.Rtp.CodecUtil { if (payload_type.name != null) { s.set("encoding-name", typeof(string), payload_type.name.up()); } + if (incoming) { + foreach (JingleRtp.RtcpFeedback rtcp_fb in payload_type.rtcp_fbs) { + if (rtcp_fb.subtype == null) { + s.set(@"rtcp-fb-$(rtcp_fb.type_)", typeof(bool), true); + } else { + s.set(@"rtcp-fb-$(rtcp_fb.type_)-$(rtcp_fb.subtype)", typeof(bool), true); + } + } + } return caps; } @@ -122,32 +131,82 @@ public class Dino.Plugins.Rtp.CodecUtil { return new string[0]; } - public static string? get_encode_prefix(string media, string codec, string encode) { + public static string? get_encode_prefix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { if (encode == "msdkh264enc") return "video/x-raw,format=NV12 ! "; if (encode == "vaapih264enc") return "video/x-raw,format=NV12 ! "; return null; } - public static string? get_encode_suffix(string media, string codec, string encode) { + public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { // H264 - const string h264_suffix = " ! video/x-h264,profile=constrained-baseline ! h264parse"; - if (encode == "msdkh264enc") return @" bitrate=256 rate-control=vbr target-usage=7$h264_suffix"; - if (encode == "vaapih264enc") return @" bitrate=256 quality-level=7 tune=low-power$h264_suffix"; - if (encode == "x264enc") return @" byte-stream=1 bitrate=256 profile=baseline speed-preset=ultrafast tune=zerolatency$h264_suffix"; - if (media == "video" && codec == "h264") return h264_suffix; + if (encode == "msdkh264enc") return @" rate-control=vbr"; + if (encode == "vaapih264enc") return @" tune=low-power"; + if (encode == "x264enc") return @" byte-stream=1 profile=baseline speed-preset=ultrafast tune=zerolatency"; // VP8 - if (encode == "msdkvp8enc") return " bitrate=256 rate-control=vbr target-usage=7"; - if (encode == "vaapivp8enc") return " bitrate=256 rate-control=vbr quality-level=7"; - if (encode == "vp8enc") return " target-bitrate=256000 deadline=1 error-resilient=1"; + if (encode == "msdkvp8enc") return " rate-control=vbr"; + if (encode == "vaapivp8enc") return " rate-control=vbr"; + if (encode == "vp8enc") return " deadline=1 error-resilient=1"; // OPUS - if (encode == "opusenc") return " audio-type=voice"; + if (encode == "opusenc") { + if (payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " audio-type=voice inband-fec=true"; + return " audio-type=voice"; + } return null; } - public static string? get_decode_prefix(string media, string codec, string decode) { + public static string? get_encode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + // H264 + if (media == "video" && codec == "h264") return " ! video/x-h264,profile=constrained-baseline ! h264parse"; + return null; + } + + public uint update_bitrate(string media, JingleRtp.PayloadType payload_type, Gst.Element encode_element, uint bitrate) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return 0; + string? codec = get_codec_from_payload(media, payload_type); + string? encode_name = get_encode_element_name(media, codec); + if (encode_name == null) return 0; + Gst.Element encode = encode_bin.get_by_name(@"$(encode_bin.name)_encode"); + + bitrate = uint.min(2048000, bitrate); + + switch (encode_name) { + case "msdkh264enc": + case "vaapih264enc": + case "x264enc": + case "msdkvp8enc": + case "vaapivp8enc": + bitrate = uint.min(2048000, bitrate); + encode.set("bitrate", bitrate); + return bitrate; + case "vp8enc": + bitrate = uint.min(2147483, bitrate); + encode.set("target-bitrate", bitrate * 1000); + return bitrate; + } + + return 0; + } + + public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { + return null; + } + + public static string? get_decode_args(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { + if (decode == "opusdec" && payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " use-inband-fec=true"; + if (decode == "vaapivp9dec" || decode == "vaapivp8dec" || decode == "vaapih264dec") return " max-errors=100"; + return null; + } + + public static string? get_decode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + return null; + } + + public static string? get_depay_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + if (codec == "vp8") return " wait-for-keyframe=true"; return null; } @@ -195,21 +254,24 @@ public class Dino.Plugins.Rtp.CodecUtil { unsupported_elements.add(element_name); } - public string? get_decode_bin_description(string media, string? codec, string? element_name = null, string? name = null) { + public string? get_decode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { if (codec == null) return null; string base_name = name ?? @"encode-$codec-$(Random.next_int())"; string depay = get_depay_element_name(media, codec); string decode = element_name ?? get_decode_element_name(media, codec); if (depay == null || decode == null) return null; - string decode_prefix = get_decode_prefix(media, codec, decode) ?? ""; - string resample = media == "audio" ? @" ! audioresample name=$base_name-resample" : ""; - return @"$depay name=$base_name-rtp-depay ! $decode_prefix$decode name=$base_name-decode ! $(media)convert name=$base_name-convert$resample"; + string decode_prefix = get_decode_prefix(media, codec, decode, payload_type) ?? ""; + string decode_args = get_decode_args(media, codec, decode, payload_type) ?? ""; + string decode_suffix = get_decode_suffix(media, codec, decode, payload_type) ?? ""; + string depay_args = get_depay_args(media, codec, decode, payload_type) ?? ""; + string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; + return @"$depay$depay_args name=$(base_name)_rtp_depay ! $decode_prefix$decode$decode_args name=$(base_name)_$(codec)_decode$decode_suffix ! $(media)convert name=$(base_name)_convert$resample"; } public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { string? codec = get_codec_from_payload(media, payload_type); string base_name = name ?? @"encode-$codec-$(Random.next_int())"; - string? desc = get_decode_bin_description(media, codec, null, base_name); + string? desc = get_decode_bin_description(media, codec, payload_type, null, base_name); if (desc == null) return null; debug("Pipeline to decode %s %s: %s", media, codec, desc); Gst.Element bin = Gst.parse_bin_from_description(desc, true); @@ -217,22 +279,23 @@ public class Dino.Plugins.Rtp.CodecUtil { return bin; } - public string? get_encode_bin_description(string media, string? codec, string? element_name = null, uint pt = 96, string? name = null) { + public string? get_encode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { if (codec == null) return null; - string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; string pay = get_pay_element_name(media, codec); string encode = element_name ?? get_encode_element_name(media, codec); if (pay == null || encode == null) return null; - string encode_prefix = get_encode_prefix(media, codec, encode) ?? ""; - string encode_suffix = get_encode_suffix(media, codec, encode) ?? ""; - string resample = media == "audio" ? @" ! audioresample name=$base_name-resample" : ""; - return @"$(media)convert name=$base_name-convert$resample ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; + string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? ""; + string encode_args = get_encode_args(media, codec, encode, payload_type) ?? ""; + string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? ""; + string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; + return @"$(media)convert name=$(base_name)_convert$resample ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix ! $pay pt=$(payload_type != null ? payload_type.id : 96) name=$(base_name)_rtp_pay"; } public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { string? codec = get_codec_from_payload(media, payload_type); - string base_name = name ?? @"encode-$codec-$(Random.next_int())"; - string? desc = get_encode_bin_description(media, codec, null, payload_type.id, base_name); + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? desc = get_encode_bin_description(media, codec, payload_type, null, base_name); if (desc == null) return null; debug("Pipeline to encode %s %s: %s", media, codec, desc); Gst.Element bin = Gst.parse_bin_from_description(desc, true); diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 3c9a38d2..785f853a 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -126,19 +126,20 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { element = device.create_element(id); pipe.add(element); if (is_source) { - filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); + element.@set("do-timestamp", true); + filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); filter.@set("caps", get_best_caps()); pipe.add(filter); element.link(filter); if (media == "audio" && plugin.echoprobe != null) { - dsp = Gst.ElementFactory.make("webrtcdsp", @"$id-dsp"); + dsp = Gst.ElementFactory.make("webrtcdsp", @"dsp_$id"); if (dsp != null) { dsp.@set("probe", plugin.echoprobe.name); pipe.add(dsp); filter.link(dsp); } } - tee = Gst.ElementFactory.make("tee", @"$id-tee"); + tee = Gst.ElementFactory.make("tee", @"tee_$id"); tee.@set("allow-not-linked", true); pipe.add(tee); (dsp ?? filter).link(tee); @@ -148,7 +149,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { element.@set("sync", false); } if (is_sink && media == "audio") { - filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); + filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); filter.@set("caps", get_best_caps()); pipe.add(filter); if (plugin.echoprobe != null) { diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index 231a9dde..52cc1880 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -63,7 +63,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { return supported; } - private async bool supports(string media, JingleRtp.PayloadType payload_type) { + private async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) { string codec = CodecUtil.get_codec_from_payload(media, payload_type); if (codec == null) return false; if (unsupported_codecs.contains(codec)) return false; @@ -77,7 +77,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { return false; } - string encode_bin = codec_util.get_encode_bin_description(media, codec, encode_element); + string encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element); while (!(yield pipeline_works(media, encode_bin))) { debug("%s not suited for encoding %s", encode_element, codec); codec_util.mark_element_unsupported(encode_element); @@ -87,11 +87,11 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { unsupported_codecs.add(codec); return false; } - encode_bin = codec_util.get_encode_bin_description(media, codec, encode_element); + encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element); } debug("using %s to encode %s", encode_element, codec); - string decode_bin = codec_util.get_decode_bin_description(media, codec, decode_element); + string decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element); while (!(yield pipeline_works(media, @"$encode_bin ! $decode_bin"))) { debug("%s not suited for decoding %s", decode_element, codec); codec_util.mark_element_unsupported(decode_element); @@ -101,7 +101,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { unsupported_codecs.add(codec); return false; } - decode_bin = codec_util.get_decode_bin_description(media, codec, decode_element); + decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element); } debug("using %s to decode %s", decode_element, codec); @@ -109,8 +109,21 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { return true; } + public override bool is_header_extension_supported(string media, JingleRtp.HeaderExtension ext) { + if (media == "video" && ext.uri == "urn:3gpp:video-orientation") return true; + return false; + } + + public override Gee.List get_suggested_header_extensions(string media) { + Gee.List exts = new ArrayList(); + if (media == "video") { + exts.add(new JingleRtp.HeaderExtension(1, "urn:3gpp:video-orientation")); + } + return exts; + } + public async void add_if_supported(Gee.List list, string media, JingleRtp.PayloadType payload_type) { - if (yield supports(media, payload_type)) { + if (yield is_payload_supported(media, payload_type)) { list.add(payload_type); } } @@ -118,58 +131,34 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { public override async Gee.List get_supported_payloads(string media) { Gee.List list = new ArrayList(JingleRtp.PayloadType.equals_func); if (media == "audio") { - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 2, - clockrate = 48000, - name = "opus", - id = 99 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 1, - clockrate = 32000, - name = "speex", - id = 100 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 1, - clockrate = 16000, - name = "speex", - id = 101 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 1, - clockrate = 8000, - name = "speex", - id = 102 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 1, - clockrate = 8000, - name = "PCMU", - id = 0 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 1, - clockrate = 8000, - name = "PCMA", - id = 8 - }); + var opus = new JingleRtp.PayloadType() { channels = 2, clockrate = 48000, name = "opus", id = 99 }; + opus.parameters["useinbandfec"] = "1"; + var speex32 = new JingleRtp.PayloadType() { channels = 1, clockrate = 32000, name = "speex", id = 100 }; + var speex16 = new JingleRtp.PayloadType() { channels = 1, clockrate = 16000, name = "speex", id = 101 }; + var speex8 = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "speex", id = 102 }; + var pcmu = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMU", id = 0 }; + var pcma = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMA", id = 8 }; + yield add_if_supported(list, media, opus); + yield add_if_supported(list, media, speex32); + yield add_if_supported(list, media, speex16); + yield add_if_supported(list, media, speex8); + yield add_if_supported(list, media, pcmu); + yield add_if_supported(list, media, pcma); } else if (media == "video") { - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - clockrate = 90000, - name = "H264", - id = 96 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - clockrate = 90000, - name = "VP9", - id = 97 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - clockrate = 90000, - name = "VP8", - id = 98 - }); + var h264 = new JingleRtp.PayloadType() { clockrate = 90000, name = "H264", id = 96 }; + var vp9 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP9", id = 97 }; + var vp8 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP8", id = 98 }; + var rtcp_fbs = new ArrayList(); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("goog-remb")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("ccm", "fir")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack", "pli")); + h264.rtcp_fbs.add_all(rtcp_fbs); + vp9.rtcp_fbs.add_all(rtcp_fbs); + vp8.rtcp_fbs.add_all(rtcp_fbs); + yield add_if_supported(list, media, h264); + yield add_if_supported(list, media, vp9); + yield add_if_supported(list, media, vp8); } else { warning("Unsupported media type: %s", media); } @@ -179,11 +168,15 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { public override async JingleRtp.PayloadType? pick_payload_type(string media, Gee.List payloads) { if (media == "audio") { foreach (JingleRtp.PayloadType type in payloads) { - if (yield supports(media, type)) return type; + if (yield is_payload_supported(media, type)) return adjust_payload_type(media, type.clone()); } } else if (media == "video") { + // We prefer H.264 (best support for hardware acceleration and good overall codec quality) + JingleRtp.PayloadType? h264 = payloads.first_match((it) => it.name.up() == "H264"); + if (h264 != null && yield is_payload_supported(media, h264)) return adjust_payload_type(media, h264.clone()); + // Take first of the list that we do support otherwise foreach (JingleRtp.PayloadType type in payloads) { - if (yield supports(media, type)) return type; + if (yield is_payload_supported(media, type)) return adjust_payload_type(media, type.clone()); } } else { warning("Unsupported media type: %s", media); @@ -191,6 +184,28 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { return null; } + public JingleRtp.PayloadType adjust_payload_type(string media, JingleRtp.PayloadType type) { + var iter = type.rtcp_fbs.iterator(); + while (iter.next()) { + var fb = iter.@get(); + switch (fb.type_) { + case "goog-remb": + if (fb.subtype != null) iter.remove(); + break; + case "ccm": + if (fb.subtype != "fir") iter.remove(); + break; + case "nack": + if (fb.subtype != null && fb.subtype != "pli") iter.remove(); + break; + default: + iter.remove(); + break; + } + } + return type; + } + public override JingleRtp.Stream create_stream(Jingle.Content content) { return plugin.open_stream(content); } diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index 40ad1e0f..f0ad7db2 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -65,6 +65,9 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } rtpbin.pad_added.connect(on_rtp_pad_added); rtpbin.@set("latency", 100); + rtpbin.@set("do-lost", true); + rtpbin.@set("do-sync-event", true); + rtpbin.@set("drop-on-latency", true); rtpbin.connect("signal::request-pt-map", request_pt_map, this); pipe.add(rtpbin); @@ -160,6 +163,17 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { case Gst.MessageType.QOS: // Ignore break; + case Gst.MessageType.LATENCY: + if (message.src != null && message.src.name != null && message.src is Gst.Element) { + Gst.Query latency_query = new Gst.Query.latency(); + if (((Gst.Element)message.src).query(latency_query)) { + bool live; + Gst.ClockTime min_latency, max_latency; + latency_query.parse_latency(out live, out min_latency, out max_latency); + debug("Latency message from %s: live=%s, min_latency=%s, max_latency=%s", message.src.name, live.to_string(), min_latency.to_string(), max_latency.to_string()); + } + } + break; default: debug("Pipe bus message: %s", message.type.to_string()); break; diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 3a63f3fa..23634aa3 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -19,9 +19,12 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private Gst.App.Src recv_rtp; private Gst.App.Src recv_rtcp; private Gst.Element encode; + private Gst.RTP.BasePayload encode_pay; private Gst.Element decode; + private Gst.RTP.BaseDepayload decode_depay; private Gst.Element input; private Gst.Element output; + private Gst.Element session; private Device _input_device; public Device input_device { get { return _input_device; } set { @@ -85,15 +88,15 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } // Create app elements - send_rtp = Gst.ElementFactory.make("appsink", @"rtp-sink-$rtpid") as Gst.App.Sink; + send_rtp = Gst.ElementFactory.make("appsink", @"rtp_sink_$rtpid") as Gst.App.Sink; send_rtp.async = false; - send_rtp.caps = CodecUtil.get_caps(media, payload_type); + send_rtp.caps = CodecUtil.get_caps(media, payload_type, false); send_rtp.emit_signals = true; send_rtp.sync = false; send_rtp.new_sample.connect(on_new_sample); pipe.add(send_rtp); - send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp-sink-$rtpid") as Gst.App.Sink; + send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp_sink_$rtpid") as Gst.App.Sink; send_rtcp.async = false; send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); send_rtcp.emit_signals = true; @@ -101,14 +104,14 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { send_rtcp.new_sample.connect(on_new_sample); pipe.add(send_rtcp); - recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp-src-$rtpid") as Gst.App.Src; - recv_rtp.caps = CodecUtil.get_caps(media, payload_type); + recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp_src_$rtpid") as Gst.App.Src; + recv_rtp.caps = CodecUtil.get_caps(media, payload_type, true); recv_rtp.do_timestamp = true; recv_rtp.format = Gst.Format.TIME; recv_rtp.is_live = true; pipe.add(recv_rtp); - recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp-src-$rtpid") as Gst.App.Src; + recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp_src_$rtpid") as Gst.App.Src; recv_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); recv_rtcp.do_timestamp = true; recv_rtcp.format = Gst.Format.TIME; @@ -122,7 +125,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad); // Connect input - encode = codec_util.get_encode_bin(media, payload_type, @"encode-$rtpid"); + encode = codec_util.get_encode_bin(media, payload_type, @"encode_$rtpid"); + encode_pay = (Gst.RTP.BasePayload)((Gst.Bin)encode).get_by_name(@"encode_$(rtpid)_rtp_pay"); pipe.add(encode); send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid"); encode.get_static_pad("src").link(send_rtp_sink_pad); @@ -131,7 +135,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } // Connect output - decode = codec_util.get_decode_bin(media, payload_type, @"decode-$rtpid"); + decode = codec_util.get_decode_bin(media, payload_type, @"decode_$rtpid"); + decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)encode).get_by_name(@"decode_$(rtpid)_rtp_depay"); pipe.add(decode); if (output != null) { decode.link(output); @@ -144,6 +149,110 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { created = true; push_recv_data = true; plugin.unpause(); + + GLib.Signal.emit_by_name(rtpbin, "get-session", rtpid, out session); + if (session != null && payload_type.rtcp_fbs.any_match((it) => it.type_ == "goog-remb")) { + Object internal_session; + session.@get("internal-session", out internal_session); + if (internal_session != null) { + internal_session.connect("signal::on-feedback-rtcp", on_feedback_rtcp, this); + } + Timeout.add(1000, () => remb_adjust()); + } + if (media == "video") { + codec_util.update_bitrate(media, payload_type, encode, 256); + } + } + + private uint remb = 256; + private int last_packets_lost = -1; + private uint64 last_packets_received; + private uint64 last_octets_received; + private bool remb_adjust() { + unowned Gst.Structure? stats; + if (session == null) { + debug("Session for %u finished, turning off remb adjustment", rtpid); + return Source.REMOVE; + } + session.get("stats", out stats); + if (stats == null) { + warning("No stats for session %u", rtpid); + return Source.REMOVE; + } + unowned ValueArray? source_stats; + stats.get("source-stats", typeof(ValueArray), out source_stats); + if (source_stats == null) { + warning("No source-stats for session %u", rtpid); + return Source.REMOVE; + } + foreach (Value value in source_stats.values) { + unowned Gst.Structure source_stat = (Gst.Structure) value.get_boxed(); + uint ssrc; + if (!source_stat.get_uint("ssrc", out ssrc)) continue; + if (ssrc.to_string() == participant_ssrc) { + int packets_lost; + uint64 packets_received, octets_received; + source_stat.get_int("packets-lost", out packets_lost); + source_stat.get_uint64("packets-received", out packets_received); + source_stat.get_uint64("octets-received", out octets_received); + int new_lost = packets_lost - last_packets_lost; + uint64 new_received = packets_received - last_packets_received; + uint64 new_octets = octets_received - last_octets_received; + if (new_received == 0) continue; + last_packets_lost = packets_lost; + last_packets_received = packets_received; + last_octets_received = octets_received; + double loss_rate = (double)new_lost / (double)(new_lost + new_received); + if (new_lost <= 0 || loss_rate < 0.02) { + remb = (uint)(1.08 * (double)remb); + } else if (loss_rate > 0.1) { + remb = (uint)((1.0 - 0.5 * loss_rate) * (double)remb); + } + remb = uint.max(remb, (uint)((new_octets * 8) / 1000)); + remb = uint.max(16, remb); // Never go below 16 + uint8[] data = new uint8[] { + 143, 206, 0, 5, + 0, 0, 0, 0, + 0, 0, 0, 0, + 'R', 'E', 'M', 'B', + 1, 0, 0, 0, + 0, 0, 0, 0 + }; + data[4] = (uint8)((encode_pay.ssrc >> 24) & 0xff); + data[5] = (uint8)((encode_pay.ssrc >> 16) & 0xff); + data[6] = (uint8)((encode_pay.ssrc >> 8) & 0xff); + data[7] = (uint8)(encode_pay.ssrc & 0xff); + uint8 br_exp = 0; + uint32 br_mant = remb * 1000; + uint8 bits = (uint8)Math.log2(br_mant); + if (bits > 16) { + br_exp = (uint8)bits - 16; + br_mant = br_mant >> br_exp; + } + data[17] = (uint8)((br_exp << 2) | ((br_mant >> 16) & 0x3)); + data[18] = (uint8)((br_mant >> 8) & 0xff); + data[19] = (uint8)(br_mant & 0xff); + data[20] = (uint8)((ssrc >> 24) & 0xff); + data[21] = (uint8)((ssrc >> 16) & 0xff); + data[22] = (uint8)((ssrc >> 8) & 0xff); + data[23] = (uint8)(ssrc & 0xff); + encrypt_and_send_rtcp(data); + } + } + return Source.CONTINUE; + } + + private static void on_feedback_rtcp(Gst.Element session, uint type, uint fbtype, uint sender_ssrc, uint media_ssrc, Gst.Buffer? fci, Stream self) { + if (type == 206 && fbtype == 15 && fci != null && sender_ssrc.to_string() == self.participant_ssrc) { + // https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03 + uint8[] data; + fci.extract_dup(0, fci.get_size(), out data); + if (data[0] != 'R' || data[1] != 'E' || data[2] != 'M' || data[3] != 'B') return; + uint8 br_exp = data[5] >> 2; + uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7]; + uint bitrate = (br_mant << br_exp) / 1000; + self.codec_util.update_bitrate(self.media, self.payload_type, self.encode, bitrate * 8); + } } private void prepare_local_crypto() { @@ -167,22 +276,26 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { if (crypto_session.has_encrypt) { data = crypto_session.encrypt_rtp(data); } - on_send_rtp_data(new Bytes.take(data)); + on_send_rtp_data(new Bytes.take((owned) data)); } else if (sink == send_rtcp) { - if (crypto_session.has_encrypt) { - data = crypto_session.encrypt_rtcp(data); - } - if (rtcp_mux) { - on_send_rtp_data(new Bytes.take(data)); - } else { - on_send_rtcp_data(new Bytes.take(data)); - } + encrypt_and_send_rtcp((owned) data); } else { warning("unknown sample"); } return Gst.FlowReturn.OK; } + private void encrypt_and_send_rtcp(owned uint8[] data) { + if (crypto_session.has_encrypt) { + data = crypto_session.encrypt_rtcp(data); + } + if (rtcp_mux) { + on_send_rtp_data(new Bytes.take((owned) data)); + } else { + on_send_rtcp_data(new Bytes.take((owned) data)); + } + } + private static Gst.PadProbeReturn drop_probe() { return Gst.PadProbeReturn.DROP; } @@ -211,6 +324,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { encode.get_static_pad("src").unlink(send_rtp_sink_pad); pipe.remove(encode); encode = null; + encode_pay = null; // Disconnect RTP sending if (send_rtp_src_pad != null) { @@ -243,6 +357,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { decode.set_state(Gst.State.NULL); pipe.remove(decode); decode = null; + decode_depay = null; output = null; // Disconnect output device @@ -276,6 +391,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { send_rtcp_src_pad = null; send_rtp_src_pad = null; recv_rtp_src_pad = null; + + session = null; } private void prepare_remote_crypto() { @@ -285,6 +402,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } } + private uint16 previous_video_orientation_degree = uint16.MAX; + public signal void video_orientation_changed(uint16 degree); + public override void on_recv_rtp_data(Bytes bytes) { if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) { on_recv_rtcp_data(bytes); @@ -301,6 +421,33 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } if (push_recv_data) { Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); + Gst.RTP.Buffer rtp_buffer; + if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { + if (rtp_buffer.get_extension()) { + Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation"); + if (ext != null) { + unowned uint8[] extension_data; + if (rtp_buffer.get_extension_onebyte_header(ext.id, 0, out extension_data) && extension_data.length == 1) { + bool camera = (extension_data[0] & 0x8) > 0; + bool flip = (extension_data[0] & 0x4) > 0; + uint8 rotation = extension_data[0] & 0x3; + uint16 rotation_degree = uint16.MAX; + switch(rotation) { + case 0: rotation_degree = 0; break; + case 1: rotation_degree = 90; break; + case 2: rotation_degree = 180; break; + case 3: rotation_degree = 270; break; + } + if (rotation_degree != previous_video_orientation_degree) { + video_orientation_changed(rotation_degree); + previous_video_orientation_degree = rotation_degree; + } + } + } + } + rtp_buffer.unmap(); + } + // FIXME: VAPI file in Vala < 0.49.1 has a bug that results in broken ownership of buffer in push_buffer() // We workaround by using the plain signal. The signal unfortunately will cause an unnecessary copy of // the underlying buffer, so and some point we should move over to the new version (once we require @@ -449,6 +596,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public class Dino.Plugins.Rtp.VideoStream : Stream { private Gee.List outputs = new ArrayList(); private Gst.Element output_tee; + private Gst.Element rotate; + private ulong video_orientation_changed_handler; public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { base(plugin, content); @@ -456,11 +605,15 @@ public class Dino.Plugins.Rtp.VideoStream : Stream { } public override void create() { + video_orientation_changed_handler = video_orientation_changed.connect(on_video_orientation_changed); plugin.pause(); - output_tee = Gst.ElementFactory.make("tee", null); + rotate = Gst.ElementFactory.make("videoflip", @"video_rotate_$rtpid"); + pipe.add(rotate); + output_tee = Gst.ElementFactory.make("tee", @"video_tee_$rtpid"); output_tee.@set("allow-not-linked", true); pipe.add(output_tee); - add_output(output_tee); + rotate.link(output_tee); + add_output(rotate); base.create(); foreach (Gst.Element output in outputs) { output_tee.link(output); @@ -468,19 +621,44 @@ public class Dino.Plugins.Rtp.VideoStream : Stream { plugin.unpause(); } + private void on_video_orientation_changed(uint16 degree) { + if (rotate != null) { + switch (degree) { + case 0: + rotate.@set("method", 0); + break; + case 90: + rotate.@set("method", 1); + break; + case 180: + rotate.@set("method", 2); + break; + case 270: + rotate.@set("method", 3); + break; + } + } + } + public override void destroy() { foreach (Gst.Element output in outputs) { output_tee.unlink(output); } base.destroy(); + rotate.set_locked_state(true); + rotate.set_state(Gst.State.NULL); + rotate.unlink(output_tee); + pipe.remove(rotate); + rotate = null; output_tee.set_locked_state(true); output_tee.set_state(Gst.State.NULL); pipe.remove(output_tee); output_tee = null; + disconnect(video_orientation_changed_handler); } public override void add_output(Gst.Element element) { - if (element == output_tee) { + if (element == output_tee || element == rotate) { base.add_output(element); return; } @@ -491,7 +669,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream { } public override void remove_output(Gst.Element element) { - if (element == output_tee) { + if (element == output_tee || element == rotate) { base.remove_output(element); return; } diff --git a/plugins/rtp/src/video_widget.vala b/plugins/rtp/src/video_widget.vala index fa5ba138..351069a7 100644 --- a/plugins/rtp/src/video_widget.vala +++ b/plugins/rtp/src/video_widget.vala @@ -19,7 +19,7 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge this.plugin = plugin; id = last_id++; - element = Gst.ElementFactory.make("gtksink", @"video-widget-$id"); + element = Gst.ElementFactory.make("gtksink", @"video_widget_$id"); if (element != null) { Gtk.Widget widget; element.@get("widget", out widget); @@ -51,8 +51,8 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge if (connected_stream == null) return; plugin.pause(); pipe.add(element); - convert = Gst.parse_bin_from_description(@"videoconvert name=video-widget-$id-convert", true); - convert.name = @"video-widget-$id-prepare"; + convert = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true); + convert.name = @"video_widget_$(id)_prepare"; pipe.add(convert); convert.link(element); connected_stream.add_output(convert); @@ -68,8 +68,8 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge if (connected_device == null) return; plugin.pause(); pipe.add(element); - convert = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video-widget-$id-flip ! videoconvert name=video-widget-$id-convert", true); - convert.name = @"video-widget-$id-prepare"; + convert = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); + convert.name = @"video_widget_$(id)_prepare"; pipe.add(convert); convert.link(element); connected_device.link_source().link(convert); diff --git a/plugins/rtp/vapi/gstreamer-rtp-1.0.vapi b/plugins/rtp/vapi/gstreamer-rtp-1.0.vapi new file mode 100644 index 00000000..30490896 --- /dev/null +++ b/plugins/rtp/vapi/gstreamer-rtp-1.0.vapi @@ -0,0 +1,625 @@ +// Fixme: This is fetched from development code of Vala upstream which fixed a few bugs. +/* gstreamer-rtp-1.0.vapi generated by vapigen, do not modify. */ + +[CCode (cprefix = "Gst", gir_namespace = "GstRtp", gir_version = "1.0", lower_case_cprefix = "gst_")] +namespace Gst { + namespace RTCP { + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTCPBuffer")] + public struct Buffer { + public weak Gst.Buffer buffer; + public bool add_packet (Gst.RTCP.Type type, Gst.RTCP.Packet packet); + public bool get_first_packet (Gst.RTCP.Packet packet); + public uint get_packet_count (); + public static bool map (Gst.Buffer buffer, Gst.MapFlags flags, out Gst.RTCP.Buffer rtcp); + public static Gst.Buffer @new (uint mtu); + public static Gst.Buffer new_copy_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + public static Gst.Buffer new_take_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] owned uint8[] data); + public bool unmap (); + public static bool validate (Gst.Buffer buffer); + public static bool validate_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + [Version (since = "1.6")] + public static bool validate_data_reduced ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + [Version (since = "1.6")] + public static bool validate_reduced (Gst.Buffer buffer); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTCPPacket")] + public struct Packet { + public weak Gst.RTCP.Buffer? rtcp; + public uint offset; + [Version (since = "1.10")] + public bool add_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + public bool add_rb (uint32 ssrc, uint8 fractionlost, int32 packetslost, uint32 exthighestseq, uint32 jitter, uint32 lsr, uint32 dlsr); + [Version (since = "1.10")] + public uint8 app_get_data (); + [Version (since = "1.10")] + public uint16 app_get_data_length (); + [Version (since = "1.10")] + public unowned string app_get_name (); + [Version (since = "1.10")] + public uint32 app_get_ssrc (); + [Version (since = "1.10")] + public uint8 app_get_subtype (); + [Version (since = "1.10")] + public bool app_set_data_length (uint16 wordlen); + [Version (since = "1.10")] + public void app_set_name (string name); + [Version (since = "1.10")] + public void app_set_ssrc (uint32 ssrc); + [Version (since = "1.10")] + public void app_set_subtype (uint8 subtype); + public bool bye_add_ssrc (uint32 ssrc); + public bool bye_add_ssrcs ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint32[] ssrc); + public uint32 bye_get_nth_ssrc (uint nth); + public string bye_get_reason (); + public uint8 bye_get_reason_len (); + public uint bye_get_ssrc_count (); + public bool bye_set_reason (string reason); + [Version (since = "1.10")] + public bool copy_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] out uint8[] data); + public uint8 fb_get_fci (); + public uint16 fb_get_fci_length (); + public uint32 fb_get_media_ssrc (); + public uint32 fb_get_sender_ssrc (); + public Gst.RTCP.FBType fb_get_type (); + public bool fb_set_fci_length (uint16 wordlen); + public void fb_set_media_ssrc (uint32 ssrc); + public void fb_set_sender_ssrc (uint32 ssrc); + public void fb_set_type (Gst.RTCP.FBType type); + public uint8 get_count (); + public uint16 get_length (); + public bool get_padding (); + [Version (since = "1.10")] + public bool get_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] out unowned uint8[] data); + [Version (since = "1.10")] + public uint16 get_profile_specific_ext_length (); + public void get_rb (uint nth, out uint32 ssrc, out uint8 fractionlost, out int32 packetslost, out uint32 exthighestseq, out uint32 jitter, out uint32 lsr, out uint32 dlsr); + public uint get_rb_count (); + public Gst.RTCP.Type get_type (); + public bool move_to_next (); + public bool remove (); + public uint32 rr_get_ssrc (); + public void rr_set_ssrc (uint32 ssrc); + public bool sdes_add_entry (Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] uint8[] data); + public bool sdes_add_item (uint32 ssrc); + public bool sdes_copy_entry (out Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] out uint8[] data); + public bool sdes_first_entry (); + public bool sdes_first_item (); + public bool sdes_get_entry (out Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] out unowned uint8[] data); + public uint sdes_get_item_count (); + public uint32 sdes_get_ssrc (); + public bool sdes_next_entry (); + public bool sdes_next_item (); + public void set_rb (uint nth, uint32 ssrc, uint8 fractionlost, int32 packetslost, uint32 exthighestseq, uint32 jitter, uint32 lsr, uint32 dlsr); + public void sr_get_sender_info (out uint32 ssrc, out uint64 ntptime, out uint32 rtptime, out uint32 packet_count, out uint32 octet_count); + public void sr_set_sender_info (uint32 ssrc, uint64 ntptime, uint32 rtptime, uint32 packet_count, uint32 octet_count); + [Version (since = "1.16")] + public bool xr_first_rb (); + [Version (since = "1.16")] + public uint16 xr_get_block_length (); + [Version (since = "1.16")] + public Gst.RTCP.XRType xr_get_block_type (); + [Version (since = "1.16")] + public bool xr_get_dlrr_block (uint nth, out uint32 ssrc, out uint32 last_rr, out uint32 delay); + [Version (since = "1.16")] + public bool xr_get_prt_by_seq (uint16 seq, out uint32 receipt_time); + [Version (since = "1.16")] + public bool xr_get_prt_info (out uint32 ssrc, out uint8 thinning, out uint16 begin_seq, out uint16 end_seq); + [Version (since = "1.16")] + public bool xr_get_rle_info (out uint32 ssrc, out uint8 thinning, out uint16 begin_seq, out uint16 end_seq, out uint32 chunk_count); + [Version (since = "1.16")] + public bool xr_get_rle_nth_chunk (uint nth, out uint16 chunk); + [Version (since = "1.16")] + public bool xr_get_rrt (out uint64 timestamp); + [Version (since = "1.16")] + public uint32 xr_get_ssrc (); + [Version (since = "1.16")] + public bool xr_get_summary_info (out uint32 ssrc, out uint16 begin_seq, out uint16 end_seq); + [Version (since = "1.16")] + public bool xr_get_summary_jitter (out uint32 min_jitter, out uint32 max_jitter, out uint32 mean_jitter, out uint32 dev_jitter); + [Version (since = "1.16")] + public bool xr_get_summary_pkt (out uint32 lost_packets, out uint32 dup_packets); + [Version (since = "1.16")] + public bool xr_get_summary_ttl (out bool is_ipv4, out uint8 min_ttl, out uint8 max_ttl, out uint8 mean_ttl, out uint8 dev_ttl); + [Version (since = "1.16")] + public bool xr_get_voip_burst_metrics (out uint8 burst_density, out uint8 gap_density, out uint16 burst_duration, out uint16 gap_duration); + [Version (since = "1.16")] + public bool xr_get_voip_configuration_params (out uint8 gmin, out uint8 rx_config); + [Version (since = "1.16")] + public bool xr_get_voip_delay_metrics (out uint16 roundtrip_delay, out uint16 end_system_delay); + [Version (since = "1.16")] + public bool xr_get_voip_jitter_buffer_params (out uint16 jb_nominal, out uint16 jb_maximum, out uint16 jb_abs_max); + [Version (since = "1.16")] + public bool xr_get_voip_metrics_ssrc (out uint32 ssrc); + [Version (since = "1.16")] + public bool xr_get_voip_packet_metrics (out uint8 loss_rate, out uint8 discard_rate); + [Version (since = "1.16")] + public bool xr_get_voip_quality_metrics (out uint8 r_factor, out uint8 ext_r_factor, out uint8 mos_lq, out uint8 mos_cq); + [Version (since = "1.16")] + public bool xr_get_voip_signal_metrics (out uint8 signal_level, out uint8 noise_level, out uint8 rerl, out uint8 gmin); + [Version (since = "1.16")] + public bool xr_next_rb (); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_", type_id = "gst_rtcpfb_type_get_type ()")] + [GIR (name = "RTCPFBType")] + public enum FBType { + FB_TYPE_INVALID, + RTPFB_TYPE_NACK, + RTPFB_TYPE_TMMBR, + RTPFB_TYPE_TMMBN, + RTPFB_TYPE_RTCP_SR_REQ, + RTPFB_TYPE_TWCC, + PSFB_TYPE_PLI, + PSFB_TYPE_SLI, + PSFB_TYPE_RPSI, + PSFB_TYPE_AFB, + PSFB_TYPE_FIR, + PSFB_TYPE_TSTR, + PSFB_TYPE_TSTN, + PSFB_TYPE_VBCN + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_SDES_", type_id = "gst_rtcpsdes_type_get_type ()")] + [GIR (name = "RTCPSDESType")] + public enum SDESType { + INVALID, + END, + CNAME, + NAME, + EMAIL, + PHONE, + LOC, + TOOL, + NOTE, + PRIV; + [CCode (cname = "gst_rtcp_sdes_name_to_type")] + public static Gst.RTCP.SDESType from_string (string name); + [CCode (cname = "gst_rtcp_sdes_type_to_name")] + public unowned string to_string (); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_TYPE_", type_id = "gst_rtcp_type_get_type ()")] + [GIR (name = "RTCPType")] + public enum Type { + INVALID, + SR, + RR, + SDES, + BYE, + APP, + RTPFB, + PSFB, + XR + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_XR_TYPE_", type_id = "gst_rtcpxr_type_get_type ()")] + [GIR (name = "RTCPXRType")] + [Version (since = "1.16")] + public enum XRType { + INVALID, + LRLE, + DRLE, + PRT, + RRT, + DLRR, + SSUMM, + VOIP_METRICS + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_BYE_SSRC_COUNT")] + public const int MAX_BYE_SSRC_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_RB_COUNT")] + public const int MAX_RB_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_SDES")] + public const int MAX_SDES; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_SDES_ITEM_COUNT")] + public const int MAX_SDES_ITEM_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_REDUCED_SIZE_VALID_MASK")] + public const int REDUCED_SIZE_VALID_MASK; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VALID_MASK")] + public const int VALID_MASK; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VALID_VALUE")] + public const int VALID_VALUE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VERSION")] + public const int VERSION; + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static uint64 ntp_to_unix (uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static uint64 unix_to_ntp (uint64 unixtime); + } + namespace RTP { + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_audio_payload_get_type ()")] + [GIR (name = "RTPBaseAudioPayload")] + public class BaseAudioPayload : Gst.RTP.BasePayload { + public Gst.ClockTime base_ts; + public int frame_duration; + public int frame_size; + public int sample_size; + [CCode (has_construct_function = false)] + protected BaseAudioPayload (); + public Gst.FlowReturn flush (uint payload_len, Gst.ClockTime timestamp); + public Gst.Base.Adapter get_adapter (); + public Gst.FlowReturn push ([CCode (array_length_cname = "payload_len", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, Gst.ClockTime timestamp); + public void set_frame_based (); + public void set_frame_options (int frame_duration, int frame_size); + public void set_sample_based (); + public void set_sample_options (int sample_size); + public void set_samplebits_options (int sample_size); + [NoAccessorMethod] + public bool buffer_list { get; set; } + } + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_depayload_get_type ()")] + [GIR (name = "RTPBaseDepayload")] + public abstract class BaseDepayload : Gst.Element { + public uint clock_rate; + public bool need_newsegment; + public weak Gst.Segment segment; + public weak Gst.Pad sinkpad; + public weak Gst.Pad srcpad; + [CCode (has_construct_function = false)] + protected BaseDepayload (); + [NoWrapper] + public virtual bool handle_event (Gst.Event event); + [Version (since = "1.16")] + public bool is_source_info_enabled (); + [NoWrapper] + public virtual bool packet_lost (Gst.Event event); + [NoWrapper] + public virtual Gst.Buffer process (Gst.Buffer @in); + [NoWrapper] + public virtual Gst.Buffer process_rtp_packet (Gst.RTP.Buffer rtp_buffer); + public Gst.FlowReturn push (Gst.Buffer out_buf); + public Gst.FlowReturn push_list (Gst.BufferList out_list); + [NoWrapper] + public virtual bool set_caps (Gst.Caps caps); + [Version (since = "1.16")] + public void set_source_info_enabled (bool enable); + [NoAccessorMethod] + [Version (since = "1.20")] + public bool auto_header_extension { get; set; } + [NoAccessorMethod] + [Version (since = "1.18")] + public int max_reorder { get; set; } + [NoAccessorMethod] + [Version (since = "1.16")] + public bool source_info { get; set; } + [NoAccessorMethod] + public Gst.Structure stats { owned get; } + [Version (since = "1.20")] + public signal void add_extension (owned Gst.RTP.HeaderExtension ext); + [Version (since = "1.20")] + public signal void clear_extensions (); + [Version (since = "1.20")] + public signal Gst.RTP.HeaderExtension request_extension (uint ext_id, string? ext_uri); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_payload_get_type ()")] + [GIR (name = "RTPBasePayload")] + public abstract class BasePayload : Gst.Element { + [CCode (has_construct_function = false)] + protected BasePayload (); + [Version (since = "1.16")] + public Gst.Buffer allocate_output_buffer (uint payload_len, uint8 pad_len, uint8 csrc_count); + [NoWrapper] + public virtual Gst.Caps get_caps (Gst.Pad pad, Gst.Caps filter); + [Version (since = "1.16")] + public uint get_source_count (Gst.Buffer buffer); + [NoWrapper] + public virtual Gst.FlowReturn handle_buffer (Gst.Buffer buffer); + public bool is_filled (uint size, Gst.ClockTime duration); + [Version (since = "1.16")] + public bool is_source_info_enabled (); + public Gst.FlowReturn push (Gst.Buffer buffer); + public Gst.FlowReturn push_list (Gst.BufferList list); + [NoWrapper] + public virtual bool query (Gst.Pad pad, Gst.Query query); + [NoWrapper] + public virtual bool set_caps (Gst.Caps caps); + public void set_options (string media, bool @dynamic, string encoding_name, uint32 clock_rate); + [Version (since = "1.20")] + public bool set_outcaps_structure (Gst.Structure? s); + [Version (since = "1.16")] + public void set_source_info_enabled (bool enable); + [NoWrapper] + public virtual bool sink_event (Gst.Event event); + [NoWrapper] + public virtual bool src_event (Gst.Event event); + [NoAccessorMethod] + [Version (since = "1.20")] + public bool auto_header_extension { get; set; } + [NoAccessorMethod] + public int64 max_ptime { get; set; } + [NoAccessorMethod] + public int64 min_ptime { get; set; } + [NoAccessorMethod] + public uint mtu { get; set; } + [NoAccessorMethod] + [Version (since = "1.16")] + public bool onvif_no_rate_control { get; set; } + [NoAccessorMethod] + public bool perfect_rtptime { get; set; } + [NoAccessorMethod] + public uint pt { get; set; } + [NoAccessorMethod] + public int64 ptime_multiple { get; set; } + [NoAccessorMethod] + [Version (since = "1.18")] + public bool scale_rtptime { get; set; } + [NoAccessorMethod] + public uint seqnum { get; } + [NoAccessorMethod] + public int seqnum_offset { get; set; } + [NoAccessorMethod] + [Version (since = "1.16")] + public bool source_info { get; set; } + [NoAccessorMethod] + public uint ssrc { get; set; } + [NoAccessorMethod] + public Gst.Structure stats { owned get; } + [NoAccessorMethod] + public uint timestamp { get; } + [NoAccessorMethod] + public uint timestamp_offset { get; set; } + [Version (since = "1.20")] + public signal void add_extension (owned Gst.RTP.HeaderExtension ext); + [Version (since = "1.20")] + public signal void clear_extensions (); + [Version (since = "1.20")] + public signal Gst.RTP.HeaderExtension request_extension (uint ext_id, string ext_uri); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_header_extension_get_type ()")] + [GIR (name = "RTPHeaderExtension")] + [Version (since = "1.20")] + public abstract class HeaderExtension : Gst.Element { + public uint ext_id; + [CCode (has_construct_function = false)] + protected HeaderExtension (); + public static Gst.RTP.HeaderExtension? create_from_uri (string uri); + public uint get_id (); + public virtual size_t get_max_size (Gst.Buffer input_meta); + public string get_sdp_caps_field_name (); + public virtual Gst.RTP.HeaderExtensionFlags get_supported_flags (); + public unowned string get_uri (); + public virtual bool read (Gst.RTP.HeaderExtensionFlags read_flags, [CCode (array_length_cname = "size", array_length_pos = 2.5, array_length_type = "gsize", type = "const guint8*")] uint8[] data, Gst.Buffer buffer); + public virtual bool set_attributes_from_caps (Gst.Caps caps); + public bool set_attributes_from_caps_simple_sdp (Gst.Caps caps); + public virtual bool set_caps_from_attributes (Gst.Caps caps); + public bool set_caps_from_attributes_simple_sdp (Gst.Caps caps); + public void set_id (uint ext_id); + public virtual bool set_non_rtp_sink_caps (Gst.Caps caps); + [CCode (cname = "gst_rtp_header_extension_class_set_uri")] + public class void set_uri (string uri); + public void set_wants_update_non_rtp_src_caps (bool state); + public virtual bool update_non_rtp_src_caps (Gst.Caps caps); + public virtual size_t write (Gst.Buffer input_meta, Gst.RTP.HeaderExtensionFlags write_flags, Gst.Buffer output, [CCode (array_length_cname = "size", array_length_pos = 4.1, array_length_type = "gsize", type = "guint8*")] uint8[] data); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTPBuffer")] + public struct Buffer { + public weak Gst.Buffer buffer; + public uint state; + [CCode (array_length = false)] + public weak void* data[4]; + [CCode (array_length = false)] + public weak size_t size[4]; + public bool add_extension_onebyte_header (uint8 id, [CCode (array_length_cname = "size", array_length_pos = 2.1, array_length_type = "guint")] uint8[] data); + public bool add_extension_twobytes_header (uint8 appbits, uint8 id, [CCode (array_length_cname = "size", array_length_pos = 3.1, array_length_type = "guint")] uint8[] data); + [CCode (cname = "gst_buffer_add_rtp_source_meta")] + [Version (since = "1.16")] + public static unowned Gst.RTP.SourceMeta? add_rtp_source_meta (Gst.Buffer buffer, uint32? ssrc, uint32? csrc, uint csrc_count); + public static void allocate_data (Gst.Buffer buffer, uint payload_len, uint8 pad_len, uint8 csrc_count); + public static uint calc_header_len (uint8 csrc_count); + public static uint calc_packet_len (uint payload_len, uint8 pad_len, uint8 csrc_count); + public static uint calc_payload_len (uint packet_len, uint8 pad_len, uint8 csrc_count); + public static int compare_seqnum (uint16 seqnum1, uint16 seqnum2); + public static uint32 default_clock_rate (uint8 payload_type); + public static uint64 ext_timestamp (ref uint64 exttimestamp, uint32 timestamp); + public uint32 get_csrc (uint8 idx); + public uint8 get_csrc_count (); + public bool get_extension (); + [Version (since = "1.2")] + public GLib.Bytes get_extension_bytes (out uint16 bits); + public bool get_extension_data (out uint16 bits, [CCode (array_length = false)] out unowned uint8[] data, out uint wordlen); + public bool get_extension_onebyte_header (uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 3.1, array_length_type = "guint")] out unowned uint8[] data); + [Version (since = "1.18")] + public static bool get_extension_onebyte_header_from_bytes (GLib.Bytes bytes, uint16 bit_pattern, uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 5.1, array_length_type = "guint")] out unowned uint8[] data); + public bool get_extension_twobytes_header (out uint8 appbits, uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 4.1, array_length_type = "guint")] out unowned uint8[] data); + public uint get_header_len (); + public bool get_marker (); + public uint get_packet_len (); + public bool get_padding (); + [CCode (array_length = false)] + public unowned uint8[] get_payload (); + public Gst.Buffer get_payload_buffer (); + [Version (since = "1.2")] + public GLib.Bytes get_payload_bytes (); + public uint get_payload_len (); + public Gst.Buffer get_payload_subbuffer (uint offset, uint len); + public uint8 get_payload_type (); + [CCode (cname = "gst_buffer_get_rtp_source_meta")] + [Version (since = "1.16")] + public static unowned Gst.RTP.SourceMeta? get_rtp_source_meta (Gst.Buffer buffer); + public uint16 get_seq (); + public uint32 get_ssrc (); + public uint32 get_timestamp (); + public uint8 get_version (); + public static bool map (Gst.Buffer buffer, Gst.MapFlags flags, out Gst.RTP.Buffer rtp); + public static Gst.Buffer new_allocate (uint payload_len, uint8 pad_len, uint8 csrc_count); + public static Gst.Buffer new_allocate_len (uint packet_len, uint8 pad_len, uint8 csrc_count); + public static Gst.Buffer new_copy_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "gsize")] uint8[] data); + public static Gst.Buffer new_take_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "gsize")] owned uint8[] data); + public void pad_to (uint len); + public void set_csrc (uint8 idx, uint32 csrc); + public void set_extension (bool extension); + public bool set_extension_data (uint16 bits, uint16 length); + public void set_marker (bool marker); + public void set_packet_len (uint len); + public void set_padding (bool padding); + public void set_payload_type (uint8 payload_type); + public void set_seq (uint16 seq); + public void set_ssrc (uint32 ssrc); + public void set_timestamp (uint32 timestamp); + public void set_version (uint8 version); + public void unmap (); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTPPayloadInfo")] + public struct PayloadInfo { + public uint8 payload_type; + public weak string media; + public weak string encoding_name; + public uint clock_rate; + public weak string encoding_parameters; + public uint bitrate; + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTPSourceMeta")] + [Version (since = "1.16")] + public struct SourceMeta { + public Gst.Meta meta; + public uint32 ssrc; + public bool ssrc_valid; + [CCode (array_length = false)] + public weak uint32 csrc[15]; + public uint csrc_count; + public bool append_csrc ([CCode (array_length_cname = "csrc_count", array_length_pos = 1.1, array_length_type = "guint", type = "const guint32*")] uint32[] csrc); + public uint get_source_count (); + public bool set_ssrc (uint32? ssrc); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_BUFFER_FLAG_", type_id = "gst_rtp_buffer_flags_get_type ()")] + [Flags] + [GIR (name = "RTPBufferFlags")] + [Version (since = "1.10")] + public enum BufferFlags { + RETRANSMISSION, + REDUNDANT, + LAST + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_BUFFER_MAP_FLAG_", type_id = "gst_rtp_buffer_map_flags_get_type ()")] + [Flags] + [GIR (name = "RTPBufferMapFlags")] + [Version (since = "1.6.1")] + public enum BufferMapFlags { + SKIP_PADDING, + LAST + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_HEADER_EXTENSION_", type_id = "gst_rtp_header_extension_flags_get_type ()")] + [Flags] + [GIR (name = "RTPHeaderExtensionFlags")] + [Version (since = "1.20")] + public enum HeaderExtensionFlags { + ONE_BYTE, + TWO_BYTE + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_PAYLOAD_", type_id = "gst_rtp_payload_get_type ()")] + [GIR (name = "RTPPayload")] + public enum Payload { + PCMU, + @1016, + G721, + GSM, + G723, + DVI4_8000, + DVI4_16000, + LPC, + PCMA, + G722, + L16_STEREO, + L16_MONO, + QCELP, + CN, + MPA, + G728, + DVI4_11025, + DVI4_22050, + G729, + CELLB, + JPEG, + NV, + H261, + MPV, + MP2T, + H263; + public const string @1016_STRING; + public const string CELLB_STRING; + public const string CN_STRING; + public const string DVI4_11025_STRING; + public const string DVI4_16000_STRING; + public const string DVI4_22050_STRING; + public const string DVI4_8000_STRING; + public const string DYNAMIC_STRING; + public const string G721_STRING; + public const string G722_STRING; + public const int G723_53; + public const string G723_53_STRING; + public const int G723_63; + public const string G723_63_STRING; + public const string G723_STRING; + public const string G728_STRING; + public const string G729_STRING; + public const string GSM_STRING; + public const string H261_STRING; + public const string H263_STRING; + public const string JPEG_STRING; + public const string L16_MONO_STRING; + public const string L16_STEREO_STRING; + public const string LPC_STRING; + public const string MP2T_STRING; + public const string MPA_STRING; + public const string MPV_STRING; + public const string NV_STRING; + public const string PCMA_STRING; + public const string PCMU_STRING; + public const string QCELP_STRING; + public const int TS41; + public const string TS41_STRING; + public const int TS48; + public const string TS48_STRING; + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_PROFILE_", type_id = "gst_rtp_profile_get_type ()")] + [GIR (name = "RTPProfile")] + [Version (since = "1.6")] + public enum Profile { + UNKNOWN, + AVP, + SAVP, + AVPF, + SAVPF + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_BASE")] + public const string HDREXT_BASE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_ELEMENT_CLASS")] + [Version (since = "1.20")] + public const string HDREXT_ELEMENT_CLASS; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_56")] + public const string HDREXT_NTP_56; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_56_SIZE")] + public const int HDREXT_NTP_56_SIZE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_64")] + public const string HDREXT_NTP_64; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_64_SIZE")] + public const int HDREXT_NTP_64_SIZE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HEADER_EXTENSION_URI_METADATA_KEY")] + [Version (since = "1.20")] + public const string HEADER_EXTENSION_URI_METADATA_KEY; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_SOURCE_META_MAX_CSRC_COUNT")] + public const int SOURCE_META_MAX_CSRC_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_VERSION")] + public const int VERSION; + [CCode (cheader_filename = "gst/rtp/rtp.h")] + [Version (since = "1.20")] + public static GLib.List get_header_extension_list (); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_get_ntp_56 ([CCode (array_length_cname = "size", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, out uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_get_ntp_64 ([CCode (array_length_cname = "size", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, out uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_set_ntp_56 (void* data, uint size, uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_set_ntp_64 (void* data, uint size, uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static unowned Gst.RTP.PayloadInfo? payload_info_for_name (string media, string encoding_name); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static unowned Gst.RTP.PayloadInfo? payload_info_for_pt (uint8 payload_type); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static GLib.Type source_meta_api_get_type (); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static unowned Gst.MetaInfo? source_meta_get_info (); + } +} diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index d6f1acd2..d4440169 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -17,6 +17,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { public bool encryption_required { get; private set; default = false; } public PayloadType? agreed_payload_type { get; private set; } public Gee.List payload_types = new ArrayList(PayloadType.equals_func); + public Gee.List header_extensions = new ArrayList(); public Gee.List remote_cryptos = new ArrayList(); public Crypto? local_crypto = null; public Crypto? remote_crypto = null; @@ -54,9 +55,12 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { this.remote_cryptos.add(Crypto.parse(crypto)); } } - foreach (StanzaNode payloadType in node.get_subnodes("payload-type")) { + foreach (StanzaNode payloadType in node.get_subnodes(PayloadType.NAME)) { this.payload_types.add(PayloadType.parse(payloadType)); } + foreach (StanzaNode subnode in node.get_subnodes(HeaderExtension.NAME, HeaderExtension.NS_URI)) { + this.header_extensions.add(HeaderExtension.parse(subnode)); + } } public async void handle_proposed_content(XmppStream stream, Jingle.Session session, Jingle.Content content) { @@ -66,6 +70,11 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { content.reject(); return; } + // Drop unsupported header extensions + var iter = header_extensions.iterator(); + while(iter.next()) { + if (!parent.is_header_extension_supported(media, iter.@get())) iter.remove(); + } remote_crypto = parent.pick_remote_crypto(remote_cryptos); if (local_crypto == null && remote_crypto != null) { local_crypto = parent.pick_local_crypto(remote_crypto); @@ -151,7 +160,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { Gee.List crypto_nodes = description_node.get_deep_subnodes("encryption", "crypto"); if (crypto_nodes.size == 0) { - warning("Counterpart didn't include any cryptos"); + debug("Counterpart didn't include any cryptos"); if (encryption_required) { return; } @@ -182,6 +191,9 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { ret.put_node(payload_type.to_xml()); } } + foreach (HeaderExtension ext in header_extensions) { + ret.put_node(ext.to_xml()); + } if (local_crypto != null) { ret.put_node(new StanzaNode.build("encryption", NS_URI) .put_node(local_crypto.to_xml())); @@ -191,4 +203,28 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { } return ret; } +} + +public class Xmpp.Xep.JingleRtp.HeaderExtension { + public const string NS_URI = "urn:xmpp:jingle:apps:rtp:rtp-hdrext:0"; + public const string NAME = "rtp-hdrext"; + + public uint8 id { get; private set; } + public string uri { get; private set; } + + public HeaderExtension(uint8 id, string uri) { + this.id = id; + this.uri = uri; + } + + public static HeaderExtension parse(StanzaNode node) { + return new HeaderExtension((uint8) node.get_attribute_int("id"), node.get_attribute("uri")); + } + + public StanzaNode to_xml() { + return new StanzaNode.build(NAME, NS_URI) + .add_self_xmlns() + .put_attribute("id", id.to_string()) + .put_attribute("uri", uri); + } } \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala index 3adad114..6eb6289b 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala @@ -24,6 +24,8 @@ public abstract class Module : XmppStreamModule { public abstract Crypto? pick_remote_crypto(Gee.List cryptos); public abstract Crypto? pick_local_crypto(Crypto? remote); public abstract Stream create_stream(Jingle.Content content); + public abstract bool is_header_extension_supported(string media, HeaderExtension ext); + public abstract Gee.List get_suggested_header_extensions(string media); public abstract void close_stream(Stream stream); public async Jingle.Session start_call(XmppStream stream, Jid receiver_full_jid, bool video, string? sid = null) throws Jingle.Error { @@ -40,6 +42,7 @@ public abstract class Module : XmppStreamModule { // Create audio content Parameters audio_content_parameters = new Parameters(this, "audio", yield get_supported_payloads("audio")); audio_content_parameters.local_crypto = generate_local_crypto(); + audio_content_parameters.header_extensions.add_all(get_suggested_header_extensions("audio")); Jingle.Transport? audio_transport = yield jingle_module.select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (audio_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable audio transports"); @@ -57,6 +60,7 @@ public abstract class Module : XmppStreamModule { // Create video content Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); video_content_parameters.local_crypto = generate_local_crypto(); + video_content_parameters.header_extensions.add_all(get_suggested_header_extensions("video")); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (video_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); @@ -98,6 +102,7 @@ public abstract class Module : XmppStreamModule { // Content for video does not yet exist -> create it Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); video_content_parameters.local_crypto = generate_local_crypto(); + video_content_parameters.header_extensions.add_all(get_suggested_header_extensions("video")); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (video_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala index 452f1d65..faba38c9 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala @@ -3,6 +3,8 @@ using Xmpp; using Xmpp.Xep; public class Xmpp.Xep.JingleRtp.PayloadType { + public const string NAME = "payload-type"; + public uint8 id { get; set; } public string? name { get; set; } public uint8 channels { get; set; default = 1; } @@ -10,6 +12,7 @@ public class Xmpp.Xep.JingleRtp.PayloadType { public uint32 maxptime { get; set; } public uint32 ptime { get; set; } public Map parameters = new HashMap(); + public Gee.List rtcp_fbs = new ArrayList(); public static PayloadType parse(StanzaNode node) { PayloadType payloadType = new PayloadType(); @@ -22,11 +25,14 @@ public class Xmpp.Xep.JingleRtp.PayloadType { foreach (StanzaNode parameter in node.get_subnodes("parameter")) { payloadType.parameters[parameter.get_attribute("name")] = parameter.get_attribute("value"); } + foreach (StanzaNode subnode in node.get_subnodes(RtcpFeedback.NAME, RtcpFeedback.NS_URI)) { + payloadType.rtcp_fbs.add(RtcpFeedback.parse(subnode)); + } return payloadType; } public StanzaNode to_xml() { - StanzaNode node = new StanzaNode.build("payload-type", NS_URI) + StanzaNode node = new StanzaNode.build(NAME, NS_URI) .put_attribute("id", id.to_string()); if (channels != 1) node.put_attribute("channels", channels.to_string()); if (clockrate != 0) node.put_attribute("clockrate", clockrate.to_string()); @@ -38,9 +44,25 @@ public class Xmpp.Xep.JingleRtp.PayloadType { .put_attribute("name", parameter) .put_attribute("value", parameters[parameter])); } + foreach (RtcpFeedback rtcp_fb in rtcp_fbs) { + node.put_node(rtcp_fb.to_xml()); + } return node; } + public PayloadType clone() { + PayloadType clone = new PayloadType(); + clone.id = id; + clone.name = name; + clone.channels = channels; + clone.clockrate = clockrate; + clone.maxptime = maxptime; + clone.ptime = ptime; + clone.parameters.set_all(parameters); + clone.rtcp_fbs.add_all(rtcp_fbs); + return clone; + } + public static bool equals_func(PayloadType a, PayloadType b) { return a.id == b.id && a.name == b.name && @@ -49,4 +71,29 @@ public class Xmpp.Xep.JingleRtp.PayloadType { a.maxptime == b.maxptime && a.ptime == b.ptime; } +} + +public class Xmpp.Xep.JingleRtp.RtcpFeedback { + public const string NS_URI = "urn:xmpp:jingle:apps:rtp:rtcp-fb:0"; + public const string NAME = "rtcp-fb"; + + public string type_ { get; private set; } + public string? subtype { get; private set; } + + public RtcpFeedback(string type, string? subtype = null) { + this.type_ = type; + this.subtype = subtype; + } + + public static RtcpFeedback parse(StanzaNode node) { + return new RtcpFeedback(node.get_attribute("type"), node.get_attribute("subtype")); + } + + public StanzaNode to_xml() { + StanzaNode node = new StanzaNode.build(NAME, NS_URI) + .add_self_xmlns() + .put_attribute("type", type_); + if (subtype != null) node.put_attribute("subtype", subtype); + return node; + } } \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala index adae11f5..65be8a0a 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala @@ -33,6 +33,13 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { } return null; }} + public Gee.List? header_extensions { get { + var content_params = content.content_params; + if (content_params is Parameters) { + return ((Parameters)content_params).header_extensions; + } + return null; + }} public bool sending { get { return content.session.senders_include_us(content.senders); }} From 5d85b6cdb0165d863aadd25d9a73707b8f5cc83e Mon Sep 17 00:00:00 2001 From: fiaxh Date: Sat, 17 Apr 2021 14:50:31 +0200 Subject: [PATCH 46/56] Handle non-existant call support --- libdino/src/plugin/interfaces.vala | 2 + libdino/src/service/calls.vala | 38 +++++++++++++++++-- .../ui/conversation_titlebar/call_entry.vala | 13 +++++-- plugins/rtp/src/codec_util.vala | 1 + plugins/rtp/src/module.vala | 6 +-- plugins/rtp/src/plugin.vala | 16 ++++++++ 6 files changed, 65 insertions(+), 11 deletions(-) diff --git a/libdino/src/plugin/interfaces.vala b/libdino/src/plugin/interfaces.vala index 8be77895..97951850 100644 --- a/libdino/src/plugin/interfaces.vala +++ b/libdino/src/plugin/interfaces.vala @@ -85,6 +85,8 @@ public abstract interface ConversationAdditionPopulator : ConversationItemPopula } public abstract interface VideoCallPlugin : Object { + + public abstract bool supports(string media); // Video widget public abstract VideoCallWidget? create_widget(WidgetType type); diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index b7374607..1d47823d 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -83,7 +83,7 @@ namespace Dino { we_should_send_video[call] = video; we_should_send_audio[call] = true; - if (yield has_jmi_resources(conversation)) { + if (has_jmi_resources(conversation)) { XmppStream? stream = stream_interactor.get_stream(conversation.account); jmi_call[conversation.account] = call; jmi_video[conversation.account] = video; @@ -245,8 +245,28 @@ namespace Dino { // If video_feed == null && !mute we're trying to mute a non-existant feed. It will be muted as soon as it is created. } - public async bool can_do_calls(Conversation conversation) { - return (yield get_call_resources(conversation)).size > 0 || yield has_jmi_resources(conversation); + public async bool can_do_audio_calls_async(Conversation conversation) { + if (!can_do_audio_calls()) return false; + return (yield get_call_resources(conversation)).size > 0 || has_jmi_resources(conversation); + } + + private bool can_do_audio_calls() { + Plugins.VideoCallPlugin? plugin = Application.get_default().plugin_registry.video_call_plugin; + if (plugin == null) return false; + + return plugin.supports("audio"); + } + + public async bool can_do_video_calls_async(Conversation conversation) { + if (!can_do_video_calls()) return false; + return (yield get_call_resources(conversation)).size > 0 || has_jmi_resources(conversation); + } + + private bool can_do_video_calls() { + Plugins.VideoCallPlugin? plugin = Application.get_default().plugin_registry.video_call_plugin; + if (plugin == null) return false; + + return plugin.supports("video"); } private async Gee.List get_call_resources(Conversation conversation) { @@ -266,7 +286,7 @@ namespace Dino { return ret; } - private async bool has_jmi_resources(Conversation conversation) { + private bool has_jmi_resources(Conversation conversation) { int64 jmi_resources = db.entity.select() .with(db.entity.jid_id, "=", db.get_jid_id(conversation.counterpart)) .join_with(db.entity_feature, db.entity.caps_hash, db.entity_feature.entity) @@ -289,6 +309,11 @@ namespace Dino { } private void on_incoming_call(Account account, Xep.Jingle.Session session) { + if (!can_do_audio_calls()) { + warning("Incoming call but no call support detected. Ignoring."); + return; + } + bool counterpart_wants_video = false; foreach (Xep.Jingle.Content content in session.contents) { Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; @@ -550,6 +575,11 @@ namespace Dino { Xep.JingleMessageInitiation.Module mi_module = stream_interactor.module_manager.get_module(account, Xep.JingleMessageInitiation.Module.IDENTITY); mi_module.session_proposed.connect((from, to, sid, descriptions) => { + if (!can_do_audio_calls()) { + warning("Incoming call but no call support detected. Ignoring."); + return; + } + bool audio_requested = descriptions.any_match((description) => description.ns_uri == Xep.JingleRtp.NS_URI && description.get_attribute("media") == "audio"); bool video_requested = descriptions.any_match((description) => description.ns_uri == Xep.JingleRtp.NS_URI && description.get_attribute("media") == "video"); if (!audio_requested && !video_requested) return; diff --git a/main/src/ui/conversation_titlebar/call_entry.vala b/main/src/ui/conversation_titlebar/call_entry.vala index 5e28ecbe..e1d10e5c 100644 --- a/main/src/ui/conversation_titlebar/call_entry.vala +++ b/main/src/ui/conversation_titlebar/call_entry.vala @@ -34,6 +34,9 @@ namespace Dino.Ui { private StreamInteractor stream_interactor; private Conversation conversation; + private ModelButton audio_button = new ModelButton() { text="Audio call", visible=true }; + private ModelButton video_button = new ModelButton() { text="Video call", visible=true }; + public CallButton(StreamInteractor stream_interactor) { this.stream_interactor = stream_interactor; @@ -42,7 +45,6 @@ namespace Dino.Ui { Gtk.PopoverMenu popover_menu = new Gtk.PopoverMenu(); Box box = new Box(Orientation.VERTICAL, 0) { margin=10, visible=true }; - ModelButton audio_button = new ModelButton() { text="Audio call", visible=true }; audio_button.clicked.connect(() => { stream_interactor.get_module(Calls.IDENTITY).initiate_call.begin(conversation, false, (_, res) => { Call call = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); @@ -50,7 +52,7 @@ namespace Dino.Ui { }); }); box.add(audio_button); - ModelButton video_button = new ModelButton() { text="Video call", visible=true }; + video_button.clicked.connect(() => { stream_interactor.get_module(Calls.IDENTITY).initiate_call.begin(conversation, true, (_, res) => { Call call = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); @@ -116,9 +118,12 @@ namespace Dino.Ui { private async void update_visibility() { if (conversation.type_ == Conversation.Type.CHAT) { Conversation conv_bak = conversation; - bool can_do_calls = yield stream_interactor.get_module(Calls.IDENTITY).can_do_calls(conversation); + bool audio_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_audio_calls_async(conversation); + bool video_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_audio_calls_async(conversation); if (conv_bak != conversation) return; - visible = can_do_calls; + + visible = audio_works; + video_button.visible = video_works; } else { visible = false; } diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index 7537c11d..6a2438f1 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -225,6 +225,7 @@ public class Dino.Plugins.Rtp.CodecUtil { } public string? get_encode_element_name(string media, string? codec) { + if (!is_element_supported(get_pay_element_name(media, codec))) return null; foreach (string candidate in get_encode_candidates(media, codec)) { if (is_element_supported(candidate)) return candidate; } diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index 52cc1880..13a21cd8 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -64,13 +64,13 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { } private async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) { - string codec = CodecUtil.get_codec_from_payload(media, payload_type); + string? codec = CodecUtil.get_codec_from_payload(media, payload_type); if (codec == null) return false; if (unsupported_codecs.contains(codec)) return false; if (supported_codecs.contains(codec)) return true; - string encode_element = codec_util.get_encode_element_name(media, codec); - string decode_element = codec_util.get_decode_element_name(media, codec); + string? encode_element = codec_util.get_encode_element_name(media, codec); + string? decode_element = codec_util.get_decode_element_name(media, codec); if (encode_element == null || decode_element == null) { debug("No suitable encoder or decoder found for %s", codec); unsupported_codecs.add(codec); diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index f0ad7db2..d43588b4 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -278,6 +278,22 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { Gst.deinit(); } + public bool supports(string media) { + if (rtpbin == null) return false; + + if (media == "audio") { + if (get_devices("audio", false).is_empty) return false; + if (get_devices("audio", true).is_empty) return false; + } + + if (media == "video") { + if (Gst.ElementFactory.make("gtksink", null) == null) return false; + if (get_devices("video", false).is_empty) return false; + } + + return true; + } + public VideoCallWidget? create_widget(WidgetType type) { if (type == WidgetType.GTK) { return new VideoWidget(this); From 421f43dd8bd993eb88581e1b5011cc061ceb4fc8 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Sun, 25 Apr 2021 19:49:10 +0200 Subject: [PATCH 47/56] Add support for OMEMO call encryption --- libdino/src/service/calls.vala | 48 ++- libdino/src/service/connection_manager.vala | 4 +- main/src/ui/call_window/call_bottom_bar.vala | 17 +- plugins/ice/src/transport_parameters.vala | 15 +- plugins/omemo/CMakeLists.txt | 4 +- .../src/dtls_srtp_verification_draft.vala | 195 +++++++++++ plugins/omemo/src/jingle/jet_omemo.vala | 78 ++--- plugins/omemo/src/logic/decrypt.vala | 210 ++++++++++++ plugins/omemo/src/logic/encrypt.vala | 131 ++++++++ plugins/omemo/src/logic/encrypt_state.vala | 24 -- plugins/omemo/src/logic/manager.vala | 16 +- plugins/omemo/src/logic/trust_manager.vala | 302 +----------------- plugins/omemo/src/plugin.vala | 22 +- plugins/omemo/src/protocol/stream_module.vala | 6 +- xmpp-vala/CMakeLists.txt | 3 + xmpp-vala/src/module/iq/module.vala | 5 + .../src/module/xep/0166_jingle/content.vala | 3 +- .../xep/0166_jingle/content_transport.vala | 2 +- .../module/xep/0166_jingle/jingle_module.vala | 4 +- .../src/module/xep/0166_jingle/session.vala | 10 +- .../0167_jingle_rtp/content_parameters.vala | 3 +- .../jingle_ice_udp_module.vala | 2 +- .../transport_parameters.vala | 6 +- .../xep/0260_jingle_socks5_bytestreams.vala | 2 +- .../xep/0261_jingle_in_band_bytestreams.vala | 2 +- .../xep/0353_jingle_message_initiation.vala | 2 + .../xep/0384_omemo/omemo_decryptor.vala | 62 ++++ .../xep/0384_omemo/omemo_encryptor.vala | 116 +++++++ 28 files changed, 859 insertions(+), 435 deletions(-) create mode 100644 plugins/omemo/src/dtls_srtp_verification_draft.vala create mode 100644 plugins/omemo/src/logic/decrypt.vala create mode 100644 plugins/omemo/src/logic/encrypt.vala delete mode 100644 plugins/omemo/src/logic/encrypt_state.vala create mode 100644 xmpp-vala/src/module/xep/0384_omemo/omemo_decryptor.vala create mode 100644 xmpp-vala/src/module/xep/0384_omemo/omemo_encryptor.vala diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index 1d47823d..3615e24f 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -40,8 +40,8 @@ namespace Dino { private HashMap video_content_parameter = new HashMap(Call.hash_func, Call.equals_func); private HashMap audio_content = new HashMap(Call.hash_func, Call.equals_func); private HashMap video_content = new HashMap(Call.hash_func, Call.equals_func); - private HashMap video_encryption = new HashMap(Call.hash_func, Call.equals_func); - private HashMap audio_encryption = new HashMap(Call.hash_func, Call.equals_func); + private HashMap> video_encryptions = new HashMap>(Call.hash_func, Call.equals_func); + private HashMap> audio_encryptions = new HashMap>(Call.hash_func, Call.equals_func); public static void start(StreamInteractor stream_interactor, Database db) { Calls m = new Calls(stream_interactor, db); @@ -498,24 +498,46 @@ namespace Dino { } if (media == "audio") { - audio_encryption[call] = content.encryption; + audio_encryptions[call] = content.encryptions; } else if (media == "video") { - video_encryption[call] = content.encryption; + video_encryptions[call] = content.encryptions; } - if ((audio_encryption.has_key(call) && audio_encryption[call] == null) || (video_encryption.has_key(call) && video_encryption[call] == null)) { + if ((audio_encryptions.has_key(call) && audio_encryptions[call].is_empty) || (video_encryptions.has_key(call) && video_encryptions[call].is_empty)) { call.encryption = Encryption.NONE; encryption_updated(call, null); return; } - Xep.Jingle.ContentEncryption encryption = audio_encryption[call] ?? video_encryption[call]; - if (encryption.encryption_ns == Xep.JingleIceUdp.DTLS_NS_URI) { - call.encryption = Encryption.DTLS_SRTP; - } else if (encryption.encryption_name == "SRTP") { - call.encryption = Encryption.SRTP; + HashMap encryptions = audio_encryptions[call] ?? video_encryptions[call]; + + Xep.Jingle.ContentEncryption? omemo_encryption = null, dtls_encryption = null, srtp_encryption = null; + foreach (string encr_name in encryptions.keys) { + if (video_encryptions.has_key(call) && !video_encryptions[call].has_key(encr_name)) continue; + + var encryption = encryptions[encr_name]; + if (encryption.encryption_ns == "http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification") { + omemo_encryption = encryption; + } else if (encryption.encryption_ns == Xep.JingleIceUdp.DTLS_NS_URI) { + dtls_encryption = encryption; + } else if (encryption.encryption_name == "SRTP") { + srtp_encryption = encryption; + } + } + + if (omemo_encryption != null && dtls_encryption != null) { + call.encryption = Encryption.OMEMO; + encryption_updated(call, omemo_encryption); + } else if (dtls_encryption != null) { + call.encryption = Encryption.DTLS_SRTP; + encryption_updated(call, dtls_encryption); + } else if (srtp_encryption != null) { + call.encryption = Encryption.SRTP; + encryption_updated(call, srtp_encryption); + } else { + call.encryption = Encryption.NONE; + encryption_updated(call, null); } - encryption_updated(call, encryption); } private void remove_call_from_datastructures(Call call) { @@ -533,8 +555,8 @@ namespace Dino { video_content_parameter.unset(call); audio_content.unset(call); video_content.unset(call); - audio_encryption.unset(call); - video_encryption.unset(call); + audio_encryptions.unset(call); + video_encryptions.unset(call); } private void on_account_added(Account account) { diff --git a/libdino/src/service/connection_manager.vala b/libdino/src/service/connection_manager.vala index 1439c6f3..0eb6a6f5 100644 --- a/libdino/src/service/connection_manager.vala +++ b/libdino/src/service/connection_manager.vala @@ -350,7 +350,9 @@ public class ConnectionManager : Object { foreach (Account account in connections.keys) { try { make_offline(account); - yield connections[account].stream.disconnect(); + if (connections[account].stream != null) { + yield connections[account].stream.disconnect(); + } } catch (Error e) { debug("Error disconnecting stream %p: %s", connections[account].stream, e.message); } diff --git a/main/src/ui/call_window/call_bottom_bar.vala b/main/src/ui/call_window/call_bottom_bar.vala index c6375ea2..a9fee8c3 100644 --- a/main/src/ui/call_window/call_bottom_bar.vala +++ b/main/src/ui/call_window/call_bottom_bar.vala @@ -100,16 +100,25 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { encryption_button.get_style_context().add_class("unencrypted"); popover.add(new Label("This call isn't encrypted.") { margin=10, visible=true } ); + } else if (encryption.encryption_name == "OMEMO") { + encryption_image.set_from_icon_name("changes-prevent-symbolic", IconSize.BUTTON); + encryption_button.get_style_context().remove_class("unencrypted"); + + popover.add(new Label("This call is encrypted with OMEMO.") { margin=10, visible=true } ); } else { encryption_image.set_from_icon_name("changes-prevent-symbolic", IconSize.BUTTON); encryption_button.get_style_context().remove_class("unencrypted"); Grid encryption_info_grid = new Grid() { margin=10, row_spacing=3, column_spacing=5, visible=true }; encryption_info_grid.attach(new Label("This call is end-to-end encrypted.") { use_markup=true, xalign=0, visible=true }, 1, 1, 2, 1); - encryption_info_grid.attach(new Label("Peer key") { xalign=0, visible=true }, 1, 2, 1, 1); - encryption_info_grid.attach(new Label("Your key") { xalign=0, visible=true }, 1, 3, 1, 1); - encryption_info_grid.attach(new Label("" + format_fingerprint(encryption.peer_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 2, 1, 1); - encryption_info_grid.attach(new Label("" + format_fingerprint(encryption.our_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 3, 1, 1); + if (encryption.peer_key.length > 0) { + encryption_info_grid.attach(new Label("Peer key") { xalign=0, visible=true }, 1, 2, 1, 1); + encryption_info_grid.attach(new Label("" + format_fingerprint(encryption.peer_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 2, 1, 1); + } + if (encryption.our_key.length > 0) { + encryption_info_grid.attach(new Label("Your key") { xalign=0, visible=true }, 1, 3, 1, 1); + encryption_info_grid.attach(new Label("" + format_fingerprint(encryption.our_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 3, 1, 1); + } popover.add(encryption_info_grid); } diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index 52451fcf..38652952 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -77,7 +77,10 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport own_setup = "actpass"; dtls_srtp_handler.mode = DtlsSrtp.Mode.SERVER; dtls_srtp_handler.setup_dtls_connection.begin((_, res) => { - this.content.encryption = dtls_srtp_handler.setup_dtls_connection.end(res) ?? this.content.encryption; + var content_encryption = dtls_srtp_handler.setup_dtls_connection.end(res); + if (content_encryption != null) { + this.content.encryptions[content_encryption.encryption_ns] = content_encryption; + } }); } } @@ -157,7 +160,10 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport dtls_srtp_handler.mode = DtlsSrtp.Mode.CLIENT; dtls_srtp_handler.stop_dtls_connection(); dtls_srtp_handler.setup_dtls_connection.begin((_, res) => { - this.content.encryption = dtls_srtp_handler.setup_dtls_connection.end(res) ?? this.content.encryption; + var content_encryption = dtls_srtp_handler.setup_dtls_connection.end(res); + if (content_encryption != null) { + this.content.encryptions[content_encryption.encryption_ns] = content_encryption; + } }); } } else { @@ -225,7 +231,10 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport may_consider_ready(stream_id, component_id); if (incoming && dtls_srtp_handler != null && !dtls_srtp_handler.ready && is_component_ready(agent, stream_id, component_id) && dtls_srtp_handler.mode == DtlsSrtp.Mode.CLIENT) { dtls_srtp_handler.setup_dtls_connection.begin((_, res) => { - this.content.encryption = dtls_srtp_handler.setup_dtls_connection.end(res) ?? this.content.encryption; + Jingle.ContentEncryption? encryption = dtls_srtp_handler.setup_dtls_connection.end(res); + if (encryption != null) { + this.content.encryptions[encryption.encryption_ns] = encryption; + } }); } } diff --git a/plugins/omemo/CMakeLists.txt b/plugins/omemo/CMakeLists.txt index c7a45069..944fc649 100644 --- a/plugins/omemo/CMakeLists.txt +++ b/plugins/omemo/CMakeLists.txt @@ -29,6 +29,7 @@ compile_gresources( vala_precompile(OMEMO_VALA_C SOURCES + src/dtls_srtp_verification_draft.vala src/plugin.vala src/register_plugin.vala src/trust_level.vala @@ -39,7 +40,8 @@ SOURCES src/jingle/jet_omemo.vala src/logic/database.vala - src/logic/encrypt_state.vala + src/logic/decrypt.vala + src/logic/encrypt.vala src/logic/manager.vala src/logic/pre_key_store.vala src/logic/session_store.vala diff --git a/plugins/omemo/src/dtls_srtp_verification_draft.vala b/plugins/omemo/src/dtls_srtp_verification_draft.vala new file mode 100644 index 00000000..e2441670 --- /dev/null +++ b/plugins/omemo/src/dtls_srtp_verification_draft.vala @@ -0,0 +1,195 @@ +using Signal; +using Gee; +using Xmpp; + +namespace Dino.Plugins.Omemo.DtlsSrtpVerificationDraft { + public const string NS_URI = "http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification"; + + public class StreamModule : XmppStreamModule { + + public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "dtls_srtp_omemo_verification_draft"); + + private VerificationSendListener send_listener = new VerificationSendListener(); + private HashMap device_id_by_jingle_sid = new HashMap(); + private HashMap> content_names_by_jingle_sid = new HashMap>(); + + private void on_preprocess_incoming_iq_set_get(XmppStream stream, Xmpp.Iq.Stanza iq) { + if (iq.type_ != Iq.Stanza.TYPE_SET) return; + + Gee.List content_nodes = iq.stanza.get_deep_subnodes(Xep.Jingle.NS_URI + ":jingle", Xep.Jingle.NS_URI + ":content"); + if (content_nodes.size == 0) return; + + string? jingle_sid = iq.stanza.get_deep_attribute(Xep.Jingle.NS_URI + ":jingle", "sid"); + if (jingle_sid == null) return; + + Xep.Omemo.OmemoDecryptor decryptor = stream.get_module(Xep.Omemo.OmemoDecryptor.IDENTITY); + + foreach (StanzaNode content_node in content_nodes) { + string? content_name = content_node.get_attribute("name"); + if (content_name == null) continue; + StanzaNode? transport_node = content_node.get_subnode("transport", Xep.JingleIceUdp.NS_URI); + if (transport_node == null) continue; + StanzaNode? fingerprint_node = transport_node.get_subnode("fingerprint", NS_URI); + if (fingerprint_node == null) continue; + StanzaNode? encrypted_node = fingerprint_node.get_subnode("encrypted", Omemo.NS_URI); + if (encrypted_node == null) continue; + + Xep.Omemo.ParsedData? parsed_data = decryptor.parse_node(encrypted_node); + if (parsed_data == null || parsed_data.ciphertext == null) continue; + + if (device_id_by_jingle_sid.has_key(jingle_sid) && device_id_by_jingle_sid[jingle_sid] != parsed_data.sid) { + warning("Expected DTLS fingerprint to be OMEMO encrypted from %s %d, but it was from %d", iq.from.to_string(), device_id_by_jingle_sid[jingle_sid], parsed_data.sid); + } + + foreach (Bytes encr_key in parsed_data.our_potential_encrypted_keys.keys) { + parsed_data.is_prekey = parsed_data.our_potential_encrypted_keys[encr_key]; + parsed_data.encrypted_key = encr_key.get_data(); + + try { + uint8[] key = decryptor.decrypt_key(parsed_data, iq.from.bare_jid); + string cleartext = decryptor.decrypt(parsed_data.ciphertext, key, parsed_data.iv); + + StanzaNode new_fingerprint_node = new StanzaNode.build("fingerprint", Xep.JingleIceUdp.DTLS_NS_URI).add_self_xmlns() + .put_node(new StanzaNode.text(cleartext)); + string? hash_attr = fingerprint_node.get_attribute("hash", NS_URI); + string? setup_attr = fingerprint_node.get_attribute("setup", NS_URI); + if (hash_attr != null) new_fingerprint_node.put_attribute("hash", hash_attr); + if (setup_attr != null) new_fingerprint_node.put_attribute("setup", setup_attr); + transport_node.put_node(new_fingerprint_node); + + device_id_by_jingle_sid[jingle_sid] = parsed_data.sid; + if (!content_names_by_jingle_sid.has_key(content_name)) { + content_names_by_jingle_sid[content_name] = new ArrayList(); + } + content_names_by_jingle_sid[content_name].add(content_name); + + stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.begin(jingle_sid, (_, res) => { + Xep.Jingle.Session? session = stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.end(res); + if (session != null) print(@"$(session.contents_map.has_key(content_name))\n"); + if (session == null || !session.contents_map.has_key(content_name)) return; + var encryption = new OmemoContentEncryption() { encryption_ns=NS_URI, encryption_name="OMEMO", our_key=new uint8[0], peer_key=new uint8[0], peer_device_id=device_id_by_jingle_sid[jingle_sid] }; + session.contents_map[content_name].encryptions[NS_URI] = encryption; + + if (iq.stanza.get_deep_attribute(Xep.Jingle.NS_URI + ":jingle", "action") == "session-accept") { + session.additional_content_add_incoming.connect(on_content_add_received); + } + }); + + break; + } catch (Error e) { + debug("Decrypting message from %s/%d failed: %s", iq.from.bare_jid.to_string(), parsed_data.sid, e.message); + } + } + } + } + + private void on_preprocess_outgoing_iq_set_get(XmppStream stream, Xmpp.Iq.Stanza iq) { + if (iq.type_ != Iq.Stanza.TYPE_SET) return; + + StanzaNode? jingle_node = iq.stanza.get_subnode("jingle", Xep.Jingle.NS_URI); + if (jingle_node == null) return; + + string? sid = jingle_node.get_attribute("sid", Xep.Jingle.NS_URI); + if (sid == null || !device_id_by_jingle_sid.has_key(sid)) return; + + Gee.List content_nodes = jingle_node.get_subnodes("content", Xep.Jingle.NS_URI); + if (content_nodes.size == 0) return; + + foreach (StanzaNode content_node in content_nodes) { + StanzaNode? transport_node = content_node.get_subnode("transport", Xep.JingleIceUdp.NS_URI); + if (transport_node == null) continue; + StanzaNode? fingerprint_node = transport_node.get_subnode("fingerprint", Xep.JingleIceUdp.DTLS_NS_URI); + if (fingerprint_node == null) continue; + string fingerprint = fingerprint_node.get_deep_string_content(); + + Xep.Omemo.OmemoEncryptor encryptor = stream.get_module(Xep.Omemo.OmemoEncryptor.IDENTITY); + Xep.Omemo.EncryptionData enc_data = encryptor.encrypt_plaintext(fingerprint); + encryptor.encrypt_key(enc_data, iq.to.bare_jid, device_id_by_jingle_sid[sid]); + + StanzaNode new_fingerprint_node = new StanzaNode.build("fingerprint", NS_URI).add_self_xmlns().put_node(enc_data.get_encrypted_node()); + string? hash_attr = fingerprint_node.get_attribute("hash", Xep.JingleIceUdp.DTLS_NS_URI); + string? setup_attr = fingerprint_node.get_attribute("setup", Xep.JingleIceUdp.DTLS_NS_URI); + if (hash_attr != null) new_fingerprint_node.put_attribute("hash", hash_attr); + if (setup_attr != null) new_fingerprint_node.put_attribute("setup", setup_attr); + transport_node.put_node(new_fingerprint_node); + + transport_node.sub_nodes.remove(fingerprint_node); + } + } + + private void on_message_received(XmppStream stream, Xmpp.MessageStanza message) { + StanzaNode? proceed_node = message.stanza.get_subnode("proceed", Xep.JingleMessageInitiation.NS_URI); + if (proceed_node == null) return; + + string? jingle_sid = proceed_node.get_attribute("id"); + if (jingle_sid == null) return; + + StanzaNode? device_node = proceed_node.get_subnode("device", NS_URI); + if (device_node == null) return; + + int device_id = device_node.get_attribute_int("id", -1); + if (device_id == -1) return; + + device_id_by_jingle_sid[jingle_sid] = device_id; + } + + private void on_session_initiate_received(XmppStream stream, Xep.Jingle.Session session) { + if (device_id_by_jingle_sid.has_key(session.sid)) { + foreach (Xep.Jingle.Content content in session.contents) { + on_content_add_received(stream, content); + } + } + session.additional_content_add_incoming.connect(on_content_add_received); + } + + private void on_content_add_received(XmppStream stream, Xep.Jingle.Content content) { + if (!content_names_by_jingle_sid.has_key(content.session.sid) || content_names_by_jingle_sid[content.session.sid].contains(content.content_name)) { + var encryption = new OmemoContentEncryption() { encryption_ns=NS_URI, encryption_name="OMEMO", our_key=new uint8[0], peer_key=new uint8[0], peer_device_id=device_id_by_jingle_sid[content.session.sid] }; + content.encryptions[encryption.encryption_ns] = encryption; + } + } + + public override void attach(XmppStream stream) { + stream.get_module(Xmpp.MessageModule.IDENTITY).received_message.connect(on_message_received); + stream.get_module(Xmpp.MessageModule.IDENTITY).send_pipeline.connect(send_listener); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_incoming_iq_set_get.connect(on_preprocess_incoming_iq_set_get); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_outgoing_iq_set_get.connect(on_preprocess_outgoing_iq_set_get); + stream.get_module(Xep.Jingle.Module.IDENTITY).session_initiate_received.connect(on_session_initiate_received); + } + + public override void detach(XmppStream stream) { + stream.get_module(Xmpp.MessageModule.IDENTITY).received_message.disconnect(on_message_received); + stream.get_module(Xmpp.MessageModule.IDENTITY).send_pipeline.disconnect(send_listener); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_incoming_iq_set_get.disconnect(on_preprocess_incoming_iq_set_get); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_outgoing_iq_set_get.disconnect(on_preprocess_outgoing_iq_set_get); + stream.get_module(Xep.Jingle.Module.IDENTITY).session_initiate_received.disconnect(on_session_initiate_received); + } + + public override string get_ns() { return NS_URI; } + + public override string get_id() { return IDENTITY.id; } + } + + public class VerificationSendListener : StanzaListener { + + private const string[] after_actions_const = {}; + + public override string action_group { get { return "REWRITE_NODES"; } } + public override string[] after_actions { get { return after_actions_const; } } + + public override async bool run(XmppStream stream, MessageStanza message) { + StanzaNode? proceed_node = message.stanza.get_subnode("proceed", Xep.JingleMessageInitiation.NS_URI); + if (proceed_node == null) return false; + + StanzaNode device_node = new StanzaNode.build("device", NS_URI).add_self_xmlns() + .put_attribute("id", stream.get_module(Omemo.StreamModule.IDENTITY).store.local_registration_id.to_string()); + proceed_node.put_node(device_node); + return false; + } + } + + public class OmemoContentEncryption : Xep.Jingle.ContentEncryption { + public int peer_device_id { get; set; } + } +} + diff --git a/plugins/omemo/src/jingle/jet_omemo.vala b/plugins/omemo/src/jingle/jet_omemo.vala index 14307be2..afcdfcd6 100644 --- a/plugins/omemo/src/jingle/jet_omemo.vala +++ b/plugins/omemo/src/jingle/jet_omemo.vala @@ -7,18 +7,15 @@ using Xmpp; using Xmpp.Xep; namespace Dino.Plugins.JetOmemo { + private const string NS_URI = "urn:xmpp:jingle:jet-omemo:0"; private const string AES_128_GCM_URI = "urn:xmpp:ciphers:aes-128-gcm-nopadding"; + public class Module : XmppStreamModule, Jet.EnvelopEncoding { public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "0396_jet_omemo"); - private Omemo.Plugin plugin; const uint KEY_SIZE = 16; const uint IV_SIZE = 12; - public Module(Omemo.Plugin plugin) { - this.plugin = plugin; - } - public override void attach(XmppStream stream) { if (stream.get_module(Jet.Module.IDENTITY) != null) { stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); @@ -44,71 +41,38 @@ public class Module : XmppStreamModule, Jet.EnvelopEncoding { } public Jet.TransportSecret decode_envolop(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, StanzaNode security) throws Jingle.IqError { - Store store = stream.get_module(Omemo.StreamModule.IDENTITY).store; StanzaNode? encrypted = security.get_subnode("encrypted", Omemo.NS_URI); if (encrypted == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: missing encrypted element"); - StanzaNode? header = encrypted.get_subnode("header", Omemo.NS_URI); - if (header == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: missing header element"); - string? iv_node = header.get_deep_string_content("iv"); - if (header == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: missing iv element"); - uint8[] iv = Base64.decode((!)iv_node); - foreach (StanzaNode key_node in header.get_subnodes("key")) { - if (key_node.get_attribute_int("rid") == store.local_registration_id) { - string? key_node_content = key_node.get_string_content(); - uint8[] key; - Address address = new Address(peer_full_jid.bare_jid.to_string(), header.get_attribute_int("sid")); - if (key_node.get_attribute_bool("prekey")) { - PreKeySignalMessage msg = Omemo.Plugin.get_context().deserialize_pre_key_signal_message(Base64.decode((!)key_node_content)); - SessionCipher cipher = store.create_session_cipher(address); - key = cipher.decrypt_pre_key_signal_message(msg); - } else { - SignalMessage msg = Omemo.Plugin.get_context().deserialize_signal_message(Base64.decode((!)key_node_content)); - SessionCipher cipher = store.create_session_cipher(address); - key = cipher.decrypt_signal_message(msg); - } - address.device_id = 0; // TODO: Hack to have address obj live longer + Xep.Omemo.OmemoDecryptor decryptor = stream.get_module(Xep.Omemo.OmemoDecryptor.IDENTITY); - uint8[] authtag = null; - if (key.length >= 32) { - int authtaglength = key.length - 16; - authtag = new uint8[authtaglength]; - uint8[] new_key = new uint8[16]; - Memory.copy(authtag, (uint8*)key + 16, 16); - Memory.copy(new_key, key, 16); - key = new_key; - } - // TODO: authtag? - return new Jet.TransportSecret(key, iv); + Xmpp.Xep.Omemo.ParsedData? data = decryptor.parse_node(encrypted); + if (data == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: bad encrypted element"); + + foreach (Bytes encr_key in data.our_potential_encrypted_keys.keys) { + data.is_prekey = data.our_potential_encrypted_keys[encr_key]; + data.encrypted_key = encr_key.get_data(); + + try { + uint8[] key = decryptor.decrypt_key(data, peer_full_jid.bare_jid); + return new Jet.TransportSecret(key, data.iv); + } catch (GLib.Error e) { + debug("Decrypting JET key from %s/%d failed: %s", peer_full_jid.bare_jid.to_string(), data.sid, e.message); } } throw new Jingle.IqError.NOT_ACCEPTABLE("Not encrypted for targeted device"); } public void encode_envelop(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, Jet.SecurityParameters security_params, StanzaNode security) { - ArrayList accounts = plugin.app.stream_interactor.get_accounts(); Store store = stream.get_module(Omemo.StreamModule.IDENTITY).store; - Account? account = null; - foreach (Account compare in accounts) { - if (compare.bare_jid.equals_bare(local_full_jid)) { - account = compare; - break; - } - } - if (account == null) { - // TODO - critical("Sending from offline account %s", local_full_jid.to_string()); - } - StanzaNode header_node; - StanzaNode encrypted_node = new StanzaNode.build("encrypted", Omemo.NS_URI).add_self_xmlns() - .put_node(header_node = new StanzaNode.build("header", Omemo.NS_URI) - .put_attribute("sid", store.local_registration_id.to_string()) - .put_node(new StanzaNode.build("iv", Omemo.NS_URI) - .put_node(new StanzaNode.text(Base64.encode(security_params.secret.initialization_vector))))); + var encryption_data = new Xep.Omemo.EncryptionData(store.local_registration_id); + encryption_data.iv = security_params.secret.initialization_vector; + encryption_data.keytag = security_params.secret.transport_key; + Xep.Omemo.OmemoEncryptor encryptor = stream.get_module(Xep.Omemo.OmemoEncryptor.IDENTITY); + encryptor.encrypt_key_to_recipient(stream, encryption_data, peer_full_jid.bare_jid); - plugin.trust_manager.encrypt_key(header_node, security_params.secret.transport_key, local_full_jid.bare_jid, new ArrayList.wrap(new Jid[] {peer_full_jid.bare_jid}), stream, account); - security.put_node(encrypted_node); + security.put_node(encryption_data.get_encrypted_node()); } public override string get_ns() { return NS_URI; } diff --git a/plugins/omemo/src/logic/decrypt.vala b/plugins/omemo/src/logic/decrypt.vala new file mode 100644 index 00000000..3cdacbf7 --- /dev/null +++ b/plugins/omemo/src/logic/decrypt.vala @@ -0,0 +1,210 @@ +using Dino.Entities; +using Qlite; +using Gee; +using Signal; +using Xmpp; + +namespace Dino.Plugins.Omemo { + + public class OmemoDecryptor : Xep.Omemo.OmemoDecryptor { + + private Account account; + private Store store; + private Database db; + private StreamInteractor stream_interactor; + private TrustManager trust_manager; + + public override uint32 own_device_id { get { return store.local_registration_id; }} + + public OmemoDecryptor(Account account, StreamInteractor stream_interactor, TrustManager trust_manager, Database db, Store store) { + this.account = account; + this.stream_interactor = stream_interactor; + this.trust_manager = trust_manager; + this.db = db; + this.store = store; + } + + public bool decrypt_message(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) { + StanzaNode? encrypted_node = stanza.stanza.get_subnode("encrypted", NS_URI); + if (encrypted_node == null || MessageFlag.get_flag(stanza) != null || stanza.from == null) return false; + + if (message.body == null && Xep.ExplicitEncryption.get_encryption_tag(stanza) == NS_URI) { + message.body = "[This message is OMEMO encrypted]"; // TODO temporary + } + if (!Plugin.ensure_context()) return false; + int identity_id = db.identity.get_id(conversation.account.id); + + MessageFlag flag = new MessageFlag(); + stanza.add_flag(flag); + + Xep.Omemo.ParsedData? data = parse_node(encrypted_node); + if (data == null || data.ciphertext == null) return false; + + + foreach (Bytes encr_key in data.our_potential_encrypted_keys.keys) { + data.is_prekey = data.our_potential_encrypted_keys[encr_key]; + data.encrypted_key = encr_key.get_data(); + Gee.List possible_jids = get_potential_message_jids(message, data, identity_id); + if (possible_jids.size == 0) { + debug("Received message from unknown entity with device id %d", data.sid); + } + + foreach (Jid possible_jid in possible_jids) { + try { + uint8[] key = decrypt_key(data, possible_jid); + string cleartext = arr_to_str(aes_decrypt(Cipher.AES_GCM_NOPADDING, key, data.iv, data.ciphertext)); + + // If we figured out which real jid a message comes from due to decryption working, save it + if (conversation.type_ == Conversation.Type.GROUPCHAT && message.real_jid == null) { + message.real_jid = possible_jid; + } + + trust_manager.message_device_id_map[message] = data.sid; + message.body = cleartext; + message.encryption = Encryption.OMEMO; + return true; + } catch (Error e) { + debug("Decrypting message from %s/%d failed: %s", possible_jid.to_string(), data.sid, e.message); + } + } + } + + if ( + encrypted_node.get_deep_string_content("payload") != null && // Ratchet forwarding doesn't contain payload and might not include us, which is ok + data.our_potential_encrypted_keys.size == 0 && // The message was not encrypted to us + stream_interactor.module_manager.get_module(message.account, StreamModule.IDENTITY).store.local_registration_id != data.sid // Message from this device. Never encrypted to itself. + ) { + db.identity_meta.update_last_message_undecryptable(identity_id, data.sid, message.time); + trust_manager.bad_message_state_updated(conversation.account, message.from, data.sid); + } + + debug("Received OMEMO encryped message that could not be decrypted."); + return false; + } + + public Gee.List get_potential_message_jids(Entities.Message message, Xmpp.Xep.Omemo.ParsedData data, int identity_id) { + Gee.List possible_jids = new ArrayList(); + if (message.type_ == Message.Type.CHAT) { + possible_jids.add(message.from.bare_jid); + } else { + if (message.real_jid != null) { + possible_jids.add(message.real_jid.bare_jid); + } else if (data.is_prekey) { + // pre key messages do store the identity key, so we can use that to find the real jid + PreKeySignalMessage msg = Plugin.get_context().deserialize_pre_key_signal_message(data.encrypted_key); + string identity_key = Base64.encode(msg.identity_key.serialize()); + foreach (Row row in db.identity_meta.get_with_device_id(identity_id, data.sid).with(db.identity_meta.identity_key_public_base64, "=", identity_key)) { + try { + possible_jids.add(new Jid(row[db.identity_meta.address_name])); + } catch (InvalidJidError e) { + warning("Ignoring invalid jid from database: %s", e.message); + } + } + } else { + // If we don't know the device name (MUC history w/o MAM), test decryption with all keys with fitting device id + foreach (Row row in db.identity_meta.get_with_device_id(identity_id, data.sid)) { + try { + possible_jids.add(new Jid(row[db.identity_meta.address_name])); + } catch (InvalidJidError e) { + warning("Ignoring invalid jid from database: %s", e.message); + } + } + } + } + return possible_jids; + } + + public override uint8[] decrypt_key(Xmpp.Xep.Omemo.ParsedData data, Jid from_jid) throws GLib.Error { + int sid = data.sid; + uint8[] ciphertext = data.ciphertext; + uint8[] encrypted_key = data.encrypted_key; + + Address address = new Address(from_jid.to_string(), sid); + uint8[] key; + + if (data.is_prekey) { + int identity_id = db.identity.get_id(account.id); + PreKeySignalMessage msg = Plugin.get_context().deserialize_pre_key_signal_message(encrypted_key); + string identity_key = Base64.encode(msg.identity_key.serialize()); + + bool ok = update_db_for_prekey(identity_id, identity_key, from_jid, sid); + if (!ok) return null; + + debug("Starting new session for decryption with device from %s/%d", from_jid.to_string(), sid); + SessionCipher cipher = store.create_session_cipher(address); + key = cipher.decrypt_pre_key_signal_message(msg); + // TODO: Finish session + } else { + debug("Continuing session for decryption with device from %s/%d", from_jid.to_string(), sid); + SignalMessage msg = Plugin.get_context().deserialize_signal_message(encrypted_key); + SessionCipher cipher = store.create_session_cipher(address); + key = cipher.decrypt_signal_message(msg); + } + + if (key.length >= 32) { + int authtaglength = key.length - 16; + uint8[] new_ciphertext = new uint8[ciphertext.length + authtaglength]; + uint8[] new_key = new uint8[16]; + Memory.copy(new_ciphertext, ciphertext, ciphertext.length); + Memory.copy((uint8*)new_ciphertext + ciphertext.length, (uint8*)key + 16, authtaglength); + Memory.copy(new_key, key, 16); + data.ciphertext = new_ciphertext; + key = new_key; + } + + return key; + } + + public override string decrypt(uint8[] ciphertext, uint8[] key, uint8[] iv) throws GLib.Error { + return arr_to_str(aes_decrypt(Cipher.AES_GCM_NOPADDING, key, iv, ciphertext)); + } + + private bool update_db_for_prekey(int identity_id, string identity_key, Jid from_jid, int sid) { + Row? device = db.identity_meta.get_device(identity_id, from_jid.to_string(), sid); + if (device != null && device[db.identity_meta.identity_key_public_base64] != null) { + if (device[db.identity_meta.identity_key_public_base64] != identity_key) { + critical("Tried to use a different identity key for a known device id."); + return false; + } + } else { + debug("Learn new device from incoming message from %s/%d", from_jid.to_string(), sid); + bool blind_trust = db.trust.get_blind_trust(identity_id, from_jid.to_string(), true); + if (db.identity_meta.insert_device_session(identity_id, from_jid.to_string(), sid, identity_key, blind_trust ? TrustLevel.TRUSTED : TrustLevel.UNKNOWN) < 0) { + critical("Failed learning a device."); + return false; + } + + XmppStream? stream = stream_interactor.get_stream(account); + if (device == null && stream != null) { + stream.get_module(StreamModule.IDENTITY).request_user_devicelist.begin(stream, from_jid); + } + } + return true; + } + + private string arr_to_str(uint8[] arr) { + // null-terminate the array + uint8[] rarr = new uint8[arr.length+1]; + Memory.copy(rarr, arr, arr.length); + return (string)rarr; + } + } + + public class DecryptMessageListener : MessageListener { + public string[] after_actions_const = new string[]{ }; + public override string action_group { get { return "DECRYPT"; } } + public override string[] after_actions { get { return after_actions_const; } } + + private HashMap decryptors; + + public DecryptMessageListener(HashMap decryptors) { + this.decryptors = decryptors; + } + + public override async bool run(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) { + decryptors[message.account].decrypt_message(message, stanza, conversation); + return false; + } + } +} + diff --git a/plugins/omemo/src/logic/encrypt.vala b/plugins/omemo/src/logic/encrypt.vala new file mode 100644 index 00000000..cd994c3a --- /dev/null +++ b/plugins/omemo/src/logic/encrypt.vala @@ -0,0 +1,131 @@ +using Gee; +using Signal; +using Dino.Entities; +using Xmpp; +using Xmpp.Xep.Omemo; + +namespace Dino.Plugins.Omemo { + + public class OmemoEncryptor : Xep.Omemo.OmemoEncryptor { + + private Account account; + private Store store; + private TrustManager trust_manager; + + public override uint32 own_device_id { get { return store.local_registration_id; }} + + public OmemoEncryptor(Account account, TrustManager trust_manager, Store store) { + this.account = account; + this.trust_manager = trust_manager; + this.store = store; + } + + public override Xep.Omemo.EncryptionData encrypt_plaintext(string plaintext) throws GLib.Error { + const uint KEY_SIZE = 16; + const uint IV_SIZE = 12; + + //Create a key and use it to encrypt the message + uint8[] key = new uint8[KEY_SIZE]; + Plugin.get_context().randomize(key); + uint8[] iv = new uint8[IV_SIZE]; + Plugin.get_context().randomize(iv); + + uint8[] aes_encrypt_result = aes_encrypt(Cipher.AES_GCM_NOPADDING, key, iv, plaintext.data); + uint8[] ciphertext = aes_encrypt_result[0:aes_encrypt_result.length - 16]; + uint8[] tag = aes_encrypt_result[aes_encrypt_result.length - 16:aes_encrypt_result.length]; + uint8[] keytag = new uint8[key.length + tag.length]; + Memory.copy(keytag, key, key.length); + Memory.copy((uint8*)keytag + key.length, tag, tag.length); + + var ret = new Xep.Omemo.EncryptionData(own_device_id); + ret.ciphertext = ciphertext; + ret.keytag = keytag; + ret.iv = iv; + return ret; + } + + public EncryptState encrypt(MessageStanza message, Jid self_jid, Gee.List recipients, XmppStream stream) { + + EncryptState status = new EncryptState(); + if (!Plugin.ensure_context()) return status; + if (message.to == null) return status; + + try { + EncryptionData enc_data = encrypt_plaintext(message.body); + status = encrypt_key_to_recipients(enc_data, self_jid, recipients, stream); + + message.stanza.put_node(enc_data.get_encrypted_node()); + Xep.ExplicitEncryption.add_encryption_tag_to_message(message, NS_URI, "OMEMO"); + message.body = "[This message is OMEMO encrypted]"; + status.encrypted = true; + } catch (Error e) { + warning(@"Signal error while encrypting message: $(e.message)\n"); + message.body = "[OMEMO encryption failed]"; + status.encrypted = false; + } + return status; + } + + internal EncryptState encrypt_key_to_recipients(EncryptionData enc_data, Jid self_jid, Gee.List recipients, XmppStream stream) throws Error { + EncryptState status = new EncryptState(); + + //Check we have the bundles and device lists needed to send the message + if (!trust_manager.is_known_address(account, self_jid)) return status; + status.own_list = true; + status.own_devices = trust_manager.get_trusted_devices(account, self_jid).size; + status.other_waiting_lists = 0; + status.other_devices = 0; + foreach (Jid recipient in recipients) { + if (!trust_manager.is_known_address(account, recipient)) { + status.other_waiting_lists++; + } + if (status.other_waiting_lists > 0) return status; + status.other_devices += trust_manager.get_trusted_devices(account, recipient).size; + } + if (status.own_devices == 0 || status.other_devices == 0) return status; + + + //Encrypt the key for each recipient's device individually + foreach (Jid recipient in recipients) { + EncryptionResult enc_res = encrypt_key_to_recipient(stream, enc_data, recipient); + status.add_result(enc_res, false); + } + + // Encrypt the key for each own device + EncryptionResult enc_res = encrypt_key_to_recipient(stream, enc_data, self_jid); + status.add_result(enc_res, true); + + return status; + } + + public override EncryptionResult encrypt_key_to_recipient(XmppStream stream, Xep.Omemo.EncryptionData enc_data, Jid recipient) throws GLib.Error { + var result = new EncryptionResult(); + StreamModule module = stream.get_module(StreamModule.IDENTITY); + + foreach(int32 device_id in trust_manager.get_trusted_devices(account, recipient)) { + if (module.is_ignored_device(recipient, device_id)) { + result.lost++; + continue; + } + try { + encrypt_key(enc_data, recipient, device_id); + result.success++; + } catch (Error e) { + if (e.code == ErrorCode.UNKNOWN) result.unknown++; + else result.failure++; + } + } + return result; + } + + public override void encrypt_key(Xep.Omemo.EncryptionData encryption_data, Jid jid, int32 device_id) throws GLib.Error { + Address address = new Address(jid.to_string(), device_id); + SessionCipher cipher = store.create_session_cipher(address); + CiphertextMessage device_key = cipher.encrypt(encryption_data.keytag); + address.device_id = 0; + debug("Created encrypted key for %s/%d", jid.to_string(), device_id); + + encryption_data.add_device_key(device_id, device_key.serialized, device_key.type == CiphertextType.PREKEY); + } + } +} \ No newline at end of file diff --git a/plugins/omemo/src/logic/encrypt_state.vala b/plugins/omemo/src/logic/encrypt_state.vala deleted file mode 100644 index fd72faf4..00000000 --- a/plugins/omemo/src/logic/encrypt_state.vala +++ /dev/null @@ -1,24 +0,0 @@ -namespace Dino.Plugins.Omemo { - -public class EncryptState { - public bool encrypted { get; internal set; } - public int other_devices { get; internal set; } - public int other_success { get; internal set; } - public int other_lost { get; internal set; } - public int other_unknown { get; internal set; } - public int other_failure { get; internal set; } - public int other_waiting_lists { get; internal set; } - - public int own_devices { get; internal set; } - public int own_success { get; internal set; } - public int own_lost { get; internal set; } - public int own_unknown { get; internal set; } - public int own_failure { get; internal set; } - public bool own_list { get; internal set; } - - public string to_string() { - return @"EncryptState (encrypted=$encrypted, other=(devices=$other_devices, success=$other_success, lost=$other_lost, unknown=$other_unknown, failure=$other_failure, waiting_lists=$other_waiting_lists, own=(devices=$own_devices, success=$own_success, lost=$own_lost, unknown=$own_unknown, failure=$own_failure, list=$own_list))"; - } -} - -} diff --git a/plugins/omemo/src/logic/manager.vala b/plugins/omemo/src/logic/manager.vala index 64b117c7..5552e212 100644 --- a/plugins/omemo/src/logic/manager.vala +++ b/plugins/omemo/src/logic/manager.vala @@ -13,11 +13,12 @@ public class Manager : StreamInteractionModule, Object { private StreamInteractor stream_interactor; private Database db; private TrustManager trust_manager; + private HashMap encryptors; private Map message_states = new HashMap(Entities.Message.hash_func, Entities.Message.equals_func); private class MessageState { public Entities.Message msg { get; private set; } - public EncryptState last_try { get; private set; } + public Xep.Omemo.EncryptState last_try { get; private set; } public int waiting_other_sessions { get; set; } public int waiting_own_sessions { get; set; } public bool waiting_own_devicelist { get; set; } @@ -26,11 +27,11 @@ public class Manager : StreamInteractionModule, Object { public bool will_send_now { get; private set; } public bool active_send_attempt { get; set; } - public MessageState(Entities.Message msg, EncryptState last_try) { + public MessageState(Entities.Message msg, Xep.Omemo.EncryptState last_try) { update_from_encrypt_status(msg, last_try); } - public void update_from_encrypt_status(Entities.Message msg, EncryptState new_try) { + public void update_from_encrypt_status(Entities.Message msg, Xep.Omemo.EncryptState new_try) { this.msg = msg; this.last_try = new_try; this.waiting_other_sessions = new_try.other_unknown; @@ -59,10 +60,11 @@ public class Manager : StreamInteractionModule, Object { } } - private Manager(StreamInteractor stream_interactor, Database db, TrustManager trust_manager) { + private Manager(StreamInteractor stream_interactor, Database db, TrustManager trust_manager, HashMap encryptors) { this.stream_interactor = stream_interactor; this.db = db; this.trust_manager = trust_manager; + this.encryptors = encryptors; stream_interactor.stream_negotiated.connect(on_stream_negotiated); stream_interactor.get_module(MessageProcessor.IDENTITY).pre_message_send.connect(on_pre_message_send); @@ -125,7 +127,7 @@ public class Manager : StreamInteractionModule, Object { } //Attempt to encrypt the message - EncryptState enc_state = trust_manager.encrypt(message_stanza, conversation.account.bare_jid, recipients, stream, conversation.account); + Xep.Omemo.EncryptState enc_state = encryptors[conversation.account].encrypt(message_stanza, conversation.account.bare_jid, recipients, stream); MessageState state; lock (message_states) { if (message_states.has_key(message)) { @@ -411,8 +413,8 @@ public class Manager : StreamInteractionModule, Object { return true; // TODO wait for stream? } - public static void start(StreamInteractor stream_interactor, Database db, TrustManager trust_manager) { - Manager m = new Manager(stream_interactor, db, trust_manager); + public static void start(StreamInteractor stream_interactor, Database db, TrustManager trust_manager, HashMap encryptors) { + Manager m = new Manager(stream_interactor, db, trust_manager, encryptors); stream_interactor.add_module(m); } } diff --git a/plugins/omemo/src/logic/trust_manager.vala b/plugins/omemo/src/logic/trust_manager.vala index 1e61b201..20076a43 100644 --- a/plugins/omemo/src/logic/trust_manager.vala +++ b/plugins/omemo/src/logic/trust_manager.vala @@ -12,18 +12,15 @@ public class TrustManager { private StreamInteractor stream_interactor; private Database db; - private DecryptMessageListener decrypt_message_listener; private TagMessageListener tag_message_listener; - private HashMap message_device_id_map = new HashMap(Message.hash_func, Message.equals_func); + public HashMap message_device_id_map = new HashMap(Message.hash_func, Message.equals_func); public TrustManager(StreamInteractor stream_interactor, Database db) { this.stream_interactor = stream_interactor; this.db = db; - decrypt_message_listener = new DecryptMessageListener(stream_interactor, this, db, message_device_id_map); tag_message_listener = new TagMessageListener(stream_interactor, this, db, message_device_id_map); - stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(decrypt_message_listener); stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(tag_message_listener); } @@ -69,127 +66,6 @@ public class TrustManager { } } - private StanzaNode create_encrypted_key_node(uint8[] key, Address address, Store store) throws GLib.Error { - SessionCipher cipher = store.create_session_cipher(address); - CiphertextMessage device_key = cipher.encrypt(key); - debug("Created encrypted key for %s/%d", address.name, address.device_id); - StanzaNode key_node = new StanzaNode.build("key", NS_URI) - .put_attribute("rid", address.device_id.to_string()) - .put_node(new StanzaNode.text(Base64.encode(device_key.serialized))); - if (device_key.type == CiphertextType.PREKEY) key_node.put_attribute("prekey", "true"); - return key_node; - } - - internal EncryptState encrypt_key(StanzaNode header_node, uint8[] keytag, Jid self_jid, Gee.List recipients, XmppStream stream, Account account) throws Error { - EncryptState status = new EncryptState(); - StreamModule module = stream.get_module(StreamModule.IDENTITY); - - //Check we have the bundles and device lists needed to send the message - if (!is_known_address(account, self_jid)) return status; - status.own_list = true; - status.own_devices = get_trusted_devices(account, self_jid).size; - status.other_waiting_lists = 0; - status.other_devices = 0; - foreach (Jid recipient in recipients) { - if (!is_known_address(account, recipient)) { - status.other_waiting_lists++; - } - if (status.other_waiting_lists > 0) return status; - status.other_devices += get_trusted_devices(account, recipient).size; - } - if (status.own_devices == 0 || status.other_devices == 0) return status; - - - //Encrypt the key for each recipient's device individually - Address address = new Address("", 0); - foreach (Jid recipient in recipients) { - foreach(int32 device_id in get_trusted_devices(account, recipient)) { - if (module.is_ignored_device(recipient, device_id)) { - status.other_lost++; - continue; - } - try { - address.name = recipient.bare_jid.to_string(); - address.device_id = (int) device_id; - StanzaNode key_node = create_encrypted_key_node(keytag, address, module.store); - header_node.put_node(key_node); - status.other_success++; - } catch (Error e) { - if (e.code == ErrorCode.UNKNOWN) status.other_unknown++; - else status.other_failure++; - } - } - } - - // Encrypt the key for each own device - address.name = self_jid.bare_jid.to_string(); - foreach(int32 device_id in get_trusted_devices(account, self_jid)) { - if (module.is_ignored_device(self_jid, device_id)) { - status.own_lost++; - continue; - } - if (device_id != module.store.local_registration_id) { - address.device_id = (int) device_id; - try { - StanzaNode key_node = create_encrypted_key_node(keytag, address, module.store); - header_node.put_node(key_node); - status.own_success++; - } catch (Error e) { - if (e.code == ErrorCode.UNKNOWN) status.own_unknown++; - else status.own_failure++; - } - } - } - - return status; - } - - public EncryptState encrypt(MessageStanza message, Jid self_jid, Gee.List recipients, XmppStream stream, Account account) { - const uint KEY_SIZE = 16; - const uint IV_SIZE = 12; - EncryptState status = new EncryptState(); - if (!Plugin.ensure_context()) return status; - if (message.to == null) return status; - - StreamModule module = stream.get_module(StreamModule.IDENTITY); - - try { - //Create a key and use it to encrypt the message - uint8[] key = new uint8[KEY_SIZE]; - Plugin.get_context().randomize(key); - uint8[] iv = new uint8[IV_SIZE]; - Plugin.get_context().randomize(iv); - - uint8[] aes_encrypt_result = aes_encrypt(Cipher.AES_GCM_NOPADDING, key, iv, message.body.data); - uint8[] ciphertext = aes_encrypt_result[0:aes_encrypt_result.length-16]; - uint8[] tag = aes_encrypt_result[aes_encrypt_result.length-16:aes_encrypt_result.length]; - uint8[] keytag = new uint8[key.length + tag.length]; - Memory.copy(keytag, key, key.length); - Memory.copy((uint8*)keytag + key.length, tag, tag.length); - - StanzaNode header_node; - StanzaNode encrypted_node = new StanzaNode.build("encrypted", NS_URI).add_self_xmlns() - .put_node(header_node = new StanzaNode.build("header", NS_URI) - .put_attribute("sid", module.store.local_registration_id.to_string()) - .put_node(new StanzaNode.build("iv", NS_URI) - .put_node(new StanzaNode.text(Base64.encode(iv))))) - .put_node(new StanzaNode.build("payload", NS_URI) - .put_node(new StanzaNode.text(Base64.encode(ciphertext)))); - - status = encrypt_key(header_node, keytag, self_jid, recipients, stream, account); - - message.stanza.put_node(encrypted_node); - Xep.ExplicitEncryption.add_encryption_tag_to_message(message, NS_URI, "OMEMO"); - message.body = "[This message is OMEMO encrypted]"; - status.encrypted = true; - } catch (Error e) { - warning(@"Signal error while encrypting message: $(e.message)\n"); - message.body = "[OMEMO encryption failed]"; - status.encrypted = false; - } - return status; - } - public bool is_known_address(Account account, Jid jid) { int identity_id = db.identity.get_id(account.id); if (identity_id < 0) return false; @@ -260,182 +136,6 @@ public class TrustManager { return false; } } - - private class DecryptMessageListener : MessageListener { - public string[] after_actions_const = new string[]{ }; - public override string action_group { get { return "DECRYPT"; } } - public override string[] after_actions { get { return after_actions_const; } } - - private StreamInteractor stream_interactor; - private TrustManager trust_manager; - private Database db; - private HashMap message_device_id_map; - - public DecryptMessageListener(StreamInteractor stream_interactor, TrustManager trust_manager, Database db, HashMap message_device_id_map) { - this.stream_interactor = stream_interactor; - this.trust_manager = trust_manager; - this.db = db; - this.message_device_id_map = message_device_id_map; - } - - public override async bool run(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) { - StreamModule module = stream_interactor.module_manager.get_module(conversation.account, StreamModule.IDENTITY); - Store store = module.store; - - StanzaNode? _encrypted = stanza.stanza.get_subnode("encrypted", NS_URI); - if (_encrypted == null || MessageFlag.get_flag(stanza) != null || stanza.from == null) return false; - StanzaNode encrypted = (!)_encrypted; - if (message.body == null && Xep.ExplicitEncryption.get_encryption_tag(stanza) == NS_URI) { - message.body = "[This message is OMEMO encrypted]"; // TODO temporary - }; - if (!Plugin.ensure_context()) return false; - int identity_id = db.identity.get_id(conversation.account.id); - MessageFlag flag = new MessageFlag(); - stanza.add_flag(flag); - StanzaNode? _header = encrypted.get_subnode("header"); - if (_header == null) return false; - StanzaNode header = (!)_header; - int sid = header.get_attribute_int("sid"); - if (sid <= 0) return false; - - var our_nodes = new ArrayList(); - foreach (StanzaNode key_node in header.get_subnodes("key")) { - debug("Is ours? %d =? %u", key_node.get_attribute_int("rid"), store.local_registration_id); - if (key_node.get_attribute_int("rid") == store.local_registration_id) { - our_nodes.add(key_node); - } - } - - string? payload = encrypted.get_deep_string_content("payload"); - string? iv_node = header.get_deep_string_content("iv"); - - foreach (StanzaNode key_node in our_nodes) { - string? key_node_content = key_node.get_string_content(); - if (payload == null || iv_node == null || key_node_content == null) continue; - uint8[] key; - uint8[] ciphertext = Base64.decode((!)payload); - uint8[] iv = Base64.decode((!)iv_node); - Gee.List possible_jids = new ArrayList(); - if (conversation.type_ == Conversation.Type.CHAT) { - possible_jids.add(stanza.from.bare_jid); - } else { - Jid? real_jid = message.real_jid; - if (real_jid != null) { - possible_jids.add(real_jid.bare_jid); - } else if (key_node.get_attribute_bool("prekey")) { - // pre key messages do store the identity key, so we can use that to find the real jid - PreKeySignalMessage msg = Plugin.get_context().deserialize_pre_key_signal_message(Base64.decode((!)key_node_content)); - string identity_key = Base64.encode(msg.identity_key.serialize()); - foreach (Row row in db.identity_meta.get_with_device_id(identity_id, sid).with(db.identity_meta.identity_key_public_base64, "=", identity_key)) { - try { - possible_jids.add(new Jid(row[db.identity_meta.address_name])); - } catch (InvalidJidError e) { - warning("Ignoring invalid jid from database: %s", e.message); - } - } - if (possible_jids.size != 1) { - continue; - } - } else { - // If we don't know the device name (MUC history w/o MAM), test decryption with all keys with fitting device id - foreach (Row row in db.identity_meta.get_with_device_id(identity_id, sid)) { - try { - possible_jids.add(new Jid(row[db.identity_meta.address_name])); - } catch (InvalidJidError e) { - warning("Ignoring invalid jid from database: %s", e.message); - } - } - } - } - - if (possible_jids.size == 0) { - debug("Received message from unknown entity with device id %d", sid); - } - - foreach (Jid possible_jid in possible_jids) { - try { - Address address = new Address(possible_jid.to_string(), sid); - if (key_node.get_attribute_bool("prekey")) { - Row? device = db.identity_meta.get_device(identity_id, possible_jid.to_string(), sid); - PreKeySignalMessage msg = Plugin.get_context().deserialize_pre_key_signal_message(Base64.decode((!)key_node_content)); - string identity_key = Base64.encode(msg.identity_key.serialize()); - if (device != null && device[db.identity_meta.identity_key_public_base64] != null) { - if (device[db.identity_meta.identity_key_public_base64] != identity_key) { - critical("Tried to use a different identity key for a known device id."); - continue; - } - } else { - debug("Learn new device from incoming message from %s/%d", possible_jid.to_string(), sid); - bool blind_trust = db.trust.get_blind_trust(identity_id, possible_jid.to_string(), true); - if (db.identity_meta.insert_device_session(identity_id, possible_jid.to_string(), sid, identity_key, blind_trust ? TrustLevel.TRUSTED : TrustLevel.UNKNOWN) < 0) { - critical("Failed learning a device."); - continue; - } - XmppStream? stream = stream_interactor.get_stream(conversation.account); - if (device == null && stream != null) { - module.request_user_devicelist.begin(stream, possible_jid); - } - } - debug("Starting new session for decryption with device from %s/%d", possible_jid.to_string(), sid); - SessionCipher cipher = store.create_session_cipher(address); - key = cipher.decrypt_pre_key_signal_message(msg); - // TODO: Finish session - } else { - debug("Continuing session for decryption with device from %s/%d", possible_jid.to_string(), sid); - SignalMessage msg = Plugin.get_context().deserialize_signal_message(Base64.decode((!)key_node_content)); - SessionCipher cipher = store.create_session_cipher(address); - key = cipher.decrypt_signal_message(msg); - } - //address.device_id = 0; // TODO: Hack to have address obj live longer - - if (key.length >= 32) { - int authtaglength = key.length - 16; - uint8[] new_ciphertext = new uint8[ciphertext.length + authtaglength]; - uint8[] new_key = new uint8[16]; - Memory.copy(new_ciphertext, ciphertext, ciphertext.length); - Memory.copy((uint8*)new_ciphertext + ciphertext.length, (uint8*)key + 16, authtaglength); - Memory.copy(new_key, key, 16); - ciphertext = new_ciphertext; - key = new_key; - } - - message.body = arr_to_str(aes_decrypt(Cipher.AES_GCM_NOPADDING, key, iv, ciphertext)); - message_device_id_map[message] = address.device_id; - message.encryption = Encryption.OMEMO; - flag.decrypted = true; - } catch (Error e) { - debug("Decrypting message from %s/%d failed: %s", possible_jid.to_string(), sid, e.message); - continue; - } - - // If we figured out which real jid a message comes from due to decryption working, save it - if (conversation.type_ == Conversation.Type.GROUPCHAT && message.real_jid == null) { - message.real_jid = possible_jid; - } - return false; - } - } - - if ( - payload != null && // Ratchet forwarding doesn't contain payload and might not include us, which is ok - our_nodes.size == 0 && // The message was not encrypted to us - module.store.local_registration_id != sid // Message from this device. Never encrypted to itself. - ) { - db.identity_meta.update_last_message_undecryptable(identity_id, sid, message.time); - trust_manager.bad_message_state_updated(conversation.account, message.from, sid); - } - - debug("Received OMEMO encryped message that could not be decrypted."); - return false; - } - - private string arr_to_str(uint8[] arr) { - // null-terminate the array - uint8[] rarr = new uint8[arr.length+1]; - Memory.copy(rarr, arr, arr.length); - return (string)rarr; - } - } } } diff --git a/plugins/omemo/src/plugin.vala b/plugins/omemo/src/plugin.vala index e739fc4d..7a0304d1 100644 --- a/plugins/omemo/src/plugin.vala +++ b/plugins/omemo/src/plugin.vala @@ -1,3 +1,4 @@ +using Gee; using Dino.Entities; extern const string GETTEXT_PACKAGE; @@ -20,6 +21,7 @@ public class Plugin : RootInterface, Object { } return true; } catch (Error e) { + warning("Error initializing Signal Context %s", e.message); return false; } } @@ -33,6 +35,9 @@ public class Plugin : RootInterface, Object { public DeviceNotificationPopulator device_notification_populator; public OwnNotifications own_notifications; public TrustManager trust_manager; + public DecryptMessageListener decrypt_message_listener; + public HashMap decryptors = new HashMap(Account.hash_func, Account.equals_func); + public HashMap encryptors = new HashMap(Account.hash_func, Account.equals_func); public void registered(Dino.Application app) { ensure_context(); @@ -43,22 +48,33 @@ public class Plugin : RootInterface, Object { this.contact_details_provider = new ContactDetailsProvider(this); this.device_notification_populator = new DeviceNotificationPopulator(this, this.app.stream_interactor); this.trust_manager = new TrustManager(this.app.stream_interactor, this.db); + this.app.plugin_registry.register_encryption_list_entry(list_entry); this.app.plugin_registry.register_account_settings_entry(settings_entry); this.app.plugin_registry.register_contact_details_entry(contact_details_provider); this.app.plugin_registry.register_notification_populator(device_notification_populator); this.app.plugin_registry.register_conversation_addition_populator(new BadMessagesPopulator(this.app.stream_interactor, this)); + this.app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => { - list.add(new StreamModule()); - list.add(new JetOmemo.Module(this)); + Signal.Store signal_store = Plugin.get_context().create_store(); + list.add(new StreamModule(signal_store)); + decryptors[account] = new OmemoDecryptor(account, app.stream_interactor, trust_manager, db, signal_store); + list.add(decryptors[account]); + encryptors[account] = new OmemoEncryptor(account, trust_manager,signal_store); + list.add(encryptors[account]); + list.add(new JetOmemo.Module()); + list.add(new DtlsSrtpVerificationDraft.StreamModule()); this.own_notifications = new OwnNotifications(this, this.app.stream_interactor, account); }); + decrypt_message_listener = new DecryptMessageListener(decryptors); + app.stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(decrypt_message_listener); + app.stream_interactor.get_module(FileManager.IDENTITY).add_file_decryptor(new OmemoFileDecryptor()); app.stream_interactor.get_module(FileManager.IDENTITY).add_file_encryptor(new OmemoFileEncryptor()); JingleFileHelperRegistry.instance.add_encryption_helper(Encryption.OMEMO, new JetOmemo.EncryptionHelper(app.stream_interactor)); - Manager.start(this.app.stream_interactor, db, trust_manager); + Manager.start(this.app.stream_interactor, db, trust_manager, encryptors); SimpleAction own_keys_action = new SimpleAction("own-keys", VariantType.INT32); own_keys_action.activate.connect((variant) => { diff --git a/plugins/omemo/src/protocol/stream_module.vala b/plugins/omemo/src/protocol/stream_module.vala index e4a2733c..39d9c448 100644 --- a/plugins/omemo/src/protocol/stream_module.vala +++ b/plugins/omemo/src/protocol/stream_module.vala @@ -25,10 +25,8 @@ public class StreamModule : XmppStreamModule { public signal void bundle_fetched(Jid jid, int device_id, Bundle bundle); public signal void bundle_fetch_failed(Jid jid, int device_id); - public StreamModule() { - if (Plugin.ensure_context()) { - this.store = Plugin.get_context().create_store(); - } + public StreamModule(Store store) { + this.store = store; } public override void attach(XmppStream stream) { diff --git a/xmpp-vala/CMakeLists.txt b/xmpp-vala/CMakeLists.txt index 3aa10caf..bf8f0068 100644 --- a/xmpp-vala/CMakeLists.txt +++ b/xmpp-vala/CMakeLists.txt @@ -109,6 +109,9 @@ SOURCES "src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala" "src/module/xep/0176_jingle_ice_udp/transport_parameters.vala" + "src/module/xep/0384_omemo/omemo_encryptor.vala" + "src/module/xep/0384_omemo/omemo_decryptor.vala" + "src/module/xep/0184_message_delivery_receipts.vala" "src/module/xep/0191_blocking_command.vala" "src/module/xep/0198_stream_management.vala" diff --git a/xmpp-vala/src/module/iq/module.vala b/xmpp-vala/src/module/iq/module.vala index 56605d01..17cd3f0d 100644 --- a/xmpp-vala/src/module/iq/module.vala +++ b/xmpp-vala/src/module/iq/module.vala @@ -6,6 +6,9 @@ namespace Xmpp.Iq { public class Module : XmppStreamNegotiationModule { public static ModuleIdentity IDENTITY = new ModuleIdentity(NS_URI, "iq_module"); + public signal void preprocess_incoming_iq_set_get(XmppStream stream, Stanza iq_stanza); + public signal void preprocess_outgoing_iq_set_get(XmppStream stream, Stanza iq_stanza); + private HashMap responseListeners = new HashMap(); private HashMap> namespaceRegistrants = new HashMap>(); @@ -23,6 +26,7 @@ namespace Xmpp.Iq { public delegate void OnResult(XmppStream stream, Iq.Stanza iq); public void send_iq(XmppStream stream, Iq.Stanza iq, owned OnResult? listener = null) { + preprocess_outgoing_iq_set_get(stream, iq); stream.write(iq.stanza); if (listener != null) { responseListeners[iq.id] = new ResponseListener((owned) listener); @@ -70,6 +74,7 @@ namespace Xmpp.Iq { } else { Gee.List children = node.get_all_subnodes(); if (children.size == 1 && namespaceRegistrants.has_key(children[0].ns_uri)) { + preprocess_incoming_iq_set_get(stream, iq); Gee.List handlers = namespaceRegistrants[children[0].ns_uri]; foreach (Handler handler in handlers) { if (iq.type_ == Iq.Stanza.TYPE_GET) { diff --git a/xmpp-vala/src/module/xep/0166_jingle/content.vala b/xmpp-vala/src/module/xep/0166_jingle/content.vala index beb12183..befe02f4 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/content.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/content.vala @@ -34,9 +34,8 @@ public class Xmpp.Xep.Jingle.Content : Object { public weak Session session; public Map component_connections = new HashMap(); // TODO private - public ContentEncryption? encryption { get; set; } + public HashMap encryptions = new HashMap(); - // INITIATE_SENT | INITIATE_RECEIVED | CONNECTING public Set tried_transport_methods = new HashSet(); diff --git a/xmpp-vala/src/module/xep/0166_jingle/content_transport.vala b/xmpp-vala/src/module/xep/0166_jingle/content_transport.vala index cd74c836..2697a01c 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/content_transport.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/content_transport.vala @@ -21,7 +21,7 @@ namespace Xmpp.Xep.Jingle { public abstract uint8 components { get; } public abstract void set_content(Content content); - public abstract StanzaNode to_transport_stanza_node(); + public abstract StanzaNode to_transport_stanza_node(string action_type); public abstract void handle_transport_accept(StanzaNode transport) throws IqError; public abstract void handle_transport_info(StanzaNode transport) throws IqError; public abstract void create_transport_connection(XmppStream stream, Content content); diff --git a/xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala b/xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala index 7314ca6c..186848f6 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/jingle_module.vala @@ -3,7 +3,7 @@ using Xmpp; namespace Xmpp.Xep.Jingle { - internal const string NS_URI = "urn:xmpp:jingle:1"; + public const string NS_URI = "urn:xmpp:jingle:1"; private const string ERROR_NS_URI = "urn:xmpp:jingle:errors:1"; // This module can only be attached to one stream at a time. @@ -131,7 +131,7 @@ namespace Xmpp.Xep.Jingle { .put_attribute("name", content.content_name) .put_attribute("senders", content.senders.to_string()) .put_node(content.content_params.get_description_node()) - .put_node(content.transport_params.to_transport_stanza_node()); + .put_node(content.transport_params.to_transport_stanza_node("session-initiate")); if (content.security_params != null) { content_node.put_node(content.security_params.to_security_stanza_node(stream, my_jid, receiver_full_jid)); } diff --git a/xmpp-vala/src/module/xep/0166_jingle/session.vala b/xmpp-vala/src/module/xep/0166_jingle/session.vala index 5fe89415..4d04c8d5 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/session.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/session.vala @@ -221,7 +221,7 @@ public class Xmpp.Xep.Jingle.Session : Object { .put_attribute("name", content.content_name) .put_attribute("senders", content.senders.to_string()) .put_node(content.content_params.get_description_node()) - .put_node(content.transport_params.to_transport_stanza_node())); + .put_node(content.transport_params.to_transport_stanza_node("content-add"))); Iq.Stanza iq = new Iq.Stanza.set(content_add_node) { to=peer_full_jid }; yield stream.get_module(Iq.Module.IDENTITY).send_iq_async(stream, iq); @@ -343,7 +343,7 @@ public class Xmpp.Xep.Jingle.Session : Object { .put_attribute("name", content.content_name) .put_attribute("senders", content.senders.to_string()) .put_node(content.content_params.get_description_node()) - .put_node(content.transport_params.to_transport_stanza_node()); + .put_node(content.transport_params.to_transport_stanza_node("session-accept")); jingle.put_node(content_node); } @@ -379,7 +379,7 @@ public class Xmpp.Xep.Jingle.Session : Object { .put_attribute("name", content.content_name) .put_attribute("senders", content.senders.to_string()) .put_node(content.content_params.get_description_node()) - .put_node(content.transport_params.to_transport_stanza_node())); + .put_node(content.transport_params.to_transport_stanza_node("content-accept"))); Iq.Stanza iq = new Iq.Stanza.set(content_accept_node) { to=peer_full_jid }; stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); @@ -477,7 +477,7 @@ public class Xmpp.Xep.Jingle.Session : Object { .put_node(new StanzaNode.build("content", NS_URI) .put_attribute("creator", "initiator") .put_attribute("name", content.content_name) - .put_node(transport_params.to_transport_stanza_node()) + .put_node(transport_params.to_transport_stanza_node("transport-accept")) ); Iq.Stanza iq_response = new Iq.Stanza.set(jingle_response) { to=peer_full_jid }; stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq_response); @@ -493,7 +493,7 @@ public class Xmpp.Xep.Jingle.Session : Object { .put_node(new StanzaNode.build("content", NS_URI) .put_attribute("creator", "initiator") .put_attribute("name", content.content_name) - .put_node(transport_params.to_transport_stanza_node()) + .put_node(transport_params.to_transport_stanza_node("transport-replace")) ); Iq.Stanza iq = new Iq.Stanza.set(jingle) { to=peer_full_jid }; stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index d4440169..344fe8b8 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -133,7 +133,8 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { local_crypto = null; } if (remote_crypto != null && local_crypto != null) { - content.encryption = new Xmpp.Xep.Jingle.ContentEncryption() { encryption_ns = "", encryption_name = "SRTP", our_key=local_crypto.key, peer_key=remote_crypto.key }; + var content_encryption = new Xmpp.Xep.Jingle.ContentEncryption() { encryption_ns = "", encryption_name = "SRTP", our_key=local_crypto.key, peer_key=remote_crypto.key }; + content.encryptions[content_encryption.encryption_name] = content_encryption; } this.stream = parent.create_stream(content); diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala index 5211e3a9..87c010dd 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala @@ -4,7 +4,7 @@ using Xmpp; namespace Xmpp.Xep.JingleIceUdp { -private const string NS_URI = "urn:xmpp:jingle:transports:ice-udp:1"; +public const string NS_URI = "urn:xmpp:jingle:transports:ice-udp:1"; public const string DTLS_NS_URI = "urn:xmpp:jingle:apps:dtls:0"; public abstract class Module : XmppStreamModule, Jingle.Transport { diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala index ed0fab50..83da296b 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala @@ -65,13 +65,13 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T this.content = null; } - public StanzaNode to_transport_stanza_node() { + public StanzaNode to_transport_stanza_node(string action_type) { var node = new StanzaNode.build("transport", NS_URI) .add_self_xmlns() .put_attribute("ufrag", local_ufrag) .put_attribute("pwd", local_pwd); - if (own_fingerprint != null) { + if (own_fingerprint != null && action_type != "transport-info") { var fingerprint_node = new StanzaNode.build("fingerprint", DTLS_NS_URI) .add_self_xmlns() .put_attribute("hash", "sha-256") @@ -137,7 +137,7 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T private void check_send_transport_info() { if (this.content != null && unsent_local_candidates.size > 0) { - content.send_transport_info(to_transport_stanza_node()); + content.send_transport_info(to_transport_stanza_node("transport-info")); } } diff --git a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala index 1c4e0d38..6edebbbc 100644 --- a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala +++ b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala @@ -391,7 +391,7 @@ class Parameters : Jingle.TransportParameters, Object { } - public StanzaNode to_transport_stanza_node() { + public StanzaNode to_transport_stanza_node(string action_type) { StanzaNode transport = new StanzaNode.build("transport", NS_URI) .add_self_xmlns() .put_attribute("dstaddr", local_dstaddr); diff --git a/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala b/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala index 5bb71831..f7c77544 100644 --- a/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala +++ b/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala @@ -73,7 +73,7 @@ class Parameters : Jingle.TransportParameters, Object { } - public StanzaNode to_transport_stanza_node() { + public StanzaNode to_transport_stanza_node(string action_type) { return new StanzaNode.build("transport", NS_URI) .add_self_xmlns() .put_attribute("block-size", block_size.to_string()) diff --git a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala index 08e803a2..e26be515 100644 --- a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala +++ b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala @@ -102,10 +102,12 @@ namespace Xmpp.Xep.JingleMessageInitiation { } public override void attach(XmppStream stream) { + stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); stream.get_module(MessageModule.IDENTITY).received_message.connect(on_received_message); } public override void detach(XmppStream stream) { + stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI); stream.get_module(MessageModule.IDENTITY).received_message.disconnect(on_received_message); } diff --git a/xmpp-vala/src/module/xep/0384_omemo/omemo_decryptor.vala b/xmpp-vala/src/module/xep/0384_omemo/omemo_decryptor.vala new file mode 100644 index 00000000..8e3213ae --- /dev/null +++ b/xmpp-vala/src/module/xep/0384_omemo/omemo_decryptor.vala @@ -0,0 +1,62 @@ +using Gee; +using Xmpp.Xep; +using Xmpp; + +namespace Xmpp.Xep.Omemo { + + public abstract class OmemoDecryptor : XmppStreamModule { + + public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "0384_omemo_decryptor"); + + public abstract uint32 own_device_id { get; } + + public abstract string decrypt(uint8[] ciphertext, uint8[] key, uint8[] iv) throws GLib.Error; + + public abstract uint8[] decrypt_key(ParsedData data, Jid from_jid) throws GLib.Error; + + public ParsedData? parse_node(StanzaNode encrypted_node) { + ParsedData ret = new ParsedData(); + + StanzaNode? header_node = encrypted_node.get_subnode("header"); + if (header_node == null) return null; + + ret.sid = header_node.get_attribute_int("sid", -1); + if (ret.sid == -1) return null; + + string? payload_str = encrypted_node.get_deep_string_content("payload"); + if (payload_str != null) ret.ciphertext = Base64.decode(payload_str); + + string? iv_str = header_node.get_deep_string_content("iv"); + if (iv_str == null) return null; + ret.iv = Base64.decode(iv_str); + + foreach (StanzaNode key_node in header_node.get_subnodes("key")) { + debug("Is ours? %d =? %u", key_node.get_attribute_int("rid"), own_device_id); + if (key_node.get_attribute_int("rid") == own_device_id) { + string? key_node_content = key_node.get_string_content(); + if (key_node_content == null) continue; + uchar[] encrypted_key = Base64.decode(key_node_content); + ret.our_potential_encrypted_keys[new Bytes.take(encrypted_key)] = key_node.get_attribute_bool("prekey"); + } + } + + return ret; + } + + public override void attach(XmppStream stream) { } + public override void detach(XmppStream stream) { } + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } + } + + public class ParsedData { + public int sid; + public uint8[] ciphertext; + public uint8[] iv; + public uchar[] encrypted_key; + public bool is_prekey; + + public HashMap our_potential_encrypted_keys = new HashMap(); + } +} + diff --git a/xmpp-vala/src/module/xep/0384_omemo/omemo_encryptor.vala b/xmpp-vala/src/module/xep/0384_omemo/omemo_encryptor.vala new file mode 100644 index 00000000..6509bfe3 --- /dev/null +++ b/xmpp-vala/src/module/xep/0384_omemo/omemo_encryptor.vala @@ -0,0 +1,116 @@ +using Gee; +using Xmpp.Xep; +using Xmpp; + +namespace Xmpp.Xep.Omemo { + + public const string NS_URI = "eu.siacs.conversations.axolotl"; + public const string NODE_DEVICELIST = NS_URI + ".devicelist"; + public const string NODE_BUNDLES = NS_URI + ".bundles"; + public const string NODE_VERIFICATION = NS_URI + ".verification"; + + public abstract class OmemoEncryptor : XmppStreamModule { + + public static Xmpp.ModuleIdentity IDENTITY = new Xmpp.ModuleIdentity(NS_URI, "0384_omemo_encryptor"); + + public abstract uint32 own_device_id { get; } + + public abstract EncryptionData encrypt_plaintext(string plaintext) throws GLib.Error; + + public abstract void encrypt_key(Xep.Omemo.EncryptionData encryption_data, Jid jid, int32 device_id) throws GLib.Error; + + public abstract EncryptionResult encrypt_key_to_recipient(XmppStream stream, Xep.Omemo.EncryptionData enc_data, Jid recipient) throws GLib.Error; + + public override void attach(XmppStream stream) { } + public override void detach(XmppStream stream) { } + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } + } + + public class EncryptionData { + public uint32 own_device_id; + public uint8[] ciphertext; + public uint8[] keytag; + public uint8[] iv; + + public Gee.List key_nodes = new ArrayList(); + + public EncryptionData(uint32 own_device_id) { + this.own_device_id = own_device_id; + } + + public void add_device_key(int device_id, uint8[] device_key, bool prekey) { + StanzaNode key_node = new StanzaNode.build("key", NS_URI) + .put_attribute("rid", device_id.to_string()) + .put_node(new StanzaNode.text(Base64.encode(device_key))); + if (prekey) { + key_node.put_attribute("prekey", "true"); + } + key_nodes.add(key_node); + } + + public StanzaNode get_encrypted_node() { + StanzaNode encrypted_node = new StanzaNode.build("encrypted", NS_URI).add_self_xmlns(); + + StanzaNode header_node = new StanzaNode.build("header", NS_URI) + .put_attribute("sid", own_device_id.to_string()) + .put_node(new StanzaNode.build("iv", NS_URI).put_node(new StanzaNode.text(Base64.encode(iv)))); + encrypted_node.put_node(header_node); + + if (ciphertext != null) { + StanzaNode payload_node = new StanzaNode.build("payload", NS_URI) + .put_node(new StanzaNode.text(Base64.encode(ciphertext))); + encrypted_node.put_node(payload_node); + } + + foreach (StanzaNode key_node in key_nodes) { + header_node.put_node(key_node); + } + + return encrypted_node; + } + } + + public class EncryptionResult { + public int lost { get; internal set; } + public int success { get; internal set; } + public int unknown { get; internal set; } + public int failure { get; internal set; } + } + + public class EncryptState { + public bool encrypted { get; internal set; } + public int other_devices { get; internal set; } + public int other_success { get; internal set; } + public int other_lost { get; internal set; } + public int other_unknown { get; internal set; } + public int other_failure { get; internal set; } + public int other_waiting_lists { get; internal set; } + + public int own_devices { get; internal set; } + public int own_success { get; internal set; } + public int own_lost { get; internal set; } + public int own_unknown { get; internal set; } + public int own_failure { get; internal set; } + public bool own_list { get; internal set; } + + public void add_result(EncryptionResult enc_res, bool own) { + if (own) { + own_lost += enc_res.lost; + own_success += enc_res.success; + own_unknown += enc_res.unknown; + own_failure += enc_res.failure; + } else { + other_lost += enc_res.lost; + other_success += enc_res.success; + other_unknown += enc_res.unknown; + other_failure += enc_res.failure; + } + } + + public string to_string() { + return @"EncryptState (encrypted=$encrypted, other=(devices=$other_devices, success=$other_success, lost=$other_lost, unknown=$other_unknown, failure=$other_failure, waiting_lists=$other_waiting_lists, own=(devices=$own_devices, success=$own_success, lost=$own_lost, unknown=$own_unknown, failure=$own_failure, list=$own_list))"; + } + } +} + From 4c6664a365bb64904078c07c588f129456583457 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Thu, 29 Apr 2021 15:03:37 +0200 Subject: [PATCH 48/56] Improve JMI handling (type=chat, filter message sender) --- libdino/src/service/calls.vala | 12 ++++++- .../xep/0353_jingle_message_initiation.vala | 35 ++++++------------- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index 3615e24f..26a0492a 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -185,8 +185,8 @@ namespace Dino { if (stream == null) return; string sid = sid_by_call[call.account][call]; - stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_reject_to_self(stream, sid); stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_reject_to_peer(stream, call.counterpart, sid); + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_reject_to_self(stream, sid); remove_call_from_datastructures(call); } } @@ -632,6 +632,11 @@ namespace Dino { mi_module.session_rejected.connect((from, to, sid) => { if (!call_by_sid[account].has_key(sid)) return; Call call = call_by_sid[account][sid]; + + bool outgoing_reject = call.direction == Call.DIRECTION_OUTGOING && from.equals_bare(call.counterpart); + bool incoming_reject = call.direction == Call.DIRECTION_INCOMING && from.equals_bare(account.bare_jid); + if (!(outgoing_reject || incoming_reject)) return; + call.state = Call.State.DECLINED; remove_call_from_datastructures(call); call_terminated(call, null, null); @@ -639,6 +644,11 @@ namespace Dino { mi_module.session_retracted.connect((from, to, sid) => { if (!call_by_sid[account].has_key(sid)) return; Call call = call_by_sid[account][sid]; + + bool outgoing_retract = call.direction == Call.DIRECTION_OUTGOING && from.equals_bare(call.counterpart); + bool incoming_retract = call.direction == Call.DIRECTION_INCOMING && from.equals_bare(account.bare_jid); + if (!(outgoing_retract || incoming_retract)) return; + call.state = Call.State.MISSED; remove_call_from_datastructures(call); call_terminated(call, null, null); diff --git a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala index e26be515..71e16a95 100644 --- a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala +++ b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala @@ -17,47 +17,35 @@ namespace Xmpp.Xep.JingleMessageInitiation { propose_node.put_node(desc_node); } - MessageStanza accepted_message = new MessageStanza() { to=to }; + MessageStanza accepted_message = new MessageStanza() { to=to, type_=MessageStanza.TYPE_CHAT }; accepted_message.stanza.put_node(propose_node); stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); } public void send_session_retract_to_peer(XmppStream stream, Jid to, string sid) { - MessageStanza retract_message = new MessageStanza() { to=to }; - retract_message.stanza.put_node( - new StanzaNode.build("retract", NS_URI).add_self_xmlns() - .put_attribute("id", sid, NS_URI)); - stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, retract_message); + send_jmi_message(stream, "retract", to, sid); } public void send_session_accept_to_self(XmppStream stream, string sid) { - MessageStanza accepted_message = new MessageStanza() { to=Bind.Flag.get_my_jid(stream).bare_jid }; - accepted_message.stanza.put_node( - new StanzaNode.build("accept", NS_URI).add_self_xmlns() - .put_attribute("id", sid, NS_URI)); - stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); + send_jmi_message(stream, "accept", Bind.Flag.get_my_jid(stream).bare_jid, sid); } public void send_session_reject_to_self(XmppStream stream, string sid) { - MessageStanza accepted_message = new MessageStanza() { to=Bind.Flag.get_my_jid(stream).bare_jid }; - accepted_message.stanza.put_node( - new StanzaNode.build("reject", NS_URI).add_self_xmlns() - .put_attribute("id", sid, NS_URI)); - stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); + send_jmi_message(stream, "reject", Bind.Flag.get_my_jid(stream).bare_jid, sid); } public void send_session_proceed_to_peer(XmppStream stream, Jid to, string sid) { - MessageStanza accepted_message = new MessageStanza() { to=to }; - accepted_message.stanza.put_node( - new StanzaNode.build("proceed", NS_URI).add_self_xmlns() - .put_attribute("id", sid, NS_URI)); - stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); + send_jmi_message(stream, "proceed", to, sid); } public void send_session_reject_to_peer(XmppStream stream, Jid to, string sid) { - MessageStanza accepted_message = new MessageStanza() { to=to }; + send_jmi_message(stream, "reject", to, sid); + } + + private void send_jmi_message(XmppStream stream, string name, Jid to, string sid) { + MessageStanza accepted_message = new MessageStanza() { to=to, type_=MessageStanza.TYPE_CHAT }; accepted_message.stanza.put_node( - new StanzaNode.build("reject", NS_URI).add_self_xmlns() + new StanzaNode.build(name, NS_URI).add_self_xmlns() .put_attribute("id", sid, NS_URI)); stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); } @@ -95,7 +83,6 @@ namespace Xmpp.Xep.JingleMessageInitiation { session_retracted(message.from, message.to, mi_node.get_attribute("id")); break; case "reject": - if (!message.from.equals_bare(Bind.Flag.get_my_jid(stream))) return; session_rejected(message.from, message.to, mi_node.get_attribute("id")); break; } From 6b976cdb6604f6f27b72f7397b38d45dd4f916c6 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Thu, 29 Apr 2021 15:29:41 +0200 Subject: [PATCH 49/56] Adjust JMI vs direct calling order --- libdino/src/service/calls.vala | 38 +++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index 26a0492a..d535dfca 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -83,7 +83,19 @@ namespace Dino { we_should_send_video[call] = video; we_should_send_audio[call] = true; - if (has_jmi_resources(conversation)) { + Gee.List call_resources = yield get_call_resources(conversation); + + bool do_jmi = false; + Jid? jid_for_direct = null; + if (yield contains_jmi_resources(conversation.account, call_resources)) { + do_jmi = true; + } else if (!call_resources.is_empty) { + jid_for_direct = call_resources[0]; + } else if (has_jmi_resources(conversation)) { + do_jmi = true; + } + + if (do_jmi) { XmppStream? stream = stream_interactor.get_stream(conversation.account); jmi_call[conversation.account] = call; jmi_video[conversation.account] = video; @@ -92,19 +104,14 @@ namespace Dino { call_by_sid[call.account][jmi_sid[conversation.account]] = call; var descriptions = new ArrayList(); - descriptions.add(new StanzaNode.build("description", "urn:xmpp:jingle:apps:rtp:1").add_self_xmlns().put_attribute("media", "audio")); + descriptions.add(new StanzaNode.build("description", Xep.JingleRtp.NS_URI).add_self_xmlns().put_attribute("media", "audio")); if (video) { - descriptions.add(new StanzaNode.build("description", "urn:xmpp:jingle:apps:rtp:1").add_self_xmlns().put_attribute("media", "video")); + descriptions.add(new StanzaNode.build("description", Xep.JingleRtp.NS_URI).add_self_xmlns().put_attribute("media", "video")); } stream.get_module(Xmpp.Xep.JingleMessageInitiation.Module.IDENTITY).send_session_propose_to_peer(stream, conversation.counterpart, jmi_sid[call.account], descriptions); - } else { - Gee.List call_resources = yield get_call_resources(conversation); - if (call_resources.size == 0) { - warning("No call resources"); - return null; - } - yield call_resource(conversation.account, call_resources[0], call, video); + } else if (jid_for_direct != null) { + yield call_resource(conversation.account, jid_for_direct, call, video); } conversation.last_active = call.time; @@ -286,6 +293,17 @@ namespace Dino { return ret; } + private async bool contains_jmi_resources(Account account, Gee.List full_jids) { + XmppStream? stream = stream_interactor.get_stream(account); + if (stream == null) return false; + + foreach (Jid full_jid in full_jids) { + bool does_jmi = yield stream_interactor.get_module(EntityInfo.IDENTITY).has_feature(account, full_jid, Xep.JingleMessageInitiation.NS_URI); + if (does_jmi) return true; + } + return false; + } + private bool has_jmi_resources(Conversation conversation) { int64 jmi_resources = db.entity.select() .with(db.entity.jid_id, "=", db.get_jid_id(conversation.counterpart)) From 23ffd37dded3bf872e42d7a00727ab3c4d105a97 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 1 May 2021 15:19:05 +0200 Subject: [PATCH 50/56] Echo Cancellation --- CMakeLists.txt | 6 +- cmake/FindGstAudio.cmake | 14 ++ cmake/FindWebRTCAudioProcessing.cmake | 12 ++ plugins/rtp/CMakeLists.txt | 20 ++- plugins/rtp/src/device.vala | 30 ++-- plugins/rtp/src/plugin.vala | 5 +- plugins/rtp/src/voice_processor.vala | 176 +++++++++++++++++++++ plugins/rtp/src/voice_processor_native.cpp | 141 +++++++++++++++++ 8 files changed, 385 insertions(+), 19 deletions(-) create mode 100644 cmake/FindGstAudio.cmake create mode 100644 cmake/FindWebRTCAudioProcessing.cmake create mode 100644 plugins/rtp/src/voice_processor.vala create mode 100644 plugins/rtp/src/voice_processor_native.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f480b0b2..b3bd35cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,11 +2,11 @@ cmake_minimum_required(VERSION 3.3) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) include(ComputeVersion) if (NOT VERSION_FOUND) - project(Dino LANGUAGES C) + project(Dino LANGUAGES C CXX) elseif (VERSION_IS_RELEASE) - project(Dino VERSION ${VERSION_FULL} LANGUAGES C) + project(Dino VERSION ${VERSION_FULL} LANGUAGES C CXX) else () - project(Dino LANGUAGES C) + project(Dino LANGUAGES C CXX) set(PROJECT_VERSION ${VERSION_FULL}) endif () diff --git a/cmake/FindGstAudio.cmake b/cmake/FindGstAudio.cmake new file mode 100644 index 00000000..d5fc5dfb --- /dev/null +++ b/cmake/FindGstAudio.cmake @@ -0,0 +1,14 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(GstAudio + PKG_CONFIG_NAME gstreamer-audio-1.0 + LIB_NAMES gstaudio + LIB_DIR_HINTS gstreamer-1.0 + INCLUDE_NAMES gst/audio/audio.h + INCLUDE_DIR_SUFFIXES gstreamer-1.0 gstreamer-1.0/include gstreamer-audio-1.0 gstreamer-audio-1.0/include + DEPENDS Gst +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GstAudio + REQUIRED_VARS GstAudio_LIBRARY + VERSION_VAR GstAudio_VERSION) diff --git a/cmake/FindWebRTCAudioProcessing.cmake b/cmake/FindWebRTCAudioProcessing.cmake new file mode 100644 index 00000000..5f17805d --- /dev/null +++ b/cmake/FindWebRTCAudioProcessing.cmake @@ -0,0 +1,12 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(WebRTCAudioProcessing + PKG_CONFIG_NAME webrtc-audio-processing + LIB_NAMES webrtc_audio_processing + INCLUDE_NAMES webrtc/modules/audio_processing/include/audio_processing.h + INCLUDE_DIR_SUFFIXES webrtc-audio-processing webrtc_audio_processing +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(WebRTCAudioProcessing + REQUIRED_VARS WebRTCAudioProcessing_LIBRARY + VERSION_VAR WebRTCAudioProcessing_VERSION) diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 92ec1b97..b19c8a8f 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -1,4 +1,5 @@ find_package(GstRtp REQUIRED) +find_package(WebRTCAudioProcessing 0.2) find_packages(RTP_PACKAGES REQUIRED Gee GLib @@ -8,12 +9,26 @@ find_packages(RTP_PACKAGES REQUIRED GTK3 Gst GstApp + GstAudio ) if(Gst_VERSION VERSION_GREATER "1.16") set(RTP_DEFINITIONS GST_1_16) endif() +if(WebRTCAudioProcessing_VERSION GREATER "0.4") + message(WARNING "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") + unset(WebRTCAudioProcessing_FOUND) +endif() + +if(WebRTCAudioProcessing_FOUND) + set(RTP_DEFINITIONS ${RTP_DEFINITIONS} WITH_VOICE_PROCESSOR) + set(RTP_VOICE_PROCESSOR_VALA src/voice_processor.vala) + set(RTP_VOICE_PROCESSOR_CXX src/voice_processor_native.cpp) +else() + message(WARNING "WebRTCAudioProcessing not found, build without voice pre-processing!") +endif() + vala_precompile(RTP_VALA_C SOURCES src/codec_util.vala @@ -23,6 +38,7 @@ SOURCES src/stream.vala src/video_widget.vala src/register_plugin.vala + ${RTP_VOICE_PROCESSOR_VALA} CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi @@ -36,8 +52,8 @@ DEFINITIONS ) add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) -add_library(rtp SHARED ${RTP_VALA_C}) -target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0) +add_library(rtp SHARED ${RTP_VALA_C} ${RTP_VOICE_PROCESSOR_CXX}) +target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 webrtc-audio-processing) set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 785f853a..f8894502 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -37,6 +37,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { private Gst.Element dsp; private Gst.Element mixer; private Gst.Element filter; + private Gst.Element rate; private int links = 0; public Device(Plugin plugin, Gst.Device device) { @@ -132,12 +133,10 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { pipe.add(filter); element.link(filter); if (media == "audio" && plugin.echoprobe != null) { - dsp = Gst.ElementFactory.make("webrtcdsp", @"dsp_$id"); - if (dsp != null) { - dsp.@set("probe", plugin.echoprobe.name); - pipe.add(dsp); - filter.link(dsp); - } + dsp = new VoiceProcessor(plugin.echoprobe, element as Gst.Audio.StreamVolume); + dsp.name = @"dsp_$id"; + pipe.add(dsp); + filter.link(dsp); } tee = Gst.ElementFactory.make("tee", @"tee_$id"); tee.@set("allow-not-linked", true); @@ -153,7 +152,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { filter.@set("caps", get_best_caps()); pipe.add(filter); if (plugin.echoprobe != null) { - filter.link(plugin.echoprobe); + rate = Gst.ElementFactory.make("audiorate", @"rate_$id"); + rate.@set("tolerance", 100000000); + pipe.add(rate); + filter.link(rate); + rate.link(plugin.echoprobe); plugin.echoprobe.link(element); } else { filter.link(element); @@ -184,14 +187,17 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { if (filter != null) { filter.set_locked_state(true); filter.set_state(Gst.State.NULL); - if (plugin.echoprobe != null) { - filter.unlink(plugin.echoprobe); - } else { - filter.unlink(element); - } + filter.unlink(rate ?? ((Gst.Element)plugin.echoprobe) ?? element); pipe.remove(filter); filter = null; } + if (rate != null) { + rate.set_locked_state(true); + rate.set_state(Gst.State.NULL); + rate.unlink(plugin.echoprobe); + pipe.remove(rate); + rate = null; + } if (plugin.echoprobe != null) { plugin.echoprobe.unlink(element); } diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index d43588b4..e3d5ee41 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -8,7 +8,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { public Gst.DeviceMonitor device_monitor { get; private set; } public Gst.Pipeline pipe { get; private set; } public Gst.Bin rtpbin { get; private set; } - public Gst.Element echoprobe { get; private set; } + public EchoProbe echoprobe { get; private set; } private Gee.List streams = new ArrayList(); private Gee.List devices = new ArrayList(); @@ -72,7 +72,8 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { pipe.add(rtpbin); // Audio echo probe - echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); +// echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); + echoprobe = new EchoProbe(); if (echoprobe != null) pipe.add(echoprobe); // Pipeline diff --git a/plugins/rtp/src/voice_processor.vala b/plugins/rtp/src/voice_processor.vala new file mode 100644 index 00000000..e6dc7e8f --- /dev/null +++ b/plugins/rtp/src/voice_processor.vala @@ -0,0 +1,176 @@ +using Gst; + +namespace Dino.Plugins.Rtp { +public static extern Buffer adjust_to_running_time(Base.Transform transform, Buffer buf); +} + +public class Dino.Plugins.Rtp.EchoProbe : Audio.Filter { + private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + public Audio.Info audio_info { get; private set; } + public signal void on_new_buffer(Buffer buffer); + private uint period_samples; + private uint period_size; + private Base.Adapter adapter = new Base.Adapter(); + + static construct { + add_static_pad_template(sink_template); + add_static_pad_template(src_template); + set_static_metadata("Acoustic Echo Canceller probe", "Generic/Audio", "Gathers playback buffers for echo cancellation", "Dino Team "); + } + + construct { + set_passthrough(true); + } + + public override bool setup(Audio.Info info) { + audio_info = info; + period_samples = info.rate / 100; // 10ms buffers + period_size = period_samples * info.bpf; + return true; + } + + + public override FlowReturn transform_ip(Buffer buf) { + lock (adapter) { + adapter.push(adjust_to_running_time(this, buf)); + while (adapter.available() > period_size) { + on_new_buffer(adapter.take_buffer(period_size)); + } + } + return FlowReturn.OK; + } + + public override bool stop() { + adapter.clear(); + return true; + } +} + +public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter { + private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + public Audio.Info audio_info { get; private set; } + private ulong process_outgoing_buffer_handler_id; + private uint adjust_delay_timeout_id; + private uint period_samples; + private uint period_size; + private Base.Adapter adapter = new Base.Adapter(); + private EchoProbe? echo_probe; + private Audio.StreamVolume? stream_volume; + private ClockTime last_reverse; + private void* native; + + static construct { + add_static_pad_template(sink_template); + add_static_pad_template(src_template); + set_static_metadata("Voice Processor (AGC, AEC, filters, etc.)", "Generic/Audio", "Pre-processes voice with WebRTC Audio Processing Library", "Dino Team "); + } + + construct { + set_passthrough(false); + } + + public VoiceProcessor(EchoProbe? echo_probe = null, Audio.StreamVolume? stream_volume = null) { + this.echo_probe = echo_probe; + this.stream_volume = stream_volume; + } + + private static extern void* init_native(int stream_delay); + private static extern void setup_native(void* native); + private static extern void destroy_native(void* native); + private static extern void analyze_reverse_stream(void* native, Audio.Info info, Buffer buffer); + private static extern void process_stream(void* native, Audio.Info info, Buffer buffer); + private static extern void adjust_stream_delay(void* native); + private static extern void notify_gain_level(void* native, int gain_level); + private static extern int get_suggested_gain_level(void* native); + private static extern bool get_stream_has_voice(void* native); + + public override bool setup(Audio.Info info) { + debug("VoiceProcessor.setup(%s)", info.to_caps().to_string()); + audio_info = info; + period_samples = info.rate / 100; // 10ms buffers + period_size = period_samples * info.bpf; + adapter.clear(); + setup_native(native); + return true; + } + + public override bool start() { + native = init_native(150); + if (process_outgoing_buffer_handler_id == 0 && echo_probe != null) { + process_outgoing_buffer_handler_id = echo_probe.on_new_buffer.connect(process_outgoing_buffer); + } + if (stream_volume == null && sinkpad.get_peer() != null && sinkpad.get_peer().get_parent_element() is Audio.StreamVolume) { + stream_volume = sinkpad.get_peer().get_parent_element() as Audio.StreamVolume; + } + return true; + } + + private bool adjust_delay() { + if (native != null) { + adjust_stream_delay(native); + return Source.CONTINUE; + } else { + adjust_delay_timeout_id = 0; + return Source.REMOVE; + } + } + + private void process_outgoing_buffer(Buffer buffer) { + if (buffer.pts != uint64.MAX) { + last_reverse = buffer.pts; + } + analyze_reverse_stream(native, echo_probe.audio_info, buffer); + if (adjust_delay_timeout_id == 0 && echo_probe != null) { + adjust_delay_timeout_id = Timeout.add(5000, adjust_delay); + } + } + + public override FlowReturn submit_input_buffer(bool is_discont, Buffer input) { + lock (adapter) { + if (is_discont) { + adapter.clear(); + } + adapter.push(adjust_to_running_time(this, input)); + } + return FlowReturn.OK; + } + + public override FlowReturn generate_output(out Buffer output_buffer) { + lock (adapter) { + if (adapter.available() >= period_size) { + output_buffer = (Gst.Buffer) adapter.take_buffer(period_size).make_writable(); + int old_gain_level = 0; + if (stream_volume != null) { + old_gain_level = (int) (stream_volume.get_volume(Audio.StreamVolumeFormat.LINEAR) * 255.0); + notify_gain_level(native, old_gain_level); + } + process_stream(native, audio_info, output_buffer); + if (stream_volume != null) { + int new_gain_level = get_suggested_gain_level(native); + if (old_gain_level != new_gain_level) { + debug("Gain: %i -> %i", old_gain_level, new_gain_level); + stream_volume.set_volume(Audio.StreamVolumeFormat.LINEAR, ((double)new_gain_level) / 255.0); + } + } + } + } + return FlowReturn.OK; + } + + public override bool stop() { + if (process_outgoing_buffer_handler_id != 0) { + echo_probe.disconnect(process_outgoing_buffer_handler_id); + process_outgoing_buffer_handler_id = 0; + } + if (adjust_delay_timeout_id != 0) { + Source.remove(adjust_delay_timeout_id); + adjust_delay_timeout_id = 0; + } + adapter.clear(); + destroy_native(native); + native = null; + return true; + } +} \ No newline at end of file diff --git a/plugins/rtp/src/voice_processor_native.cpp b/plugins/rtp/src/voice_processor_native.cpp new file mode 100644 index 00000000..9b3292b8 --- /dev/null +++ b/plugins/rtp/src/voice_processor_native.cpp @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include + +#define SAMPLE_RATE 48000 +#define SAMPLE_CHANNELS 1 + +struct _DinoPluginsRtpVoiceProcessorNative { + webrtc::AudioProcessing *apm; + gint stream_delay; +}; + +extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) { + GstBuffer *copy = gst_buffer_copy(buffer); + GST_BUFFER_PTS(copy) = gst_segment_to_running_time(&transform->segment, GST_FORMAT_TIME, GST_BUFFER_PTS(buffer)); + return copy; +} + +extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay) { + _DinoPluginsRtpVoiceProcessorNative *native = new _DinoPluginsRtpVoiceProcessorNative(); + webrtc::Config config; + config.Set(new webrtc::ExtendedFilter(true)); + config.Set(new webrtc::ExperimentalAgc(true, 85)); + native->apm = webrtc::AudioProcessing::Create(config); + native->stream_delay = stream_delay; + return native; +} + +extern "C" void dino_plugins_rtp_voice_processor_setup_native(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + webrtc::ProcessingConfig pconfig; + pconfig.streams[webrtc::ProcessingConfig::kInputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kOutputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kReverseInputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kReverseOutputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + apm->Initialize(pconfig); + apm->high_pass_filter()->Enable(true); + apm->echo_cancellation()->enable_drift_compensation(false); + apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kModerateSuppression); + apm->echo_cancellation()->enable_delay_logging(true); + apm->echo_cancellation()->Enable(true); + apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kModerate); + apm->noise_suppression()->Enable(true); + apm->gain_control()->set_analog_level_limits(0, 255); + apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveAnalog); + apm->gain_control()->set_target_level_dbfs(3); + apm->gain_control()->set_compression_gain_db(9); + apm->gain_control()->enable_limiter(true); + apm->gain_control()->Enable(true); + apm->voice_detection()->set_likelihood(webrtc::VoiceDetection::Likelihood::kLowLikelihood); + apm->voice_detection()->Enable(true); +} + +extern "C" void +dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); + webrtc::AudioProcessing *apm = native->apm; + + GstAudioBuffer audio_buffer; + gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READ); + + webrtc::AudioFrame frame; + frame.num_channels_ = info->channels; + frame.sample_rate_hz_ = info->rate; + frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf; + memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); + + int err = apm->AnalyzeReverseStream(&frame); + if (err < 0) g_warning("ProcessReverseStream %i", err); + + gst_audio_buffer_unmap(&audio_buffer); +} + +extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + apm->gain_control()->set_stream_analog_level(gain_level); +} + +extern "C" gint dino_plugins_rtp_voice_processor_get_suggested_gain_level(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + return apm->gain_control()->stream_analog_level(); +} + +extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + return apm->voice_detection()->stream_has_voice(); +} + +extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + int median, std; + float fraction_poor_delays; + apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays); + if (fraction_poor_delays < 0) return; + g_debug("voice_processor_native.cpp: Stream delay metrics: %i %i %f", median, std, fraction_poor_delays); + if (fraction_poor_delays > 0.5) { + native->stream_delay = std::max(0, native->stream_delay + std::min(-10, std::max(median, 10))); + g_debug("Adjusted stream delay %i", native->stream_delay); + } +} + +extern "C" void +dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); + webrtc::AudioProcessing *apm = native->apm; + + GstAudioBuffer audio_buffer; + gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READWRITE); + + webrtc::AudioFrame frame; + frame.num_channels_ = info->channels; + frame.sample_rate_hz_ = info->rate; + frame.samples_per_channel_ = info->rate / 100; + memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); + + apm->set_stream_delay_ms(native->stream_delay); + int err = apm->ProcessStream(&frame); + if (err >= 0) memcpy(audio_buffer.planes[0], frame.data_, frame.samples_per_channel_ * info->bpf); + if (err < 0) g_warning("ProcessStream %i", err); + + gst_audio_buffer_unmap(&audio_buffer); +} + +extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + delete native; +} \ No newline at end of file From d388525fc69ab688a90f19d2d2499e0f6f10f573 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 1 May 2021 16:00:37 +0200 Subject: [PATCH 51/56] Correctly handle missing webrtc-audio-processing --- .github/workflows/build.yml | 2 +- plugins/rtp/src/device.vala | 4 +++- plugins/rtp/src/plugin.vala | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ce12d441..60d587e9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,7 @@ jobs: steps: - uses: actions/checkout@v2 - run: sudo apt-get update - - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-3-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libgspell-1-dev libnice-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libsrtp2-dev + - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-3-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libgspell-1-dev libnice-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libsrtp2-dev libwebrtc-audio-processing-dev - run: ./configure --with-tests --with-libsignal-in-tree - run: make - run: build/xmpp-vala-test diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index f8894502..3c650ad6 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -132,12 +132,14 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { filter.@set("caps", get_best_caps()); pipe.add(filter); element.link(filter); +#if WITH_VOICE_PROCESSOR if (media == "audio" && plugin.echoprobe != null) { - dsp = new VoiceProcessor(plugin.echoprobe, element as Gst.Audio.StreamVolume); + dsp = new VoiceProcessor(plugin.echoprobe as EchoProbe, element as Gst.Audio.StreamVolume); dsp.name = @"dsp_$id"; pipe.add(dsp); filter.link(dsp); } +#endif tee = Gst.ElementFactory.make("tee", @"tee_$id"); tee.@set("allow-not-linked", true); pipe.add(tee); diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index e3d5ee41..f575a7d0 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -8,7 +8,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { public Gst.DeviceMonitor device_monitor { get; private set; } public Gst.Pipeline pipe { get; private set; } public Gst.Bin rtpbin { get; private set; } - public EchoProbe echoprobe { get; private set; } + public Gst.Element echoprobe { get; private set; } private Gee.List streams = new ArrayList(); private Gee.List devices = new ArrayList(); @@ -71,10 +71,11 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { rtpbin.connect("signal::request-pt-map", request_pt_map, this); pipe.add(rtpbin); +#if WITH_VOICE_PROCESSOR // Audio echo probe -// echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); echoprobe = new EchoProbe(); if (echoprobe != null) pipe.add(echoprobe); +#endif // Pipeline pipe.auto_flush_bus = true; From 0409f554268c0e8f24e23e471a94de4d3a035ff1 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 1 May 2021 17:27:55 +0200 Subject: [PATCH 52/56] Fix webcam framerate selection --- plugins/rtp/src/device.vala | 33 ++++++++++++++++++++-- plugins/rtp/src/plugin.vala | 6 +--- plugins/rtp/src/voice_processor_native.cpp | 6 ++-- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 3c650ad6..e25271b1 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -87,6 +87,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1"); } else if (media == "video" && device.caps.get_size() > 0) { int best_index = 0; + Value? best_fraction = null; int best_fps = 0; int best_width = 0; int best_height = 0; @@ -94,7 +95,28 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { unowned Gst.Structure? that = device.caps.get_structure(i); if (!that.has_name("video/x-raw")) continue; int num = 0, den = 0, width = 0, height = 0; - if (!that.has_field("framerate") || !that.get_fraction("framerate", out num, out den)) continue; + if (!that.has_field("framerate")) continue; + Value framerate = that.get_value("framerate"); + if (framerate.type() == typeof(Gst.Fraction)) { + num = Gst.Value.get_fraction_numerator(framerate); + den = Gst.Value.get_fraction_denominator(framerate); + } else if (framerate.type() == typeof(Gst.ValueList)) { + for(uint j = 0; j < Gst.ValueList.get_size(framerate); j++) { + Value fraction = Gst.ValueList.get_value(framerate, j); + int in_num = Gst.Value.get_fraction_numerator(fraction); + int in_den = Gst.Value.get_fraction_denominator(fraction); + int fps = den > 0 ? (num/den) : 0; + int in_fps = in_den > 0 ? (in_num/in_den) : 0; + if (in_fps > fps) { + best_fraction = fraction; + num = in_num; + den = in_den; + } + } + } else { + debug("Unknown type for framerate: %s", framerate.type_name()); + } + if (den == 0) continue; if (!that.has_field("width") || !that.get_int("width", out width)) continue; if (!that.has_field("height") || !that.get_int("height", out height)) continue; int fps = num/den; @@ -105,7 +127,14 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { best_index = i; } } - return caps_copy_nth(device.caps, best_index); + Gst.Caps res = caps_copy_nth(device.caps, best_index); + unowned Gst.Structure? that = res.get_structure(0); + Value framerate = that.get_value("framerate"); + if (framerate.type() == typeof(Gst.ValueList)) { + that.set_value("framerate", best_fraction); + } + debug("Selected caps %s", res.to_string()); + return res; } else if (device.caps.get_size() > 0) { return caps_copy_nth(device.caps, 0); } else { diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index f575a7d0..19a266b1 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -136,11 +136,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { pipe.set_state(Gst.State.PLAYING); break; case Gst.MessageType.STATE_CHANGED: - Gst.State new_state; - message.parse_state_changed(null, out new_state, null); - if (message.src is Gst.Element) { - debug("%s changed state to %s", ((Gst.Element)message.src).name, new_state.to_string()); - } + // Ignore break; case Gst.MessageType.STREAM_STATUS: Gst.StreamStatusType status; diff --git a/plugins/rtp/src/voice_processor_native.cpp b/plugins/rtp/src/voice_processor_native.cpp index 9b3292b8..00f719e1 100644 --- a/plugins/rtp/src/voice_processor_native.cpp +++ b/plugins/rtp/src/voice_processor_native.cpp @@ -75,7 +75,7 @@ dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAud memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); int err = apm->AnalyzeReverseStream(&frame); - if (err < 0) g_warning("ProcessReverseStream %i", err); + if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err); gst_audio_buffer_unmap(&audio_buffer); } @@ -108,7 +108,7 @@ extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *nativ g_debug("voice_processor_native.cpp: Stream delay metrics: %i %i %f", median, std, fraction_poor_delays); if (fraction_poor_delays > 0.5) { native->stream_delay = std::max(0, native->stream_delay + std::min(-10, std::max(median, 10))); - g_debug("Adjusted stream delay %i", native->stream_delay); + g_debug("voice_processor_native.cpp: Adjusted stream delay %i", native->stream_delay); } } @@ -130,7 +130,7 @@ dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo * apm->set_stream_delay_ms(native->stream_delay); int err = apm->ProcessStream(&frame); if (err >= 0) memcpy(audio_buffer.planes[0], frame.data_, frame.samples_per_channel_ * info->bpf); - if (err < 0) g_warning("ProcessStream %i", err); + if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err); gst_audio_buffer_unmap(&audio_buffer); } From 7d2e64769067c1b47e0500f6456dd7e6f4eb435a Mon Sep 17 00:00:00 2001 From: fiaxh Date: Thu, 29 Apr 2021 15:56:22 +0200 Subject: [PATCH 53/56] Improve call wording, cleanup --- main/src/ui/call_window/call_bottom_bar.vala | 1 - main/src/ui/call_window/call_window.vala | 4 +- .../call_window/call_window_controller.vala | 30 ++--- .../call_widget.vala | 2 +- .../ui/conversation_titlebar/call_entry.vala | 5 +- .../src/dtls_srtp_verification_draft.vala | 1 - plugins/rtp/src/module.vala | 12 +- plugins/rtp/src/stream.vala | 4 +- .../src/module/xep/0166_jingle/component.vala | 12 +- .../src/module/xep/0166_jingle/content.vala | 4 +- .../0167_jingle_rtp/jingle_rtp_module.vala | 18 +-- .../transport_parameters.vala | 2 +- .../module/xep/0234_jingle_file_transfer.vala | 20 +-- .../xep/0260_jingle_socks5_bytestreams.vala | 118 +++++++++--------- .../xep/0261_jingle_in_band_bytestreams.vala | 2 +- 15 files changed, 125 insertions(+), 110 deletions(-) diff --git a/main/src/ui/call_window/call_bottom_bar.vala b/main/src/ui/call_window/call_bottom_bar.vala index a9fee8c3..a3e4b93b 100644 --- a/main/src/ui/call_window/call_bottom_bar.vala +++ b/main/src/ui/call_window/call_bottom_bar.vala @@ -25,7 +25,6 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { private MenuButton video_settings_button = new MenuButton() { halign=Align.END, valign=Align.END }; public VideoSettingsPopover? video_settings_popover; - private EventBox encryption_event_box = new EventBox() { visible=true }; private MenuButton encryption_button = new MenuButton() { relief=ReliefStyle.NONE, height_request=30, width_request=30, margin_start=20, margin_bottom=25, halign=Align.START, valign=Align.END }; private Image encryption_image = new Image.from_icon_name("changes-allow-symbolic", IconSize.BUTTON) { visible=true }; diff --git a/main/src/ui/call_window/call_window.vala b/main/src/ui/call_window/call_window.vala index 572f73b6..3b3d4dc2 100644 --- a/main/src/ui/call_window/call_window.vala +++ b/main/src/ui/call_window/call_window.vala @@ -158,13 +158,13 @@ namespace Dino.Ui { public void set_status(string state) { switch (state) { case "requested": - header_bar.subtitle = _("Sending a call request…"); + header_bar.subtitle = _("Calling…"); break; case "ringing": header_bar.subtitle = _("Ringing…"); break; case "establishing": - header_bar.subtitle = _("Establishing a (peer-to-peer) connection…"); + header_bar.subtitle = _("Connecting…"); break; default: header_bar.subtitle = null; diff --git a/main/src/ui/call_window/call_window_controller.vala b/main/src/ui/call_window/call_window_controller.vala index 616e341d..0a223d72 100644 --- a/main/src/ui/call_window/call_window_controller.vala +++ b/main/src/ui/call_window/call_window_controller.vala @@ -3,8 +3,6 @@ using Gtk; public class Dino.Ui.CallWindowController : Object { - public signal void terminated(); - private CallWindow call_window; private Call call; private Conversation conversation; @@ -40,8 +38,16 @@ public class Dino.Ui.CallWindowController : Object { call_window.set_status("requested"); } - call_window.bottom_bar.hang_up.connect(end_call); - call_window.destroy.connect(end_call); + call_window.bottom_bar.hang_up.connect(() => { + calls.end_call(conversation, call); + call_window.close(); + call_window.destroy(); + this.dispose(); + }); + call_window.destroy.connect(() => { + calls.end_call(conversation, call); + this.dispose(); + }); call_window.bottom_bar.notify["audio-enabled"].connect(() => { calls.mute_own_audio(call, !call_window.bottom_bar.audio_enabled); @@ -116,16 +122,6 @@ public class Dino.Ui.CallWindowController : Object { this.window_width = this.call_window.get_allocated_width(); } - private void end_call() { - call.notify["state"].disconnect(on_call_state_changed); - calls.call_terminated.disconnect(on_call_terminated); - - calls.end_call(conversation, call); - call_window.close(); - call_window.destroy(); - terminated(); - } - private void on_call_state_changed() { if (call.state == Call.State.IN_PROGRESS) { call_window.set_status(""); @@ -234,4 +230,10 @@ public class Dino.Ui.CallWindowController : Object { call_window.unset_own_video(); } } + + public override void dispose() { + base.dispose(); + call.notify["state"].disconnect(on_call_state_changed); + calls.call_terminated.disconnect(on_call_terminated); + } } \ No newline at end of file diff --git a/main/src/ui/conversation_content_view/call_widget.vala b/main/src/ui/conversation_content_view/call_widget.vala index 66788e28..74525d11 100644 --- a/main/src/ui/conversation_content_view/call_widget.vala +++ b/main/src/ui/conversation_content_view/call_widget.vala @@ -154,7 +154,7 @@ namespace Dino.Ui { case Call.State.FAILED: image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); title_label.label = _("Call failed"); - subtitle_label.label = "This call failed to establish"; + subtitle_label.label = "Call failed to establish"; break; } } diff --git a/main/src/ui/conversation_titlebar/call_entry.vala b/main/src/ui/conversation_titlebar/call_entry.vala index e1d10e5c..9353f631 100644 --- a/main/src/ui/conversation_titlebar/call_entry.vala +++ b/main/src/ui/conversation_titlebar/call_entry.vala @@ -92,9 +92,6 @@ namespace Dino.Ui { call_window.present(); update_button_state(); - call_controller.terminated.connect(() => { - update_button_state(); - }); } public new void set_conversation(Conversation conversation) { @@ -119,7 +116,7 @@ namespace Dino.Ui { if (conversation.type_ == Conversation.Type.CHAT) { Conversation conv_bak = conversation; bool audio_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_audio_calls_async(conversation); - bool video_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_audio_calls_async(conversation); + bool video_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_video_calls_async(conversation); if (conv_bak != conversation) return; visible = audio_works; diff --git a/plugins/omemo/src/dtls_srtp_verification_draft.vala b/plugins/omemo/src/dtls_srtp_verification_draft.vala index e2441670..66a31954 100644 --- a/plugins/omemo/src/dtls_srtp_verification_draft.vala +++ b/plugins/omemo/src/dtls_srtp_verification_draft.vala @@ -65,7 +65,6 @@ namespace Dino.Plugins.Omemo.DtlsSrtpVerificationDraft { stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.begin(jingle_sid, (_, res) => { Xep.Jingle.Session? session = stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.end(res); - if (session != null) print(@"$(session.contents_map.has_key(content_name))\n"); if (session == null || !session.contents_map.has_key(content_name)) return; var encryption = new OmemoContentEncryption() { encryption_ns=NS_URI, encryption_name="OMEMO", our_key=new uint8[0], peer_key=new uint8[0], peer_device_id=device_id_by_jingle_sid[jingle_sid] }; session.contents_map[content_name].encryptions[NS_URI] = encryption; diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index 13a21cd8..19a7501d 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -216,9 +216,9 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { } public override JingleRtp.Crypto? generate_local_crypto() { - uint8[] keyAndSalt = new uint8[30]; - Crypto.randomize(keyAndSalt); - return JingleRtp.Crypto.create(JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_80, keyAndSalt); + uint8[] key_and_salt = new uint8[30]; + Crypto.randomize(key_and_salt); + return JingleRtp.Crypto.create(JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_80, key_and_salt); } public override JingleRtp.Crypto? pick_remote_crypto(Gee.List cryptos) { @@ -230,8 +230,8 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { public override JingleRtp.Crypto? pick_local_crypto(JingleRtp.Crypto? remote) { if (remote == null || !remote.is_valid) return null; - uint8[] keyAndSalt = new uint8[30]; - Crypto.randomize(keyAndSalt); - return remote.rekey(keyAndSalt); + uint8[] key_and_salt = new uint8[30]; + Crypto.randomize(key_and_salt); + return remote.rekey(key_and_salt); } } \ No newline at end of file diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 23634aa3..bd8a279f 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -256,7 +256,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private void prepare_local_crypto() { - if (local_crypto != null && !crypto_session.has_encrypt) { + if (local_crypto != null && local_crypto.is_valid && !crypto_session.has_encrypt) { crypto_session.set_encryption_key(local_crypto.crypto_suite, local_crypto.key, local_crypto.salt); debug("Setting up encryption with key params %s", local_crypto.key_params); } @@ -396,7 +396,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private void prepare_remote_crypto() { - if (remote_crypto != null && !crypto_session.has_decrypt) { + if (remote_crypto != null && remote_crypto.is_valid && !crypto_session.has_decrypt) { crypto_session.set_decryption_key(remote_crypto.crypto_suite, remote_crypto.key, remote_crypto.salt); debug("Setting up decryption with key params %s", remote_crypto.key_params); } diff --git a/xmpp-vala/src/module/xep/0166_jingle/component.vala b/xmpp-vala/src/module/xep/0166_jingle/component.vala index 544bcd69..5d573522 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/component.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/component.vala @@ -31,7 +31,11 @@ namespace Xmpp.Xep.Jingle { protected Gee.Promise promise = new Gee.Promise(); private string? terminated = null; - public async void init(IOStream stream) { + public async void set_stream(IOStream? stream) { + if (stream == null) { + promise.set_exception(new IOError.FAILED("Jingle connection failed")); + return; + } assert(!this.stream.ready); promise.set_value(stream); if (terminated != null) { @@ -39,11 +43,17 @@ namespace Xmpp.Xep.Jingle { } } + public void set_error(GLib.Error? e) { + promise.set_exception(e); + } + public override async void terminate(bool we_terminated, string? reason_name = null, string? reason_text = null) { if (terminated == null) { terminated = (reason_name ?? "") + " - " + (reason_text ?? "") + @"we terminated? $we_terminated"; if (stream.ready) { yield stream.value.close_async(); + } else { + promise.set_exception(new IOError.FAILED("Jingle connection failed")); } } } diff --git a/xmpp-vala/src/module/xep/0166_jingle/content.vala b/xmpp-vala/src/module/xep/0166_jingle/content.vala index befe02f4..41310aeb 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/content.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/content.vala @@ -36,7 +36,7 @@ public class Xmpp.Xep.Jingle.Content : Object { public HashMap encryptions = new HashMap(); - public Set tried_transport_methods = new HashSet(); + private Set tried_transport_methods = new HashSet(); public Content.initiate_sent(string content_name, Senders senders, @@ -109,7 +109,7 @@ public class Xmpp.Xep.Jingle.Content : Object { transport_params.dispose(); foreach (ComponentConnection connection in component_connections.values) { - connection.terminate(we_terminated, reason_name, reason_text); + connection.terminate.begin(we_terminated, reason_name, reason_text); } } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala index 6eb6289b..6b55cbe6 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala @@ -83,7 +83,7 @@ public abstract class Module : XmppStreamModule { } } - public async Jingle.Content add_outgoing_video_content(XmppStream stream, Jingle.Session session) { + public async Jingle.Content add_outgoing_video_content(XmppStream stream, Jingle.Session session) throws Jingle.Error { Jid my_jid = session.local_full_jid; Jid receiver_full_jid = session.peer_full_jid; @@ -168,7 +168,7 @@ public class Crypto { public string? session_params { get; private set; } public string tag { get; private set; } - public uint8[] key_and_salt { owned get { + public uint8[]? key_and_salt { owned get { if (!key_params.has_prefix("inline:")) return null; int endIndex = key_params.index_of("|"); if (endIndex < 0) endIndex = key_params.length; @@ -221,30 +221,30 @@ public class Crypto { case AES_CM_128_HMAC_SHA1_80: case AES_CM_128_HMAC_SHA1_32: case F8_128_HMAC_SHA1_80: - return key_and_salt.length == 30; + return key_and_salt != null && key_and_salt.length == 30; } return false; }} - public uint8[] key { owned get { - uint8[] key_and_salt = key_and_salt; + public uint8[]? key { owned get { + uint8[]? key_and_salt = key_and_salt; switch(crypto_suite) { case AES_CM_128_HMAC_SHA1_80: case AES_CM_128_HMAC_SHA1_32: case F8_128_HMAC_SHA1_80: - if (key_and_salt.length >= 16) return key_and_salt[0:16]; + if (key_and_salt != null && key_and_salt.length >= 16) return key_and_salt[0:16]; break; } return null; }} - public uint8[] salt { owned get { - uint8[] keyAndSalt = key_and_salt; + public uint8[]? salt { owned get { + uint8[]? key_and_salt = key_and_salt; switch(crypto_suite) { case AES_CM_128_HMAC_SHA1_80: case AES_CM_128_HMAC_SHA1_32: case F8_128_HMAC_SHA1_80: - if (keyAndSalt.length >= 30) return keyAndSalt[16:30]; + if (key_and_salt != null && key_and_salt.length >= 30) return key_and_salt[16:30]; break; } return null; diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala index 83da296b..07b599ee 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala @@ -152,7 +152,7 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T return sb.str; } - private uint8[] fingerprint_to_bytes(string? fingerprint_) { + private uint8[]? fingerprint_to_bytes(string? fingerprint_) { if (fingerprint_ == null) return null; string fingerprint = fingerprint_.replace(":", "").up(); diff --git a/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala b/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala index 07b158bc..4581019f 100644 --- a/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala +++ b/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala @@ -268,13 +268,19 @@ public class FileTransfer : Object { content.accept(); Jingle.StreamingConnection connection = content.component_connections.values.to_array()[0] as Jingle.StreamingConnection; - IOStream? io_stream = yield connection.stream.wait_async(); - FileTransferInputStream ft_stream = new FileTransferInputStream(io_stream.input_stream, size); - io_stream.output_stream.close(); - ft_stream.closed.connect(() => { - session.terminate(Jingle.ReasonElement.SUCCESS, null, null); - }); - this.stream = ft_stream; + try { + IOStream io_stream = yield connection.stream.wait_async(); + FileTransferInputStream ft_stream = new FileTransferInputStream(io_stream.input_stream, size); + io_stream.output_stream.close(); + ft_stream.closed.connect(() => { + session.terminate(Jingle.ReasonElement.SUCCESS, null, null); + }); + this.stream = ft_stream; + } catch (FutureError.EXCEPTION e) { + warning("Error accepting Jingle file-transfer: %s", connection.stream.exception.message); + } catch (FutureError e) { + warning("FutureError accepting Jingle file-transfer: %s", e.message); + } } public void reject(XmppStream stream) { diff --git a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala index 6edebbbc..47c243e8 100644 --- a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala +++ b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala @@ -28,6 +28,7 @@ public class Module : Jingle.Transport, XmppStreamModule { public string ns_uri { get { return NS_URI; } } public Jingle.TransportType type_ { get { return Jingle.TransportType.STREAMING; } } public int priority { get { return 1; } } + private Gee.List get_proxies(XmppStream stream) { Gee.List result = new ArrayList(); int i = 1 << 15; @@ -37,6 +38,7 @@ public class Module : Jingle.Transport, XmppStreamModule { } return result; } + private Gee.List start_local_listeners(XmppStream stream, Jid local_full_jid, string dstaddr, out LocalListener? local_listener) { Gee.List result = new ArrayList(); SocketListener listener = new SocketListener(); @@ -62,15 +64,17 @@ public class Module : Jingle.Transport, XmppStreamModule { } return result; } + private void select_candidates(XmppStream stream, Jid local_full_jid, string dstaddr, Parameters result) { result.local_candidates.add_all(get_proxies(stream)); - //result.local_candidates.add_all(start_local_listeners(stream, local_full_jid, dstaddr, out result.listener)); + result.local_candidates.add_all(start_local_listeners(stream, local_full_jid, dstaddr, out result.listener)); result.local_candidates.sort((c1, c2) => { if (c1.priority < c2.priority) { return 1; } if (c1.priority > c2.priority) { return -1; } return 0; }); } + public Jingle.TransportParameters create_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid) { assert(components == 1); Parameters result = new Parameters.create(local_full_jid, peer_full_jid, random_uuid()); @@ -78,6 +82,7 @@ public class Module : Jingle.Transport, XmppStreamModule { select_candidates(stream, local_full_jid, dstaddr, result); return result; } + public Jingle.TransportParameters parse_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { Parameters result = Parameters.parse(local_full_jid, peer_full_jid, transport); string dstaddr = calculate_dstaddr(result.sid, local_full_jid, peer_full_jid); @@ -146,6 +151,7 @@ public class Candidate : Socks5Bytestreams.Proxy { public Candidate.build(string cid, string host, Jid jid, int port, int local_priority, CandidateType type) { this(cid, host, jid, port, type.type_preference() + local_priority, type); } + public Candidate.proxy(string cid, Socks5Bytestreams.Proxy proxy, int local_priority) { this.build(cid, proxy.host, proxy.jid, proxy.port, local_priority, CandidateType.PROXY); } @@ -170,6 +176,7 @@ public class Candidate : Socks5Bytestreams.Proxy { return new Candidate(cid, host, jid, port, priority, type); } + public StanzaNode to_xml() { return new StanzaNode.build("candidate", NS_URI) .put_attribute("cid", cid) @@ -210,6 +217,7 @@ class LocalListener { this.inner = inner; this.dstaddr = dstaddr; } + public LocalListener.empty() { this.inner = null; this.dstaddr = ""; @@ -233,6 +241,7 @@ class LocalListener { handle_conn.begin(((StringWrapper)cid).str, conn); } } + async void handle_conn(string cid, SocketConnection conn) { conn.socket.timeout = NEGOTIATION_TIMEOUT; size_t read; @@ -418,39 +427,39 @@ class Parameters : Jingle.TransportParameters, Object { } public void handle_transport_info(StanzaNode transport) throws Jingle.IqError { - StanzaNode? candidate_error = transport.get_subnode("candidate-error", NS_URI); - StanzaNode? candidate_used = transport.get_subnode("candidate-used", NS_URI); - StanzaNode? activated = transport.get_subnode("activated", NS_URI); - StanzaNode? proxy_error = transport.get_subnode("proxy-error", NS_URI); - int num_children = 0; - if (candidate_error != null) { num_children += 1; } - if (candidate_used != null) { num_children += 1; } - if (activated != null) { num_children += 1; } - if (proxy_error != null) { num_children += 1; } - if (num_children == 0) { - throw new Jingle.IqError.UNSUPPORTED_INFO("unknown transport-info"); - } else if (num_children > 1) { - throw new Jingle.IqError.BAD_REQUEST("transport-info with more than one child"); + ArrayList socks5_nodes = new ArrayList(); + foreach (StanzaNode node in transport.sub_nodes) { + if (node.ns_uri == NS_URI) socks5_nodes.add(node); } - if (candidate_error != null) { - handle_remote_candidate(null); - } - if (candidate_used != null) { - string? cid = candidate_used.get_attribute("cid"); - if (cid == null) { - throw new Jingle.IqError.BAD_REQUEST("missing cid"); - } - handle_remote_candidate(cid); - } - if (activated != null) { - string? cid = activated.get_attribute("cid"); - if (cid == null) { - throw new Jingle.IqError.BAD_REQUEST("missing cid"); - } - handle_activated(cid); - } - if (proxy_error != null) { - handle_proxy_error(); + if (socks5_nodes.is_empty) { warning("No socks5 subnodes in transport node"); return; } + if (socks5_nodes.size > 1) { warning("Too many socks5 subnodes in transport node"); return; } + + StanzaNode node = socks5_nodes[0]; + + switch (node.name) { + case "activated": + string? cid = node.get_attribute("cid"); + if (cid == null) { + throw new Jingle.IqError.BAD_REQUEST("missing cid"); + } + handle_activated(cid); + break; + case "candidate-used": + string? cid = node.get_attribute("cid"); + if (cid == null) { + throw new Jingle.IqError.BAD_REQUEST("missing cid"); + } + handle_remote_candidate(cid); + break; + case "candidate-error": + handle_remote_candidate(null); + break; + case "proxy-error": + handle_proxy_error(); + break; + default: + warning("Unknown transport-info: %s", transport.to_string()); + break; } } @@ -499,32 +508,22 @@ class Parameters : Jingle.TransportParameters, Object { return; } - Candidate? remote = remote_selected_candidate; - Candidate? local = local_selected_candidate; - - int num_candidates = 0; - if (remote != null) { num_candidates += 1; } - if (local != null) { num_candidates += 1; } - - if (num_candidates == 0) { - // Notify Jingle of the failed transport. - content_set_transport_connection(null); + if (remote_selected_candidate == null && local_selected_candidate == null) { + content_set_transport_connection_error(new IOError.FAILED("No candidates")); return; } bool remote_wins; - if (num_candidates == 1) { - remote_wins = remote != null; - } else { - if (local.priority < remote.priority) { - remote_wins = true; - } else if (local.priority > remote.priority) { - remote_wins = false; - } else { + if (remote_selected_candidate != null && local_selected_candidate != null) { + if (local_selected_candidate.priority == remote_selected_candidate.priority) { // equal priority -> XEP-0260 says that the candidate offered // by the initiator wins, so the one that the remote chose remote_wins = role == Jingle.Role.INITIATOR; + } else { + remote_wins = local_selected_candidate.priority < remote_selected_candidate.priority; } + } else { + remote_wins = remote_selected_candidate != null; } if (!remote_wins) { @@ -545,8 +544,7 @@ class Parameters : Jingle.TransportParameters, Object { } SocketConnection? conn = listener.get_connection(remote_selected_candidate.cid); if (conn == null) { - // Remote hasn't actually connected to us?! - content_set_transport_connection(null); + content_set_transport_connection_error(new IOError.FAILED("Remote hasn't actually connected to us?!")); return; } content_set_transport_connection(conn); @@ -569,7 +567,7 @@ class Parameters : Jingle.TransportParameters, Object { if (!waiting_for_activation_error) { content_set_transport_connection(conn); } else { - content_set_transport_connection(null); + content_set_transport_connection_error(new IOError.FAILED("waiting_for_activation_error")); } } @@ -620,7 +618,7 @@ class Parameters : Jingle.TransportParameters, Object { .put_attribute("sid", sid) .put_node(new StanzaNode.build("proxy-error", NS_URI)) ); - content_set_transport_connection(null); + content_set_transport_connection_error(new IOError.FAILED("Connect to local candidate error: %s", e.message)); } } @@ -745,15 +743,19 @@ class Parameters : Jingle.TransportParameters, Object { private Jingle.StreamingConnection connection = new Jingle.StreamingConnection(); - private void content_set_transport_connection(IOStream? ios) { - IOStream? iostream = ios; + private void content_set_transport_connection(IOStream ios) { + IOStream iostream = ios; Jingle.Content? strong_content = content; if (strong_content == null) return; if (strong_content.security_params != null) { iostream = strong_content.security_params.wrap_stream(iostream); } - connection.init.begin(iostream); + connection.set_stream.begin(iostream); + } + + private void content_set_transport_connection_error(Error e) { + connection.set_error(e); } public void create_transport_connection(XmppStream stream, Jingle.Content content) { diff --git a/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala b/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala index f7c77544..09eaf711 100644 --- a/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala +++ b/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala @@ -98,7 +98,7 @@ class Parameters : Jingle.TransportParameters, Object { if (content.security_params != null) { iostream = content.security_params.wrap_stream(iostream); } - connection.init.begin(iostream); + connection.set_stream.begin(iostream); debug("set transport conn ibb"); content.set_transport_connection(connection, 1); } From 0ad968df367f5a44c568329834115018866ff8b9 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Fri, 30 Apr 2021 21:37:02 +0200 Subject: [PATCH 54/56] Use the same DTLS fingerprint in all contents. Display audio+video enc keys in UI if they differ. --- libdino/src/service/calls.vala | 22 +++++-- main/src/ui/call_window/call_bottom_bar.vala | 60 +++++++++++-------- .../call_window/call_window_controller.vala | 4 +- plugins/ice/src/dtls_srtp.vala | 35 +++++++---- plugins/ice/src/module.vala | 17 +++++- plugins/ice/src/transport_parameters.vala | 8 +-- 6 files changed, 96 insertions(+), 50 deletions(-) diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index d535dfca..a44b59fd 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -14,7 +14,7 @@ namespace Dino { public signal void counterpart_ringing(Call call); public signal void counterpart_sends_video_updated(Call call, bool mute); public signal void info_received(Call call, Xep.JingleRtp.CallSessionInfo session_info); - public signal void encryption_updated(Call call, Xep.Jingle.ContentEncryption? encryption); + public signal void encryption_updated(Call call, Xep.Jingle.ContentEncryption? audio_encryption, Xep.Jingle.ContentEncryption? video_encryption, bool same); public signal void stream_created(Call call, string media); @@ -523,7 +523,7 @@ namespace Dino { if ((audio_encryptions.has_key(call) && audio_encryptions[call].is_empty) || (video_encryptions.has_key(call) && video_encryptions[call].is_empty)) { call.encryption = Encryption.NONE; - encryption_updated(call, null); + encryption_updated(call, null, null, true); return; } @@ -545,16 +545,26 @@ namespace Dino { if (omemo_encryption != null && dtls_encryption != null) { call.encryption = Encryption.OMEMO; - encryption_updated(call, omemo_encryption); + Xep.Jingle.ContentEncryption? video_encryption = video_encryptions.has_key(call) ? video_encryptions[call]["http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification"] : null; + omemo_encryption.peer_key = dtls_encryption.peer_key; + omemo_encryption.our_key = dtls_encryption.our_key; + encryption_updated(call, omemo_encryption, video_encryption, true); } else if (dtls_encryption != null) { call.encryption = Encryption.DTLS_SRTP; - encryption_updated(call, dtls_encryption); + Xep.Jingle.ContentEncryption? video_encryption = video_encryptions.has_key(call) ? video_encryptions[call][Xep.JingleIceUdp.DTLS_NS_URI] : null; + bool same = true; + if (video_encryption != null && dtls_encryption.peer_key.length == video_encryption.peer_key.length) { + for (int i = 0; i < dtls_encryption.peer_key.length; i++) { + if (dtls_encryption.peer_key[i] != video_encryption.peer_key[i]) { same = false; break; } + } + } + encryption_updated(call, dtls_encryption, video_encryption, same); } else if (srtp_encryption != null) { call.encryption = Encryption.SRTP; - encryption_updated(call, srtp_encryption); + encryption_updated(call, srtp_encryption, video_encryptions[call]["SRTP"], false); } else { call.encryption = Encryption.NONE; - encryption_updated(call, null); + encryption_updated(call, null, null, true); } } diff --git a/main/src/ui/call_window/call_bottom_bar.vala b/main/src/ui/call_window/call_bottom_bar.vala index a3e4b93b..64b157dd 100644 --- a/main/src/ui/call_window/call_bottom_bar.vala +++ b/main/src/ui/call_window/call_bottom_bar.vala @@ -89,42 +89,54 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { this.get_style_context().add_class("call-bottom-bar"); } - public void set_encryption(Xmpp.Xep.Jingle.ContentEncryption? encryption) { + public void set_encryption(Xmpp.Xep.Jingle.ContentEncryption? audio_encryption, Xmpp.Xep.Jingle.ContentEncryption? video_encryption, bool same) { encryption_button.visible = true; Popover popover = new Popover(encryption_button); - - if (encryption == null) { + if (audio_encryption == null) { encryption_image.set_from_icon_name("changes-allow-symbolic", IconSize.BUTTON); encryption_button.get_style_context().add_class("unencrypted"); popover.add(new Label("This call isn't encrypted.") { margin=10, visible=true } ); - } else if (encryption.encryption_name == "OMEMO") { - encryption_image.set_from_icon_name("changes-prevent-symbolic", IconSize.BUTTON); - encryption_button.get_style_context().remove_class("unencrypted"); - - popover.add(new Label("This call is encrypted with OMEMO.") { margin=10, visible=true } ); - } else { - encryption_image.set_from_icon_name("changes-prevent-symbolic", IconSize.BUTTON); - encryption_button.get_style_context().remove_class("unencrypted"); - - Grid encryption_info_grid = new Grid() { margin=10, row_spacing=3, column_spacing=5, visible=true }; - encryption_info_grid.attach(new Label("This call is end-to-end encrypted.") { use_markup=true, xalign=0, visible=true }, 1, 1, 2, 1); - if (encryption.peer_key.length > 0) { - encryption_info_grid.attach(new Label("Peer key") { xalign=0, visible=true }, 1, 2, 1, 1); - encryption_info_grid.attach(new Label("" + format_fingerprint(encryption.peer_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 2, 1, 1); - } - if (encryption.our_key.length > 0) { - encryption_info_grid.attach(new Label("Your key") { xalign=0, visible=true }, 1, 3, 1, 1); - encryption_info_grid.attach(new Label("" + format_fingerprint(encryption.our_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 3, 1, 1); - } - - popover.add(encryption_info_grid); + return; } + encryption_image.set_from_icon_name("changes-prevent-symbolic", IconSize.BUTTON); + encryption_button.get_style_context().remove_class("unencrypted"); + + Box box = new Box(Orientation.VERTICAL, 5) { margin=10, visible=true }; + if (audio_encryption.encryption_name == "OMEMO") { + box.add(new Label("This call is encrypted with OMEMO.") { use_markup=true, xalign=0, visible=true } ); + } else { + box.add(new Label("This call is end-to-end encrypted.") { use_markup=true, xalign=0, visible=true }); + } + + if (same) { + box.add(create_media_encryption_grid(audio_encryption)); + } else { + box.add(new Label("Audio") { use_markup=true, xalign=0, visible=true }); + box.add(create_media_encryption_grid(audio_encryption)); + box.add(new Label("Video") { use_markup=true, xalign=0, visible=true }); + box.add(create_media_encryption_grid(video_encryption)); + } + popover.add(box); + encryption_button.set_popover(popover); } + private Grid create_media_encryption_grid(Xmpp.Xep.Jingle.ContentEncryption? encryption) { + Grid ret = new Grid() { row_spacing=3, column_spacing=5, visible=true }; + if (encryption.peer_key.length > 0) { + ret.attach(new Label("Peer call key") { xalign=0, visible=true }, 1, 2, 1, 1); + ret.attach(new Label("" + format_fingerprint(encryption.peer_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 2, 1, 1); + } + if (encryption.our_key.length > 0) { + ret.attach(new Label("Your call key") { xalign=0, visible=true }, 1, 3, 1, 1); + ret.attach(new Label("" + format_fingerprint(encryption.our_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 3, 1, 1); + } + return ret; + } + public AudioSettingsPopover? show_audio_device_choices(bool show) { audio_settings_button.visible = show; if (audio_settings_popover != null) audio_settings_popover.visible = false; diff --git a/main/src/ui/call_window/call_window_controller.vala b/main/src/ui/call_window/call_window_controller.vala index 0a223d72..7e5920ce 100644 --- a/main/src/ui/call_window/call_window_controller.vala +++ b/main/src/ui/call_window/call_window_controller.vala @@ -76,9 +76,9 @@ public class Dino.Ui.CallWindowController : Object { call_window.set_status("ringing"); } }); - calls.encryption_updated.connect((call, encryption) => { + calls.encryption_updated.connect((call, audio_encryption, video_encryption, same) => { if (!this.call.equals(call)) return; - call_window.bottom_bar.set_encryption(encryption); + call_window.bottom_bar.set_encryption(audio_encryption, video_encryption, same); }); own_video.resolution_changed.connect((width, height) => { diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index f5ef830a..0254351d 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -2,10 +2,10 @@ using GnuTLS; namespace Dino.Plugins.Ice.DtlsSrtp { -public static Handler setup() throws GLib.Error { - var obj = new Handler(); - obj.generate_credentials(); - return obj; +public class CredentialsCapsule { + public uint8[] own_fingerprint; + public X509.Certificate[] own_cert; + public X509.PrivateKey private_key; } public class Handler { @@ -21,8 +21,7 @@ public class Handler { public uint8[] peer_fingerprint { get; set; } public string peer_fp_algo { get; set; } - private X509.Certificate[] own_cert; - private X509.PrivateKey private_key; + private CredentialsCapsule credentials; private Cond buffer_cond = Cond(); private Mutex buffer_mutex = Mutex(); private Gee.LinkedList buffer_queue = new Gee.LinkedList(); @@ -33,6 +32,11 @@ public class Handler { private Crypto.Srtp.Session srtp_session = new Crypto.Srtp.Session(); + public Handler.with_cert(CredentialsCapsule creds) { + this.credentials = creds; + this.own_fingerprint = creds.own_fingerprint; + } + public uint8[]? process_incoming_data(uint component_id, uint8[] data) { if (srtp_session.has_decrypt) { try { @@ -78,10 +82,10 @@ public class Handler { buffer_mutex.unlock(); } - internal void generate_credentials() throws GLib.Error { + internal static CredentialsCapsule generate_credentials() throws GLib.Error { int err = 0; - private_key = X509.PrivateKey.create(); + X509.PrivateKey private_key = X509.PrivateKey.create(); err = private_key.generate(PKAlgorithm.RSA, 2048); throw_if_error(err); @@ -99,8 +103,15 @@ public class Handler { cert.sign(cert, private_key); - own_fingerprint = get_fingerprint(cert, DigestAlgorithm.SHA256); - own_cert = new X509.Certificate[] { (owned)cert }; + uint8[] own_fingerprint = get_fingerprint(cert, DigestAlgorithm.SHA256); + X509.Certificate[] own_cert = new X509.Certificate[] { (owned)cert }; + + var creds = new CredentialsCapsule(); + creds.own_fingerprint = own_fingerprint; + creds.own_cert = (owned) own_cert; + creds.private_key = (owned) private_key; + + return creds; } public void stop_dtls_connection() { @@ -129,7 +140,7 @@ public class Handler { debug("Setting up DTLS connection. We're %s", mode.to_string()); CertificateCredentials cert_cred = CertificateCredentials.create(); - int err = cert_cred.set_x509_key(own_cert, private_key); + int err = cert_cred.set_x509_key(credentials.own_cert, credentials.private_key); throw_if_error(err); Session? session = Session.create(server_or_client | InitFlags.DATAGRAM); @@ -200,7 +211,7 @@ public class Handler { srtp_session.set_encryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, client_key.extract(), client_salt.extract()); srtp_session.set_decryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, server_key.extract(), server_salt.extract()); } - return new Xmpp.Xep.Jingle.ContentEncryption() { encryption_ns=Xmpp.Xep.JingleIceUdp.DTLS_NS_URI, encryption_name = "DTLS-SRTP", our_key=own_fingerprint, peer_key=peer_fingerprint }; + return new Xmpp.Xep.Jingle.ContentEncryption() { encryption_ns=Xmpp.Xep.JingleIceUdp.DTLS_NS_URI, encryption_name = "DTLS-SRTP", our_key=credentials.own_fingerprint, peer_key=peer_fingerprint }; } private static ssize_t pull_function(void* transport_ptr, uint8[] buffer) { diff --git a/plugins/ice/src/module.vala b/plugins/ice/src/module.vala index e961ffb6..2645d7dc 100644 --- a/plugins/ice/src/module.vala +++ b/plugins/ice/src/module.vala @@ -10,6 +10,7 @@ public class Dino.Plugins.Ice.Module : JingleIceUdp.Module { public Xep.ExternalServiceDiscovery.Service? turn_service = null; private weak Nice.Agent? agent; + private HashMap cerds = new HashMap(); private Nice.Agent get_agent() { Nice.Agent? agent = this.agent; @@ -29,11 +30,23 @@ public class Dino.Plugins.Ice.Module : JingleIceUdp.Module { } public override Jingle.TransportParameters create_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid) { - return new TransportParameters(get_agent(), turn_service, turn_ip, components, local_full_jid, peer_full_jid); + DtlsSrtp.CredentialsCapsule? cred = get_create_credentials(local_full_jid, peer_full_jid); + return new TransportParameters(get_agent(), cred, turn_service, turn_ip, components, local_full_jid, peer_full_jid); } public override Jingle.TransportParameters parse_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { - return new TransportParameters(get_agent(), turn_service, turn_ip, components, local_full_jid, peer_full_jid, transport); + DtlsSrtp.CredentialsCapsule? cred = get_create_credentials(local_full_jid, peer_full_jid); + return new TransportParameters(get_agent(), cred, turn_service, turn_ip, components, local_full_jid, peer_full_jid, transport); + } + + private DtlsSrtp.CredentialsCapsule? get_create_credentials(Jid local_full_jid, Jid peer_full_jid) { + string from_to_id = local_full_jid.to_string() + peer_full_jid.to_string(); + try { + if (!cerds.has_key(from_to_id)) cerds[from_to_id] = DtlsSrtp.Handler.generate_credentials(); + } catch (Error e) { + warning("Error creating dtls credentials: %s", e.message); + } + return cerds[from_to_id]; } private void agent_unweak() { diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index 38652952..62c04906 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -60,13 +60,13 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport } } - public TransportParameters(Nice.Agent agent, Xep.ExternalServiceDiscovery.Service? turn_service, string? turn_ip, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode? node = null) { + public TransportParameters(Nice.Agent agent, DtlsSrtp.CredentialsCapsule? credentials, Xep.ExternalServiceDiscovery.Service? turn_service, string? turn_ip, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode? node = null) { base(components, local_full_jid, peer_full_jid, node); this.we_want_connection = (node == null); this.agent = agent; if (this.peer_fingerprint != null || !incoming) { - dtls_srtp_handler = setup_dtls(this); + dtls_srtp_handler = setup_dtls(this, credentials); own_fingerprint = dtls_srtp_handler.own_fingerprint; if (incoming) { own_setup = "active"; @@ -113,9 +113,9 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport agent.gather_candidates(stream_id); } - private static DtlsSrtp.Handler setup_dtls(TransportParameters tp) { + private static DtlsSrtp.Handler setup_dtls(TransportParameters tp, DtlsSrtp.CredentialsCapsule credentials) { var weak_self = WeakRef(tp); - DtlsSrtp.Handler dtls_srtp = DtlsSrtp.setup(); + DtlsSrtp.Handler dtls_srtp = new DtlsSrtp.Handler.with_cert(credentials); dtls_srtp.send_data.connect((data) => { TransportParameters self = (TransportParameters) weak_self.get(); if (self != null) self.agent.send(self.stream_id, 1, data); From 8044b546d0ac15d34a3e6499b9c0d55d3d8f9c94 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 2 May 2021 00:34:17 +0200 Subject: [PATCH 55/56] Support voice processing on GStreamer 0.14 --- plugins/rtp/CMakeLists.txt | 7 ++-- plugins/rtp/src/voice_processor.vala | 2 +- plugins/rtp/src/voice_processor_native.cpp | 37 +++++++++++++--------- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index b19c8a8f..52419425 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -17,7 +17,7 @@ if(Gst_VERSION VERSION_GREATER "1.16") endif() if(WebRTCAudioProcessing_VERSION GREATER "0.4") - message(WARNING "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") + message(STATUS "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") unset(WebRTCAudioProcessing_FOUND) endif() @@ -25,8 +25,9 @@ if(WebRTCAudioProcessing_FOUND) set(RTP_DEFINITIONS ${RTP_DEFINITIONS} WITH_VOICE_PROCESSOR) set(RTP_VOICE_PROCESSOR_VALA src/voice_processor.vala) set(RTP_VOICE_PROCESSOR_CXX src/voice_processor_native.cpp) + set(RTP_VOICE_PROCESSOR_LIB webrtc-audio-processing) else() - message(WARNING "WebRTCAudioProcessing not found, build without voice pre-processing!") + message(STATUS "WebRTCAudioProcessing not found, build without voice pre-processing!") endif() vala_precompile(RTP_VALA_C @@ -53,7 +54,7 @@ DEFINITIONS add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) add_library(rtp SHARED ${RTP_VALA_C} ${RTP_VOICE_PROCESSOR_CXX}) -target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 webrtc-audio-processing) +target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 ${RTP_VOICE_PROCESSOR_LIB}) set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/src/voice_processor.vala b/plugins/rtp/src/voice_processor.vala index e6dc7e8f..66e95d72 100644 --- a/plugins/rtp/src/voice_processor.vala +++ b/plugins/rtp/src/voice_processor.vala @@ -123,7 +123,7 @@ public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter { } analyze_reverse_stream(native, echo_probe.audio_info, buffer); if (adjust_delay_timeout_id == 0 && echo_probe != null) { - adjust_delay_timeout_id = Timeout.add(5000, adjust_delay); + adjust_delay_timeout_id = Timeout.add(1000, adjust_delay); } } diff --git a/plugins/rtp/src/voice_processor_native.cpp b/plugins/rtp/src/voice_processor_native.cpp index 00f719e1..8a052cf8 100644 --- a/plugins/rtp/src/voice_processor_native.cpp +++ b/plugins/rtp/src/voice_processor_native.cpp @@ -11,6 +11,8 @@ struct _DinoPluginsRtpVoiceProcessorNative { webrtc::AudioProcessing *apm; gint stream_delay; + gint last_median; + gint last_poor_delays; }; extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) { @@ -26,6 +28,8 @@ extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay) config.Set(new webrtc::ExperimentalAgc(true, 85)); native->apm = webrtc::AudioProcessing::Create(config); native->stream_delay = stream_delay; + native->last_median = 0; + native->last_poor_delays = 0; return native; } @@ -65,19 +69,19 @@ dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAud webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); webrtc::AudioProcessing *apm = native->apm; - GstAudioBuffer audio_buffer; - gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READ); + GstMapInfo map; + gst_buffer_map(buffer, &map, GST_MAP_READ); webrtc::AudioFrame frame; frame.num_channels_ = info->channels; frame.sample_rate_hz_ = info->rate; frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf; - memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); + memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); int err = apm->AnalyzeReverseStream(&frame); if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err); - gst_audio_buffer_unmap(&audio_buffer); + gst_buffer_unmap(buffer, &map); } extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) { @@ -101,14 +105,17 @@ extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *nati extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) { _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; webrtc::AudioProcessing *apm = native->apm; - int median, std; + int median, std, poor_delays; float fraction_poor_delays; apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays); - if (fraction_poor_delays < 0) return; - g_debug("voice_processor_native.cpp: Stream delay metrics: %i %i %f", median, std, fraction_poor_delays); - if (fraction_poor_delays > 0.5) { - native->stream_delay = std::max(0, native->stream_delay + std::min(-10, std::max(median, 10))); - g_debug("voice_processor_native.cpp: Adjusted stream delay %i", native->stream_delay); + poor_delays = (int)(fraction_poor_delays * 100.0); + if (fraction_poor_delays < 0 || (native->last_median == median && native->last_poor_delays == poor_delays)) return; + g_debug("voice_processor_native.cpp: Stream delay metrics: median=%i std=%i poor_delays=%i%%", median, std, poor_delays); + native->last_median = median; + native->last_poor_delays = poor_delays; + if (poor_delays > 90) { + native->stream_delay = std::min(std::max(0, native->stream_delay + std::min(48, std::max(median, -48))), 384); + g_debug("voice_processor_native.cpp: set stream_delay=%i", native->stream_delay); } } @@ -118,21 +125,21 @@ dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo * webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); webrtc::AudioProcessing *apm = native->apm; - GstAudioBuffer audio_buffer; - gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READWRITE); + GstMapInfo map; + gst_buffer_map(buffer, &map, GST_MAP_READWRITE); webrtc::AudioFrame frame; frame.num_channels_ = info->channels; frame.sample_rate_hz_ = info->rate; frame.samples_per_channel_ = info->rate / 100; - memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); + memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); apm->set_stream_delay_ms(native->stream_delay); int err = apm->ProcessStream(&frame); - if (err >= 0) memcpy(audio_buffer.planes[0], frame.data_, frame.samples_per_channel_ * info->bpf); + if (err >= 0) memcpy(map.data, frame.data_, frame.samples_per_channel_ * info->bpf); if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err); - gst_audio_buffer_unmap(&audio_buffer); + gst_buffer_unmap(buffer, &map); } extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) { From 90f9ecf62b2ebfef14de2874e7942552409632bf Mon Sep 17 00:00:00 2001 From: fiaxh Date: Mon, 3 May 2021 13:17:17 +0200 Subject: [PATCH 56/56] Calls: Indicate whether OMEMO key is verified --- libdino/src/plugin/interfaces.vala | 10 +++ libdino/src/plugin/registry.vala | 8 ++ libdino/src/service/calls.vala | 4 +- main/CMakeLists.txt | 1 + main/src/ui/call_window/call_bottom_bar.vala | 64 +-------------- .../call_window/call_encryption_button.vala | 77 +++++++++++++++++++ .../call_window/call_window_controller.vala | 17 +++- plugins/omemo/CMakeLists.txt | 1 + .../src/dtls_srtp_verification_draft.vala | 7 +- plugins/omemo/src/logic/decrypt.vala | 3 +- plugins/omemo/src/plugin.vala | 6 +- .../omemo/src/ui/call_encryption_entry.vala | 57 ++++++++++++++ 12 files changed, 181 insertions(+), 74 deletions(-) create mode 100644 main/src/ui/call_window/call_encryption_button.vala create mode 100644 plugins/omemo/src/ui/call_encryption_entry.vala diff --git a/libdino/src/plugin/interfaces.vala b/libdino/src/plugin/interfaces.vala index 97951850..eadbb085 100644 --- a/libdino/src/plugin/interfaces.vala +++ b/libdino/src/plugin/interfaces.vala @@ -29,6 +29,16 @@ public interface EncryptionListEntry : Object { public abstract Object? get_encryption_icon(Entities.Conversation conversation, ContentItem content_item); } +public interface CallEncryptionEntry : Object { + public abstract CallEncryptionWidget? get_widget(Account account, Xmpp.Xep.Jingle.ContentEncryption encryption); +} + +public interface CallEncryptionWidget : Object { + public abstract string? get_title(); + public abstract bool show_keys(); + public abstract string? get_icon_name(); +} + public abstract class AccountSettingsEntry : Object { public abstract string id { get; } public virtual Priority priority { get { return Priority.DEFAULT; } } diff --git a/libdino/src/plugin/registry.vala b/libdino/src/plugin/registry.vala index 27d72b80..e28c4de7 100644 --- a/libdino/src/plugin/registry.vala +++ b/libdino/src/plugin/registry.vala @@ -4,6 +4,7 @@ namespace Dino.Plugins { public class Registry { internal ArrayList encryption_list_entries = new ArrayList(); + internal HashMap call_encryption_entries = new HashMap(); internal ArrayList account_settings_entries = new ArrayList(); internal ArrayList contact_details_entries = new ArrayList(); internal Map text_commands = new HashMap(); @@ -25,6 +26,13 @@ public class Registry { } } + public bool register_call_entryption_entry(string ns, CallEncryptionEntry entry) { + lock (call_encryption_entries) { + call_encryption_entries[ns] = entry; + } + return true; + } + public bool register_account_settings_entry(AccountSettingsEntry entry) { lock(account_settings_entries) { foreach(var e in account_settings_entries) { diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index a44b59fd..4c3bbea7 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -75,7 +75,7 @@ namespace Dino { call.account = conversation.account; call.counterpart = conversation.counterpart; call.ourpart = conversation.account.full_jid; - call.time = call.local_time = new DateTime.now_utc(); + call.time = call.local_time = call.end_time = new DateTime.now_utc(); call.state = Call.State.RINGING; stream_interactor.get_module(CallStore.IDENTITY).add_call(call, conversation); @@ -380,7 +380,7 @@ namespace Dino { call.counterpart = from; } call.account = account; - call.time = call.local_time = new DateTime.now_utc(); + call.time = call.local_time = call.end_time = new DateTime.now_utc(); call.state = Call.State.RINGING; Conversation conversation = stream_interactor.get_module(ConversationManager.IDENTITY).create_conversation(call.counterpart.bare_jid, account, Conversation.Type.CHAT); diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 69992f06..4891abb0 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -139,6 +139,7 @@ SOURCES src/ui/call_window/audio_settings_popover.vala src/ui/call_window/call_bottom_bar.vala + src/ui/call_window/call_encryption_button.vala src/ui/call_window/call_window.vala src/ui/call_window/call_window_controller.vala src/ui/call_window/video_settings_popover.vala diff --git a/main/src/ui/call_window/call_bottom_bar.vala b/main/src/ui/call_window/call_bottom_bar.vala index 64b157dd..8a0604b3 100644 --- a/main/src/ui/call_window/call_bottom_bar.vala +++ b/main/src/ui/call_window/call_bottom_bar.vala @@ -25,8 +25,7 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { private MenuButton video_settings_button = new MenuButton() { halign=Align.END, valign=Align.END }; public VideoSettingsPopover? video_settings_popover; - private MenuButton encryption_button = new MenuButton() { relief=ReliefStyle.NONE, height_request=30, width_request=30, margin_start=20, margin_bottom=25, halign=Align.START, valign=Align.END }; - private Image encryption_image = new Image.from_icon_name("changes-allow-symbolic", IconSize.BUTTON) { visible=true }; + public CallEntryptionButton encryption_button = new CallEntryptionButton() { relief=ReliefStyle.NONE, height_request=30, width_request=30, margin_start=20, margin_bottom=25, halign=Align.START, valign=Align.END }; private Label label = new Label("") { margin=20, halign=Align.CENTER, valign=Align.CENTER, wrap=true, wrap_mode=Pango.WrapMode.WORD_CHAR, hexpand=true, visible=true }; private Stack stack = new Stack() { visible=true }; @@ -35,8 +34,6 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { Object(orientation:Orientation.HORIZONTAL, spacing:0); Overlay default_control = new Overlay() { visible=true }; - encryption_button.add(encryption_image); - encryption_button.get_style_context().add_class("encryption-box"); default_control.add_overlay(encryption_button); Box main_buttons = new Box(Orientation.HORIZONTAL, 20) { margin_start=40, margin_end=40, margin=20, halign=Align.CENTER, hexpand=true, visible=true }; @@ -89,54 +86,6 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { this.get_style_context().add_class("call-bottom-bar"); } - public void set_encryption(Xmpp.Xep.Jingle.ContentEncryption? audio_encryption, Xmpp.Xep.Jingle.ContentEncryption? video_encryption, bool same) { - encryption_button.visible = true; - - Popover popover = new Popover(encryption_button); - if (audio_encryption == null) { - encryption_image.set_from_icon_name("changes-allow-symbolic", IconSize.BUTTON); - encryption_button.get_style_context().add_class("unencrypted"); - - popover.add(new Label("This call isn't encrypted.") { margin=10, visible=true } ); - return; - } - - encryption_image.set_from_icon_name("changes-prevent-symbolic", IconSize.BUTTON); - encryption_button.get_style_context().remove_class("unencrypted"); - - Box box = new Box(Orientation.VERTICAL, 5) { margin=10, visible=true }; - if (audio_encryption.encryption_name == "OMEMO") { - box.add(new Label("This call is encrypted with OMEMO.") { use_markup=true, xalign=0, visible=true } ); - } else { - box.add(new Label("This call is end-to-end encrypted.") { use_markup=true, xalign=0, visible=true }); - } - - if (same) { - box.add(create_media_encryption_grid(audio_encryption)); - } else { - box.add(new Label("Audio") { use_markup=true, xalign=0, visible=true }); - box.add(create_media_encryption_grid(audio_encryption)); - box.add(new Label("Video") { use_markup=true, xalign=0, visible=true }); - box.add(create_media_encryption_grid(video_encryption)); - } - popover.add(box); - - encryption_button.set_popover(popover); - } - - private Grid create_media_encryption_grid(Xmpp.Xep.Jingle.ContentEncryption? encryption) { - Grid ret = new Grid() { row_spacing=3, column_spacing=5, visible=true }; - if (encryption.peer_key.length > 0) { - ret.attach(new Label("Peer call key") { xalign=0, visible=true }, 1, 2, 1, 1); - ret.attach(new Label("" + format_fingerprint(encryption.peer_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 2, 1, 1); - } - if (encryption.our_key.length > 0) { - ret.attach(new Label("Your call key") { xalign=0, visible=true }, 1, 3, 1, 1); - ret.attach(new Label("" + format_fingerprint(encryption.our_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 3, 1, 1); - } - return ret; - } - public AudioSettingsPopover? show_audio_device_choices(bool show) { audio_settings_button.visible = show; if (audio_settings_popover != null) audio_settings_popover.visible = false; @@ -212,15 +161,4 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { public bool is_menu_active() { return video_settings_button.active || audio_settings_button.active || encryption_button.active; } - - private string format_fingerprint(uint8[] fingerprint) { - var sb = new StringBuilder(); - for (int i = 0; i < fingerprint.length; i++) { - sb.append("%02x".printf(fingerprint[i])); - if (i < fingerprint.length - 1) { - sb.append(":"); - } - } - return sb.str; - } } \ No newline at end of file diff --git a/main/src/ui/call_window/call_encryption_button.vala b/main/src/ui/call_window/call_encryption_button.vala new file mode 100644 index 00000000..1d785d51 --- /dev/null +++ b/main/src/ui/call_window/call_encryption_button.vala @@ -0,0 +1,77 @@ +using Dino.Entities; +using Gtk; +using Pango; + +public class Dino.Ui.CallEntryptionButton : MenuButton { + + private Image encryption_image = new Image.from_icon_name("changes-allow-symbolic", IconSize.BUTTON) { visible=true }; + + construct { + add(encryption_image); + get_style_context().add_class("encryption-box"); + this.set_popover(popover); + } + + public void set_icon(bool encrypted, string? icon_name) { + this.visible = true; + + if (encrypted) { + encryption_image.set_from_icon_name(icon_name ?? "changes-prevent-symbolic", IconSize.BUTTON); + get_style_context().remove_class("unencrypted"); + } else { + encryption_image.set_from_icon_name(icon_name ?? "changes-allow-symbolic", IconSize.BUTTON); + get_style_context().add_class("unencrypted"); + } + } + + public void set_info(string? title, bool show_keys, Xmpp.Xep.Jingle.ContentEncryption? audio_encryption, Xmpp.Xep.Jingle.ContentEncryption? video_encryption) { + Popover popover = new Popover(this); + this.set_popover(popover); + + if (audio_encryption == null) { + popover.add(new Label("This call is unencrypted.") { margin=10, visible=true } ); + return; + } + if (title != null && !show_keys) { + popover.add(new Label(title) { use_markup=true, margin=10, visible=true } ); + return; + } + + Box box = new Box(Orientation.VERTICAL, 10) { margin=10, visible=true }; + box.add(new Label("%s".printf(title ?? "This call is end-to-end encrypted.")) { use_markup=true, xalign=0, visible=true }); + + if (video_encryption == null) { + box.add(create_media_encryption_grid(audio_encryption)); + } else { + box.add(new Label("Audio") { use_markup=true, xalign=0, visible=true }); + box.add(create_media_encryption_grid(audio_encryption)); + box.add(new Label("Video") { use_markup=true, xalign=0, visible=true }); + box.add(create_media_encryption_grid(video_encryption)); + } + popover.add(box); + } + + private Grid create_media_encryption_grid(Xmpp.Xep.Jingle.ContentEncryption? encryption) { + Grid ret = new Grid() { row_spacing=3, column_spacing=5, visible=true }; + if (encryption.peer_key.length > 0) { + ret.attach(new Label("Peer call key") { xalign=0, visible=true }, 1, 2, 1, 1); + ret.attach(new Label("" + format_fingerprint(encryption.peer_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 2, 1, 1); + } + if (encryption.our_key.length > 0) { + ret.attach(new Label("Your call key") { xalign=0, visible=true }, 1, 3, 1, 1); + ret.attach(new Label("" + format_fingerprint(encryption.our_key) + "") { use_markup=true, max_width_chars=25, ellipsize=EllipsizeMode.MIDDLE, xalign=0, hexpand=true, visible=true }, 2, 3, 1, 1); + } + return ret; + } + + private string format_fingerprint(uint8[] fingerprint) { + var sb = new StringBuilder(); + for (int i = 0; i < fingerprint.length; i++) { + sb.append("%02x".printf(fingerprint[i])); + if (i < fingerprint.length - 1) { + sb.append(":"); + } + } + return sb.str; + } +} \ No newline at end of file diff --git a/main/src/ui/call_window/call_window_controller.vala b/main/src/ui/call_window/call_window_controller.vala index 7e5920ce..b07b41b1 100644 --- a/main/src/ui/call_window/call_window_controller.vala +++ b/main/src/ui/call_window/call_window_controller.vala @@ -78,7 +78,22 @@ public class Dino.Ui.CallWindowController : Object { }); calls.encryption_updated.connect((call, audio_encryption, video_encryption, same) => { if (!this.call.equals(call)) return; - call_window.bottom_bar.set_encryption(audio_encryption, video_encryption, same); + + string? title = null; + string? icon_name = null; + bool show_keys = true; + Plugins.Registry registry = Dino.Application.get_default().plugin_registry; + Plugins.CallEncryptionEntry? encryption_entry = audio_encryption != null ? registry.call_encryption_entries[audio_encryption.encryption_ns] : null; + if (encryption_entry != null) { + Plugins.CallEncryptionWidget? encryption_widgets = encryption_entry.get_widget(call.account, audio_encryption); + if (encryption_widgets != null) { + title = encryption_widgets.get_title(); + icon_name = encryption_widgets.get_icon_name(); + show_keys = encryption_widgets.show_keys(); + } + } + call_window.bottom_bar.encryption_button.set_info(title, show_keys, audio_encryption, same ? null :video_encryption); + call_window.bottom_bar.encryption_button.set_icon(audio_encryption != null, icon_name); }); own_video.resolution_changed.connect((width, height) => { diff --git a/plugins/omemo/CMakeLists.txt b/plugins/omemo/CMakeLists.txt index 944fc649..195001cb 100644 --- a/plugins/omemo/CMakeLists.txt +++ b/plugins/omemo/CMakeLists.txt @@ -55,6 +55,7 @@ SOURCES src/ui/account_settings_entry.vala src/ui/account_settings_widget.vala src/ui/bad_messages_populator.vala + src/ui/call_encryption_entry.vala src/ui/contact_details_provider.vala src/ui/contact_details_dialog.vala src/ui/device_notification_populator.vala diff --git a/plugins/omemo/src/dtls_srtp_verification_draft.vala b/plugins/omemo/src/dtls_srtp_verification_draft.vala index 66a31954..5fc9b339 100644 --- a/plugins/omemo/src/dtls_srtp_verification_draft.vala +++ b/plugins/omemo/src/dtls_srtp_verification_draft.vala @@ -66,7 +66,7 @@ namespace Dino.Plugins.Omemo.DtlsSrtpVerificationDraft { stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.begin(jingle_sid, (_, res) => { Xep.Jingle.Session? session = stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.end(res); if (session == null || !session.contents_map.has_key(content_name)) return; - var encryption = new OmemoContentEncryption() { encryption_ns=NS_URI, encryption_name="OMEMO", our_key=new uint8[0], peer_key=new uint8[0], peer_device_id=device_id_by_jingle_sid[jingle_sid] }; + var encryption = new OmemoContentEncryption() { encryption_ns=NS_URI, encryption_name="OMEMO", our_key=new uint8[0], peer_key=new uint8[0], sid=device_id_by_jingle_sid[jingle_sid], jid=iq.from.bare_jid }; session.contents_map[content_name].encryptions[NS_URI] = encryption; if (iq.stanza.get_deep_attribute(Xep.Jingle.NS_URI + ":jingle", "action") == "session-accept") { @@ -143,7 +143,7 @@ namespace Dino.Plugins.Omemo.DtlsSrtpVerificationDraft { private void on_content_add_received(XmppStream stream, Xep.Jingle.Content content) { if (!content_names_by_jingle_sid.has_key(content.session.sid) || content_names_by_jingle_sid[content.session.sid].contains(content.content_name)) { - var encryption = new OmemoContentEncryption() { encryption_ns=NS_URI, encryption_name="OMEMO", our_key=new uint8[0], peer_key=new uint8[0], peer_device_id=device_id_by_jingle_sid[content.session.sid] }; + var encryption = new OmemoContentEncryption() { encryption_ns=NS_URI, encryption_name="OMEMO", our_key=new uint8[0], peer_key=new uint8[0], sid=device_id_by_jingle_sid[content.session.sid], jid=content.peer_full_jid.bare_jid }; content.encryptions[encryption.encryption_ns] = encryption; } } @@ -188,7 +188,8 @@ namespace Dino.Plugins.Omemo.DtlsSrtpVerificationDraft { } public class OmemoContentEncryption : Xep.Jingle.ContentEncryption { - public int peer_device_id { get; set; } + public Jid jid { get; set; } + public int sid { get; set; } } } diff --git a/plugins/omemo/src/logic/decrypt.vala b/plugins/omemo/src/logic/decrypt.vala index 3cdacbf7..cfbb9c58 100644 --- a/plugins/omemo/src/logic/decrypt.vala +++ b/plugins/omemo/src/logic/decrypt.vala @@ -59,9 +59,10 @@ namespace Dino.Plugins.Omemo { message.real_jid = possible_jid; } - trust_manager.message_device_id_map[message] = data.sid; message.body = cleartext; message.encryption = Encryption.OMEMO; + + trust_manager.message_device_id_map[message] = data.sid; return true; } catch (Error e) { debug("Decrypting message from %s/%d failed: %s", possible_jid.to_string(), data.sid, e.message); diff --git a/plugins/omemo/src/plugin.vala b/plugins/omemo/src/plugin.vala index 7a0304d1..643428a8 100644 --- a/plugins/omemo/src/plugin.vala +++ b/plugins/omemo/src/plugin.vala @@ -35,7 +35,6 @@ public class Plugin : RootInterface, Object { public DeviceNotificationPopulator device_notification_populator; public OwnNotifications own_notifications; public TrustManager trust_manager; - public DecryptMessageListener decrypt_message_listener; public HashMap decryptors = new HashMap(Account.hash_func, Account.equals_func); public HashMap encryptors = new HashMap(Account.hash_func, Account.equals_func); @@ -54,6 +53,7 @@ public class Plugin : RootInterface, Object { this.app.plugin_registry.register_contact_details_entry(contact_details_provider); this.app.plugin_registry.register_notification_populator(device_notification_populator); this.app.plugin_registry.register_conversation_addition_populator(new BadMessagesPopulator(this.app.stream_interactor, this)); + this.app.plugin_registry.register_call_entryption_entry(DtlsSrtpVerificationDraft.NS_URI, new CallEncryptionEntry(db)); this.app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => { Signal.Store signal_store = Plugin.get_context().create_store(); @@ -67,9 +67,7 @@ public class Plugin : RootInterface, Object { this.own_notifications = new OwnNotifications(this, this.app.stream_interactor, account); }); - decrypt_message_listener = new DecryptMessageListener(decryptors); - app.stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(decrypt_message_listener); - + app.stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(new DecryptMessageListener(decryptors)); app.stream_interactor.get_module(FileManager.IDENTITY).add_file_decryptor(new OmemoFileDecryptor()); app.stream_interactor.get_module(FileManager.IDENTITY).add_file_encryptor(new OmemoFileEncryptor()); JingleFileHelperRegistry.instance.add_encryption_helper(Encryption.OMEMO, new JetOmemo.EncryptionHelper(app.stream_interactor)); diff --git a/plugins/omemo/src/ui/call_encryption_entry.vala b/plugins/omemo/src/ui/call_encryption_entry.vala new file mode 100644 index 00000000..69b7b686 --- /dev/null +++ b/plugins/omemo/src/ui/call_encryption_entry.vala @@ -0,0 +1,57 @@ +using Dino.Entities; +using Gtk; +using Qlite; +using Xmpp; + +namespace Dino.Plugins.Omemo { + + public class CallEncryptionEntry : Plugins.CallEncryptionEntry, Object { + private Database db; + + public CallEncryptionEntry(Database db) { + this.db = db; + } + + public Plugins.CallEncryptionWidget? get_widget(Account account, Xmpp.Xep.Jingle.ContentEncryption encryption) { + DtlsSrtpVerificationDraft.OmemoContentEncryption? omemo_encryption = encryption as DtlsSrtpVerificationDraft.OmemoContentEncryption; + if (omemo_encryption == null) return null; + + int identity_id = db.identity.get_id(account.id); + Row? device = db.identity_meta.get_device(identity_id, omemo_encryption.jid.to_string(), omemo_encryption.sid); + if (device == null) return null; + TrustLevel trust = (TrustLevel) device[db.identity_meta.trust_level]; + + return new CallEncryptionWidget(trust); + } + } + + public class CallEncryptionWidget : Plugins.CallEncryptionWidget, Object { + + string? title = null; + string? icon = null; + bool should_show_keys = false; + + public CallEncryptionWidget(TrustLevel trust) { + if (trust == TrustLevel.VERIFIED) { + title = "This call is encrypted and verified with OMEMO."; + icon = "dino-security-high-symbolic"; + should_show_keys = false; + } else { + title = "This call is encrypted with OMEMO."; + should_show_keys = true; + } + } + + public string? get_title() { + return title; + } + + public string? get_icon_name() { + return icon; + } + + public bool show_keys() { + return should_show_keys; + } + } +}