anotherim/anotherim-desktop
2
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

RTP: Encode with device

Browse source
This commit is contained in:
Marvin W 2021-11-09 22:06:47 +01:00 committed by fiaxh
parent 083f73b0ca
commit b593aa05ef
2 changed files with 416 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

335
plugins/rtp/src/device.vala
View file

@ -1,5 +1,9 @@
using Xmpp.Xep.JingleRtp;
using Gee;
public class Dino.Plugins.Rtp.Device : MediaDevice, Object { public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
public Plugin plugin { get; private set; } public Plugin plugin { get; private set; }
public CodecUtil codec_util { get { return plugin.codec_util; } }
public Gst.Device device { get; private set; } public Gst.Device device { get; private set; }
private string device_name; private string device_name;
@ -17,28 +21,45 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
return plugin.pipe; return plugin.pipe;
}} }}
public string? media { get { public string? media { get {
if (device.device_class.has_prefix("Audio/")) { if (device.has_classes("Audio")) {
return "audio"; return "audio";
} else if (device.device_class.has_prefix("Video/")) { } else if (device.has_classes("Video")) {
return "video"; return "video";
} else { } else {
return null; return null;
} }
}} }}
public bool is_source { get { public bool is_source { get {
return device.device_class.has_suffix("/Source"); return device.has_classes("Source");
}} }}
public bool is_sink { get { public bool is_sink { get {
return device.device_class.has_suffix("/Sink"); return device.has_classes("Sink");
}} }}
private Gst.Element element; private Gst.Element element;
private Gst.Element tee; private Gst.Element tee;
private Gst.Element dsp; private Gst.Element dsp;
private Gst.Element mixer; private Gst.Base.Aggregator mixer;
private Gst.Element filter; private Gst.Element filter;
private Gst.Element rate; private int links;
private int links = 0;
// Codecs
private Gee.Map<PayloadType, Gst.Element> codecs = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func);
private Gee.Map<PayloadType, Gst.Element> codec_tees = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func);
private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloaders = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func);
private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloader_tees = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func);
private Gee.Map<PayloadType, Gee.Map<uint, uint>> payloader_links = new HashMap<PayloadType, Gee.Map<uint, uint>>(PayloadType.hash_func, PayloadType.equals_func);
private Gee.Map<PayloadType, Gee.List<CodecBitrate>> codec_bitrates = new HashMap<PayloadType, Gee.List<CodecBitrate>>(PayloadType.hash_func, PayloadType.equals_func);
private class CodecBitrate {
public uint bitrate;
public int64 timestamp;
public CodecBitrate(uint bitrate) {
this.bitrate = bitrate;
this.timestamp = get_monotonic_time();
}
}
public Device(Plugin plugin, Gst.Device device) { public Device(Plugin plugin, Gst.Device device) {
this.plugin = plugin; this.plugin = plugin;
@ -57,25 +78,154 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
} }
public Gst.Element? link_sink() { public Gst.Element? link_sink() {
if (!is_sink) return null;
if (element == null) create(); if (element == null) create();
links++; links++;
if (mixer != null) return mixer; if (mixer != null) {
if (is_sink && media == "audio") return filter; Gst.Element rate = Gst.ElementFactory.make("audiorate", @"$(id)_rate_$(Random.next_int())");
pipe.add(rate);
rate.link(mixer);
return rate;
}
if (media == "audio") return filter;
return element; return element;
} }
public Gst.Element? link_source() { public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = Random.next_int(), int seqnum_offset = -1) {
if (!is_source) return null;
if (element == null) create(); if (element == null) create();
links++; links++;
if (payload_type != null && tee != null) {
bool new_codec = false;
string? codec = CodecUtil.get_codec_from_payload(media, payload_type);
if (!codecs.has_key(payload_type)) {
codecs[payload_type] = codec_util.get_encode_bin_without_payloader(media, payload_type, @"$(id)_$(codec)_encoder");
pipe.add(codecs[payload_type]);
new_codec = true;
}
if (!codec_tees.has_key(payload_type)) {
codec_tees[payload_type] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_tee");
codec_tees[payload_type].@set("allow-not-linked", true);
pipe.add(codec_tees[payload_type]);
codecs[payload_type].link(codec_tees[payload_type]);
}
if (!payloaders.has_key(payload_type)) {
payloaders[payload_type] = new HashMap<uint, Gst.Element>();
}
if (!payloaders[payload_type].has_key(ssrc)) {
payloaders[payload_type][ssrc] = codec_util.get_payloader_bin(media, payload_type, @"$(id)_$(codec)_$(ssrc)");
var payload = (Gst.RTP.BasePayload) ((Gst.Bin) payloaders[payload_type][ssrc]).get_by_name(@"$(id)_$(codec)_$(ssrc)_rtp_pay");
payload.ssrc = ssrc;
payload.seqnum_offset = seqnum_offset;
pipe.add(payloaders[payload_type][ssrc]);
codec_tees[payload_type].link(payloaders[payload_type][ssrc]);
}
if (!payloader_tees.has_key(payload_type)) {
payloader_tees[payload_type] = new HashMap<uint, Gst.Element>();
}
if (!payloader_tees[payload_type].has_key(ssrc)) {
payloader_tees[payload_type][ssrc] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_$(ssrc)_tee");
payloader_tees[payload_type][ssrc].@set("allow-not-linked", true);
pipe.add(payloader_tees[payload_type][ssrc]);
payloaders[payload_type][ssrc].link(payloader_tees[payload_type][ssrc]);
}
if (!payloader_links.has_key(payload_type)) {
payloader_links[payload_type] = new HashMap<uint, uint>();
}
if (!payloader_links[payload_type].has_key(ssrc)) {
payloader_links[payload_type][ssrc] = 1;
} else {
payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] + 1;
}
if (new_codec) {
tee.link(codecs[payload_type]);
}
return payloader_tees[payload_type][ssrc];
}
if (tee != null) return tee; if (tee != null) return tee;
return element; return element;
} }
public void unlink() { public void update_bitrate(PayloadType payload_type, uint bitrate) {
if (codecs.has_key(payload_type)) {
lock(codec_bitrates);
if (!codec_bitrates.has_key(payload_type)) {
codec_bitrates[payload_type] = new ArrayList<CodecBitrate>();
}
codec_bitrates[payload_type].add(new CodecBitrate(bitrate));
var remove = new ArrayList<CodecBitrate>();
foreach (CodecBitrate rate in codec_bitrates[payload_type]) {
if (rate.timestamp < get_monotonic_time() - 5000000L) {
remove.add(rate);
continue;
}
if (rate.bitrate < bitrate) {
bitrate = rate.bitrate;
}
}
codec_bitrates[payload_type].remove_all(remove);
codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate);
unlock(codec_bitrates);
}
}
public void unlink(Gst.Element? link = null) {
if (links <= 0) { if (links <= 0) {
critical("Link count below zero."); critical("Link count below zero.");
return; return;
} }
if (link != null && is_source && tee != null) {
PayloadType payload_type = payloader_tees.first_match((entry) => entry.value.any_match((entry) => entry.value == link)).key;
uint ssrc = payloader_tees[payload_type].first_match((entry) => entry.value == link).key;
payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] - 1;
if (payloader_links[payload_type][ssrc] == 0) {
plugin.pause();
codec_tees[payload_type].unlink(payloaders[payload_type][ssrc]);
payloaders[payload_type][ssrc].set_locked_state(true);
payloaders[payload_type][ssrc].set_state(Gst.State.NULL);
payloaders[payload_type][ssrc].unlink(payloader_tees[payload_type][ssrc]);
pipe.remove(payloaders[payload_type][ssrc]);
payloaders[payload_type].unset(ssrc);
payloader_tees[payload_type][ssrc].set_locked_state(true);
payloader_tees[payload_type][ssrc].set_state(Gst.State.NULL);
pipe.remove(payloader_tees[payload_type][ssrc]);
payloader_tees[payload_type].unset(ssrc);
payloader_links[payload_type].unset(ssrc);
plugin.unpause();
}
if (payloader_links[payload_type].size == 0) {
plugin.pause();
tee.unlink(codecs[payload_type]);
codecs[payload_type].set_locked_state(true);
codecs[payload_type].set_state(Gst.State.NULL);
codecs[payload_type].unlink(codec_tees[payload_type]);
pipe.remove(codecs[payload_type]);
codecs.unset(payload_type);
codec_tees[payload_type].set_locked_state(true);
codec_tees[payload_type].set_state(Gst.State.NULL);
pipe.remove(codec_tees[payload_type]);
codec_tees.unset(payload_type);
payloaders.unset(payload_type);
payloader_tees.unset(payload_type);
payloader_links.unset(payload_type);
plugin.unpause();
}
}
if (link != null && is_sink && mixer != null) {
plugin.pause();
link.set_locked_state(true);
Gst.Base.AggregatorPad mixer_sink_pad = (Gst.Base.AggregatorPad) link.get_static_pad("src").get_peer();
link.get_static_pad("src").unlink(mixer_sink_pad);
mixer_sink_pad.set_active(false);
link.set_state(Gst.State.NULL);
pipe.remove(link);
mixer.release_request_pad(mixer_sink_pad);
plugin.unpause();
}
links--; links--;
if (links == 0) { if (links == 0) {
destroy(); destroy();
@ -150,15 +300,59 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
return target; return target;
} }
private static Gst.PadProbeReturn log_probe(Gst.Pad pad, Gst.PadProbeInfo info) {
if ((info.type & Gst.PadProbeType.EVENT_DOWNSTREAM) > 0) {
debug("%s.%s probed downstream event %s", pad.get_parent_element().name, pad.name, info.get_event().type.get_name());
}
if ((info.type & Gst.PadProbeType.EVENT_UPSTREAM) > 0) {
var event = info.get_event();
if (event.type == Gst.EventType.RECONFIGURE) return Gst.PadProbeReturn.DROP;
if (event.type == Gst.EventType.QOS) {
Gst.QOSType qos_type;
double proportion;
Gst.ClockTimeDiff diff;
Gst.ClockTime timestamp;
event.parse_qos(out qos_type, out proportion, out diff, out timestamp);
debug("%s.%s probed qos event: type: %s, proportion: %f, diff: %lli, timestamp: %llu", pad.get_parent_element().name, pad.name, @"$qos_type", proportion, diff, timestamp);
} else {
debug("%s.%s probed upstream event %s", pad.get_parent_element().name, pad.name, event.type.get_name());
}
}
if ((info.type & Gst.PadProbeType.QUERY_DOWNSTREAM) > 0) {
debug("%s.%s probed downstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name());
}
if ((info.type & Gst.PadProbeType.QUERY_UPSTREAM) > 0) {
debug("%s.%s probed upstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name());
}
if ((info.type & Gst.PadProbeType.BUFFER) > 0) {
uint id = pad.get_data("no_buffer_probe_timeout");
if (id != 0) {
Source.remove(id);
}
string name = @"$(pad.get_parent_element().name).$(pad.name)";
id = Timeout.add_seconds(1, () => {
debug("%s probed no buffer for 1 second", name);
return Source.REMOVE;
});
pad.set_data("no_buffer_probe_timeout", id);
}
return Gst.PadProbeReturn.PASS;
}
private void create() { private void create() {
debug("Creating device %s", id); debug("Creating device %s", id);
plugin.pause(); plugin.pause();
element = device.create_element(id); element = device.create_element(id);
if (is_sink) {
element.@set("async", false);
element.@set("sync", false);
}
pipe.add(element); pipe.add(element);
if (is_source) { if (is_source) {
element.@set("do-timestamp", true); element.@set("do-timestamp", true);
filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
filter.@set("caps", get_best_caps()); filter.@set("caps", get_best_caps());
filter.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe);
pipe.add(filter); pipe.add(filter);
element.link(filter); element.link(filter);
#if WITH_VOICE_PROCESSOR #if WITH_VOICE_PROCESSOR
@ -174,22 +368,18 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
pipe.add(tee); pipe.add(tee);
(dsp ?? filter).link(tee); (dsp ?? filter).link(tee);
} }
if (is_sink) {
element.@set("async", false);
element.@set("sync", false);
}
if (is_sink && media == "audio") { if (is_sink && media == "audio") {
filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); mixer = (Gst.Base.Aggregator) Gst.ElementFactory.make("audiomixer", @"mixer_$id");
filter.@set("caps", get_best_caps()); pipe.add(mixer);
pipe.add(filter); mixer.link(pipe);
if (plugin.echoprobe != null) { if (plugin.echoprobe != null && !plugin.echoprobe.get_static_pad("src").is_linked()) {
rate = Gst.ElementFactory.make("audiorate", @"rate_$id"); mixer.link(plugin.echoprobe);
rate.@set("tolerance", 100000000);
pipe.add(rate);
filter.link(rate);
rate.link(plugin.echoprobe);
plugin.echoprobe.link(element); plugin.echoprobe.link(element);
} else { } else {
filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
filter.@set("caps", get_best_caps());
pipe.add(filter);
mixer.link(filter);
filter.link(element); filter.link(element);
} }
} }
@ -197,38 +387,25 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
} }
private void destroy() { private void destroy() {
if (mixer != null) { if (is_sink) {
if (is_sink && media == "audio" && plugin.echoprobe != null) { if (mixer != null) {
plugin.echoprobe.unlink(mixer); int linked_sink_pads = 0;
mixer.foreach_sink_pad((_, pad) => {
if (pad.is_linked()) linked_sink_pads++;
return true;
});
if (linked_sink_pads > 0) {
warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads);
}
mixer.unlink(plugin.echoprobe ?? element);
} }
int linked_sink_pads = 0;
mixer.foreach_sink_pad((_, pad) => {
if (pad.is_linked()) linked_sink_pads++;
return true;
});
if (linked_sink_pads > 0) {
warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads);
}
mixer.set_locked_state(true);
mixer.set_state(Gst.State.NULL);
mixer.unlink(element);
pipe.remove(mixer);
mixer = null;
} else if (is_sink && media == "audio") {
if (filter != null) { if (filter != null) {
filter.set_locked_state(true); filter.set_locked_state(true);
filter.set_state(Gst.State.NULL); filter.set_state(Gst.State.NULL);
filter.unlink(rate ?? ((Gst.Element)plugin.echoprobe) ?? element); filter.unlink(element);
pipe.remove(filter); pipe.remove(filter);
filter = null; filter = null;
} }
if (rate != null) {
rate.set_locked_state(true);
rate.set_state(Gst.State.NULL);
rate.unlink(plugin.echoprobe);
pipe.remove(rate);
rate = null;
}
if (plugin.echoprobe != null) { if (plugin.echoprobe != null) {
plugin.echoprobe.unlink(element); plugin.echoprobe.unlink(element);
} }
@ -239,34 +416,42 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
else if (is_source) element.unlink(tee); else if (is_source) element.unlink(tee);
pipe.remove(element); pipe.remove(element);
element = null; element = null;
if (filter != null) { if (mixer != null) {
filter.set_locked_state(true); mixer.set_locked_state(true);
filter.set_state(Gst.State.NULL); mixer.set_state(Gst.State.NULL);
filter.unlink(dsp ?? tee); pipe.remove(mixer);
pipe.remove(filter); mixer = null;
filter = null;
} }
if (dsp != null) { if (is_source) {
dsp.set_locked_state(true); if (filter != null) {
dsp.set_state(Gst.State.NULL); filter.set_locked_state(true);
dsp.unlink(tee); filter.set_state(Gst.State.NULL);
pipe.remove(dsp); filter.unlink(dsp ?? tee);
dsp = null; pipe.remove(filter);
} filter = null;
if (tee != null) { }
int linked_src_pads = 0; if (dsp != null) {
tee.foreach_src_pad((_, pad) => { dsp.set_locked_state(true);
if (pad.is_linked()) linked_src_pads++; dsp.set_state(Gst.State.NULL);
return true; dsp.unlink(tee);
}); pipe.remove(dsp);
if (linked_src_pads != 0) { dsp = null;
warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); }
if (tee != null) {
int linked_src_pads = 0;
tee.foreach_src_pad((_, pad) => {
if (pad.is_linked()) linked_src_pads++;
return true;
});
if (linked_src_pads != 0) {
warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads);
}
tee.set_locked_state(true);
tee.set_state(Gst.State.NULL);
pipe.remove(tee);
tee = null;
} }
tee.set_locked_state(true);
tee.set_state(Gst.State.NULL);
pipe.remove(tee);
tee = null;
} }
debug("Destroyed device %s", id); debug("Destroyed device %s", id);
} }
} }

258
plugins/rtp/src/stream.vala
View file

@ -18,22 +18,19 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
private Gst.App.Sink send_rtcp; private Gst.App.Sink send_rtcp;
private Gst.App.Src recv_rtp; private Gst.App.Src recv_rtp;
private Gst.App.Src recv_rtcp; private Gst.App.Src recv_rtcp;
private Gst.Element encode;
private Gst.RTP.BasePayload encode_pay;
private Gst.Element decode; private Gst.Element decode;
private Gst.RTP.BaseDepayload decode_depay; private Gst.RTP.BaseDepayload decode_depay;
private Gst.Element input; private Gst.Element input;
private Gst.Pad input_pad;
private Gst.Element output; private Gst.Element output;
private Gst.Element session; private Gst.Element session;
private Device _input_device; private Device _input_device;
public Device input_device { get { return _input_device; } set { public Device input_device { get { return _input_device; } set {
if (!paused) { if (!paused) {
if (this._input_device != null) { var input = this.input;
this._input_device.unlink(); set_input(value != null ? value.link_source(payload_type, our_ssrc, next_seqnum_offset) : null);
this._input_device = null; if (this._input_device != null) this._input_device.unlink(input);
}
set_input(value != null ? value.link_source() : null);
} }
this._input_device = value; this._input_device = value;
}} }}
@ -47,7 +44,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
public bool created { get; private set; default = false; } public bool created { get; private set; default = false; }
public bool paused { get; private set; default = false; } public bool paused { get; private set; default = false; }
private bool push_recv_data = false; private bool push_recv_data = false;
private string participant_ssrc = null; private uint our_ssrc = Random.next_int();
private int next_seqnum_offset = -1;
private uint32 participant_ssrc = 0;
private Gst.Pad recv_rtcp_sink_pad; private Gst.Pad recv_rtcp_sink_pad;
private Gst.Pad recv_rtp_sink_pad; private Gst.Pad recv_rtp_sink_pad;
@ -93,7 +92,10 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
send_rtp.caps = CodecUtil.get_caps(media, payload_type, false); send_rtp.caps = CodecUtil.get_caps(media, payload_type, false);
send_rtp.emit_signals = true; send_rtp.emit_signals = true;
send_rtp.sync = false; send_rtp.sync = false;
send_rtp.drop = true;
send_rtp.wait_on_eos = false;
send_rtp.new_sample.connect(on_new_sample); send_rtp.new_sample.connect(on_new_sample);
send_rtp.connect("signal::eos", on_eos_static, this);
pipe.add(send_rtp); pipe.add(send_rtp);
send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp_sink_$rtpid") as Gst.App.Sink; send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp_sink_$rtpid") as Gst.App.Sink;
@ -101,7 +103,10 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp");
send_rtcp.emit_signals = true; send_rtcp.emit_signals = true;
send_rtcp.sync = false; send_rtcp.sync = false;
send_rtcp.drop = true;
send_rtcp.wait_on_eos = false;
send_rtcp.new_sample.connect(on_new_sample); send_rtcp.new_sample.connect(on_new_sample);
send_rtcp.connect("signal::eos", on_eos_static, this);
pipe.add(send_rtcp); pipe.add(send_rtcp);
recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp_src_$rtpid") as Gst.App.Src; recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp_src_$rtpid") as Gst.App.Src;
@ -125,18 +130,15 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad); recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad);
// Connect input // Connect input
encode = codec_util.get_encode_bin(media, payload_type, @"encode_$rtpid");
encode_pay = (Gst.RTP.BasePayload)((Gst.Bin)encode).get_by_name(@"encode_$(rtpid)_rtp_pay");
pipe.add(encode);
send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid"); send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid");
encode.get_static_pad("src").link(send_rtp_sink_pad);
if (input != null) { if (input != null) {
input.link(encode); input_pad = input.get_request_pad(@"src_$rtpid");
input_pad.link(send_rtp_sink_pad);
} }
// Connect output // Connect output
decode = codec_util.get_decode_bin(media, payload_type, @"decode_$rtpid"); decode = codec_util.get_decode_bin(media, payload_type, @"decode_$rtpid");
decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)encode).get_by_name(@"decode_$(rtpid)_rtp_depay"); decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)decode).get_by_name(@"decode_$(rtpid)_rtp_depay");
pipe.add(decode); pipe.add(decode);
if (output != null) { if (output != null) {
decode.link(output); decode.link(output);
@ -159,8 +161,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
} }
Timeout.add(1000, () => remb_adjust()); Timeout.add(1000, () => remb_adjust());
} }
if (media == "video") { if (input_device != null && media == "video") {
codec_util.update_bitrate(media, payload_type, encode, 256); input_device.update_bitrate(payload_type, 256);
} }
} }
@ -185,11 +187,14 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
warning("No source-stats for session %u", rtpid); warning("No source-stats for session %u", rtpid);
return Source.REMOVE; return Source.REMOVE;
} }
if (input_device == null) return Source.CONTINUE;
foreach (Value value in source_stats.values) { foreach (Value value in source_stats.values) {
unowned Gst.Structure source_stat = (Gst.Structure) value.get_boxed(); unowned Gst.Structure source_stat = (Gst.Structure) value.get_boxed();
uint ssrc; uint32 ssrc;
if (!source_stat.get_uint("ssrc", out ssrc)) continue; if (!source_stat.get_uint("ssrc", out ssrc)) continue;
if (ssrc.to_string() == participant_ssrc) { if (ssrc == participant_ssrc) {
int packets_lost; int packets_lost;
uint64 packets_received, octets_received; uint64 packets_received, octets_received;
source_stat.get_int("packets-lost", out packets_lost); source_stat.get_int("packets-lost", out packets_lost);
@ -218,10 +223,10 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
1, 0, 0, 0, 1, 0, 0, 0,
0, 0, 0, 0 0, 0, 0, 0
}; };
data[4] = (uint8)((encode_pay.ssrc >> 24) & 0xff); data[4] = (uint8)((our_ssrc >> 24) & 0xff);
data[5] = (uint8)((encode_pay.ssrc >> 16) & 0xff); data[5] = (uint8)((our_ssrc >> 16) & 0xff);
data[6] = (uint8)((encode_pay.ssrc >> 8) & 0xff); data[6] = (uint8)((our_ssrc >> 8) & 0xff);
data[7] = (uint8)(encode_pay.ssrc & 0xff); data[7] = (uint8)(our_ssrc & 0xff);
uint8 br_exp = 0; uint8 br_exp = 0;
uint32 br_mant = remb * 1000; uint32 br_mant = remb * 1000;
uint8 bits = (uint8)Math.log2(br_mant); uint8 bits = (uint8)Math.log2(br_mant);
@ -243,7 +248,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
} }
private static void on_feedback_rtcp(Gst.Element session, uint type, uint fbtype, uint sender_ssrc, uint media_ssrc, Gst.Buffer? fci, Stream self) { private static void on_feedback_rtcp(Gst.Element session, uint type, uint fbtype, uint sender_ssrc, uint media_ssrc, Gst.Buffer? fci, Stream self) {
if (type == 206 && fbtype == 15 && fci != null && sender_ssrc.to_string() == self.participant_ssrc) { if (self.input_device != null && self.media == "video" && type == 206 && fbtype == 15 && fci != null && sender_ssrc == self.participant_ssrc) {
// https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03 // https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03
uint8[] data; uint8[] data;
fci.extract_dup(0, fci.get_size(), out data); fci.extract_dup(0, fci.get_size(), out data);
@ -251,7 +256,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
uint8 br_exp = data[5] >> 2; uint8 br_exp = data[5] >> 2;
uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7]; uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7];
uint bitrate = (br_mant << br_exp) / 1000; uint bitrate = (br_mant << br_exp) / 1000;
self.codec_util.update_bitrate(self.media, self.payload_type, self.encode, bitrate * 8); self.input_device.update_bitrate(self.payload_type, bitrate * 8);
} }
} }
@ -267,20 +272,30 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
debug("Sink is null"); debug("Sink is null");
return Gst.FlowReturn.EOS; return Gst.FlowReturn.EOS;
} }
if (sink != send_rtp && sink != send_rtcp) {
warning("unknown sample");
return Gst.FlowReturn.NOT_SUPPORTED;
}
Gst.Sample sample = sink.pull_sample(); Gst.Sample sample = sink.pull_sample();
Gst.Buffer buffer = sample.get_buffer(); Gst.Buffer buffer = sample.get_buffer();
uint8[] data; uint8[] data;
buffer.extract_dup(0, buffer.get_size(), out data); buffer.extract_dup(0, buffer.get_size(), out data);
prepare_local_crypto(); prepare_local_crypto();
if (sink == send_rtp) { if (sink == send_rtp) {
Gst.RTP.Buffer rtp_buffer;
if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) {
if (our_ssrc != rtp_buffer.get_ssrc()) {
warning("Sending buffer with SSRC %u when our ssrc is %u", rtp_buffer.get_ssrc(), our_ssrc);
}
next_seqnum_offset = rtp_buffer.get_seq() + 1;
rtp_buffer.unmap();
}
if (crypto_session.has_encrypt) { if (crypto_session.has_encrypt) {
data = crypto_session.encrypt_rtp(data); data = crypto_session.encrypt_rtp(data);
} }
on_send_rtp_data(new Bytes.take((owned) data)); on_send_rtp_data(new Bytes.take((owned) data));
} else if (sink == send_rtcp) { } else if (sink == send_rtcp) {
encrypt_and_send_rtcp((owned) data); encrypt_and_send_rtcp((owned) data);
} else {
warning("unknown sample");
} }
return Gst.FlowReturn.OK; return Gst.FlowReturn.OK;
} }
@ -300,41 +315,59 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
return Gst.PadProbeReturn.DROP; return Gst.PadProbeReturn.DROP;
} }
public override void destroy() { private static void on_eos_static(Gst.App.Sink sink, Stream self) {
// Stop network communication debug("EOS on %s", sink.name);
push_recv_data = false; if (sink == self.send_rtp) {
recv_rtp.end_of_stream(); Idle.add(() => { self.on_send_rtp_eos(); return Source.REMOVE; });
recv_rtcp.end_of_stream(); } else if (sink == self.send_rtcp) {
send_rtp.new_sample.disconnect(on_new_sample); Idle.add(() => { self.on_send_rtcp_eos(); return Source.REMOVE; });
send_rtcp.new_sample.disconnect(on_new_sample);
// Disconnect input device
if (input != null) {
input.unlink(encode);
input = null;
}
if (this._input_device != null) {
if (!paused) this._input_device.unlink();
this._input_device = null;
} }
}
// Disconnect encode private void on_send_rtp_eos() {
encode.set_locked_state(true);
encode.set_state(Gst.State.NULL);
encode.get_static_pad("src").unlink(send_rtp_sink_pad);
pipe.remove(encode);
encode = null;
encode_pay = null;
// Disconnect RTP sending
if (send_rtp_src_pad != null) { if (send_rtp_src_pad != null) {
send_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe);
send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink")); send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink"));
send_rtp_src_pad = null;
} }
send_rtp.set_locked_state(true); send_rtp.set_locked_state(true);
send_rtp.set_state(Gst.State.NULL); send_rtp.set_state(Gst.State.NULL);
pipe.remove(send_rtp); pipe.remove(send_rtp);
send_rtp = null; send_rtp = null;
debug("Stopped sending RTP for %u", rtpid);
}
private void on_send_rtcp_eos() {
send_rtcp.set_locked_state(true);
send_rtcp.set_state(Gst.State.NULL);
pipe.remove(send_rtcp);
send_rtcp = null;
debug("Stopped sending RTCP for %u", rtpid);
}
public override void destroy() {
// Stop network communication
push_recv_data = false;
if (recv_rtp != null) recv_rtp.end_of_stream();
if (recv_rtcp != null) recv_rtcp.end_of_stream();
if (send_rtp != null) send_rtp.new_sample.disconnect(on_new_sample);
if (send_rtcp != null) send_rtcp.new_sample.disconnect(on_new_sample);
// Disconnect input device
if (input != null) {
input_pad.unlink(send_rtp_sink_pad);
input.release_request_pad(input_pad);
input_pad = null;
}
if (this._input_device != null) {
if (!paused) this._input_device.unlink(input);
this._input_device = null;
this.input = null;
}
// Inject EOS
if (send_rtp_sink_pad != null) {
send_rtp_sink_pad.send_event(new Gst.Event.eos());
}
// Disconnect decode // Disconnect decode
if (recv_rtp_src_pad != null) { if (recv_rtp_src_pad != null) {
@ -342,57 +375,63 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
recv_rtp_src_pad.unlink(decode.get_static_pad("sink")); recv_rtp_src_pad.unlink(decode.get_static_pad("sink"));
} }
// Disconnect RTP receiving
recv_rtp.set_locked_state(true);
recv_rtp.set_state(Gst.State.NULL);
recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad);
pipe.remove(recv_rtp);
recv_rtp = null;
// Disconnect output // Disconnect output
if (output != null) { if (output != null) {
decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe);
decode.unlink(output); decode.unlink(output);
} }
decode.set_locked_state(true);
decode.set_state(Gst.State.NULL);
pipe.remove(decode);
decode = null;
decode_depay = null;
output = null;
// Disconnect output device // Disconnect output device
if (this._output_device != null) { if (this._output_device != null) {
this._output_device.unlink(); this._output_device.unlink(output);
this._output_device = null; this._output_device = null;
} }
output = null;
// Disconnect RTCP receiving // Destroy decode
recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad); if (decode != null) {
recv_rtcp.set_locked_state(true); decode.set_locked_state(true);
recv_rtcp.set_state(Gst.State.NULL); decode.set_state(Gst.State.NULL);
pipe.remove(recv_rtcp); pipe.remove(decode);
recv_rtcp = null; decode = null;
decode_depay = null;
}
// Disconnect RTCP sending // Disconnect and remove RTP input
send_rtcp_src_pad.unlink(send_rtcp.get_static_pad("sink")); if (recv_rtp != null) {
send_rtcp.set_locked_state(true); recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad);
send_rtcp.set_state(Gst.State.NULL); recv_rtp.set_locked_state(true);
pipe.remove(send_rtcp); recv_rtp.set_state(Gst.State.NULL);
send_rtcp = null; pipe.remove(recv_rtp);
recv_rtp = null;
}
// Disconnect and remove RTCP input
if (recv_rtcp != null) {
recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad);
recv_rtcp.set_locked_state(true);
recv_rtcp.set_state(Gst.State.NULL);
pipe.remove(recv_rtcp);
recv_rtcp = null;
}
// Release rtp pads // Release rtp pads
rtpbin.release_request_pad(send_rtp_sink_pad); if (send_rtp_sink_pad != null) {
send_rtp_sink_pad = null; rtpbin.release_request_pad(send_rtp_sink_pad);
rtpbin.release_request_pad(recv_rtp_sink_pad); send_rtp_sink_pad = null;
recv_rtp_sink_pad = null; }
rtpbin.release_request_pad(recv_rtcp_sink_pad); if (recv_rtp_sink_pad != null) {
recv_rtcp_sink_pad = null; rtpbin.release_request_pad(recv_rtp_sink_pad);
rtpbin.release_request_pad(send_rtcp_src_pad); recv_rtp_sink_pad = null;
send_rtcp_src_pad = null; }
send_rtp_src_pad = null; if (send_rtcp_src_pad != null) {
recv_rtp_src_pad = null; rtpbin.release_request_pad(send_rtcp_src_pad);
send_rtcp_src_pad = null;
session = null; }
if (recv_rtcp_sink_pad != null) {
rtpbin.release_request_pad(recv_rtcp_sink_pad);
recv_rtcp_sink_pad = null;
}
} }
private void prepare_remote_crypto() { private void prepare_remote_crypto() {
@ -502,10 +541,10 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
debug("RTCP is ready, resending rtcp: %s", rtp_sent.to_string()); debug("RTCP is ready, resending rtcp: %s", rtp_sent.to_string());
} }
public void on_ssrc_pad_added(string ssrc, Gst.Pad pad) { public void on_ssrc_pad_added(uint32 ssrc, Gst.Pad pad) {
debug("New ssrc %s with pad %s", ssrc, pad.name); debug("New ssrc %u with pad %s", ssrc, pad.name);
if (participant_ssrc != null && participant_ssrc != ssrc) { if (participant_ssrc != 0 && participant_ssrc != ssrc) {
warning("Got second ssrc on stream (old: %s, new: %s), ignoring", participant_ssrc, ssrc); warning("Got second ssrc on stream (old: %u, new: %u), ignoring", participant_ssrc, ssrc);
return; return;
} }
participant_ssrc = ssrc; participant_ssrc = ssrc;
@ -534,7 +573,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
private void set_input_and_pause(Gst.Element? input, bool paused) { private void set_input_and_pause(Gst.Element? input, bool paused) {
if (created && this.input != null) { if (created && this.input != null) {
this.input.unlink(encode); this.input_pad.unlink(send_rtp_sink_pad);
this.input.release_request_pad(this.input_pad);
this.input_pad = null;
this.input = null; this.input = null;
} }
@ -543,28 +584,41 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
if (created && sending && !paused && input != null) { if (created && sending && !paused && input != null) {
plugin.pause(); plugin.pause();
input.link(encode); input_pad = input.get_request_pad(@"src_$rtpid");
input_pad.link(send_rtp_sink_pad);
plugin.unpause(); plugin.unpause();
} }
} }
public void pause() { public void pause() {
if (paused) return; if (paused) return;
var input = this.input;
set_input_and_pause(null, true); set_input_and_pause(null, true);
if (input_device != null) input_device.unlink(); if (input != null && input_device != null) input_device.unlink(input);
} }
public void unpause() { public void unpause() {
if (!paused) return; if (!paused) return;
set_input_and_pause(input_device != null ? input_device.link_source() : null, false); set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset) : null, false);
}
public uint get_participant_ssrc(Xmpp.Jid participant) {
if (participant.equals(content.session.peer_full_jid)) {
return participant_ssrc;
}
return 0;
} }
ulong block_probe_handler_id = 0; ulong block_probe_handler_id = 0;
public virtual void add_output(Gst.Element element) { public virtual void add_output(Gst.Element element, Xmpp.Jid? participant = null) {
if (output != null) { if (output != null) {
critical("add_output() invoked more than once"); critical("add_output() invoked more than once");
return; return;
} }
if (participant != null) {
critical("add_output() invoked with participant when not supported");
return;
}
this.output = element; this.output = element;
if (created) { if (created) {
plugin.pause(); plugin.pause();
@ -586,7 +640,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
decode.unlink(element); decode.unlink(element);
} }
if (this._output_device != null) { if (this._output_device != null) {
this._output_device.unlink(); this._output_device.unlink(element);
this._output_device = null; this._output_device = null;
} }
this.output = null; this.output = null;
@ -657,7 +711,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream {
disconnect(video_orientation_changed_handler); disconnect(video_orientation_changed_handler);
} }
public override void add_output(Gst.Element element) { public override void add_output(Gst.Element element, Xmpp.Jid? participant) {
if (element == output_tee || element == rotate) { if (element == output_tee || element == rotate) {
base.add_output(element); base.add_output(element);
return; return;
@ -678,4 +732,4 @@ public class Dino.Plugins.Rtp.VideoStream : Stream {
output_tee.unlink(element); output_tee.unlink(element);
} }
} }
} }