anotherim/anotherim-desktop
2
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Fix video for cameras with rotated image

Browse source
This commit is contained in:
Marvin W 2023-04-21 17:38:15 +02:00
parent cad066628a
commit dbb8abc117
No known key found for this signature in database
GPG key ID: 072E9235DB996F2A
4 changed files with 122 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
plugins/rtp/src/device.vala
View file

@ -215,7 +215,12 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
} }
if (new_width == active_caps_width) return; if (new_width == active_caps_width) return;
int new_height = device_caps_height * new_width / device_caps_width; int new_height = device_caps_height * new_width / device_caps_width;
Gst.Caps new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null); Gst.Caps new_caps;
if (device_caps_framerate_den != 0) {
new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null);
} else {
new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, null);
}
double required_bitrate = get_target_bitrate(new_caps); double required_bitrate = get_target_bitrate(new_caps);
debug("Changing resolution width from %d to %d (requires bitrate %f, current target is %u)", active_caps_width, new_width, required_bitrate, bitrate); debug("Changing resolution width from %d to %d (requires bitrate %f, current target is %u)", active_caps_width, new_width, required_bitrate, bitrate);
if (bitrate < required_bitrate && new_width > active_caps_width) return; if (bitrate < required_bitrate && new_width > active_caps_width) return;
@ -347,7 +352,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
if (media == "audio") { if (media == "audio") {
return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1"); return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1");
} else if (media == "video" && device.caps.get_size() > 0) { } else if (media == "video" && device.caps.get_size() > 0) {
int best_index = 0; int best_index = -1;
Value? best_fraction = null; Value? best_fraction = null;
int best_fps = 0; int best_fps = 0;
int best_width = 0; int best_width = 0;
@ -390,10 +395,25 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
best_fraction = best_fraction_now; best_fraction = best_fraction_now;
} }
} }
if (best_index == -1) {
// No caps in first round, try without framerate
for (int i = 0; i < device.caps.get_size(); i++) {
unowned Gst.Structure? that = device.caps.get_structure(i);
if (!that.has_name("video/x-raw")) continue;
int width = 0, height = 0;
if (!that.has_field("width") || !that.get_int("width", out width)) continue;
if (!that.has_field("height") || !that.get_int("height", out height)) continue;
if (best_width < width || best_width == width && best_height < height) {
best_width = width;
best_height = height;
best_index = i;
}
}
}
Gst.Caps res = caps_copy_nth(device.caps, best_index); Gst.Caps res = caps_copy_nth(device.caps, best_index);
unowned Gst.Structure? that = res.get_structure(0); unowned Gst.Structure? that = res.get_structure(0);
Value framerate = that.get_value("framerate"); Value? framerate = that.get_value("framerate");
if (framerate.type() == typeof(Gst.ValueList)) { if (framerate != null && framerate.type() == typeof(Gst.ValueList) && best_fraction != null) {
that.set_value("framerate", best_fraction); that.set_value("framerate", best_fraction);
} }
debug("Selected caps %s", res.to_string()); debug("Selected caps %s", res.to_string());

4
plugins/rtp/src/plugin.vala
View file

@ -426,11 +426,11 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object {
if (media == "video") { if (media == "video") {
// Pick best FPS // Pick best FPS
int max_fps = 0; int max_fps = -1;
Device? max_fps_device = null; Device? max_fps_device = null;
foreach (Device device in devices) { foreach (Device device in devices) {
int fps = get_max_fps(device); int fps = get_max_fps(device);
if (fps > max_fps) { if (fps > max_fps || max_fps_device == null) {
max_fps = fps; max_fps = fps;
max_fps_device = device; max_fps_device = device;
} }

91
plugins/rtp/src/stream.vala
View file

@ -102,6 +102,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
send_rtp.drop = true; send_rtp.drop = true;
send_rtp.wait_on_eos = false; send_rtp.wait_on_eos = false;
send_rtp.new_sample.connect(on_new_sample); send_rtp.new_sample.connect(on_new_sample);
#if GST_1_20
send_rtp.new_serialized_event.connect(on_new_event);
#endif
send_rtp.connect("signal::eos", on_eos_static, this); send_rtp.connect("signal::eos", on_eos_static, this);
pipe.add(send_rtp); pipe.add(send_rtp);
@ -294,6 +297,60 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
} }
} }
bool flip = false;
uint8 rotation = 0;
#if GST_1_20
private bool on_new_event(Gst.App.Sink sink) {
if (sink == null || sink != send_rtp) {
return false;
}
Gst.MiniObject obj = sink.try_pull_object(0);
if (obj.type == typeof(Gst.Event)) {
unowned Gst.TagList tags;
if (((Gst.Event)obj).type == Gst.EventType.TAG) {
((Gst.Event)obj).parse_tag(out tags);
Gst.Video.OrientationMethod orientation_method;
Gst.Video.Orientation.from_tag(tags, out orientation_method);
switch (orientation_method) {
case Gst.Video.OrientationMethod.IDENTITY:
case Gst.Video.OrientationMethod.VERT:
default:
rotation = 0;
break;
case Gst.Video.OrientationMethod.@90R:
case Gst.Video.OrientationMethod.UL_LR:
rotation = 1;
break;
case Gst.Video.OrientationMethod.@180:
case Gst.Video.OrientationMethod.HORIZ:
rotation = 2;
break;
case Gst.Video.OrientationMethod.@90L:
case Gst.Video.OrientationMethod.UR_LL:
rotation = 3;
break;
}
switch (orientation_method) {
case Gst.Video.OrientationMethod.IDENTITY:
case Gst.Video.OrientationMethod.@90R:
case Gst.Video.OrientationMethod.@180:
case Gst.Video.OrientationMethod.@90L:
default:
flip = false;
break;
case Gst.Video.OrientationMethod.VERT:
case Gst.Video.OrientationMethod.UL_LR:
case Gst.Video.OrientationMethod.HORIZ:
case Gst.Video.OrientationMethod.UR_LL:
flip = true;
break;
}
}
}
return false;
}
#endif
private Gst.FlowReturn on_new_sample(Gst.App.Sink sink) { private Gst.FlowReturn on_new_sample(Gst.App.Sink sink) {
if (sink == null) { if (sink == null) {
debug("Sink is null"); debug("Sink is null");
@ -323,6 +380,24 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
#endif #endif
} }
#if GST_1_20
if (sink == send_rtp) {
Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation");
if (ext != null) {
buffer = (Gst.Buffer) buffer.make_writable();
Gst.RTP.Buffer rtp_buffer;
if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.WRITE, out rtp_buffer)) {
uint8[] extension_data = new uint8[1];
bool camera = false;
extension_data[0] = extension_data[0] | (rotation & 0x3);
if (flip) extension_data[0] = extension_data[0] | 0x4;
if (camera) extension_data[0] = extension_data[0] | 0x8;
rtp_buffer.add_extension_onebyte_header(ext.id, extension_data);
}
}
}
#endif
prepare_local_crypto(); prepare_local_crypto();
uint8[] data; uint8[] data;
@ -489,8 +564,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
} }
} }
private uint16 previous_video_orientation_degree = uint16.MAX; private uint16 previous_incoming_video_orientation_degree = uint16.MAX;
public signal void video_orientation_changed(uint16 degree); public signal void incoming_video_orientation_changed(uint16 degree);
public override void on_recv_rtp_data(Bytes bytes) { public override void on_recv_rtp_data(Bytes bytes) {
if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) { if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) {
@ -545,9 +620,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
case 2: rotation_degree = 180; break; case 2: rotation_degree = 180; break;
case 3: rotation_degree = 270; break; case 3: rotation_degree = 270; break;
} }
if (rotation_degree != previous_video_orientation_degree) { if (rotation_degree != previous_incoming_video_orientation_degree) {
video_orientation_changed(rotation_degree); incoming_video_orientation_changed(rotation_degree);
previous_video_orientation_degree = rotation_degree; previous_incoming_video_orientation_degree = rotation_degree;
} }
} }
} }
@ -723,7 +798,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream {
private Gee.List<Gst.Element> outputs = new ArrayList<Gst.Element>(); private Gee.List<Gst.Element> outputs = new ArrayList<Gst.Element>();
private Gst.Element output_tee; private Gst.Element output_tee;
private Gst.Element rotate; private Gst.Element rotate;
private ulong video_orientation_changed_handler; private ulong incoming_video_orientation_changed_handler;
public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) {
base(plugin, content); base(plugin, content);
@ -731,7 +806,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream {
} }
public override void create() { public override void create() {
video_orientation_changed_handler = video_orientation_changed.connect(on_video_orientation_changed); incoming_video_orientation_changed_handler = incoming_video_orientation_changed.connect(on_video_orientation_changed);
plugin.pause(); plugin.pause();
rotate = Gst.ElementFactory.make("videoflip", @"video_rotate_$rtpid"); rotate = Gst.ElementFactory.make("videoflip", @"video_rotate_$rtpid");
pipe.add(rotate); pipe.add(rotate);
@ -780,7 +855,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream {
output_tee.set_state(Gst.State.NULL); output_tee.set_state(Gst.State.NULL);
pipe.remove(output_tee); pipe.remove(output_tee);
output_tee = null; output_tee = null;
disconnect(video_orientation_changed_handler); disconnect(incoming_video_orientation_changed_handler);
} }
public override void add_output(Gst.Element element, Xmpp.Jid? participant) { public override void add_output(Gst.Element element, Xmpp.Jid? participant) {

12
plugins/rtp/src/video_widget.vala
View file

@ -227,9 +227,21 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWi
if (connected_device == null) return; if (connected_device == null) return;
plugin.pause(); plugin.pause();
pipe.add(sink); pipe.add(sink);
#if GST_1_20
prepare = Gst.parse_bin_from_description(@"videoflip video-direction=auto name=video_widget_$(id)_orientation ! videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true);
#else
prepare = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); prepare = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true);
#endif
prepare.name = @"video_widget_$(id)_prepare"; prepare.name = @"video_widget_$(id)_prepare";
#if GST_1_20
if (prepare is Gst.Bin) {
((Gst.Bin) prepare).get_by_name(@"video_widget_$(id)_flip").get_static_pad("sink").notify["caps"].connect(input_caps_changed);
} else {
#endif
prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed); prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed);
#if GST_1_20
}
#endif
pipe.add(prepare); pipe.add(prepare);
connected_device_element = connected_device.link_source(); connected_device_element = connected_device.link_source();
connected_device_element.link(prepare); connected_device_element.link(prepare);