From c438592ab04fb196b2a4a97c095be0d4245fb6e5 Mon Sep 17 00:00:00 2001 From: Igor Sharonov Date: Mon, 15 Apr 2024 12:21:10 +0300 Subject: [PATCH] webrtc: Replace dsp and echoprobe by gst webrtcdsp and webrtcechoprobe --- .github/workflows/build.yml | 8 +- CMakeLists.txt | 7 + README.md | 8 +- build-win64.sh | 1 - cmake/FindGstWebrtcDsp.cmake | 11 ++ cmake/FindWebRTCAudioProcessing.cmake | 12 -- configure | 6 +- meson.build | 9 +- plugins/rtp/CMakeLists.txt | 28 ++-- plugins/rtp/meson.build | 8 +- plugins/rtp/src/device.vala | 5 +- plugins/rtp/src/plugin.vala | 14 +- plugins/rtp/src/voice_processor.vala | 176 --------------------- plugins/rtp/src/voice_processor_native.cpp | 148 ----------------- 14 files changed, 61 insertions(+), 380 deletions(-) create mode 100644 cmake/FindGstWebrtcDsp.cmake delete mode 100644 cmake/FindWebRTCAudioProcessing.cmake delete mode 100644 plugins/rtp/src/voice_processor.vala delete mode 100644 plugins/rtp/src/voice_processor_native.cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 736dc8b7..a5a0e408 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,8 +9,8 @@ jobs: fetch-depth: 0 - run: sudo apt-get update - run: sudo apt-get remove libunwind-14-dev - - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-4-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libnice-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libsrtp2-dev libwebrtc-audio-processing-dev libadwaita-1-dev libsignal-protocol-c-dev libcanberra-dev - - run: ./configure --release --no-debug --with-tests --enable-plugin=notification-sound --prefix=/usr + - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-4-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libnice-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libsrtp2-dev libadwaita-1-dev libsignal-protocol-c-dev libcanberra-dev + - run: ./configure --release --no-debug --with-tests --enable-plugin=notification-sound --prefix=/usr --without-webrtc - run: cmake --build build - run: cmake --build build --target=test - name: Build DEB-package @@ -38,8 +38,8 @@ jobs: fetch-depth: 0 - run: sudo apt-get update - run: sudo apt-get remove libunwind-14-dev - - run: sudo apt-get install -y build-essential gettext libadwaita-1-dev libcanberra-dev libgcrypt20-dev libgee-0.8-dev libgpgme-dev libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev libgtk-4-dev libnice-dev libnotify-dev libqrencode-dev libsignal-protocol-c-dev libsoup2.4-dev libsqlite3-dev libsrtp2-dev libwebrtc-audio-processing-dev meson valac - - run: meson setup build -Dcrypto-backend=auto -Dplugin-ice=enabled -Duse-soup2=true + - run: sudo apt-get install -y build-essential gettext libadwaita-1-dev libcanberra-dev libgcrypt20-dev libgee-0.8-dev libgpgme-dev libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev libgtk-4-dev libnice-dev libnotify-dev libqrencode-dev libsignal-protocol-c-dev libsoup2.4-dev libsqlite3-dev libsrtp2-dev meson valac + - run: meson setup build -Duse-soup2=true -Dplugin-rtp-webrtc-audio-processing=disabled - run: meson compile -C build - run: meson test -C build build-flatpak: diff --git a/CMakeLists.txt b/CMakeLists.txt index 85097edf..3d236776 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,13 @@ endif () include(CTest) +option(PLUGIN_RTP_WEBRTC_AUDIO_PROCESSING "Use WebRTC audio processing" ON) + +# https://gitlab.kitware.com/cmake/cmake/-/issues/19804 +if (WIN32) + list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .dll) +endif() + # Prepare Plugins set(DEFAULT_PLUGINS omemo;openpgp;http-files;ice;rtp) if (WIN32) diff --git a/README.md b/README.md index 5c0b59f1..aaafd327 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Make sure to install all [dependencies](https://github.com/dino/dino/wiki/Build# If you want to use `meson` build system, follow the next instructions: - meson setup build -Dcrypto-backend=gnutls -Dplugin-ice=enabled + meson setup build meson configure --prefix $PWD/build/install --libdir lib build meson compile -C build meson install -C build @@ -85,6 +85,12 @@ If your `nice` library depends on `libsoup-2.4` (consider `ldd` output for the ` Skip `meson configure` step, if you want to install the program globally. You can specify any convenient directory in the option `--prefix` where the program will be installed. +If there is no `webrtcdsp` plugin in your system (check this by calling `gst-inspect-1.0 webrtcdsp`) you should pass extra argument: + +* `--without-webrtcdsp` for `./configure`; +* `-Dplugin-rtp-webrtc-audio-processing=disabled` for `meson`; +* `-DPLUGIN_RTP_WEBRTC_AUDIO_PROCESSING=OFF` for `cmake`. + In addition, there is a git version of this package for **Arch Linux** on [AUR](https://aur.archlinux.org/packages/dino-plus-git) Build on Windows (x86_64) diff --git a/build-win64.sh b/build-win64.sh index 1ef593a5..f0c689af 100644 --- a/build-win64.sh +++ b/build-win64.sh @@ -74,7 +74,6 @@ prepare() mingw64/mingw-w64-x86_64-nsis \ mingw64/mingw-w64-x86_64-libsignal-protocol-c \ mingw64/mingw-w64-x86_64-icu \ - mingw64/mingw-w64-x86_64-webrtc-audio-processing \ mingw64/mingw-w64-x86_64-meson \ git \ make \ diff --git a/cmake/FindGstWebrtcDsp.cmake b/cmake/FindGstWebrtcDsp.cmake new file mode 100644 index 00000000..fe4de658 --- /dev/null +++ b/cmake/FindGstWebrtcDsp.cmake @@ -0,0 +1,11 @@ +find_library(GstWebrtcDsp_LIBRARY gstwebrtcdsp PATH_SUFFIXES gstreamer-1.0) + +if(GstWebrtcDsp_LIBRARY_FOUND) + find_package(Gst) + set(GstWebrtcDsp_VERSION ${Gst_VERSION}) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GstWebrtcDsp + REQUIRED_VARS GstWebrtcDsp_LIBRARY + VERSION_VAR GstWebrtcDsp_VERSION) diff --git a/cmake/FindWebRTCAudioProcessing.cmake b/cmake/FindWebRTCAudioProcessing.cmake deleted file mode 100644 index 5f17805d..00000000 --- a/cmake/FindWebRTCAudioProcessing.cmake +++ /dev/null @@ -1,12 +0,0 @@ -include(PkgConfigWithFallback) -find_pkg_config_with_fallback(WebRTCAudioProcessing - PKG_CONFIG_NAME webrtc-audio-processing - LIB_NAMES webrtc_audio_processing - INCLUDE_NAMES webrtc/modules/audio_processing/include/audio_processing.h - INCLUDE_DIR_SUFFIXES webrtc-audio-processing webrtc_audio_processing -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(WebRTCAudioProcessing - REQUIRED_VARS WebRTCAudioProcessing_LIBRARY - VERSION_VAR WebRTCAudioProcessing_VERSION) diff --git a/configure b/configure index f3b6d4f2..24a13bc3 100755 --- a/configure +++ b/configure @@ -1,7 +1,7 @@ #!/bin/sh OPTS=`getopt -o "h" --long \ -help,fetch-only,no-debug,disable-fast-vapi,with-tests,release,with-libsoup3,\ +help,fetch-only,no-debug,disable-fast-vapi,with-tests,release,with-libsoup3,without-webrtcdsp,\ enable-plugin:,disable-plugin:,\ prefix:,program-prefix:,exec-prefix:,lib-suffix:,\ bindir:,libdir:,includedir:,datadir:,\ @@ -21,6 +21,7 @@ DISABLE_FAST_VAPI= LIB_SUFFIX= NO_DEBUG= USE_SOUP3= +PLUGIN_RTP_WEBRTC_AUDIO_PROCESSING=yes EXEC_PREFIX= BINDIR= @@ -54,6 +55,7 @@ Configuration: --no-debug Build without debug symbols --release Configure to build an optimized release version --with-libsoup3 Build with libsoup-3.0 + --without-webrtcdsp Build without WebRTC audio processing --with-tests Also build tests. Plugin configuration: @@ -109,6 +111,7 @@ while true; do --valac-flags ) VALACFLAGS="$2"; shift; shift ;; --lib-suffix ) LIB_SUFFIX="$2"; shift; shift ;; --with-libsoup3 ) USE_SOUP3=yes; shift ;; + --without-webrtcdsp ) PLUGIN_RTP_WEBRTC_AUDIO_PROCESSING=no; shift ;; --disable-fast-vapi ) DISABLE_FAST_VAPI=yes; shift ;; --no-debug ) NO_DEBUG=yes; shift ;; --release ) BUILD_TYPE=RelWithDebInfo; shift ;; @@ -219,6 +222,7 @@ cmake -G "$cmake_type" \ -DBIN_INSTALL_DIR="$BINDIR" \ -DINCLUDE_INSTALL_DIR="$INCLUDEDIR" \ -DLIB_INSTALL_DIR="$LIBDIR" \ + -DPLUGIN_RTP_WEBRTC_AUDIO_PROCESSING="$PLUGIN_RTP_WEBRTC_AUDIO_PROCESSING" \ -Wno-dev \ .. || exit 9 diff --git a/meson.build b/meson.build index 3364d3bb..6c7643e7 100644 --- a/meson.build +++ b/meson.build @@ -58,6 +58,8 @@ else libsoup_version = '3.0' endif +cc = meson.get_compiler('c') + dep_gdk_pixbuf = dependency('gdk-pixbuf-2.0') dep_gee = dependency('gee-0.8') dep_gio = dependency('gio-2.0') @@ -81,9 +83,12 @@ dep_libsrtp2 = dependency('libsrtp2', disabler: true, required: plugin_crypto) dep_libsignal_protocol_c = dependency('libsignal-protocol-c', version: ['>=2.3.2', '<2.3.4'], disabler: true, required: get_option('plugin-omemo')) dep_libsoup = dependency('libsoup-@0@'.format(libsoup_version), disabler: true, required: get_option('plugin-http-files')) dep_nice = dependency('nice', version: '>=0.1.15', disabler: true, required: get_option('plugin-ice')) -dep_m = meson.get_compiler('c').find_library('m', required: false) +dep_m = cc.find_library('m', required: false) dep_sqlite3 = dependency('sqlite3', version: '>=3.24') -dep_webrtc_audio_processing = dependency('webrtc-audio-processing', version: ['>=0.2', '<0.4'], required: get_option('plugin-rtp-webrtc-audio-processing')) + +dep_gstreamer_bad = dependency('gstreamer-plugins-bad-1.0', disabler: true, required: get_option('plugin-rtp-webrtc-audio-processing')) +gstpluginsdir = dep_gstreamer_bad.get_variable('pluginsdir') +dep_webrtcdsp = cc.find_library('gstwebrtcdsp', dirs: gstpluginsdir, disabler: true, required: get_option('plugin-rtp-webrtc-audio-processing')) prog_git = find_program('git', required: false) prog_python = python.find_installation() diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 83df5df8..be0ad926 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -1,5 +1,13 @@ find_package(GLib ${GLib_GLOBAL_VERSION} REQUIRED) -find_package(WebRTCAudioProcessing 0.2) + +set(RTP_DEFINITIONS) +set(RTP_EXTRA_OPTIONS) + +if(PLUGIN_RTP_WEBRTC_AUDIO_PROCESSING) + set(EXTRA_RTP_PACKAGES GstWebrtcDsp) + list(APPEND RTP_DEFINITIONS WITH_VOICE_PROCESSOR) +endif() + find_packages(RTP_PACKAGES REQUIRED Gee GLib @@ -11,11 +19,9 @@ find_packages(RTP_PACKAGES REQUIRED GstAudio GstRtp GstVideo + ${EXTRA_RTP_PACKAGES} ) -set(RTP_DEFINITIONS) -set(RTP_EXTRA_OPTIONS) - if(GstRtp_VERSION VERSION_GREATER_EQUAL "1.16") set(RTP_DEFINITIONS ${RTP_DEFINITIONS} GST_1_16) endif() @@ -52,20 +58,6 @@ if(RTP_ENABLE_MSDK) set(RTP_DEFINITIONS ${RTP_DEFINITIONS} ENABLE_MSDK) endif() -if(WebRTCAudioProcessing_VERSION GREATER "0.4") - message(STATUS "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") - unset(WebRTCAudioProcessing_FOUND) -endif() - -if(WebRTCAudioProcessing_FOUND) - set(RTP_DEFINITIONS ${RTP_DEFINITIONS} WITH_VOICE_PROCESSOR) - set(RTP_VOICE_PROCESSOR_VALA src/voice_processor.vala) - set(RTP_VOICE_PROCESSOR_CXX src/voice_processor_native.cpp) - set(RTP_VOICE_PROCESSOR_LIB webrtc-audio-processing) -else() - message(STATUS "WebRTCAudioProcessing not found, build without voice pre-processing!") -endif() - vala_precompile(RTP_VALA_C SOURCES src/codec_util.vala diff --git a/plugins/rtp/meson.build b/plugins/rtp/meson.build index 06cb2a1f..aad4591a 100644 --- a/plugins/rtp/meson.build +++ b/plugins/rtp/meson.build @@ -30,12 +30,8 @@ c_args = [ vala_args = [ '--vapidir', meson.current_source_dir() / 'vapi', ] -if dep_webrtc_audio_processing.found() - dependencies += [dep_webrtc_audio_processing] - sources += files( - 'src/voice_processor.vala', - 'src/voice_processor_native.cpp', - ) +if dep_webrtcdsp.found() + dependencies += [dep_webrtcdsp] vala_args += ['-D', 'WITH_VOICE_PROCESSOR'] endif if dep_gstreamer_rtp.version() == 'unknown' or dep_gstreamer_rtp.version().version_compare('>=1.16') diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index a9f9bb11..1ba29545 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -475,13 +475,12 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { Gst.Bin bin = new Gst.Bin("voiceprocessorbin"); Gst.Element converter = Gst.ElementFactory.make("audioconvert", @"dsp_convert_$id"); Gst.Element resampler = Gst.ElementFactory.make("audioresample", @"dsp_resmaple_$id"); - - Gst.Element voiceproc = new VoiceProcessor(plugin.echoprobe as EchoProbe, element as Gst.Audio.StreamVolume); - voiceproc.name = @"dsp_$id"; + Gst.Element voiceproc = Gst.ElementFactory.make("webrtcdsp", @"dsp_$id"); bin.add_many(converter, resampler, voiceproc); converter.link(resampler); resampler.link(voiceproc); + voiceproc.@set("probe", "webrtcechoprobe0"); Gst.Pad sink_pad = bin.find_unlinked_pad(Gst.PadDirection.SINK); Gst.Pad src_pad = bin.find_unlinked_pad(Gst.PadDirection.SRC); diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index 97c6fb47..c189a28b 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -59,13 +59,14 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } #if WITH_VOICE_PROCESSOR - private Gst.Element create_echo_bin(Gst.Element element) { + private Gst.Element create_echo_probe() { Gst.Bin bin = new Gst.Bin("echoprobebin"); Gst.Element converter = Gst.ElementFactory.make("audioconvert", "echo_convert_"); Gst.Element resampler = Gst.ElementFactory.make("audioresample", "echo_resample_"); + Gst.Element webrtcechoprobe = Gst.ElementFactory.make("webrtcechoprobe", "webrtcechoprobe0"); - bin.add_many(element, converter, resampler); - element.link(converter); + bin.add_many(webrtcechoprobe, converter, resampler); + webrtcechoprobe.link(converter); converter.link(resampler); Gst.Pad sink_pad = bin.find_unlinked_pad(Gst.PadDirection.SINK); @@ -100,11 +101,8 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { #if WITH_VOICE_PROCESSOR // Audio echo probe - echoprobe = new EchoProbe(); - if (echoprobe != null) { - echoprobe = create_echo_bin(echoprobe); - pipe.add(echoprobe); - } + echoprobe = create_echo_probe(); + pipe.add(echoprobe); #endif // Pipeline diff --git a/plugins/rtp/src/voice_processor.vala b/plugins/rtp/src/voice_processor.vala deleted file mode 100644 index 66e95d72..00000000 --- a/plugins/rtp/src/voice_processor.vala +++ /dev/null @@ -1,176 +0,0 @@ -using Gst; - -namespace Dino.Plugins.Rtp { -public static extern Buffer adjust_to_running_time(Base.Transform transform, Buffer buf); -} - -public class Dino.Plugins.Rtp.EchoProbe : Audio.Filter { - private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; - private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; - public Audio.Info audio_info { get; private set; } - public signal void on_new_buffer(Buffer buffer); - private uint period_samples; - private uint period_size; - private Base.Adapter adapter = new Base.Adapter(); - - static construct { - add_static_pad_template(sink_template); - add_static_pad_template(src_template); - set_static_metadata("Acoustic Echo Canceller probe", "Generic/Audio", "Gathers playback buffers for echo cancellation", "Dino Team "); - } - - construct { - set_passthrough(true); - } - - public override bool setup(Audio.Info info) { - audio_info = info; - period_samples = info.rate / 100; // 10ms buffers - period_size = period_samples * info.bpf; - return true; - } - - - public override FlowReturn transform_ip(Buffer buf) { - lock (adapter) { - adapter.push(adjust_to_running_time(this, buf)); - while (adapter.available() > period_size) { - on_new_buffer(adapter.take_buffer(period_size)); - } - } - return FlowReturn.OK; - } - - public override bool stop() { - adapter.clear(); - return true; - } -} - -public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter { - private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; - private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; - public Audio.Info audio_info { get; private set; } - private ulong process_outgoing_buffer_handler_id; - private uint adjust_delay_timeout_id; - private uint period_samples; - private uint period_size; - private Base.Adapter adapter = new Base.Adapter(); - private EchoProbe? echo_probe; - private Audio.StreamVolume? stream_volume; - private ClockTime last_reverse; - private void* native; - - static construct { - add_static_pad_template(sink_template); - add_static_pad_template(src_template); - set_static_metadata("Voice Processor (AGC, AEC, filters, etc.)", "Generic/Audio", "Pre-processes voice with WebRTC Audio Processing Library", "Dino Team "); - } - - construct { - set_passthrough(false); - } - - public VoiceProcessor(EchoProbe? echo_probe = null, Audio.StreamVolume? stream_volume = null) { - this.echo_probe = echo_probe; - this.stream_volume = stream_volume; - } - - private static extern void* init_native(int stream_delay); - private static extern void setup_native(void* native); - private static extern void destroy_native(void* native); - private static extern void analyze_reverse_stream(void* native, Audio.Info info, Buffer buffer); - private static extern void process_stream(void* native, Audio.Info info, Buffer buffer); - private static extern void adjust_stream_delay(void* native); - private static extern void notify_gain_level(void* native, int gain_level); - private static extern int get_suggested_gain_level(void* native); - private static extern bool get_stream_has_voice(void* native); - - public override bool setup(Audio.Info info) { - debug("VoiceProcessor.setup(%s)", info.to_caps().to_string()); - audio_info = info; - period_samples = info.rate / 100; // 10ms buffers - period_size = period_samples * info.bpf; - adapter.clear(); - setup_native(native); - return true; - } - - public override bool start() { - native = init_native(150); - if (process_outgoing_buffer_handler_id == 0 && echo_probe != null) { - process_outgoing_buffer_handler_id = echo_probe.on_new_buffer.connect(process_outgoing_buffer); - } - if (stream_volume == null && sinkpad.get_peer() != null && sinkpad.get_peer().get_parent_element() is Audio.StreamVolume) { - stream_volume = sinkpad.get_peer().get_parent_element() as Audio.StreamVolume; - } - return true; - } - - private bool adjust_delay() { - if (native != null) { - adjust_stream_delay(native); - return Source.CONTINUE; - } else { - adjust_delay_timeout_id = 0; - return Source.REMOVE; - } - } - - private void process_outgoing_buffer(Buffer buffer) { - if (buffer.pts != uint64.MAX) { - last_reverse = buffer.pts; - } - analyze_reverse_stream(native, echo_probe.audio_info, buffer); - if (adjust_delay_timeout_id == 0 && echo_probe != null) { - adjust_delay_timeout_id = Timeout.add(1000, adjust_delay); - } - } - - public override FlowReturn submit_input_buffer(bool is_discont, Buffer input) { - lock (adapter) { - if (is_discont) { - adapter.clear(); - } - adapter.push(adjust_to_running_time(this, input)); - } - return FlowReturn.OK; - } - - public override FlowReturn generate_output(out Buffer output_buffer) { - lock (adapter) { - if (adapter.available() >= period_size) { - output_buffer = (Gst.Buffer) adapter.take_buffer(period_size).make_writable(); - int old_gain_level = 0; - if (stream_volume != null) { - old_gain_level = (int) (stream_volume.get_volume(Audio.StreamVolumeFormat.LINEAR) * 255.0); - notify_gain_level(native, old_gain_level); - } - process_stream(native, audio_info, output_buffer); - if (stream_volume != null) { - int new_gain_level = get_suggested_gain_level(native); - if (old_gain_level != new_gain_level) { - debug("Gain: %i -> %i", old_gain_level, new_gain_level); - stream_volume.set_volume(Audio.StreamVolumeFormat.LINEAR, ((double)new_gain_level) / 255.0); - } - } - } - } - return FlowReturn.OK; - } - - public override bool stop() { - if (process_outgoing_buffer_handler_id != 0) { - echo_probe.disconnect(process_outgoing_buffer_handler_id); - process_outgoing_buffer_handler_id = 0; - } - if (adjust_delay_timeout_id != 0) { - Source.remove(adjust_delay_timeout_id); - adjust_delay_timeout_id = 0; - } - adapter.clear(); - destroy_native(native); - native = null; - return true; - } -} \ No newline at end of file diff --git a/plugins/rtp/src/voice_processor_native.cpp b/plugins/rtp/src/voice_processor_native.cpp deleted file mode 100644 index 8a052cf8..00000000 --- a/plugins/rtp/src/voice_processor_native.cpp +++ /dev/null @@ -1,148 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#define SAMPLE_RATE 48000 -#define SAMPLE_CHANNELS 1 - -struct _DinoPluginsRtpVoiceProcessorNative { - webrtc::AudioProcessing *apm; - gint stream_delay; - gint last_median; - gint last_poor_delays; -}; - -extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) { - GstBuffer *copy = gst_buffer_copy(buffer); - GST_BUFFER_PTS(copy) = gst_segment_to_running_time(&transform->segment, GST_FORMAT_TIME, GST_BUFFER_PTS(buffer)); - return copy; -} - -extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay) { - _DinoPluginsRtpVoiceProcessorNative *native = new _DinoPluginsRtpVoiceProcessorNative(); - webrtc::Config config; - config.Set(new webrtc::ExtendedFilter(true)); - config.Set(new webrtc::ExperimentalAgc(true, 85)); - native->apm = webrtc::AudioProcessing::Create(config); - native->stream_delay = stream_delay; - native->last_median = 0; - native->last_poor_delays = 0; - return native; -} - -extern "C" void dino_plugins_rtp_voice_processor_setup_native(void *native_ptr) { - _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - webrtc::AudioProcessing *apm = native->apm; - webrtc::ProcessingConfig pconfig; - pconfig.streams[webrtc::ProcessingConfig::kInputStream] = - webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); - pconfig.streams[webrtc::ProcessingConfig::kOutputStream] = - webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); - pconfig.streams[webrtc::ProcessingConfig::kReverseInputStream] = - webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); - pconfig.streams[webrtc::ProcessingConfig::kReverseOutputStream] = - webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); - apm->Initialize(pconfig); - apm->high_pass_filter()->Enable(true); - apm->echo_cancellation()->enable_drift_compensation(false); - apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kModerateSuppression); - apm->echo_cancellation()->enable_delay_logging(true); - apm->echo_cancellation()->Enable(true); - apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kModerate); - apm->noise_suppression()->Enable(true); - apm->gain_control()->set_analog_level_limits(0, 255); - apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveAnalog); - apm->gain_control()->set_target_level_dbfs(3); - apm->gain_control()->set_compression_gain_db(9); - apm->gain_control()->enable_limiter(true); - apm->gain_control()->Enable(true); - apm->voice_detection()->set_likelihood(webrtc::VoiceDetection::Likelihood::kLowLikelihood); - apm->voice_detection()->Enable(true); -} - -extern "C" void -dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { - _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); - webrtc::AudioProcessing *apm = native->apm; - - GstMapInfo map; - gst_buffer_map(buffer, &map, GST_MAP_READ); - - webrtc::AudioFrame frame; - frame.num_channels_ = info->channels; - frame.sample_rate_hz_ = info->rate; - frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf; - memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); - - int err = apm->AnalyzeReverseStream(&frame); - if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err); - - gst_buffer_unmap(buffer, &map); -} - -extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) { - _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - webrtc::AudioProcessing *apm = native->apm; - apm->gain_control()->set_stream_analog_level(gain_level); -} - -extern "C" gint dino_plugins_rtp_voice_processor_get_suggested_gain_level(void *native_ptr) { - _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - webrtc::AudioProcessing *apm = native->apm; - return apm->gain_control()->stream_analog_level(); -} - -extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *native_ptr) { - _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - webrtc::AudioProcessing *apm = native->apm; - return apm->voice_detection()->stream_has_voice(); -} - -extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) { - _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - webrtc::AudioProcessing *apm = native->apm; - int median, std, poor_delays; - float fraction_poor_delays; - apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays); - poor_delays = (int)(fraction_poor_delays * 100.0); - if (fraction_poor_delays < 0 || (native->last_median == median && native->last_poor_delays == poor_delays)) return; - g_debug("voice_processor_native.cpp: Stream delay metrics: median=%i std=%i poor_delays=%i%%", median, std, poor_delays); - native->last_median = median; - native->last_poor_delays = poor_delays; - if (poor_delays > 90) { - native->stream_delay = std::min(std::max(0, native->stream_delay + std::min(48, std::max(median, -48))), 384); - g_debug("voice_processor_native.cpp: set stream_delay=%i", native->stream_delay); - } -} - -extern "C" void -dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { - _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); - webrtc::AudioProcessing *apm = native->apm; - - GstMapInfo map; - gst_buffer_map(buffer, &map, GST_MAP_READWRITE); - - webrtc::AudioFrame frame; - frame.num_channels_ = info->channels; - frame.sample_rate_hz_ = info->rate; - frame.samples_per_channel_ = info->rate / 100; - memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); - - apm->set_stream_delay_ms(native->stream_delay); - int err = apm->ProcessStream(&frame); - if (err >= 0) memcpy(map.data, frame.data_, frame.samples_per_channel_ * info->bpf); - if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err); - - gst_buffer_unmap(buffer, &map); -} - -extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) { - _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; - delete native; -} \ No newline at end of file