diff --git a/Makefile.am b/Makefile.am index 53bd7c66aa..2789737efd 100644 --- a/Makefile.am +++ b/Makefile.am @@ -179,6 +179,10 @@ if HAVE_GUMBO CORE_CFLAGS += -DSWITCH_HAVE_GUMBO $(LIBGUMBO_CFLAGS) endif +if HAVE_FVAD +CORE_CFLAGS += -DSWITCH_HAVE_FVAD $(LIBFVAD_CFLAGS) +endif + ## ## libfreeswitch ## @@ -241,9 +245,9 @@ CORE_LIBS+=libfreeswitch_libyuv.la endif lib_LTLIBRARIES = libfreeswitch.la -libfreeswitch_la_CFLAGS = $(CORE_CFLAGS) $(SQLITE_CFLAGS) $(GUMBO_CFLAGS) $(FREETYPE_CFLAGS) $(CURL_CFLAGS) $(PCRE_CFLAGS) $(SPEEX_CFLAGS) $(LIBEDIT_CFLAGS) $(openssl_CFLAGS) $(AM_CFLAGS) +libfreeswitch_la_CFLAGS = $(CORE_CFLAGS) $(SQLITE_CFLAGS) $(GUMBO_CFLAGS) $(FVAD_CFLAGS) $(FREETYPE_CFLAGS) $(CURL_CFLAGS) $(PCRE_CFLAGS) $(SPEEX_CFLAGS) $(LIBEDIT_CFLAGS) $(openssl_CFLAGS) $(AM_CFLAGS) libfreeswitch_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS) $(PLATFORM_CORE_LDFLAGS) -no-undefined -libfreeswitch_la_LIBADD = $(CORE_LIBS) $(APR_LIBS) $(SQLITE_LIBS) $(GUMBO_LIBS) $(FREETYPE_LIBS) $(CURL_LIBS) $(PCRE_LIBS) $(SPEEX_LIBS) $(LIBEDIT_LIBS) $(openssl_LIBS) $(PLATFORM_CORE_LIBS) +libfreeswitch_la_LIBADD = $(CORE_LIBS) $(APR_LIBS) $(SQLITE_LIBS) $(GUMBO_LIBS) $(FVAD_LIBS) $(FREETYPE_LIBS) $(CURL_LIBS) $(PCRE_LIBS) $(SPEEX_LIBS) $(LIBEDIT_LIBS) $(openssl_LIBS) $(PLATFORM_CORE_LIBS) libfreeswitch_la_DEPENDENCIES = $(BUILT_SOURCES) if HAVE_PNG @@ -312,6 +316,7 @@ library_include_HEADERS = \ src/include/switch_utf8.h \ src/include/switch_msrp.h \ src/include/switch_vpx.h \ + src/include/switch_vad.h \ libs/libteletone/src/libteletone_detect.h \ libs/libteletone/src/libteletone_generate.h \ libs/libteletone/src/libteletone.h \ @@ -395,6 +400,7 @@ libfreeswitch_la_SOURCES = \ src/switch_hashtable.c\ src/switch_utf8.c \ src/switch_msrp.c \ + src/switch_vad.c \ src/switch_vpx.c \ libs/libtpl-1.5/src/tpl.c \ libs/libteletone/src/libteletone_detect.c \ diff --git a/configure.ac b/configure.ac index a52af80ccd..5345b354c1 100644 --- a/configure.ac +++ b/configure.ac @@ -1310,6 +1310,10 @@ PKG_CHECK_MODULES([GUMBO], [gumbo >= 0.10.1],[ AM_CONDITIONAL([HAVE_GUMBO],[true])],[ AC_MSG_RESULT([no]); AM_CONDITIONAL([HAVE_GUMBO],[false])]) +PKG_CHECK_MODULES([FVAD], [libfvad >= 1.0],[ + AM_CONDITIONAL([HAVE_FVAD],[true])],[ + AC_MSG_RESULT([no]); AM_CONDITIONAL([HAVE_FVAD],[false])]) + PKG_CHECK_MODULES([SQLITE], [sqlite3 >= 3.6.20]) PKG_CHECK_MODULES([CURL], [libcurl >= 7.19]) PKG_CHECK_MODULES([PCRE], [libpcre >= 7.8]) diff --git a/src/include/switch.h b/src/include/switch.h index 0b1ff4c8e9..3498aecc19 100644 --- a/src/include/switch.h +++ b/src/include/switch.h @@ -145,6 +145,7 @@ #include "switch_core_video.h" #include "switch_jitterbuffer.h" #include "switch_estimators.h" +#include "switch_vad.h" #include diff --git a/src/include/switch_types.h b/src/include/switch_types.h index e479e30bbd..a85d7be59a 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -622,6 +622,14 @@ typedef enum { } switch_vad_flag_enum_t; typedef uint32_t switch_vad_flag_t; +typedef enum { + SWITCH_VAD_STATE_NONE, + SWITCH_VAD_STATE_START_TALKING, + SWITCH_VAD_STATE_TALKING, + SWITCH_VAD_STATE_STOP_TALKING, + SWITCH_VAD_STATE_ERROR +} switch_vad_state_t; +typedef struct switch_vad_s switch_vad_t; typedef struct error_period { int64_t start; diff --git a/src/include/switch_vad.h b/src/include/switch_vad.h new file mode 100644 index 0000000000..8cc6c63f05 --- /dev/null +++ b/src/include/switch_vad.h @@ -0,0 +1,66 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2018, Anthony Minessale II + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Anthony Minessale II + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * Seven Du + * + * + * switch_vad.h VAD code with optional libfvad + * + */ +/*! + \defgroup vad1 VAD code with optional libfvad + \ingroup core1 + \{ +*/ +#ifndef FREESWITCH_VAD_H +#define FREESWITCH_VAD_H + +SWITCH_BEGIN_EXTERN_C + +SWITCH_DECLARE(switch_vad_t *) switch_vad_init(int sample_rate, int channels); + +/* + * Valid modes are -1 ("disable fvad, using native"), 0 ("quality"), 1 ("low bitrate"), 2 ("aggressive"), and 3 * ("very aggressive"). + * The default mode is -1. +*/ + +SWITCH_DECLARE(int) switch_vad_set_mode(switch_vad_t *vad, int mode); +SWITCH_DECLARE(void) switch_vad_set_param(switch_vad_t *vad, const char *key, int val); +SWITCH_DECLARE(switch_vad_state_t) switch_vad_process(switch_vad_t *vad, int16_t *data, unsigned int samples); +SWITCH_DECLARE(void) switch_vad_reset(switch_vad_t *vad); +SWITCH_DECLARE(void) switch_vad_destroy(switch_vad_t **vad); + +SWITCH_END_EXTERN_C +#endif +/* For Emacs: + * Local Variables: + * mode:c + * indent-tabs-mode:t + * tab-width:4 + * c-basic-offset:4 + * End: + * For VIM: + * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet: + */ diff --git a/src/mod/applications/mod_dptools/mod_dptools.c b/src/mod/applications/mod_dptools/mod_dptools.c index 812e56a3f1..4cc8dfab64 100644 --- a/src/mod/applications/mod_dptools/mod_dptools.c +++ b/src/mod/applications/mod_dptools/mod_dptools.c @@ -6128,6 +6128,73 @@ SWITCH_STANDARD_APP(deduplicate_dtmf_app_function) } } +SWITCH_STANDARD_APP(vad_test_function) +{ + switch_channel_t *channel = switch_core_session_get_channel(session); + switch_codec_implementation_t imp = { 0 }; + switch_vad_t *vad; + switch_frame_t *frame = { 0 }; + switch_vad_state_t vad_state; + int mode = -1; + const char *var = NULL; + int tmp; + + if (!zstr(data)) { + mode = atoi(data); + + if (mode > 3) mode = 3; + } + + switch_core_session_raw_read(session); + switch_core_session_get_read_impl(session, &imp); + + vad = switch_vad_init(imp.samples_per_second, imp.number_of_channels); + switch_assert(vad); + switch_vad_set_mode(vad, mode); + + if ((var = switch_channel_get_variable(channel, "vad_hangover_len"))) { + tmp = atoi(var); + + if (tmp > 0) switch_vad_set_param(vad, "hangover_len", tmp); + } + + if ((var = switch_channel_get_variable(channel, "vad_thresh"))) { + tmp = atoi(var); + + if (tmp > 0) switch_vad_set_param(vad, "thresh", tmp); + } + + if ((var = switch_channel_get_variable(channel, "vad_timeout_len"))) { + tmp = atoi(var); + + if (tmp > 0) switch_vad_set_param(vad, "timeout_len", tmp); + } + + while(switch_channel_ready(channel)) { + switch_core_session_read_frame(session, &frame, SWITCH_IO_FLAG_NONE, 0); + + if (switch_test_flag(frame, SFF_CNG)) { + continue; + } + + vad_state = switch_vad_process(vad, frame->data, frame->datalen / 2); + + if (vad_state == SWITCH_VAD_STATE_START_TALKING) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "START TALKING\n"); + switch_core_session_write_frame(session, frame, SWITCH_IO_FLAG_NONE, 0); + } else if (vad_state == SWITCH_VAD_STATE_STOP_TALKING) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "STOP TALKING\n"); + } else if (vad_state == SWITCH_VAD_STATE_TALKING) { + switch_core_session_write_frame(session, frame, SWITCH_IO_FLAG_NONE, 0); + } else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "vad_state: %d\n", vad_state); + } + } + + switch_vad_destroy(&vad); + switch_core_session_reset(session, SWITCH_TRUE, SWITCH_TRUE); +} + #define SPEAK_DESC "Speak text to a channel via the tts interface" #define DISPLACE_DESC "Displace audio from a file to the channels input" #define SESS_REC_DESC "Starts a background recording of the entire session" @@ -6462,6 +6529,7 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_dptools_load) SWITCH_ADD_APP(app_interface, "pickup", "Pickup", "Pickup a call", pickup_function, PICKUP_SYNTAX, SAF_SUPPORT_NOMEDIA); SWITCH_ADD_APP(app_interface, "deduplicate_dtmf", "Prevent duplicate inband + 2833 dtmf", "", deduplicate_dtmf_app_function, "[only_rtp]", SAF_SUPPORT_NOMEDIA); + SWITCH_ADD_APP(app_interface, "vad_test", "VAD test", "VAD test, mode = -1(default), 0, 1, 2, 3", vad_test_function, "[mode]", SAF_NONE); SWITCH_ADD_DIALPLAN(dp_interface, "inline", inline_dialplan_hunt); diff --git a/src/switch_vad.c b/src/switch_vad.c new file mode 100644 index 0000000000..c22ed2f352 --- /dev/null +++ b/src/switch_vad.c @@ -0,0 +1,219 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2018, Anthony Minessale II + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Anthony Minessale II + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * Seven Du + * + * + * switch_vad.c VAD code with optional libfvad + * + */ + +#include + +#ifdef SWITCH_HAVE_FVAD +#include +#endif + +struct switch_vad_s { + int talking; + int talked; + int talk_hits; + int hangover; + int hangover_len; + int divisor; + int thresh; + int timeout_len; + int timeout; + int channels; + int sample_rate; + + int _hangover_len; + int _thresh; + int _timeout_len; +#ifdef SWITCH_HAVE_FVAD + Fvad *fvad; +#endif +}; + +SWITCH_DECLARE(switch_vad_t *) switch_vad_init(int sample_rate, int channels) +{ + switch_vad_t *vad = malloc(sizeof(switch_vad_t)); + + if (!vad) return NULL; + + memset(vad, 0, sizeof(*vad)); + vad->sample_rate = sample_rate ? sample_rate : 8000; + vad->channels = channels; + vad->_hangover_len = 25; + vad->_thresh = 100; + vad->_timeout_len = 25; + + switch_vad_reset(vad); + + return vad; +} + +//Valid modes are 0 ("quality"), 1 ("low bitrate"), 2 ("aggressive"), and 3 * ("very aggressive"). The default mode is 0. +SWITCH_DECLARE(int) switch_vad_set_mode(switch_vad_t *vad, int mode) +{ +#ifdef SWITCH_HAVE_FVAD + int ret; + + if (mode < 0) return 0; + + vad->fvad = fvad_new(); + ret = fvad_set_mode(vad->fvad, mode); + fvad_set_sample_rate(vad->fvad, vad->sample_rate); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "libfvad started, mode = %d\n", mode); + return ret; +#else + return 0; +#endif +} + +SWITCH_DECLARE(void) switch_vad_set_param(switch_vad_t *vad, const char *key, int val) +{ + if (!key) return; + + if (!strcmp(key, "hangover_len")) { + vad->hangover_len = vad->_hangover_len = val; + } else if (!strcmp(key, "thresh")) { + vad->thresh = vad->_thresh = val; + } else if (!strcmp(key, "timeout_len")) { + vad->timeout = vad->timeout_len = vad->_timeout_len = val; + } +} + +SWITCH_DECLARE(void) switch_vad_reset(switch_vad_t *vad) +{ +#ifdef SWITCH_HAVE_FVAD + if (vad->fvad) { + fvad_reset(vad->fvad); + return; + } +#endif + + vad->talking = 0; + vad->talked = 0; + vad->talk_hits = 0; + vad->hangover = 0; + vad->hangover_len = vad->_hangover_len; + vad->divisor = vad->sample_rate / 8000; + vad->thresh = vad->_thresh; + vad->timeout_len = vad->_timeout_len; + vad->timeout = vad->timeout_len; +} + +SWITCH_DECLARE(switch_vad_state_t) switch_vad_process(switch_vad_t *vad, int16_t *data, unsigned int samples) +{ + int energy = 0, j = 0, count = 0; + int score = 0; + switch_vad_state_t vad_state = SWITCH_VAD_STATE_NONE; + +#ifdef SWITCH_HAVE_FVAD + if (vad->fvad) { + int ret = fvad_process(vad->fvad, data, samples); + + // printf("%d ", ret); fflush(stdout); + + score = vad->thresh + ret - 1; + } else { +#endif + + for (energy = 0, j = 0, count = 0; count < samples; count++) { + energy += abs(data[j]); + j += vad->channels; + } + + score = (uint32_t) (energy / (samples / vad->divisor)); + +#ifdef SWITCH_HAVE_FVAD + } +#endif + + // printf("%d ", score); fflush(stdout); + // printf("yay %d %d %d\n", score, vad->hangover, vad->talking); + + if (vad->talking && score < vad->thresh) { + if (vad->hangover > 0) { + vad->hangover--; + } else {// if (hangover <= 0) { + vad->talking = 0; + vad->talk_hits = 0; + vad->hangover = 0; + } + } else { + if (score >= vad->thresh) { + vad_state = vad->talking ? SWITCH_VAD_STATE_TALKING : SWITCH_VAD_STATE_START_TALKING; + vad->talking = 1; + vad->hangover = vad->hangover_len; + } + } + + // printf("WTF %d %d %d\n", score, vad->talked, vad->talking); + + if (vad->talking) { + vad->talk_hits++; + // printf("WTF %d %d %d\n", vad->talking, vad->talk_hits, vad->talked); + if (vad->talk_hits > 10) { + vad->talked = 1; + vad_state = SWITCH_VAD_STATE_TALKING; + } + } else { + vad->talk_hits = 0; + } + + if (vad->timeout > 0 && !vad->talking) { + vad->timeout--; + } + + if ((vad->talked && !vad->talking)) { + // printf("NOT TALKING ANYMORE\n"); + vad->talked = 0; + vad->timeout = vad->timeout_len; + vad_state = SWITCH_VAD_STATE_STOP_TALKING; + } else { + // if (vad->skip > 0) { + // vad->skip--; + // } + } + + if (vad_state) return vad_state; + + return SWITCH_VAD_STATE_NONE; +} + +SWITCH_DECLARE(void) switch_vad_destroy(switch_vad_t **vad) +{ + if (*vad) { + +#ifdef SWITCH_HAVE_FVAD + if ((*vad)->fvad) free ((*vad)->fvad); +#endif + + free(*vad); + *vad = NULL; + } +}