From 9d40e1397d2a41bff8fe1c56363afa82e0704024 Mon Sep 17 00:00:00 2001 From: Anthony Minessale Date: Thu, 11 Jun 2015 18:49:34 -0500 Subject: [PATCH] FS-7641 --- src/include/private/switch_core_pvt.h | 3 + src/include/switch_core.h | 5 + src/include/switch_types.h | 9 +- src/mod/endpoints/mod_sofia/sofia.c | 2 +- src/switch_core_io.c | 2 +- src/switch_core_media.c | 37 +++- src/switch_core_media_bug.c | 239 +++++++++++++++++++++++--- src/switch_core_video.c | 27 ++- src/switch_ivr_async.c | 165 ++++++++++++++++-- src/switch_ivr_bridge.c | 24 ++- 10 files changed, 454 insertions(+), 59 deletions(-) diff --git a/src/include/private/switch_core_pvt.h b/src/include/private/switch_core_pvt.h index 720c0fdfd9..66faaf4764 100644 --- a/src/include/private/switch_core_pvt.h +++ b/src/include/private/switch_core_pvt.h @@ -223,6 +223,9 @@ struct switch_media_bug { switch_frame_t *read_demux_frame; switch_queue_t *read_video_queue; switch_queue_t *write_video_queue; + switch_queue_t *spy_video_queue[2]; + switch_image_t *spy_img[2]; + switch_vid_spy_fmt_t spy_fmt; switch_thread_t *video_bug_thread; struct switch_media_bug *next; }; diff --git a/src/include/switch_core.h b/src/include/switch_core.h index a935b2570b..96afd617bf 100644 --- a/src/include/switch_core.h +++ b/src/include/switch_core.h @@ -253,6 +253,11 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_exec_all(switch_core_sessi const char *function, switch_media_bug_exec_cb_t cb, void *user_data); SWITCH_DECLARE(uint32_t) switch_core_media_bug_patch_video(switch_core_session_t *orig_session, switch_frame_t *frame); SWITCH_DECLARE(uint32_t) switch_core_media_bug_count(switch_core_session_t *orig_session, const char *function); +SWITCH_DECLARE(void) switch_media_bug_set_spy_fmt(switch_media_bug_t *bug, switch_vid_spy_fmt_t spy_fmt); +SWITCH_DECLARE(switch_status_t) switch_core_media_bug_push_spy_frame(switch_media_bug_t *bug, switch_frame_t *frame, switch_rw_t rw); +SWITCH_DECLARE(switch_status_t) switch_core_media_bug_patch_spy_frame(switch_media_bug_t *bug, switch_image_t *img, switch_rw_t rw); +SWITCH_DECLARE(switch_vid_spy_fmt_t) switch_media_bug_parse_spy_fmt(const char *name); + /*! \brief Add a media bug to the session \param session the session to add the bug to diff --git a/src/include/switch_types.h b/src/include/switch_types.h index aeea4f3040..b170414344 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -1742,7 +1742,9 @@ typedef enum { SMBF_WRITE_VIDEO_PING = (1 << 18), SMBF_READ_VIDEO_STREAM = (1 << 19), SMBF_WRITE_VIDEO_STREAM = (1 << 20), - SMBF_VIDEO_PATCH = (1 << 21) + SMBF_VIDEO_PATCH = (1 << 21), + SMBF_SPY_VIDEO_STREAM = (1 << 22), + SMBF_SPY_VIDEO_STREAM_BLEG = (1 << 23) } switch_media_bug_flag_enum_t; typedef uint32_t switch_media_bug_flag_t; @@ -2557,6 +2559,11 @@ typedef enum { SVR_FLUSH = (1 << 1) } switch_video_read_flag_t; +typedef enum { + SPY_LOWER_RIGHT_SMALL, + SPY_LOWER_RIGHT_LARGE, + SPY_DUAL_CROP +} switch_vid_spy_fmt_t; SWITCH_END_EXTERN_C #endif diff --git a/src/mod/endpoints/mod_sofia/sofia.c b/src/mod/endpoints/mod_sofia/sofia.c index b6b597f61e..7e69a1088b 100644 --- a/src/mod/endpoints/mod_sofia/sofia.c +++ b/src/mod/endpoints/mod_sofia/sofia.c @@ -8765,7 +8765,7 @@ void sofia_handle_sip_i_info(nua_t *nua, sofia_profile_t *profile, nua_handle_t switch_channel_event_set_data(channel, event); } switch_event_fire(&event); - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "dispatched freeswitch event for INFO\n"); + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG1, "dispatched freeswitch event for INFO\n"); } nua_respond(nh, SIP_200_OK, NUTAG_WITH_THIS_MSG(de->data->e_msg), TAG_END()); diff --git a/src/switch_core_io.c b/src/switch_core_io.c index 6eb6af3046..fce2f41df7 100644 --- a/src/switch_core_io.c +++ b/src/switch_core_io.c @@ -523,7 +523,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_frame(switch_core_sessi /* mux or demux to match */ if (session->read_impl.number_of_channels != read_frame->codec->implementation->number_of_channels) { uint32_t rlen = session->raw_read_frame.datalen / 2 / read_frame->codec->implementation->number_of_channels; - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_WARNING, "%s MUX READ\n", switch_channel_get_name(session->channel)); + //switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_WARNING, "%s MUX READ\n", switch_channel_get_name(session->channel)); switch_mux_channels((int16_t *) session->raw_read_frame.data, rlen, read_frame->codec->implementation->number_of_channels, session->read_impl.number_of_channels); session->raw_write_frame.datalen = rlen * 2 * session->read_impl.number_of_channels; diff --git a/src/switch_core_media.c b/src/switch_core_media.c index 2e8181cc6f..1432d30c8e 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -10394,6 +10394,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor switch_media_bug_t *bp; switch_bool_t ok = SWITCH_TRUE; int prune = 0; + int patched = 0; switch_thread_rwlock_rdlock(session->bug_rwlock); for (bp = session->bugs; bp; bp = bp->next) { @@ -10412,23 +10413,35 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor if (bp->ready && switch_test_flag(bp, SMBF_WRITE_VIDEO_STREAM)) { switch_image_t *dimg = NULL; + switch_img_copy(img, &dimg); switch_queue_push(bp->write_video_queue, dimg); - + + if (switch_core_media_bug_test_flag(bp, SMBF_SPY_VIDEO_STREAM_BLEG)) { + switch_core_media_bug_patch_spy_frame(bp, img, SWITCH_RW_WRITE); + patched = 1; + } + } - if (bp->ready && img && switch_test_flag(bp, SMBF_WRITE_VIDEO_PING)) { + if (bp->ready && img && + (switch_test_flag(bp, SMBF_WRITE_VIDEO_PING) || (switch_core_media_bug_test_flag(bp, SMBF_SPY_VIDEO_STREAM) && !patched))) { switch_frame_t bug_frame = { 0 }; bug_frame.img = img; bp->ping_frame = &bug_frame; - if (bp->callback) { + if (bp->callback && switch_test_flag(bp, SMBF_WRITE_VIDEO_PING)) { if (bp->callback(bp, bp->user_data, SWITCH_ABC_TYPE_WRITE_VIDEO_PING) == SWITCH_FALSE || (bp->stop_time && bp->stop_time <= switch_epoch_time_now(NULL))) { ok = SWITCH_FALSE; } } + + if (switch_core_media_bug_test_flag(bp, SMBF_SPY_VIDEO_STREAM_BLEG) && !patched) { + switch_core_media_bug_patch_spy_frame(bp, bp->ping_frame->img, SWITCH_RW_WRITE); + } + bp->ping_frame = NULL; } @@ -10613,6 +10626,8 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core switch_media_bug_t *bp; switch_bool_t ok = SWITCH_TRUE; int prune = 0; + int patched = 0; + switch_thread_rwlock_rdlock(session->bug_rwlock); for (bp = session->bugs; bp; bp = bp->next) { if (switch_channel_test_flag(session->channel, CF_PAUSE_BUGS) && !switch_core_media_bug_test_flag(bp, SMBF_NO_PAUSE)) { @@ -10633,17 +10648,29 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core switch_image_t *img = NULL; switch_img_copy((*frame)->img, &img); switch_queue_push(bp->read_video_queue, img); + if (switch_core_media_bug_test_flag(bp, SMBF_SPY_VIDEO_STREAM)) { + switch_core_media_bug_patch_spy_frame(bp, (*frame)->img, SWITCH_RW_READ); + patched = 1; + } } } - if (bp->ready && (*frame) && (*frame)->img && switch_test_flag(bp, SMBF_READ_VIDEO_PING)) { + if (bp->ready && (*frame) && (*frame)->img && + (switch_test_flag(bp, SMBF_READ_VIDEO_PING) || (switch_core_media_bug_test_flag(bp, SMBF_SPY_VIDEO_STREAM) && !patched))) { bp->ping_frame = *frame; - if (bp->callback) { + + if (bp->callback && switch_test_flag(bp, SMBF_READ_VIDEO_PING)) { if (bp->callback(bp, bp->user_data, SWITCH_ABC_TYPE_READ_VIDEO_PING) == SWITCH_FALSE || (bp->stop_time && bp->stop_time <= switch_epoch_time_now(NULL))) { ok = SWITCH_FALSE; } } + + if (switch_core_media_bug_test_flag(bp, SMBF_SPY_VIDEO_STREAM) && !patched) { + switch_core_media_bug_patch_spy_frame(bp, (*frame)->img, SWITCH_RW_READ); + } + + bp->ping_frame = NULL; } diff --git a/src/switch_core_media_bug.c b/src/switch_core_media_bug.c index 8c27f4beae..83e0e99816 100644 --- a/src/switch_core_media_bug.c +++ b/src/switch_core_media_bug.c @@ -391,6 +391,137 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_read(switch_media_bug_t *b return SWITCH_STATUS_SUCCESS; } +SWITCH_DECLARE(switch_vid_spy_fmt_t) switch_media_bug_parse_spy_fmt(const char *name) +{ + if (zstr(name)) goto end; + + if (!strcasecmp(name, "dual-crop")) { + return SPY_DUAL_CROP; + } + + if (!strcasecmp(name, "lower-right-large")) { + return SPY_LOWER_RIGHT_LARGE; + } + + end: + + return SPY_LOWER_RIGHT_SMALL; +} + +SWITCH_DECLARE(void) switch_media_bug_set_spy_fmt(switch_media_bug_t *bug, switch_vid_spy_fmt_t spy_fmt) +{ + bug->spy_fmt = spy_fmt; +} + +SWITCH_DECLARE(switch_status_t) switch_core_media_bug_patch_spy_frame(switch_media_bug_t *bug, switch_image_t *img, switch_rw_t rw) +{ + switch_queue_t *spy_q = NULL; + int w = 0, h = 0; + switch_status_t status; + void *pop; + int i; + + for (i = 0; i < 2; i++) { + if (!bug->spy_video_queue[i]) { + switch_queue_create(&bug->spy_video_queue[i], SWITCH_CORE_QUEUE_LEN, switch_core_session_get_pool(bug->session)); + } + } + + spy_q = bug->spy_video_queue[rw]; + + while(switch_queue_size(spy_q) > 0) { + if ((status = switch_queue_trypop(spy_q, &pop)) == SWITCH_STATUS_SUCCESS) { + switch_img_free(&bug->spy_img[rw]); + if (!(bug->spy_img[rw] = (switch_image_t *) pop)) { + break; + } + } + } + + w = img->d_w; + h = img->d_h; + + if (bug->spy_img[rw]) { + + switch (bug->spy_fmt) { + case SPY_DUAL_CROP: + { + switch_image_t *spy_tmp = NULL; + switch_image_t *img_tmp = NULL; + switch_image_t *img_dup = NULL; + int x = 0, y = 0; + float aspect169 = (float)1920 / 1080; + switch_rgb_color_t bgcolor = { 0 }; + + if ((float)w/h == aspect169) { + if ((float)bug->spy_img[rw]->d_w / bug->spy_img[rw]->d_h == aspect169) { + spy_tmp = switch_img_copy_rect(bug->spy_img[rw], bug->spy_img[rw]->d_w / 4, 0, bug->spy_img[rw]->d_w / 2, bug->spy_img[rw]->d_h); + + } else { + switch_img_copy(bug->spy_img[rw], &spy_tmp); + } + } else { + if ((float)bug->spy_img[rw]->d_w / bug->spy_img[rw]->d_h == aspect169) { + spy_tmp = switch_img_copy_rect(bug->spy_img[rw], bug->spy_img[rw]->d_w / 6, 0, bug->spy_img[rw]->d_w / 4, bug->spy_img[rw]->d_h); + } else { + spy_tmp = switch_img_copy_rect(bug->spy_img[rw], bug->spy_img[rw]->d_w / 4, 0, bug->spy_img[rw]->d_w / 2, bug->spy_img[rw]->d_h); + } + } + + switch_img_copy(img, &img_dup); + img_tmp = switch_img_copy_rect(img_dup, w / 4, 0, w / 2, h); + + switch_img_fit(&spy_tmp, w / 2, h); + switch_img_fit(&img_tmp, w / 2, h); + + switch_color_set_rgb(&bgcolor, "#000000"); + switch_img_fill(img, 0, 0, img->d_w, img->d_h, &bgcolor); + + switch_img_find_position(POS_CENTER_MID, w / 2, h, img_tmp->d_w, img_tmp->d_h, &x, &y); + switch_img_patch(img, img_tmp, x, y); + + switch_img_find_position(POS_CENTER_MID, w / 2, h, spy_tmp->d_w, spy_tmp->d_h, &x, &y); + switch_img_patch(img, spy_tmp, x + w / 2, y); + + + switch_img_free(&img_tmp); + switch_img_free(&img_dup); + switch_img_free(&spy_tmp); + } + break; + case SPY_LOWER_RIGHT_SMALL: + case SPY_LOWER_RIGHT_LARGE: + default: + { + float scaler = 0.125f; + int spyw, spyh; + + if (bug->spy_fmt == SPY_LOWER_RIGHT_LARGE) { + scaler = 0.25f; + } + + spyw = (int) (float)w * scaler; + spyh = (int) (float)h * scaler; + + if (bug->spy_img[rw]->d_w != spyw || bug->spy_img[rw]->d_h != spyh) { + switch_image_t *tmp_img = NULL; + + switch_img_scale(bug->spy_img[rw], &tmp_img, spyw, spyh); + switch_img_free(&bug->spy_img[rw]); + bug->spy_img[rw] = tmp_img; + } + + switch_img_patch(img, bug->spy_img[rw], w - spyw, h - spyh); + } + break; + } + + return SWITCH_STATUS_SUCCESS; + } + + return SWITCH_STATUS_FALSE; +} + static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *obj) { switch_media_bug_t *bug = (switch_media_bug_t *) obj; @@ -427,7 +558,7 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void * if ((status = switch_queue_pop(main_q, &pop)) == SWITCH_STATUS_SUCCESS) { switch_img_free(&img); - + if (!pop) { goto end; } @@ -447,7 +578,7 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void * w = img->d_w; h = img->d_h; - + if (other_q) { if (other_img) { if (other_img->d_w != w || other_img->d_h != h) { @@ -477,7 +608,7 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void * } switch_thread_rwlock_rdlock(bug->session->bug_rwlock); - switch_mutex_lock(bug->read_mutex); + //switch_mutex_lock(bug->read_mutex); frame.img = IMG; bug->ping_frame = &frame; if (bug->callback) { @@ -487,7 +618,7 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void * } } bug->ping_frame = NULL; - switch_mutex_unlock(bug->read_mutex); + //switch_mutex_unlock(bug->read_mutex); switch_thread_rwlock_unlock(bug->session->bug_rwlock); if (!ok) { @@ -503,13 +634,13 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void * switch_img_free(&img); switch_img_free(&other_img); - while (switch_queue_pop(main_q, &pop) == SWITCH_STATUS_SUCCESS && pop) { + while (switch_queue_trypop(main_q, &pop) == SWITCH_STATUS_SUCCESS && pop) { img = (switch_image_t *) pop; switch_img_free(&img); } if (other_q) { - while (switch_queue_pop(other_q, &pop) == SWITCH_STATUS_SUCCESS && pop) { + while (switch_queue_trypop(other_q, &pop) == SWITCH_STATUS_SUCCESS && pop) { img = (switch_image_t *) pop; switch_img_free(&img); } @@ -518,6 +649,25 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void * return NULL; } +SWITCH_DECLARE(switch_status_t) switch_core_media_bug_push_spy_frame(switch_media_bug_t *bug, switch_frame_t *frame, switch_rw_t rw) +{ + + switch_assert(bug); + switch_assert(frame); + + if (bug->spy_video_queue[rw] && frame->img) { + switch_image_t *img = NULL; + + switch_img_copy(frame->img, &img); + + if (img) { + switch_queue_push(bug->spy_video_queue[rw], img); + return SWITCH_STATUS_SUCCESS; + } + } + + return SWITCH_STATUS_FALSE; +} #define MAX_BUG_BUFFER 1024 * 512 SWITCH_DECLARE(switch_status_t) switch_core_media_bug_add(switch_core_session_t *session, @@ -639,6 +789,28 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_add(switch_core_session_t bug->thread_id = switch_thread_self(); } + if (switch_test_flag(bug, SMBF_READ_VIDEO_STREAM) || switch_test_flag(bug, SMBF_WRITE_VIDEO_STREAM) || switch_test_flag(bug, SMBF_WRITE_VIDEO_PING)) { + switch_channel_set_flag_recursive(session->channel, CF_VIDEO_DECODED_READ); + } + + if (switch_test_flag(bug, SMBF_SPY_VIDEO_STREAM) || switch_core_media_bug_test_flag(bug, SMBF_SPY_VIDEO_STREAM_BLEG)) { + switch_queue_create(&bug->spy_video_queue[0], SWITCH_CORE_QUEUE_LEN, switch_core_session_get_pool(session)); + switch_queue_create(&bug->spy_video_queue[1], SWITCH_CORE_QUEUE_LEN, switch_core_session_get_pool(session)); + } + + if ((switch_test_flag(bug, SMBF_READ_VIDEO_STREAM) || switch_test_flag(bug, SMBF_WRITE_VIDEO_STREAM))) { + switch_memory_pool_t *pool = switch_core_session_get_pool(session); + + if (switch_test_flag(bug, SMBF_READ_VIDEO_STREAM)) { + switch_queue_create(&bug->read_video_queue, SWITCH_CORE_QUEUE_LEN, pool); + } + + if (switch_test_flag(bug, SMBF_WRITE_VIDEO_STREAM)) { + switch_queue_create(&bug->write_video_queue, SWITCH_CORE_QUEUE_LEN, pool); + } + } + + if (bug->callback) { switch_bool_t result = bug->callback(bug, bug->user_data, SWITCH_ABC_TYPE_INIT); if (result == SWITCH_FALSE) { @@ -649,8 +821,18 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_add(switch_core_session_t } } - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Attaching BUG to %s\n", switch_channel_get_name(session->channel)); bug->ready = 1; + + if ((switch_test_flag(bug, SMBF_READ_VIDEO_STREAM) || switch_test_flag(bug, SMBF_WRITE_VIDEO_STREAM))) { + switch_threadattr_t *thd_attr = NULL; + switch_memory_pool_t *pool = switch_core_session_get_pool(session); + switch_threadattr_create(&thd_attr, pool); + switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); + switch_thread_create(&bug->video_bug_thread, thd_attr, video_bug_thread, bug, pool); + + } + + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Attaching BUG to %s\n", switch_channel_get_name(session->channel)); switch_thread_rwlock_wrlock(session->bug_rwlock); bug->next = session->bugs; session->bugs = bug; @@ -664,25 +846,6 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_add(switch_core_session_t switch_thread_rwlock_unlock(session->bug_rwlock); *new_bug = bug; - - - if ((switch_test_flag(bug, SMBF_READ_VIDEO_STREAM) || switch_test_flag(bug, SMBF_WRITE_VIDEO_STREAM))) { - switch_threadattr_t *thd_attr = NULL; - switch_memory_pool_t *pool = switch_core_session_get_pool(session); - - if (switch_test_flag(bug, SMBF_READ_VIDEO_STREAM)) { - switch_queue_create(&bug->read_video_queue, SWITCH_CORE_QUEUE_LEN, pool); - } - - if (switch_test_flag(bug, SMBF_WRITE_VIDEO_STREAM)) { - switch_queue_create(&bug->write_video_queue, SWITCH_CORE_QUEUE_LEN, pool); - } - - switch_threadattr_create(&thd_attr, pool); - switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); - switch_thread_create(&bug->video_bug_thread, thd_attr, video_bug_thread, bug, pool); - } - if (tap_only) { switch_set_flag(session, SSF_MEDIA_BUG_TAP_ONLY); } else { @@ -985,11 +1148,31 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_close(switch_media_bug_t * bp->callback(bp, bp->user_data, SWITCH_ABC_TYPE_CLOSE); } + if (switch_test_flag(bp, SMBF_READ_VIDEO_STREAM) || switch_test_flag(bp, SMBF_WRITE_VIDEO_STREAM) || switch_test_flag(bp, SMBF_WRITE_VIDEO_PING)) { + switch_channel_clear_flag_recursive(bp->session->channel, CF_VIDEO_DECODED_READ); + } + bp->ready = 0; + switch_img_free(&bp->spy_img[0]); + switch_img_free(&bp->spy_img[1]); + if (bp->video_bug_thread) { switch_status_t st; - + int i; + + for (i = 0; i < 2; i++) { + void *pop; + switch_image_t *img; + + if (bp->spy_video_queue[i]) { + while (switch_queue_trypop(bp->spy_video_queue[i], &pop) == SWITCH_STATUS_SUCCESS && pop) { + img = (switch_image_t *) pop; + switch_img_free(&img); + } + } + } + if (bp->read_video_queue) { switch_queue_push(bp->read_video_queue, NULL); } @@ -997,7 +1180,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_close(switch_media_bug_t * if (bp->write_video_queue) { switch_queue_push(bp->write_video_queue, NULL); } - + switch_thread_join(&st, bp->video_bug_thread); } diff --git a/src/switch_core_video.c b/src/switch_core_video.c index 212267f46d..2d401ce54b 100644 --- a/src/switch_core_video.c +++ b/src/switch_core_video.c @@ -1735,22 +1735,35 @@ SWITCH_DECLARE(switch_status_t) switch_img_fit(switch_image_t **srcP, int width, src = *srcP; - if (!src || (src->d_w <= width && src->d_h <= height)) { + if (!src || (src->d_w == width && src->d_h == height)) { return SWITCH_STATUS_SUCCESS; } new_w = src->d_w; new_h = src->d_h; - while(new_w > width || new_h > height) { - if (new_w > width) { - double m = (double) width / new_w; + if (src->d_w < width && src->d_h < height) { + float rw = (float)new_w / width; + float rh = (float)new_h / height; + + if (rw > rh) { + new_h = (int)((float)new_h / rw); new_w = width; - new_h = (int) (new_h * m); } else { - double m = (double) height / new_h; + new_w = (int)((float)new_w / rh); new_h = height; - new_w = (int) (new_w * m); + } + } else { + while(new_w > width || new_h > height) { + if (new_w > width) { + double m = (double) width / new_w; + new_w = width; + new_h = (int) (new_h * m); + } else { + double m = (double) height / new_h; + new_h = height; + new_w = (int) (new_w * m); + } } } diff --git a/src/switch_ivr_async.c b/src/switch_ivr_async.c index da4c65314d..6baefb2d55 100644 --- a/src/switch_ivr_async.c +++ b/src/switch_ivr_async.c @@ -1332,10 +1332,10 @@ static switch_bool_t record_callback(switch_media_bug_t *bug, void *user_data, s } - if (switch_core_file_has_video(rh->fh)) { + //if (switch_core_file_has_video(rh->fh)) { //switch_core_media_set_video_file(session, NULL, SWITCH_RW_READ); - switch_channel_clear_flag_recursive(session->channel, CF_VIDEO_DECODED_READ); - } + //switch_channel_clear_flag_recursive(session->channel, CF_VIDEO_DECODED_READ); + //} switch_core_file_close(rh->fh); @@ -1564,24 +1564,80 @@ struct eavesdrop_pvt { switch_core_session_t *eavesdropper; uint32_t flags; switch_frame_t demux_frame; + int set_decoded_read; + int errs; uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE]; }; +static switch_status_t video_eavesdrop_callback(switch_core_session_t *session, switch_frame_t *frame, void *user_data) +{ + switch_media_bug_t *bug = (switch_media_bug_t *) user_data; + + if (frame->img) { + if (switch_core_media_bug_test_flag(bug, SMBF_SPY_VIDEO_STREAM)) { + switch_core_media_bug_push_spy_frame(bug, frame, SWITCH_RW_READ); + } + + if (switch_core_media_bug_test_flag(bug, SMBF_SPY_VIDEO_STREAM_BLEG)) { + switch_core_media_bug_push_spy_frame(bug, frame, SWITCH_RW_WRITE); + } + } + + return SWITCH_STATUS_SUCCESS; +} static switch_bool_t eavesdrop_callback(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type) { struct eavesdrop_pvt *ep = (struct eavesdrop_pvt *) user_data; uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE]; switch_frame_t frame = { 0 }; + switch_core_session_t *session = switch_core_media_bug_get_session(bug); + switch_channel_t *e_channel = switch_core_session_get_channel(ep->eavesdropper); + int show_spy = 0; frame.data = data; frame.buflen = SWITCH_RECOMMENDED_BUFFER_SIZE; + show_spy = switch_core_media_bug_test_flag(bug, SMBF_SPY_VIDEO_STREAM) || switch_core_media_bug_test_flag(bug, SMBF_SPY_VIDEO_STREAM_BLEG); + + if (show_spy) { + if (!ep->set_decoded_read) { + ep->set_decoded_read = 1; + switch_channel_set_flag_recursive(e_channel, CF_VIDEO_DECODED_READ); + switch_core_session_request_video_refresh(ep->eavesdropper); + } + } else { + if (ep->set_decoded_read) { + ep->set_decoded_read = 0; + switch_channel_clear_flag_recursive(e_channel, CF_VIDEO_DECODED_READ); + switch_core_session_request_video_refresh(ep->eavesdropper); + } + } + switch (type) { case SWITCH_ABC_TYPE_INIT: + + if (switch_core_media_bug_test_flag(bug, SMBF_READ_VIDEO_STREAM) || + switch_core_media_bug_test_flag(bug, SMBF_WRITE_VIDEO_STREAM) || + switch_core_media_bug_test_flag(bug, SMBF_READ_VIDEO_PING)) { + switch_core_session_set_video_read_callback(ep->eavesdropper, video_eavesdrop_callback, (void *)bug); + switch_channel_set_flag_recursive(switch_core_session_get_channel(session), CF_VIDEO_DECODED_READ); + } break; case SWITCH_ABC_TYPE_CLOSE: + if (ep->set_decoded_read) { + switch_channel_clear_flag_recursive(e_channel, CF_VIDEO_DECODED_READ); + } + + if (switch_core_media_bug_test_flag(bug, SMBF_READ_VIDEO_STREAM) || + switch_core_media_bug_test_flag(bug, SMBF_WRITE_VIDEO_STREAM) || + switch_core_media_bug_test_flag(bug, SMBF_READ_VIDEO_PING)) { + switch_core_session_set_video_read_callback(ep->eavesdropper, NULL, NULL); + } + + switch_channel_clear_flag_recursive(switch_core_session_get_channel(session), CF_VIDEO_DECODED_READ); + break; case SWITCH_ABC_TYPE_WRITE: break; @@ -1644,15 +1700,31 @@ static switch_bool_t eavesdrop_callback(switch_media_bug_t *bug, void *user_data } } break; - case SWITCH_ABC_TYPE_READ_VIDEO_PING: - if (!bug->ping_frame) break; - if (ep->eavesdropper && switch_core_session_read_lock(ep->eavesdropper) == SWITCH_STATUS_SUCCESS) { - switch_channel_t *channel = switch_core_session_get_channel(ep->eavesdropper); - if (switch_channel_test_flag(channel, CF_VIDEO)) { - switch_core_session_write_video_frame(ep->eavesdropper, bug->ping_frame, SWITCH_IO_FLAG_NONE, 0); + case SWITCH_ABC_TYPE_READ_VIDEO_PING: + case SWITCH_ABC_TYPE_STREAM_VIDEO_PING: + { + + if (!bug->ping_frame || !bug->ping_frame->img) { + break; + } + + if (ep->eavesdropper && switch_core_session_read_lock(ep->eavesdropper) == SWITCH_STATUS_SUCCESS) { + if (switch_core_session_write_video_frame(ep->eavesdropper, bug->ping_frame, SWITCH_IO_FLAG_NONE, 0) != SWITCH_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error writing video to %s\n", switch_core_session_get_name(ep->eavesdropper)); + ep->errs++; + + if (ep->errs > 10) { + switch_channel_hangup(e_channel, SWITCH_CAUSE_DESTINATION_OUT_OF_ORDER); + switch_core_session_reset(ep->eavesdropper, SWITCH_TRUE, SWITCH_TRUE); + switch_core_session_rwunlock(ep->eavesdropper); + return SWITCH_FALSE; + } + } else { + ep->errs = 0; + } + switch_core_session_rwunlock(ep->eavesdropper); } - switch_core_session_rwunlock(ep->eavesdropper); } break; default: @@ -1780,6 +1852,7 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_eavesdrop_session(switch_core_session switch_caller_profile_t *cp = NULL; uint32_t sanity = 600; switch_media_bug_flag_t read_flags = 0, write_flags = 0; + const char *vval; if (!switch_channel_media_up(channel)) { goto end; @@ -1889,14 +1962,39 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_eavesdrop_session(switch_core_session switch_buffer_create_dynamic(&ep->r_buffer, 2048, 2048, 8192); switch_buffer_add_mutex(ep->r_buffer, ep->r_mutex); - if (flags & ED_BRIDGE_READ) { read_flags = SMBF_READ_STREAM | SMBF_READ_REPLACE; } + if (flags & ED_BRIDGE_WRITE) { write_flags = SMBF_WRITE_STREAM | SMBF_WRITE_REPLACE; } + if (switch_channel_test_flag(session->channel, CF_VIDEO) && switch_channel_test_flag(tsession->channel, CF_VIDEO)) { + if ((vval = switch_channel_get_variable(session->channel, "eavesdrop_show_listener_video"))) { + if (switch_true(vval) || !strcasecmp(vval, "aleg") || !strcasecmp(vval, "bleg") || !strcasecmp(vval, "both")) { + read_flags |= SMBF_SPY_VIDEO_STREAM; + } + if (switch_true(vval) || !strcasecmp(vval, "bleg") || !strcasecmp(vval, "both")) { + read_flags |= SMBF_SPY_VIDEO_STREAM_BLEG; + } + } + + if ((vval = switch_channel_get_variable(session->channel, "eavesdrop_concat_video")) && switch_true(vval)) { + read_flags |= SMBF_READ_VIDEO_STREAM; + read_flags |= SMBF_WRITE_VIDEO_STREAM; + } else { + read_flags |= SMBF_READ_VIDEO_PING; + } + } else { + read_flags &= ~SMBF_READ_VIDEO_PING; + read_flags &= ~SMBF_READ_VIDEO_STREAM; + read_flags &= ~SMBF_WRITE_VIDEO_STREAM; + read_flags &= ~SMBF_SPY_VIDEO_STREAM; + read_flags &= ~SMBF_SPY_VIDEO_STREAM_BLEG; + } + + if (switch_core_media_bug_add(tsession, "eavesdrop", uuid, eavesdrop_callback, ep, 0, read_flags | write_flags | SMBF_READ_PING | SMBF_THREAD_LOCK | SMBF_NO_PAUSE, @@ -1905,6 +2003,9 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_eavesdrop_session(switch_core_session goto end; } + if ((vval = switch_channel_get_variable(session->channel, "eavesdrop_video_spy_fmt"))) { + switch_media_bug_set_spy_fmt(bug, switch_media_bug_parse_spy_fmt(vval)); + } msg.from = __FILE__; @@ -1949,6 +2050,7 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_eavesdrop_session(switch_core_session switch_event_t *event = NULL; char *fcommand = NULL; char db[2] = ""; + int vid_bug = 0, vid_dual = 0; status = switch_core_session_read_frame(session, &read_frame, SWITCH_IO_FLAG_NONE, 0); @@ -1971,6 +2073,15 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_eavesdrop_session(switch_core_session fcommand = db; } + if (switch_core_media_bug_test_flag(bug, SMBF_READ_VIDEO_STREAM) || + switch_core_media_bug_test_flag(bug, SMBF_WRITE_VIDEO_STREAM)) { + vid_dual = 1; + } + + if (vid_dual || switch_core_media_bug_test_flag(bug, SMBF_READ_VIDEO_PING)) { + vid_bug = 1; + } + if (fcommand) { char *d; for (d = fcommand; *d; d++) { @@ -1980,18 +2091,48 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_eavesdrop_session(switch_core_session case '1': switch_set_flag(ep, ED_MUX_READ); switch_clear_flag(ep, ED_MUX_WRITE); + if (vid_bug) { + switch_core_media_bug_set_flag(bug, SMBF_SPY_VIDEO_STREAM); + switch_core_media_bug_clear_flag(bug, SMBF_SPY_VIDEO_STREAM_BLEG); + switch_core_session_request_video_refresh(tsession); + } break; case '2': switch_set_flag(ep, ED_MUX_WRITE); switch_clear_flag(ep, ED_MUX_READ); + if (vid_bug) { + switch_core_media_bug_set_flag(bug, SMBF_SPY_VIDEO_STREAM_BLEG); + switch_core_media_bug_clear_flag(bug, SMBF_SPY_VIDEO_STREAM); + switch_core_session_request_video_refresh(tsession); + } break; case '3': switch_set_flag(ep, ED_MUX_READ); switch_set_flag(ep, ED_MUX_WRITE); + if (vid_bug) { + switch_core_media_bug_set_flag(bug, SMBF_SPY_VIDEO_STREAM); + switch_core_media_bug_set_flag(bug, SMBF_SPY_VIDEO_STREAM_BLEG); + switch_core_session_request_video_refresh(tsession); + } + break; + + case '4': + switch_media_bug_set_spy_fmt(bug, switch_media_bug_parse_spy_fmt("dual-crop")); + break; + case '5': + switch_media_bug_set_spy_fmt(bug, switch_media_bug_parse_spy_fmt("lower-right-small")); + break; + case '6': + switch_media_bug_set_spy_fmt(bug, switch_media_bug_parse_spy_fmt("lower-right-large")); break; case '0': switch_clear_flag(ep, ED_MUX_READ); switch_clear_flag(ep, ED_MUX_WRITE); + if (vid_bug) { + switch_core_media_bug_clear_flag(bug, SMBF_SPY_VIDEO_STREAM); + switch_core_media_bug_clear_flag(bug, SMBF_SPY_VIDEO_STREAM_BLEG); + switch_core_session_request_video_refresh(tsession); + } break; case '*': goto end_loop; @@ -2292,7 +2433,7 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_record_session(switch_core_session_t if (switch_core_file_has_video(fh)) { //switch_core_media_set_video_file(session, fh, SWITCH_RW_READ); - switch_channel_set_flag_recursive(session->channel, CF_VIDEO_DECODED_READ); + //switch_channel_set_flag_recursive(session->channel, CF_VIDEO_DECODED_READ); if ((vval = switch_channel_get_variable(channel, "record_concat_video")) && switch_true(vval)) { flags |= SMBF_READ_VIDEO_STREAM; diff --git a/src/switch_ivr_bridge.c b/src/switch_ivr_bridge.c index 0a242b0796..129ae72e12 100644 --- a/src/switch_ivr_bridge.c +++ b/src/switch_ivr_bridge.c @@ -53,6 +53,7 @@ static void video_bridge_thread(switch_core_session_t *session, void *obj) switch_channel_t *b_channel = switch_core_session_get_channel(vh->session_b); switch_status_t status; switch_frame_t *read_frame = 0; + int set_decoded_read = 0; vh->up = 1; @@ -67,11 +68,22 @@ static void video_bridge_thread(switch_core_session_t *session, void *obj) switch_codec_t *a_codec = switch_core_session_get_video_read_codec(vh->session_a); switch_codec_t *b_codec = switch_core_session_get_video_write_codec(vh->session_b); - if ((!b_codec || !a_codec || a_codec->implementation->impl_id == b_codec->implementation->impl_id) && - !switch_channel_test_flag(b_channel, CF_VIDEO_DECODED_READ)) { - switch_channel_clear_flag(channel, CF_VIDEO_DECODED_READ); + switch_assert(a_codec); + switch_assert(b_codec); + + if (switch_channel_test_flag(channel, CF_VIDEO_DECODED_READ)) { + if (a_codec->implementation->impl_id == b_codec->implementation->impl_id && !switch_channel_test_flag(b_channel, CF_VIDEO_DECODED_READ)) { + if (set_decoded_read) { + switch_channel_clear_flag_recursive(channel, CF_VIDEO_DECODED_READ); + set_decoded_read = 0; + } + } } else { - switch_channel_set_flag(channel, CF_VIDEO_DECODED_READ); + if (a_codec->implementation->impl_id != b_codec->implementation->impl_id || + switch_channel_test_flag(b_channel, CF_VIDEO_DECODED_READ)) { + switch_channel_set_flag_recursive(channel, CF_VIDEO_DECODED_READ); + set_decoded_read = 1; + } } status = switch_core_session_read_video_frame(vh->session_a, &read_frame, SWITCH_IO_FLAG_NONE, 0); @@ -94,6 +106,10 @@ static void video_bridge_thread(switch_core_session_t *session, void *obj) } } + if (set_decoded_read) { + switch_channel_clear_flag_recursive(channel, CF_VIDEO_DECODED_READ); + } + switch_core_session_kill_channel(vh->session_b, SWITCH_SIG_BREAK); switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(vh->session_a), SWITCH_LOG_DEBUG, "%s video thread ended.\n", switch_channel_get_name(channel));