From 5dbf2b3cf7541a7163eb19ca665b79c93c5f0fc9 Mon Sep 17 00:00:00 2001 From: Anthony Minessale Date: Thu, 11 Jul 2013 17:38:24 -0500 Subject: [PATCH] refactor some video code --- libs/sofia-sip/.update | 2 +- .../sofia-sip/libsofia-sip-ua/sdp/sdp_print.c | 2 +- src/include/switch_rtp.h | 1 + src/include/switch_types.h | 1 + .../mod_conference/mod_conference.c | 83 +++-- src/switch_core_media.c | 55 +++- src/switch_core_session.c | 4 + src/switch_rtp.c | 291 +++++++++++++++--- 8 files changed, 360 insertions(+), 79 deletions(-) diff --git a/libs/sofia-sip/.update b/libs/sofia-sip/.update index c36486c61a..17253549d1 100644 --- a/libs/sofia-sip/.update +++ b/libs/sofia-sip/.update @@ -1 +1 @@ -Wed Jul 3 11:09:02 CDT 2013 +Thu Jul 11 17:38:06 CDT 2013 diff --git a/libs/sofia-sip/libsofia-sip-ua/sdp/sdp_print.c b/libs/sofia-sip/libsofia-sip-ua/sdp/sdp_print.c index c912dd3b59..b587aa5cc1 100644 --- a/libs/sofia-sip/libsofia-sip-ua/sdp/sdp_print.c +++ b/libs/sofia-sip/libsofia-sip-ua/sdp/sdp_print.c @@ -590,7 +590,7 @@ static void print_media(sdp_printer_t *p, case sdp_proto_udp: proto = "udp"; break; case sdp_proto_rtp: proto = "RTP/AVP"; break; case sdp_proto_srtp: proto = "RTP/SAVP"; break; - case sdp_proto_extended_srtp: proto = "RTP/SAVPF"; break; + //case sdp_proto_extended_srtp: proto = "RTP/SAVPF"; break; case sdp_proto_udptl: proto = "udptl"; break; case sdp_proto_msrp: proto = "TCP/MSRP"; break; case sdp_proto_msrps: proto = "TCP/TLS/MSRP"; break; diff --git a/src/include/switch_rtp.h b/src/include/switch_rtp.h index ff7079b833..f704d943d1 100644 --- a/src/include/switch_rtp.h +++ b/src/include/switch_rtp.h @@ -139,6 +139,7 @@ SWITCH_DECLARE(void) switch_rtp_shutdown(void); SWITCH_DECLARE(switch_port_t) switch_rtp_set_start_port(switch_port_t port); SWITCH_DECLARE(switch_status_t) switch_rtp_set_ssrc(switch_rtp_t *rtp_session, uint32_t ssrc); +SWITCH_DECLARE(switch_status_t) switch_rtp_set_remote_ssrc(switch_rtp_t *rtp_session, uint32_t ssrc); /*! \brief Set/Get RTP end port diff --git a/src/include/switch_types.h b/src/include/switch_types.h index 2e469de3de..315c0b2fdf 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -1326,6 +1326,7 @@ typedef enum { CF_DTLS_OK, CF_VIDEO_PASSIVE, CF_NOVIDEO, + CF_VIDEO_ECHO, /* WARNING: DO NOT ADD ANY FLAGS BELOW THIS LINE */ /* IF YOU ADD NEW ONES CHECK IF THEY SHOULD PERSIST OR ZERO THEM IN switch_core_session.c switch_core_session_request_xml() */ CF_FLAG_MAX diff --git a/src/mod/applications/mod_conference/mod_conference.c b/src/mod/applications/mod_conference/mod_conference.c index a841f2d371..bb572f4e78 100644 --- a/src/mod/applications/mod_conference/mod_conference.c +++ b/src/mod/applications/mod_conference/mod_conference.c @@ -201,7 +201,8 @@ typedef enum { CFLAG_VIDEO_BRIDGE = (1 << 14), CFLAG_AUDIO_ALWAYS = (1 << 15), CFLAG_ENDCONF_FORCED = (1 << 16), - CFLAG_RFC4579 = (1 << 17) + CFLAG_RFC4579 = (1 << 17), + CFLAG_FLOOR_CHANGE = (1 << 18) } conf_flag_t; typedef enum { @@ -394,6 +395,7 @@ typedef struct conference_relationship { struct conference_member { uint32_t id; switch_core_session_t *session; + switch_channel_t *channel; conference_obj_t *conference; switch_memory_pool_t *pool; switch_buffer_t *audio_buffer; @@ -1443,15 +1445,20 @@ static switch_status_t conference_add_member(conference_obj_t *conference, confe conference_send_presence(conference); - - channel = switch_core_session_get_channel(member->session); - switch_channel_set_flag(channel, CF_VIDEO_PASSIVE); + switch_channel_set_variable_printf(channel, "conference_member_id", "%d", member->id); switch_channel_set_variable_printf(channel, "conference_moderator", "%s", switch_test_flag(member, MFLAG_MOD) ? "true" : "false"); switch_channel_set_variable(channel, "conference_recording", conference->record_filename); switch_channel_set_variable(channel, CONFERENCE_UUID_VARIABLE, conference->uuid_str); - + + + + if (switch_channel_test_flag(channel, CF_VIDEO)) { + switch_channel_clear_flag(channel, CF_VIDEO_ECHO); + /* Tell the channel to request a fresh vid frame */ + switch_core_session_refresh_video(member->session); + } if (!switch_channel_get_variable(channel, "conference_call_key")) { char *key = switch_core_session_sprintf(member->session, "conf_%s_%s_%s", @@ -1565,6 +1572,20 @@ static switch_status_t conference_add_member(conference_obj_t *conference, confe return status; } +static void conference_set_floor_holder(conference_obj_t *conference, conference_member_t *member) +{ + + if (conference->floor_holder && conference->floor_holder != member) { + switch_channel_clear_flag(conference->floor_holder->channel, CF_VIDEO_PASSIVE); + } + + if ((conference->floor_holder = member)) { + switch_channel_set_flag(member->channel, CF_VIDEO_PASSIVE); + switch_core_session_refresh_video(conference->floor_holder->session); + switch_set_flag(conference, CFLAG_FLOOR_CHANGE); + } +} + /* Gain exclusive access and remove the member from the list */ static switch_status_t conference_del_member(conference_obj_t *conference, conference_member_t *member) { @@ -1648,7 +1669,9 @@ static switch_status_t conference_del_member(conference_obj_t *conference, confe } if (member == member->conference->floor_holder) { - member->conference->floor_holder = NULL; + //member->conference->floor_holder = NULL; + conference_set_floor_holder(member->conference, NULL); + if (test_eflag(conference, EFLAG_FLOOR_CHANGE)) { switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, CONF_EVENT_MAINT); @@ -1673,7 +1696,10 @@ static switch_status_t conference_del_member(conference_obj_t *conference, confe } } - switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE); + if (switch_channel_test_flag(channel, CF_VIDEO)) { + switch_channel_set_flag(channel, CF_VIDEO_ECHO); + switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE); + } conference_send_presence(conference); switch_channel_set_variable(channel, "conference_call_key", NULL); @@ -1839,19 +1865,15 @@ static void *SWITCH_THREAD_FUNC conference_video_thread_run(switch_thread_t *thr conference_member_t *imember; switch_frame_t *vid_frame; switch_status_t status; - int has_vid = 1, want_refresh = 0; + int want_refresh = 0; int yield = 0; switch_core_session_t *session; - switch_core_session_message_t msg = { 0 }; + char buf[65536]; conference->video_running = 1; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Video thread started for conference %s\n", conference->name); - /* Tell the channel to request a fresh vid frame */ - msg.from = __FILE__; - msg.message_id = SWITCH_MESSAGE_INDICATE_VIDEO_REFRESH_REQ; - - while (has_vid && conference->video_running == 1 && globals.running && !switch_test_flag(conference, CFLAG_DESTRUCT)) { + while (conference->video_running == 1 && globals.running && !switch_test_flag(conference, CFLAG_DESTRUCT)) { if (yield) { switch_yield(yield); yield = 0; @@ -1887,12 +1909,21 @@ static void *SWITCH_THREAD_FUNC conference_video_thread_run(switch_thread_t *thr goto do_continue; } + if (vid_frame && switch_test_flag(vid_frame, SFF_CNG)) { + yield = 10000; + goto do_continue; + } + memcpy(buf, vid_frame->packet, vid_frame->packetlen); + switch_mutex_unlock(conference->mutex); switch_mutex_lock(conference->mutex); - has_vid = 0; want_refresh = 0; + if (switch_test_flag(conference, CFLAG_FLOOR_CHANGE)) { + switch_clear_flag(conference, CFLAG_FLOOR_CHANGE); + } + for (imember = conference->members; imember; imember = imember->next) { switch_core_session_t *isession = imember->session; switch_channel_t *ichannel; @@ -1908,21 +1939,20 @@ static void *SWITCH_THREAD_FUNC conference_video_thread_run(switch_thread_t *thr switch_channel_clear_flag(ichannel, CF_VIDEO_REFRESH_REQ); } - if (imember->session && switch_channel_test_flag(ichannel, CF_VIDEO)) { - has_vid++; + if (isession && switch_channel_test_flag(ichannel, CF_VIDEO)) { + memcpy(vid_frame->packet, buf, vid_frame->packetlen); switch_core_session_write_video_frame(imember->session, vid_frame, SWITCH_IO_FLAG_NONE, 0); } switch_core_session_rwunlock(isession); } - if (want_refresh) { - switch_core_session_receive_message(session, &msg); + if (want_refresh && session) { + switch_core_session_refresh_video(session); want_refresh = 0; } do_continue: - switch_mutex_unlock(conference->mutex); } @@ -2078,7 +2108,8 @@ static void *SWITCH_THREAD_FUNC conference_thread_run(switch_thread_t *thread, v } } - conference->floor_holder = floor_holder; + //conference->floor_holder = floor_holder; + conference_set_floor_holder(conference, floor_holder); } @@ -5219,7 +5250,8 @@ static switch_status_t conf_api_sub_floor(conference_member_t *member, switch_st switch_mutex_lock(member->conference->mutex); if (member->conference->floor_holder == member) { - member->conference->floor_holder = NULL; + //member->conference->floor_holder = NULL; + conference_set_floor_holder(member->conference, NULL); if (test_eflag(member->conference, EFLAG_FLOOR_CHANGE)) { switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, CONF_EVENT_MAINT); conference_add_event_data(member->conference, event); @@ -5232,7 +5264,8 @@ static switch_status_t conf_api_sub_floor(conference_member_t *member, switch_st } } } else if (member->conference->floor_holder == NULL) { - member->conference->floor_holder = member; + //member->conference->floor_holder = member; + conference_set_floor_holder(member->conference, member); if (test_eflag(member->conference, EFLAG_FLOOR_CHANGE)) { switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, CONF_EVENT_MAINT); conference_add_event_data(member->conference, event); @@ -5272,7 +5305,8 @@ static switch_status_t conf_api_sub_enforce_floor(conference_member_t *member, s if (member->conference->floor_holder != member) { conference_member_t *old_member = member->conference->floor_holder; - member->conference->floor_holder = member; + //member->conference->floor_holder = member; + conference_set_floor_holder(member->conference, member); if (test_eflag(member->conference, EFLAG_FLOOR_CHANGE)) { switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, CONF_EVENT_MAINT); conference_add_event_data(member->conference, event); @@ -7781,6 +7815,7 @@ SWITCH_STANDARD_APP(conference_function) } member.session = session; + member.channel = switch_core_session_get_channel(session); member.pool = switch_core_session_get_pool(session); if (setup_media(&member, conference)) { diff --git a/src/switch_core_media.c b/src/switch_core_media.c index a52c9e08e7..9e21e458eb 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -126,6 +126,7 @@ typedef struct switch_rtp_engine_s { uint32_t max_missed_packets; uint32_t max_missed_hold_packets; uint32_t ssrc; + uint32_t remote_ssrc; switch_port_t remote_rtcp_port; switch_rtp_bug_flag_t rtp_bugs; @@ -1985,6 +1986,8 @@ static void check_ice(switch_media_handle_t *smh, switch_media_type_t type, sdp_ generate_local_fingerprint(smh, type); switch_channel_set_flag(smh->session->channel, CF_DTLS); + } else if (!engine->remote_ssrc && !strcasecmp(attr->a_name, "ssrc") && attr->a_value) { + engine->remote_ssrc = (uint32_t) atol(attr->a_value); #ifdef RTCP_MUX } else if (!strcasecmp(attr->a_name, "rtcp-mux")) { engine->rtcp_mux = SWITCH_TRUE; @@ -2294,7 +2297,6 @@ SWITCH_DECLARE(uint8_t) switch_core_media_negotiate_sdp(switch_core_session_t *s codec_array = smh->codecs; total_codecs = smh->mparams->num_codecs; - if (!(parser = sdp_parse(NULL, r_sdp, (int) strlen(r_sdp), 0))) { return 0; } @@ -2459,9 +2461,12 @@ SWITCH_DECLARE(uint8_t) switch_core_media_negotiate_sdp(switch_core_session_t *s got_webrtc++; switch_core_session_set_ice(session); } - + if (m->m_proto_name && !strcasecmp(m->m_proto_name, "UDP/TLS/RTP/SAVPF")) { switch_channel_set_flag(session->channel, CF_WEBRTC_MOZ); + printf("PRICK FACE 1\n"); + } else { + printf("PRICK FACE 2 [%s]\n", m->m_proto_name); } if (m->m_proto == sdp_proto_srtp || m->m_proto == sdp_proto_extended_srtp) { @@ -3790,7 +3795,8 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s Video thread started\n", switch_channel_get_name(session->channel)); switch_core_session_refresh_video(session); - + switch_channel_set_flag(channel, CF_VIDEO_ECHO); + while (switch_channel_up_nosig(channel)) { if (switch_channel_test_flag(channel, CF_VIDEO_PASSIVE)) { @@ -3812,7 +3818,6 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi status = switch_core_session_read_video_frame(session, &read_frame, SWITCH_IO_FLAG_NONE, 0); - if (!SWITCH_READ_ACCEPTABLE(status)) { switch_cond_next(); continue; @@ -3828,7 +3833,9 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi continue; } - switch_core_session_write_video_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0); + if (switch_channel_test_flag(channel, CF_VIDEO_ECHO)) { + switch_core_session_write_video_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0); + } } @@ -4092,6 +4099,9 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_activate_rtp(switch_core_sessi switch_rtp_set_ssrc(a_engine->rtp_session, a_engine->ssrc); } + if (a_engine->remote_ssrc) { + switch_rtp_set_remote_ssrc(a_engine->rtp_session, a_engine->remote_ssrc); + } switch_channel_set_flag(session->channel, CF_FS_RTP); @@ -4552,6 +4562,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_activate_rtp(switch_core_sessi switch_rtp_set_ssrc(v_engine->rtp_session, v_engine->ssrc); } + if (v_engine->remote_ssrc) { + switch_rtp_set_remote_ssrc(v_engine->rtp_session, v_engine->remote_ssrc); + } + if (v_engine->ice_in.cands[v_engine->ice_in.chosen[0]][0].ready) { gen_ice(session, SWITCH_MEDIA_TYPE_VIDEO, NULL, 0); @@ -5125,7 +5139,7 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess switch_rtp_engine_t *a_engine, *v_engine; switch_media_handle_t *smh; ice_t *ice_out; - + int vp8 = 0; switch_assert(session); @@ -5629,6 +5643,10 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess if (v_engine->codec_params.rm_encoding) { const char *of; + + if (!strcasecmp(v_engine->codec_params.rm_encoding, "VP8")) { + vp8 = v_engine->codec_params.pt; + } rate = v_engine->codec_params.rm_rate; switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=rtpmap:%d %s/%ld\n", @@ -5694,6 +5712,10 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess channels = get_channels(imp); + if (!strcasecmp(imp->iananame, "VP8")) { + vp8 = ianacode; + } + if (channels > 1) { switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=rtpmap:%d %s/%d/%d\n", ianacode, imp->iananame, imp->samples_per_second, channels); @@ -5733,7 +5755,7 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess } - if (smh->mparams->rtcp_audio_interval_msec) { + if (smh->mparams->rtcp_video_interval_msec) { if (v_engine->rtcp_mux > 0) { switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=rtcp-mux\n"); switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=rtcp:%d IN %s %s\n", v_port, family, ip); @@ -5751,7 +5773,9 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess uint32_t c2 = (2^24)*126 + (2^8)*65535 + (2^0)*(256 - 2); uint32_t c3 = (2^24)*126 + (2^8)*65534 + (2^0)*(256 - 1); uint32_t c4 = (2^24)*126 + (2^8)*65534 + (2^0)*(256 - 2); - + const char *vbw; + int bw = 256; + tmp1[10] = '\0'; tmp2[10] = '\0'; switch_stun_random_string(tmp1, 10, "0123456789"); @@ -5760,6 +5784,21 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess ice_out = &v_engine->ice_out; + if ((vbw = switch_channel_get_variable(smh->session->channel, "rtp_video_max_bandwidth"))) { + int v = atoi(vbw); + bw = v; + } + + if (bw > 0) { + switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "b=AS:%d\n", bw); + } + + + if (vp8) { + switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), + "a=rtcp-fb:%d ccm fir\n", vp8); + } + switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=ssrc:%u cname:%s\n", v_engine->ssrc, smh->cname); switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=ssrc:%u msid:%s v0\n", v_engine->ssrc, smh->msid); switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=ssrc:%u mslabel:%s\n", v_engine->ssrc, smh->msid); diff --git a/src/switch_core_session.c b/src/switch_core_session.c index 6e97391f21..8fc40f7d9d 100644 --- a/src/switch_core_session.c +++ b/src/switch_core_session.c @@ -2567,6 +2567,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_execute_application_get_flag *flags = application_interface->flags; } + if (!switch_test_flag(application_interface, SAF_SUPPORT_NOMEDIA) && (switch_channel_test_flag(session->channel, CF_VIDEO))) { + switch_core_session_refresh_video(session); + } + if (switch_channel_test_flag(session->channel, CF_PROXY_MODE) && !switch_test_flag(application_interface, SAF_SUPPORT_NOMEDIA)) { switch_ivr_media(session->uuid_str, SMF_NONE); switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Application %s Requires media on channel %s!\n", diff --git a/src/switch_rtp.c b/src/switch_rtp.c index 45ab478f96..f1ceb1d5c3 100644 --- a/src/switch_rtp.c +++ b/src/switch_rtp.c @@ -34,6 +34,8 @@ //#define RTP_DEBUG_WRITE_DELTA //#define DEBUG_MISSED_SEQ +#define FIR_COUNTDOWN 100 + #include #ifndef _MSC_VER #include @@ -116,7 +118,6 @@ typedef struct { uint8_t r3; } rtcp_fir_t; - #ifdef _MSC_VER #pragma pack(push, r1, 1) #endif @@ -264,6 +265,13 @@ static int dtls_state_dummy(switch_rtp_t *rtp_session, switch_dtls_t *dtls); dtls_state_handler_t dtls_states[DS_INVALID] = {dtls_state_handshake, dtls_state_setup, dtls_state_ready, dtls_state_dummy}; +typedef struct ts_normalize_s { + uint32_t last_ssrc; + uint32_t last_frame; + uint32_t ts; + uint32_t delta; + uint8_t m; +} ts_normalize_t; struct switch_rtp { /* @@ -281,6 +289,8 @@ struct switch_rtp { rtcp_msg_t rtcp_send_msg; rtcp_ext_msg_t rtcp_ext_send_msg; uint8_t fir_seq; + uint16_t fir_countdown; + ts_normalize_t ts_norm; switch_sockaddr_t *remote_addr, *rtcp_remote_addr; rtp_msg_t recv_msg; rtcp_msg_t rtcp_recv_msg; @@ -307,6 +317,7 @@ struct switch_rtp { uint16_t seq; uint32_t ssrc; + uint32_t remote_ssrc; int8_t sending_dtmf; uint8_t need_mark; switch_payload_t payload; @@ -698,10 +709,10 @@ static switch_status_t ice_out(switch_rtp_t *rtp_session, switch_rtp_ice_t *ice) elapsed = (unsigned int) ((switch_micro_time_now() - rtp_session->last_stun) / 1000); if (elapsed > 30000) { - - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_ERROR, "No stun for a long time (PUNT!)\n"); - status = SWITCH_STATUS_GENERR; - goto end; + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_ERROR, "No stun for a long time!\n"); + rtp_session->last_stun = switch_micro_time_now(); + //status = SWITCH_STATUS_GENERR; + //goto end; } } @@ -743,7 +754,7 @@ static switch_status_t ice_out(switch_rtp_t *rtp_session, switch_rtp_ice_t *ice) ice->sending = 3; - end: + // end: READ_DEC(rtp_session); return status; @@ -1329,20 +1340,31 @@ static void send_fir(switch_rtp_t *rtp_session) return; } + if (rtp_session->remote_ssrc == 0) { + rtp_session->remote_ssrc = rtp_session->stats.rtcp.peer_ssrc; + } + + if (rtp_session->remote_ssrc == 0) { + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Peer ssrc not known yet for FIR\n"); + return; + } + if (rtp_session->rtcp_sock_output && rtp_session->flags[SWITCH_RTP_FLAG_ENABLE_RTCP]) { rtcp_fir_t *fir = (rtcp_fir_t *) rtp_session->rtcp_ext_send_msg.body; switch_size_t rtcp_bytes; - + rtp_session->rtcp_ext_send_msg.header.version = 2; rtp_session->rtcp_ext_send_msg.header.p = 0; rtp_session->rtcp_ext_send_msg.header.fmt = 4; rtp_session->rtcp_ext_send_msg.header.pt = 206; rtp_session->rtcp_ext_send_msg.header.send_ssrc = htonl(rtp_session->ssrc); - rtp_session->rtcp_ext_send_msg.header.recv_ssrc = htonl(rtp_session->stats.rtcp.peer_ssrc); + rtp_session->rtcp_ext_send_msg.header.recv_ssrc = 0;//htonl(rtp_session->stats.rtcp.peer_ssrc); - fir->ssrc = htonl(rtp_session->stats.rtcp.peer_ssrc); - fir->seq = (uint8_t) htonl(rtp_session->fir_seq++); + //fir->ssrc = htonl(rtp_session->stats.rtcp.peer_ssrc); + fir->ssrc = htonl(rtp_session->remote_ssrc); + fir->seq = ++rtp_session->fir_seq; + fir->r1 = fir->r2 = fir->r3 = 0; switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG1, "Sending RTCP FIR %d\n", rtp_session->fir_seq); @@ -1409,6 +1431,92 @@ static void send_fir(switch_rtp_t *rtp_session) return; } + +#if 0 +static void send_pli(switch_rtp_t *rtp_session) +{ + + if (!rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && rtp_session->ice.ice_user) { + return; + } + + if (rtp_session->rtcp_sock_output && rtp_session->flags[SWITCH_RTP_FLAG_ENABLE_RTCP]) { + switch_size_t rtcp_bytes; + + rtp_session->rtcp_ext_send_msg.header.version = 2; + rtp_session->rtcp_ext_send_msg.header.p = 0; + rtp_session->rtcp_ext_send_msg.header.fmt = 1; + rtp_session->rtcp_ext_send_msg.header.pt = 206; + + rtp_session->rtcp_ext_send_msg.header.send_ssrc = htonl(rtp_session->ssrc); + rtp_session->rtcp_ext_send_msg.header.recv_ssrc = 0; + + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG1, "Sending RTCP PLI\n"); + + rtcp_bytes = sizeof(switch_rtcp_ext_hdr_t); + rtp_session->rtcp_ext_send_msg.header.length = htons((u_short)(rtcp_bytes / 4) - 1); + + +#ifdef ENABLE_SRTP + if (rtp_session->flags[SWITCH_RTP_FLAG_SECURE_SEND]) { + int sbytes = (int) rtcp_bytes; + int stat = srtp_protect_rtcp(rtp_session->send_ctx[rtp_session->srtp_idx_rtcp], &rtp_session->rtcp_ext_send_msg.header, &sbytes); + + if (stat) { + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_ERROR, "Error: SRTP RTCP protection failed with code %d\n", stat); + goto end; + } else { + rtcp_bytes = sbytes; + } + + } +#endif + +#ifdef ENABLE_ZRTP + /* ZRTP Send */ + if (zrtp_on && !rtp_session->flags[SWITCH_RTP_FLAG_PROXY_MEDIA]) { + unsigned int sbytes = (int) rtcp_bytes; + zrtp_status_t stat = zrtp_status_fail; + + stat = zrtp_process_rtcp(rtp_session->zrtp_stream, (void *) &rtp_session->rtcp_ext_send_msg, &sbytes); + + switch (stat) { + case zrtp_status_ok: + break; + case zrtp_status_drop: + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error: zRTP protection drop with code %d\n", stat); + goto end; + break; + case zrtp_status_fail: + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error: zRTP protection fail with code %d\n", stat); + break; + default: + break; + } + + rtcp_bytes = sbytes; + } +#endif + +#ifdef DEBUG_EXTRA + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_CRIT, "%s SEND %s RTCP %ld\n", + switch_core_session_get_name(rtp_session->session), + rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] ? "video" : "audio", rtcp_bytes); +#endif + if (switch_socket_sendto(rtp_session->rtcp_sock_output, rtp_session->rtcp_remote_addr, 0, (void *)&rtp_session->rtcp_ext_send_msg, &rtcp_bytes ) != SWITCH_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG,"RTCP packet not written\n"); + } else { + rtp_session->stats.inbound.period_packet_count = 0; + } + } + + + end: + + return; +} +#endif + static int check_rtcp_and_ice(switch_rtp_t *rtp_session) { int ret = 0; @@ -1442,10 +1550,6 @@ static int check_rtcp_and_ice(switch_rtp_t *rtp_session) rtcp_ok = 0; } - //if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { - // rtcp_ok = 0; - //} - if (rtp_session->rtcp_sock_output && rtcp_ok && rtp_session->flags[SWITCH_RTP_FLAG_ENABLE_RTCP] && !rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU]) { struct switch_rtcp_senderinfo *sr = (struct switch_rtcp_senderinfo*) rtp_session->rtcp_send_msg.body; const char* str_cname=NULL; @@ -2123,6 +2227,10 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_set_remote_address(switch_rtp_t *rtp_ status = enable_remote_rtcp_socket(rtp_session, err); } + if (rtp_session->flags[SWITCH_RTP_FLAG_ENABLE_RTCP] && rtp_session->flags[SWITCH_RTP_FLAG_RTCP_MUX]) { + rtp_session->rtcp_remote_addr = rtp_session->remote_addr; + } + switch_mutex_unlock(rtp_session->write_mutex); return status; @@ -2224,7 +2332,7 @@ static int dtls_state_ready(switch_rtp_t *rtp_session, switch_dtls_t *dtls) if (dtls->new_state) { if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { switch_core_session_t *other_session; - send_fir(rtp_session); + rtp_session->fir_countdown = FIR_COUNTDOWN; if (rtp_session->session && switch_core_session_get_partner(rtp_session->session, &other_session) == SWITCH_STATUS_SUCCESS) { switch_core_session_refresh_video(other_session); @@ -2682,6 +2790,13 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_set_ssrc(switch_rtp_t *rtp_session, u return SWITCH_STATUS_SUCCESS; } +SWITCH_DECLARE(switch_status_t) switch_rtp_set_remote_ssrc(switch_rtp_t *rtp_session, uint32_t ssrc) +{ + rtp_session->remote_ssrc = ssrc; + + return SWITCH_STATUS_SUCCESS; +} + SWITCH_DECLARE(switch_status_t) switch_rtp_create(switch_rtp_t **new_rtp_session, switch_payload_t payload, uint32_t samples_per_interval, @@ -2781,6 +2896,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_create(switch_rtp_t **new_rtp_session switch_rtp_clear_flag(rtp_session, SWITCH_RTP_FLAG_NOBLOCK); } + if (channel) { switch_channel_set_private(channel, "__rtcp_audio_rtp_session", rtp_session); } @@ -3221,7 +3337,10 @@ SWITCH_DECLARE(void) switch_rtp_flush(switch_rtp_t *rtp_session) SWITCH_DECLARE(void) switch_rtp_video_refresh(switch_rtp_t *rtp_session) { if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && rtp_session->ice.ice_user) { - send_fir(rtp_session); + if (!rtp_session->fir_countdown) { + //send_fir(rtp_session); + rtp_session->fir_countdown = FIR_COUNTDOWN; + } } } @@ -3827,6 +3946,7 @@ static switch_status_t read_rtp_packet(switch_rtp_t *rtp_session, switch_size_t uint32_t ts = 0; unsigned char *b = NULL; int sync = 0; + switch_time_t now; switch_assert(bytes); more: @@ -3896,15 +4016,14 @@ static switch_status_t read_rtp_packet(switch_rtp_t *rtp_session, switch_size_t } } - if (status == SWITCH_STATUS_SUCCESS && *bytes) { if (rtp_session->flags[SWITCH_RTP_FLAG_RTCP_MUX]) { *flags &= ~SFF_RTCP; - if (rtp_session->recv_msg.header.pt != rtp_session->rpayload && (!rtp_session->recv_te || - rtp_session->recv_msg.header.pt != rtp_session->recv_te) && + if (rtp_session->recv_msg.header.pt != rtp_session->rpayload && + (!rtp_session->recv_te || rtp_session->recv_msg.header.pt != rtp_session->recv_te) && (!rtp_session->cng_pt || rtp_session->recv_msg.header.pt != rtp_session->cng_pt) && - rtp_session->rtcp_recv_msg_p->header.version == 2 && rtp_session->rtcp_recv_msg_p->header.type > 199 && - rtp_session->rtcp_recv_msg_p->header.type < 208) { //rtcp muxed + rtp_session->rtcp_recv_msg_p->header.version == 2 && + rtp_session->rtcp_recv_msg_p->header.type > 199 && rtp_session->rtcp_recv_msg_p->header.type < 208) { //rtcp muxed *flags |= SFF_RTCP; return SWITCH_STATUS_SUCCESS; } @@ -3952,11 +4071,13 @@ static switch_status_t read_rtp_packet(switch_rtp_t *rtp_session, switch_size_t udptl: - ts = ntohl(rtp_session->recv_msg.header.ts); + ts = 0; rtp_session->recv_msg.ebody = NULL; + now = switch_micro_time_now(); if (*bytes) { uint16_t seq = ntohs((uint16_t) rtp_session->recv_msg.header.seq); + ts = ntohl(rtp_session->recv_msg.header.ts); if (!rtp_session->flags[SWITCH_RTP_FLAG_PROXY_MEDIA] && !rtp_session->flags[SWITCH_RTP_FLAG_UDPTL] && rtp_session->recv_msg.header.version == 2 && rtp_session->recv_msg.header.x) { /* header extensions */ @@ -3981,40 +4102,47 @@ static switch_status_t read_rtp_packet(switch_rtp_t *rtp_session, switch_size_t if (num_missed == 1) { /* We missed one packet */ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Missed one RTP frame with sequence [%d]%s. Time since last read [%ld]\n", rtp_session->last_seq+1, (flushed_packets_diff == 1) ? " (flushed by FS)" : " (missed)", - rtp_session->last_read_time ? switch_micro_time_now()-rtp_session->last_read_time : 0); + rtp_session->last_read_time ? now-rtp_session->last_read_time : 0); } else { /* We missed multiple packets */ if (flushed_packets_diff == 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Missed %ld RTP frames from sequence [%d] to [%d] (missed). Time since last read [%ld]\n", num_missed, rtp_session->last_seq+1, seq-1, - rtp_session->last_read_time ? switch_micro_time_now()-rtp_session->last_read_time : 0); + rtp_session->last_read_time ? now-rtp_session->last_read_time : 0); } else if (flushed_packets_diff == num_missed) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Missed %ld RTP frames from sequence [%d] to [%d] (flushed by FS). Time since last read [%ld]\n", num_missed, rtp_session->last_seq+1, seq-1, - rtp_session->last_read_time ? switch_micro_time_now()-rtp_session->last_read_time : 0); + rtp_session->last_read_time ? now-rtp_session->last_read_time : 0); } else if (num_missed > flushed_packets_diff) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Missed %ld RTP frames from sequence [%d] to [%d] (%ld packets flushed by FS, %ld packets missed)." " Time since last read [%ld]\n", num_missed, rtp_session->last_seq+1, seq-1, flushed_packets_diff, num_missed-flushed_packets_diff, - rtp_session->last_read_time ? switch_micro_time_now()-rtp_session->last_read_time : 0); + rtp_session->last_read_time ? now-rtp_session->last_read_time : 0); } else { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Missed %ld RTP frames from sequence [%d] to [%d] (%ld packets flushed by FS). Time since last read [%ld]\n", num_missed, rtp_session->last_seq+1, seq-1, - flushed_packets_diff, rtp_session->last_read_time ? switch_micro_time_now()-rtp_session->last_read_time : 0); + flushed_packets_diff, rtp_session->last_read_time ? now-rtp_session->last_read_time : 0); } } } #endif rtp_session->last_seq = seq; - } + - rtp_session->last_flush_packet_count = rtp_session->stats.inbound.flush_packet_count; - rtp_session->last_read_time = switch_micro_time_now(); + rtp_session->last_flush_packet_count = rtp_session->stats.inbound.flush_packet_count; + + + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && now - rtp_session->last_read_time > 500000) { + switch_rtp_video_refresh(rtp_session); + } + + rtp_session->last_read_time = now; + } if (!rtp_session->flags[SWITCH_RTP_FLAG_PROXY_MEDIA] && !rtp_session->flags[SWITCH_RTP_FLAG_UDPTL] && !rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && *bytes && (!rtp_session->recv_te || rtp_session->recv_msg.header.pt != rtp_session->recv_te) && @@ -4121,7 +4249,17 @@ static switch_status_t read_rtp_packet(switch_rtp_t *rtp_session, switch_size_t } - rtp_session->last_read_ts = ts; + if (*bytes && rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { + unsigned int diff = ts - rtp_session->last_read_ts; + + if (abs(diff) > 10000) { + switch_rtp_video_refresh(rtp_session); + } + } + + if (ts) { + rtp_session->last_read_ts = ts; + } if (rtp_session->flags[SWITCH_RTP_FLAG_BYTESWAP] && rtp_session->recv_msg.header.pt == rtp_session->rpayload) { switch_swap_linear((int16_t *)RTP_BODY(rtp_session), (int) *bytes - rtp_header_len); @@ -4251,16 +4389,13 @@ static switch_status_t read_rtcp_packet(switch_rtp_t *rtp_session, switch_size_t if (rtp_session->rtcp_dtls) { char *b = (char *) &rtp_session->rtcp_recv_msg; - //printf("RECV2 %d %ld\n", *b, *bytes); - if (*b == 0 || *b == 1) { if (rtp_session->rtcp_ice.ice_user) { handle_ice(rtp_session, &rtp_session->rtcp_ice, (void *) &rtp_session->rtcp_recv_msg, *bytes); } *bytes = 0; } - - + if (*bytes && (*b >= 20) && (*b <= 64)) { rtp_session->rtcp_dtls->bytes = *bytes; rtp_session->rtcp_dtls->data = (void *) &rtp_session->rtcp_recv_msg; @@ -4474,14 +4609,15 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_ pt = 20000; } - if ((rtp_session->ice.ice_user && rtp_session->flags[SWITCH_RTP_FLAG_VIDEO])) { - pt = 10000; - } if ((io_flags & SWITCH_IO_FLAG_NOBLOCK)) { pt = 0; } + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { + pt = 100000; + } + poll_status = switch_poll(rtp_session->read_pollfd, 1, &fdr, pt); if (rtp_session->dtmf_data.out_digit_dur > 0) { @@ -4499,6 +4635,22 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_ ret = -1; goto end; } + + + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { + //switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_CRIT, "Read bytes (%i) %ld\n", status, bytes); + + if (bytes == 0) { + if (check_rtcp_and_ice(rtp_session) == -1) { + ret = -1; + goto end; + } + // This is dumb + switch_rtp_video_refresh(rtp_session); + goto rtcp; + } + } + if ((*flags & SFF_PROXY_PACKET)) { ret = (int) bytes; goto end; @@ -4509,8 +4661,8 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_ has_rtcp = 1; goto rtcp; } - - //switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Read bytes (%i) %ld\n", status, bytes); + + } poll_loop = 0; } else { @@ -4539,9 +4691,11 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_ ret = -1; goto end; } - - } else if ((!(io_flags & SWITCH_IO_FLAG_NOBLOCK)) && - (rtp_session->dtmf_data.out_digit_dur == 0)) { + goto recvfrom; + } + + if ((!(io_flags & SWITCH_IO_FLAG_NOBLOCK)) && + (rtp_session->dtmf_data.out_digit_dur == 0)) { return_cng_frame(); } } @@ -4572,7 +4726,7 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_ if (rtcp_status == SWITCH_STATUS_SUCCESS) { switch_rtp_reset_media_timer(rtp_session); - + if (rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU] || rtp_session->rtcp_recv_msg_p->header.type == 206) { switch_channel_t *channel = switch_core_session_get_channel(rtp_session->session); const char *uuid = switch_channel_get_partner_uuid(channel); @@ -5118,6 +5272,13 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_zerocopy_read_frame(switch_rtp_t *rtp return SWITCH_STATUS_FALSE; } + if (rtp_session->fir_countdown) { + if (--rtp_session->fir_countdown == 0) { + send_fir(rtp_session); + //send_pli(rtp_session); + } + } + bytes = rtp_common_read(rtp_session, &frame->payload, &frame->flags, io_flags); frame->data = RTP_BODY(rtp_session); @@ -5311,6 +5472,45 @@ static int rtp_common_write(switch_rtp_t *rtp_session, } } + + if (switch_rtp_test_flag(rtp_session, SWITCH_RTP_FLAG_VIDEO)) { + /* Normalize the timestamps to our own base by generating a made up starting point then adding the measured deltas to that base + so if the timestamps and ssrc of the source change, it will not break the other end's jitter bufffer / decoder etc *cough* CHROME *cough* + */ + + if (!rtp_session->ts_norm.ts) { + rtp_session->ts_norm.ts = (uint32_t) rand() % 1000000 + 1; + } + + if (!rtp_session->ts_norm.last_ssrc || send_msg->header.ssrc != rtp_session->ts_norm.last_ssrc) { + if (rtp_session->ts_norm.last_ssrc) { + rtp_session->ts_norm.m = 1; + if (rtp_session->ts_norm.delta) { + rtp_session->ts_norm.ts += rtp_session->ts_norm.delta; + } + } + rtp_session->ts_norm.last_ssrc = send_msg->header.ssrc; + rtp_session->ts_norm.last_frame = ntohl(send_msg->header.ts); + } + + + if (ntohl(send_msg->header.ts) != rtp_session->ts_norm.last_frame) { + rtp_session->ts_norm.delta = ntohl(send_msg->header.ts) - rtp_session->ts_norm.last_frame; + rtp_session->ts_norm.ts += rtp_session->ts_norm.delta; + } + + rtp_session->ts_norm.last_frame = ntohl(send_msg->header.ts); + send_msg->header.ts = htonl(rtp_session->ts_norm.ts); + + if (rtp_session->ts_norm.m) { + if (send_msg->header.m) { + rtp_session->ts_norm.m = 0; + } else { + send_msg->header.m = 1; + } + } + } + send_msg->header.ssrc = htonl(rtp_session->ssrc); if (rtp_session->flags[SWITCH_RTP_FLAG_GOOGLEHACK] && rtp_session->send_msg.header.pt == 97) { @@ -5447,6 +5647,8 @@ static int rtp_common_write(switch_rtp_t *rtp_session, send = 0; } + + if (send) { send_msg->header.seq = htons(++rtp_session->seq); @@ -5549,7 +5751,6 @@ static int rtp_common_write(switch_rtp_t *rtp_session, } } - if (switch_socket_sendto(rtp_session->sock_output, rtp_session->remote_addr, 0, (void *) send_msg, &bytes) != SWITCH_STATUS_SUCCESS) { rtp_session->seq--; ret = -1;