diff --git a/src/switch_rtp.c b/src/switch_rtp.c
index a516053c3b..0b1faeaaa5 100644
--- a/src/switch_rtp.c
+++ b/src/switch_rtp.c
@@ -1184,7 +1184,7 @@ static void handle_ice(switch_rtp_t *rtp_session, switch_rtp_ice_t *ice, void *d
 				ice->last_ok = now;
 				rtp_session->wrong_addrs = 0;
 			} else {
-				if (((rtp_session->dtls && rtp_session->dtls->state != DS_READY) || !ice->ready || !ice->rready)) {
+				if (((rtp_session->dtls && rtp_session->dtls->state != DS_READY) || !ice->ready || !ice->rready) && rtp_session->wrong_addrs > 2) {
 					do_adj++;
 				} else if (rtp_session->wrong_addrs > 5 || elapsed >= 3000) {
 					do_adj++;