diff --git a/channels/chan_sip.c b/channels/chan_sip.c
index c802877ffd..2d270d33e6 100644
--- a/channels/chan_sip.c
+++ b/channels/chan_sip.c
@@ -26607,7 +26607,9 @@ static struct sip_peer *build_peer(const char *name, struct ast_variable *v, str
 		found++;
 		/* we've unlinked the peer from the peers container but not unlinked from the peers_by_ip container yet
 		  this leads to a wrong refcounter and the peer object is never destroyed */
-		ao2_t_unlink(peers_by_ip, peer, "ao2_unlink peer from peers_by_ip table");
+		if (!ast_sockaddr_isnull(&peer->addr)) {
+			ao2_t_unlink(peers_by_ip, peer, "ao2_unlink peer from peers_by_ip table");
+		}
 		if (!(peer->the_mark))
 			firstpass = 0;
 	} else {