vrrp: add vrrp_exit_timer_thread

Adding a vrrp_exit_timer_thread, to keep track of the fault exit timeout.
A callback is triggered which either brings up the instance if it
is triggered from the try_up_instance or just changes the state, if it
is triggered init time because the config is reloaded, while the instance
is in fault state because of a fault exit timer (Without fault init exit
delay, the instance would start in backup state).

When fault_init_exit_delay is modified, and the config is reloaded,
the new fault_init_exit_delay is applied in the following manner:
1. If the fault_init_exit_time has already passed, don't change it.
   The new fault_init_exit_delay will only be applied the next time the
   interface comes out of fault state.
2. If the new fault_init_exit_delay is greater than the old one, and
   the fault_init_exit_delay timer is presently running, increase the timer
   by the difference.
3. If the new fault_init_exit_delay is less than the old one, and the
   propagation_delay timer is in progress, reduce the timer by the
   difference between the two, provided the reduced time is greater than
   the present time.

Author: harisasank

keepalived/include/vrrp.h

@@ -378,6 +378,12 @@ typedef struct _vrrp_t {
 							 * remain in Fault or Init state before transitioning to
 							 * another state. 0 means no delay.
 							 */
+	thread_ref_t		fault_exit_timer_thread; /* Fault exit timer thread, that starts a timer. */
+	timeval_t		fault_exit_time;	 /* Time after which the instance moves from fault state.
+							  * used when instance is reloaded while the timer is
+							  * running */
+	void			(*fault_exit_timer_cb)(struct _vrrp_t*);
+							/* Callback that is executed at the expiry of fault_exit_timer */
 	int			state;			/* internal state (init/backup/master/fault) */
 #ifdef _WITH_SNMP_VRRP_
 	int			configured_state;	/* the configured state of the instance */

keepalived/include/vrrp_scheduler.h

@@ -73,6 +73,7 @@ extern void vrrp_gratuitous_arp_vmac_update_thread(thread_ref_t);
 extern void vrrp_arp_thread(thread_ref_t);
 extern void vrrp_gna_thread(thread_ref_t);
 extern void try_up_instance(vrrp_t *, bool);
+extern void up_instance(vrrp_t *);
 #ifdef _WITH_DUMP_THREADS_
 extern void dump_threads(void);
 #endif

keepalived/vrrp/vrrp.c

@@ -4788,6 +4788,7 @@ vrrp_complete_init(void)
 	vrrp_script_t *scr, *scr_tmp;
 	unsigned quickest_takeover;
 	unsigned vrrp_timeout_min = UINT_MAX;
+	timeval_t fault_exit_time;
 
 	/* Set defaults if not specified, depending on strict mode */
 	if (global_data->vrrp_garp_lower_prio_rep == PARAMETER_UNSET)
@@ -5018,20 +5019,33 @@ vrrp_complete_init(void)
 	if (reload) {
 		/* Now step through the old vrrp to set the status on matching new instances */
 		list_for_each_entry(old_vrrp, &old_vrrp_data->vrrp, e_list) {
-			/* We work out for ourselves if the vrrp instance
-			 * should be in fault state, so it doesn't matter
-			 * if it was before */
-			if (old_vrrp->state == VRRP_STATE_FAULT)
-				continue;
-
 			vrrp = vrrp_exist(old_vrrp, &vrrp_data->vrrp);
 			if (vrrp) {
+				if (old_vrrp->fault_exit_timer_thread)
+					thread_cancel(old_vrrp->fault_exit_timer_thread);
+
 				/* If we have detected a fault, don't override it */
 				if (vrrp->state == VRRP_STATE_FAULT || vrrp->num_script_init)
 					continue;
 
+				fault_exit_time = old_vrrp->fault_exit_time;
+				if (vrrp->fault_init_exit_delay >= old_vrrp->fault_init_exit_delay)
+					fault_exit_time = timer_add_long(old_vrrp->fault_exit_time,
+						vrrp->fault_init_exit_delay - old_vrrp->fault_init_exit_delay);
+				else
+					fault_exit_time = timer_sub_long(old_vrrp->fault_exit_time,
+						old_vrrp->fault_init_exit_delay - vrrp->fault_init_exit_delay);
+				/* We work out for ourselves if the vrrp instance
+				 * should be in fault state, except if it is in
+				 * fault state because of fault_init_exit_delay).
+				 * If the fault_exit_time after reload is already in the past, ignore it. */
+				if (old_vrrp->state == VRRP_STATE_FAULT &&
+					timercmp(&time_now, &fault_exit_time, >))
+					    continue;
+
 				vrrp->state = old_vrrp->state;
 				vrrp->wantstate = old_vrrp->state;
+				vrrp->fault_exit_time = fault_exit_time;
 			}
 		}
 

keepalived/vrrp/vrrp_scheduler.c

@@ -176,6 +176,28 @@ static struct {
 /* FAULT  */	{ {NULL}, {NULL},			{vrrp_sync_master}, {NULL} }
 };
 
+static void
+vrrp_state_set_backup(vrrp_t *vrrp)
+{
+	log_message(LOG_INFO, "(%s) VRRP moving state to backup", vrrp->iname);
+	vrrp->state = VRRP_STATE_BACK;
+}
+
+static void
+vrrp_fault_exit_timer_thread(thread_ref_t thread)
+{
+	vrrp_t *vrrp = THREAD_ARG(thread);
+
+	log_message(LOG_INFO, "(%s) VRRP fault_init_exit_delay of %g seconds complete.",
+		    vrrp->iname, vrrp->fault_init_exit_delay/TIMER_HZ_DOUBLE);
+
+	if (vrrp->fault_exit_timer_cb)
+		vrrp->fault_exit_timer_cb(vrrp);
+	vrrp->fault_exit_timer_thread = NULL;
+	vrrp->fault_exit_timer_cb = NULL;
+	vrrp->fault_exit_time.tv_sec = 0;
+}
+
 /*
  * Initialize state handling
  * --rfc2338.6.4.1
@@ -274,8 +296,15 @@ vrrp_init_state(list_head_t *l)
 			    !vrrp->num_script_init &&
 			    (!vrrp->sync || !vrrp->sync->num_member_init)) {
 				if (vrrp->state != VRRP_STATE_BACK) {
-					log_message(LOG_INFO, "(%s) Entering BACKUP STATE (init)", vrrp->iname);
-					vrrp->state = VRRP_STATE_BACK;
+					if (vrrp->state == VRRP_STATE_FAULT && timercmp(&time_now, &vrrp->fault_exit_time, <)) {
+						log_message(LOG_INFO, "(%s) Entering FAULT STATE due to exit delay time", vrrp->iname);
+						vrrp->fault_exit_timer_cb = vrrp_state_set_backup;
+						vrrp->fault_exit_timer_thread = thread_add_timer_sands(master,
+								vrrp_fault_exit_timer_thread, vrrp, &vrrp->fault_exit_time);
+					} else {
+						log_message(LOG_INFO, "(%s) Entering BACKUP STATE (init)", vrrp->iname);
+						vrrp->state = VRRP_STATE_BACK;
+					}
 				}
 			} else {
 				/* Note: if we have alpha mode scripts, we enter fault state, but don't want
@@ -324,8 +353,12 @@ vrrp_init_instance_sands(vrrp_t *vrrp)
 		else
 			vrrp->sands = timer_add_long(time_now, vrrp->ms_down_timer);
 	}
-	else if (vrrp->state == VRRP_STATE_FAULT || vrrp->state == VRRP_STATE_INIT)
-		vrrp->sands.tv_sec = TIMER_DISABLED;
+	else if (vrrp->state == VRRP_STATE_FAULT || vrrp->state == VRRP_STATE_INIT) {
+		if (timercmp(&time_now, &vrrp->fault_exit_time, <)) {
+			vrrp->sands = vrrp->fault_exit_time;
+		} else
+			vrrp->sands.tv_sec = TIMER_DISABLED;
+	}
 
 	rb_move_cached(&vrrp->rb_sands, &vrrp->sockets->rb_sands, vrrp_timer_less);
 }
@@ -688,11 +721,59 @@ vrrp_gratuitous_arp_vmac_update_thread(thread_ref_t thread)
 #endif
 
 void
-try_up_instance(vrrp_t *vrrp, bool leaving_init)
+up_instance(vrrp_t *vrrp)
 {
 	int wantstate;
 	ip_address_t ip_addr = {0};
 
+	/* If the sync group can't go to master, we must go to backup state */
+	wantstate = vrrp->wantstate;
+	if (vrrp->sync && vrrp->wantstate == VRRP_STATE_MAST && !vrrp_sync_can_goto_master(vrrp))
+		vrrp->wantstate = VRRP_STATE_BACK;
+
+	/* We can come up */
+	vrrp_state_leave_fault(vrrp);
+
+	/* If we are using unicast, the master may have lost us from its ARP cache.
+	 * We want to renew the ARP cache on the master, so that it can send adverts
+	 * to us straight away, without a delay before it sends an ARP request message
+	 * and we respond. If we don't do this, we can time out and transition to master
+	 * before the master renews its ARP entry, since the master cannot send us adverts
+	 * until it has done so. */
+	if (__test_bit(VRRP_FLAG_UNICAST, &vrrp->flags) &&
+	    vrrp->ifp &&
+	    vrrp->saddr.ss_family != AF_UNSPEC) {
+		if (__test_bit(LOG_DETAIL_BIT, &debug))
+			log_message(LOG_INFO, "%s: sending gratuitous %s for %s", vrrp->iname, vrrp->family == AF_INET ? "ARP" : "NA", inet_sockaddrtos(&vrrp->saddr));
+
+		ip_addr.ifp = IF_BASE_IFP(vrrp->ifp);
+
+		if (vrrp->saddr.ss_family == AF_INET) {
+			ip_addr.u.sin.sin_addr.s_addr = PTR_CAST(struct sockaddr_in, &vrrp->saddr)->sin_addr.s_addr;
+			send_gratuitous_arp_immediate(ip_addr.ifp, &ip_addr);
+		} else {
+			/* IPv6 */
+			ip_addr.u.sin6_addr = PTR_CAST(struct sockaddr_in6, &vrrp->saddr)->sin6_addr;
+			ndisc_send_unsolicited_na_immediate(ip_addr.ifp, &ip_addr);
+		}
+	}
+
+	vrrp_init_instance_sands(vrrp);
+	vrrp_thread_requeue_read(vrrp);
+
+	vrrp->wantstate = wantstate;
+
+	if (vrrp->sync) {
+		if (vrrp->state == VRRP_STATE_MAST)
+			vrrp_sync_master(vrrp);
+		else
+			vrrp_sync_backup(vrrp);
+	}
+}
+
+void
+try_up_instance(vrrp_t *vrrp, bool leaving_init)
+{
 	if (leaving_init) {
 		if (vrrp->num_script_if_fault)
 			return;
@@ -735,49 +816,16 @@ try_up_instance(vrrp_t *vrrp, bool leaving_init)
 			return;
 	}
 
-	/* If the sync group can't go to master, we must go to backup state */
-	wantstate = vrrp->wantstate;
-	if (vrrp->sync && vrrp->wantstate == VRRP_STATE_MAST && !vrrp_sync_can_goto_master(vrrp))
-		vrrp->wantstate = VRRP_STATE_BACK;
-
-	/* We can come up */
-	vrrp_state_leave_fault(vrrp);
-
-	/* If we are using unicast, the master may have lost us from its ARP cache.
-	 * We want to renew the ARP cache on the master, so that it can send adverts
-	 * to us straight away, without a delay before it sends an ARP request message
-	 * and we respond. If we don't do this, we can time out and transition to master
-	 * before the master renews its ARP entry, since the master cannot send us adverts
-	 * until it has done so. */
-	if (__test_bit(VRRP_FLAG_UNICAST, &vrrp->flags) &&
-	    vrrp->ifp &&
-	    vrrp->saddr.ss_family != AF_UNSPEC) {
-		if (__test_bit(LOG_DETAIL_BIT, &debug))
-			log_message(LOG_INFO, "%s: sending gratuitous %s for %s", vrrp->iname, vrrp->family == AF_INET ? "ARP" : "NA", inet_sockaddrtos(&vrrp->saddr));
-
-		ip_addr.ifp = IF_BASE_IFP(vrrp->ifp);
-
-		if (vrrp->saddr.ss_family == AF_INET) {
-			ip_addr.u.sin.sin_addr.s_addr = PTR_CAST(struct sockaddr_in, &vrrp->saddr)->sin_addr.s_addr;
-			send_gratuitous_arp_immediate(ip_addr.ifp, &ip_addr);
-		} else {
-			/* IPv6 */
-			ip_addr.u.sin6_addr = PTR_CAST(struct sockaddr_in6, &vrrp->saddr)->sin6_addr;
-			ndisc_send_unsolicited_na_immediate(ip_addr.ifp, &ip_addr);
-		}
+	if (vrrp->fault_init_exit_delay) {
+		vrrp->fault_exit_time = timer_add_long(time_now, vrrp->fault_init_exit_delay);
+		vrrp->fault_exit_timer_thread =
+			thread_add_timer_sands(master, vrrp_fault_exit_timer_thread,
+					       vrrp, &vrrp->fault_exit_time);
+		vrrp->fault_exit_timer_cb = up_instance;
+		return;
 	}
 
-	vrrp_init_instance_sands(vrrp);
-	vrrp_thread_requeue_read(vrrp);
-
-	vrrp->wantstate = wantstate;
-
-	if (vrrp->sync) {
-		if (vrrp->state == VRRP_STATE_MAST)
-			vrrp_sync_master(vrrp);
-		else
-			vrrp_sync_backup(vrrp);
-	}
+	up_instance(vrrp);
 }
 
 #ifdef _WITH_BFD_

keepalived/vrrp/vrrp_track.c

@@ -552,6 +552,12 @@ down_instance(vrrp_t *vrrp)
 		if (vrrp->sync && vrrp->sync->num_member_fault++ == 0)
 			vrrp_sync_fault(vrrp);
 	}
+	if (vrrp->fault_exit_timer_thread) {
+		thread_cancel(vrrp->fault_exit_timer_thread);
+		vrrp->fault_exit_timer_thread = NULL;
+		vrrp->fault_exit_timer_cb = NULL;
+		vrrp->fault_exit_time.tv_sec = 0;
+	}
 }
 
 /* Set effective priorty, issue message on changes */