vrrp: add fault_init_exit_delay to vrrp configuration

doc/KEEPALIVED-MIB.txt

@@ -946,6 +946,7 @@ VrrpInstanceEntry ::= SEQUENCE {
     vrrpInstanceMulticastAddressType InetAddressType,
     vrrpInstanceMulticastAddress InetAddress,
     vrrpInstanceV3ChecksumAsV2 INTEGER
+    vrrpInstanceFaultInitExitDelay Unsigned32
 }
 
 vrrpInstanceIndex OBJECT-TYPE
@@ -1251,6 +1252,15 @@ vrrpInstanceV3ChecksumAsV2 OBJECT-TYPE
         "True if VRRPv3 IPv4 checksum excludes pseudo-header."
     ::= { vrrpInstanceEntry 36 }
 
+vrrpInstanceFaultInitExitDelay OBJECT-TYPE
+    SYNTAX Unsigned32
+    UNITS "seconds"
+    MAX-ACCESS read-only
+    STATUS current
+    DESCRIPTION
+        "Delay after a fault after which the instance participates in master election. 0 means that there is no delay."
+    ::= { vrrpInstanceEntry 37 }
+
 vrrpTrackedInterfaceTable OBJECT-TYPE
     SYNTAX SEQUENCE OF VrrpTrackedInterfaceEntry
     MAX-ACCESS not-accessible
@@ -5102,6 +5112,7 @@ vrrpInstanceGroup OBJECT-GROUP
     vrrpInstanceMulticastAddressType,
     vrrpInstanceMulticastAddress,
     vrrpInstanceV3ChecksumAsV2,
+    vrrpInstanceFaultInitExitDelay,
     vrrpTrackedInterfaceName,
     vrrpTrackedInterfaceWeight,
     vrrpTrackedInterfaceWgtRev,

keepalived/include/vrrp.h

@@ -386,6 +386,22 @@ typedef struct _vrrp_t {
 							 * prio is allowed.  0 means no delay.
 							 */
 	timeval_t		preempt_time;		/* Time after which preemption can happen */
+	unsigned long		fault_init_exit_delay;  /* Additional seconds*TIMER_HZ that the instance
+							 * remains in BACKUP state after moving out of
+							 * FAULT on INIT state before transitioning to
+							 * MASTER. 0 means no delay.
+							 */
+	timeval_t		fault_init_exit_time;	/* Timestamp when the instance can become MASTER,
+							 * after fault_init_exit_delay is applied.
+							 */
+	timeval_t		block_socket_time; 	/* Ignore the messages received on the socket until
+							 * this timestamp, to implement the 
+							 * fault_init_exit_delay (+ possibly startup_delay)
+							 */
+	bool 			fault_init_delay_needed;/* Indicates that we need to apply
+							 * fault_init_exit_delay when sands_timer is
+							 * initialized.
+ 							 */
 	int			state;			/* internal state (init/backup/master/fault) */
 #ifdef _WITH_SNMP_VRRP_
 	int			configured_state;	/* the configured state of the instance */

keepalived/vrrp/vrrp.c

@@ -1973,6 +1973,13 @@ vrrp_state_leave_fault(vrrp_t * vrrp)
 	/* Set the down timer */
 	vrrp->master_adver_int = vrrp->adver_int;
 	vrrp->ms_down_timer = VRRP_MS_DOWN_TIMER(vrrp);
+	if (vrrp->state == VRRP_STATE_BACK && vrrp->fault_init_exit_delay > 0)
+		vrrp->fault_init_delay_needed = true;
+	else if (vrrp->state == VRRP_STATE_FAULT) {
+		vrrp->fault_init_delay_needed = false;
+		vrrp->fault_init_exit_time = time_now;
+		vrrp->block_socket_time = time_now;
+	}
 	vrrp_init_instance_sands(vrrp);
 	vrrp->last_transition = timer_now();
 }
@@ -3378,6 +3385,10 @@ vrrp_complete_instance(vrrp_t * vrrp)
 								, vrrp->iname);
 			vrrp->preempt_delay = false;
 		}
+		if (vrrp->fault_init_exit_delay) {
+			report_config_error(CONFIG_GENERAL_ERROR, "(%s) Warning - fault init exit delay will not work with initial state MASTER - clearing", vrrp->iname);
+			vrrp->fault_init_exit_delay = false;
+		}
 	}
 	if (vrrp->preempt_delay) {
 		if (vrrp->strict_mode) {
@@ -3393,6 +3404,12 @@ vrrp_complete_instance(vrrp_t * vrrp)
 			vrrp->preempt_delay = 0;
 		}
 	}
+	if (vrrp->fault_init_exit_delay) {
+		if (vrrp->strict_mode) {
+			report_config_error(CONFIG_GENERAL_ERROR, "(%s) fault_init_exit_delay is incompatible with strict mode - resetting", vrrp->iname);
+			vrrp->fault_init_exit_delay = 0;
+		}
+	}
 
 	if (vrrp->down_timer_adverts != VRRP_DOWN_TIMER_ADVERTS && vrrp->strict_mode) {
 		report_config_error(CONFIG_GENERAL_ERROR, "(%s) down_timer_adverts is incompatible with"
@@ -5046,12 +5063,40 @@ vrrp_complete_init(void)
 
 			vrrp = vrrp_exist(old_vrrp, &vrrp_data->vrrp);
 			if (vrrp) {
+				timeval_t time_now;
+
 				/* If we have detected a fault, don't override it */
 				if (vrrp->state == VRRP_STATE_FAULT || vrrp->num_script_init)
 					continue;
 
 				vrrp->state = old_vrrp->state;
 				vrrp->wantstate = old_vrrp->state;
+				vrrp->fault_init_exit_time = old_vrrp->fault_init_exit_time;
+				vrrp->fault_init_delay_needed = old_vrrp->fault_init_delay_needed;
+				vrrp->block_socket_time = old_vrrp->block_socket_time;
+
+				/* If there is a fault_init_exit_time in progress (less than the current time),
+				 * we update the fault_init_exit_time, to reflect the updated fault_init_exit_delay
+				 */
+				time_now = timer_now();
+				if (timercmp(&time_now, &old_vrrp->fault_init_exit_time, >=))
+					continue;
+				if (vrrp->fault_init_exit_delay == old_vrrp->fault_init_exit_delay)
+					continue;
+
+				if (vrrp->fault_init_exit_delay > old_vrrp->fault_init_exit_delay) {
+					vrrp->fault_init_exit_time = timer_add_long(vrrp->fault_init_exit_time,
+										     vrrp->fault_init_exit_delay - old_vrrp->fault_init_exit_delay);
+					vrrp->block_socket_time = timer_add_long(vrrp->block_socket_time,
+										     vrrp->fault_init_exit_delay - old_vrrp->fault_init_exit_delay);
+				} else {
+					vrrp->fault_init_exit_time = timer_sub_long(vrrp->fault_init_exit_time,
+										     old_vrrp->fault_init_exit_delay - vrrp->fault_init_exit_delay);
+					vrrp->block_socket_time = timer_sub_long(vrrp->block_socket_time,
+										     old_vrrp->fault_init_exit_delay - vrrp->fault_init_exit_delay);
+				}
+				log_message(LOG_INFO, "(%s) changing fault_init_exit_time as propagation delay is changed from %f seconds to %f seconds", vrrp->iname,
+					    (float)old_vrrp->fault_init_exit_delay/TIMER_HZ, (float)vrrp->fault_init_exit_delay/TIMER_HZ);
 			}
 		}
 

keepalived/vrrp/vrrp_data.c

@@ -787,6 +787,9 @@ dump_vrrp(FILE *fp, const vrrp_t *vrrp)
 	if (vrrp->preempt_delay)
 		conf_write(fp, "   Preempt delay = %g secs",
 		       vrrp->preempt_delay / TIMER_HZ_DOUBLE);
+	if (vrrp->fault_init_exit_delay)
+		conf_write(fp, "   Fault Init Exit delay = %g secs",
+		       vrrp->fault_init_exit_delay / TIMER_HZ_DOUBLE);
 	conf_write(fp, "   Promote_secondaries = %s", __test_bit(VRRP_FLAG_PROMOTE_SECONDARIES, &vrrp->flags) ? "enabled" : "disabled");
 #if defined _WITH_VRRP_AUTH_
 	if (vrrp->auth_type) {

keepalived/vrrp/vrrp_json.c

@@ -195,6 +195,7 @@ vrrp_json_data_dump(json_writer_t *wr, vrrp_t *vrrp)
 #endif
 	jsonw_bool_field(wr, "nopreempt", __test_bit(VRRP_FLAG_NOPREEMPT, &vrrp->flags));
 	jsonw_uint_field(wr, "preempt_delay", vrrp->preempt_delay / TIMER_HZ);
+	jsonw_uint_field(wr, "fault_init_exit_delay", vrrp->fault_init_exit_delay / TIMER_HZ);
 	jsonw_uint_field(wr, "state", vrrp->state);
 	jsonw_uint_field(wr, "wantstate", vrrp->wantstate);
 	jsonw_uint_field(wr, "version", vrrp->version);

keepalived/vrrp/vrrp_parser.c

@@ -1144,6 +1144,16 @@ vrrp_preempt_delay_handler(const vector_t *strvec)
 		current_vrrp->preempt_delay = preempt_delay;
 }
 static void
+vrrp_fault_init_exit_delay_handler(const vector_t *strvec)
+{
+	unsigned fault_init_exit_delay;
+
+	if (!read_decimal_unsigned_strvec(strvec, 1, &fault_init_exit_delay, 0, TIMER_MAX_SEC * TIMER_HZ, TIMER_HZ_DIGITS, true))
+		report_config_error(CONFIG_GENERAL_ERROR, "(%s) fault_init_exit_delay not valid! must be between 0-%u", current_vrrp->iname, TIMER_MAX_SEC);
+	else
+		current_vrrp->fault_init_exit_delay = fault_init_exit_delay;
+}
+static void
 vrrp_notify_backup_handler(const vector_t *strvec)
 {
 	if (current_vrrp->script_backup) {
@@ -2211,6 +2221,7 @@ init_vrrp_keywords(bool active)
 	install_keyword("preempt", &vrrp_preempt_handler);
 	install_keyword("nopreempt", &vrrp_nopreempt_handler);
 	install_keyword("preempt_delay", &vrrp_preempt_delay_handler);
+	install_keyword("fault_init_exit_delay", &vrrp_fault_init_exit_delay_handler);
 	install_keyword("debug", &vrrp_debug_handler);
 	install_keyword_quoted("notify_backup", &vrrp_notify_backup_handler);
 	install_keyword_quoted("notify_master", &vrrp_notify_master_handler);

keepalived/vrrp/vrrp_scheduler.c

@@ -241,6 +241,15 @@ vrrp_init_state(list_head_t *l)
 				 * very quickly (1usec) */
 				vrrp->state = VRRP_STATE_BACK;
 				vrrp->ms_down_timer = 1;
+				if (vrrp->fault_init_exit_delay > 0)
+				{
+					vrrp->fault_init_delay_needed = true;
+					vrrp->block_socket_time = timer_add_long(
+						vrrp_delayed_start_time.tv_sec ? vrrp_delayed_start_time: time_now,
+						vrrp->fault_init_exit_delay);
+					vrrp->fault_init_exit_time = timer_add_long(vrrp->block_socket_time, vrrp->ms_down_timer);
+				}
+
 			}
 
 // TODO Do we need ->	vrrp_restore_interface(vrrp, false, false);
@@ -252,6 +261,15 @@ vrrp_init_state(list_head_t *l)
 			} else
 				vrrp->ms_down_timer = VRRP_MS_DOWN_TIMER(vrrp);
 
+			if (!reload && vrrp->fault_init_exit_delay > 0)
+			{
+				vrrp->fault_init_delay_needed = true;
+				vrrp->block_socket_time = timer_add_long(
+					vrrp_delayed_start_time.tv_sec ? vrrp_delayed_start_time: time_now,
+					vrrp->fault_init_exit_delay);
+				vrrp->fault_init_exit_time = timer_add_long(vrrp->block_socket_time, vrrp->ms_down_timer);
+			}
+
 #ifdef _WITH_SNMP_RFCV3_
 			vrrp->stats->next_master_reason = VRRPV3_MASTER_REASON_MASTER_NO_RESPONSE;
 #endif
@@ -278,6 +296,7 @@ vrrp_init_state(list_head_t *l)
 				if (vrrp->state != VRRP_STATE_BACK) {
 					log_message(LOG_INFO, "(%s) Entering BACKUP STATE (init)", vrrp->iname);
 					vrrp->state = VRRP_STATE_BACK;
+					vrrp->fault_init_delay_needed = true;
 				}
 			} else {
 				/* Note: if we have alpha mode scripts, we enter fault state, but don't want
@@ -325,6 +344,18 @@ vrrp_init_instance_sands(vrrp_t *vrrp)
 			vrrp->sands = timer_add_long(vrrp_delayed_start_time, vrrp->ms_down_timer);
 		else
 			vrrp->sands = timer_add_long(time_now, vrrp->ms_down_timer);
+		if (vrrp->fault_init_delay_needed)
+		{
+			log_message(LOG_INFO, "Applied vrrp fault init exit delay of %f seconds on instance (%s)",
+				    (float)vrrp->fault_init_exit_delay/TIMER_HZ, vrrp->iname);
+			vrrp->sands = timer_add_long(vrrp->sands, vrrp->fault_init_exit_delay);
+			vrrp->fault_init_delay_needed = false;
+			vrrp->fault_init_exit_time = timer_add_long(time_now, vrrp->fault_init_exit_delay);
+			if (vrrp_delayed_start_time.tv_sec)
+				vrrp->block_socket_time = timer_add_long(vrrp_delayed_start_time, vrrp->fault_init_exit_delay);
+			else
+				vrrp->block_socket_time = timer_add_long(time_now, vrrp->fault_init_exit_delay);
+		}
 	}
 	else if (vrrp->state == VRRP_STATE_FAULT || vrrp->state == VRRP_STATE_INIT)
 		vrrp->sands.tv_sec = TIMER_DISABLED;
@@ -897,6 +928,16 @@ vrrp_dispatcher_read_timeout(sock_t *sock)
 		prev_state = vrrp->state;
 
 		if (vrrp->state == VRRP_STATE_BACK) {
+			if (vrrp->fault_init_exit_delay > 0)
+			{
+				timeval_t now = timer_now();
+				/* Do not transition to master before fault_init_exit_time */
+				if (timercmp(&vrrp->fault_init_exit_time, &now, >)) {
+					vrrp_init_instance_sands(vrrp);
+					break;
+				}
+			}
+
 			if (__test_bit(LOG_DETAIL_BIT, &debug))
 				log_message(LOG_INFO, "(%s) Receive advertisement timeout", vrrp->iname);
 			vrrp_goto_master(vrrp);
@@ -1109,6 +1150,16 @@ vrrp_dispatcher_read(sock_t *sock)
 			continue;
 		}
 
+		if (vrrp->fault_init_exit_delay > 0)
+		{
+			/* Ignore the messages received on this socket till
+			 * fault_init_exit_delay (+ possible startup_delay) have expired. */
+			set_time_now();
+			if (timercmp(&time_now, &vrrp->block_socket_time, <)) {
+				continue;
+			}
+		}
+
 		/* Save non packet data */
 		vrrp->pkt_saddr = src_addr;
 		vrrp->rx_ttl_hl = -1;           /* Default to not received */

keepalived/vrrp/vrrp_snmp.c

@@ -226,6 +226,7 @@ enum snmp_vrrp_magic {
 	VRRP_SNMP_INSTANCE_MULTICAST_ADDRESSTYPE,
 	VRRP_SNMP_INSTANCE_MULTICAST_ADDRESS,
 	VRRP_SNMP_INSTANCE_V3_CHECKSUM_AS_V2,
+	VRRP_SNMP_INSTANCE_FAULTINITEXITDELAY,
 	VRRP_SNMP_TRACKEDINTERFACE_NAME,
 	VRRP_SNMP_TRACKEDINTERFACE_WEIGHT,
 	VRRP_SNMP_TRACKEDINTERFACE_WEIGHT_REVERSE,
@@ -2236,6 +2237,9 @@ vrrp_snmp_instance(struct variable *vp, oid *name, size_t *length,
 	case VRRP_SNMP_INSTANCE_V3_CHECKSUM_AS_V2:
 		long_ret.u = (__test_bit(VRRP_FLAG_V3_CHECKSUM_AS_V2, &rt->flags)) ? 1 : 2;
 		return PTR_CAST(u_char, &long_ret);
+	case VRRP_SNMP_INSTANCE_FAULTINITEXITDELAY:
+		long_ret.u = rt->fault_init_exit_delay / TIMER_HZ;
+		return PTR_CAST(u_char, &long_ret);
 	default:
 		return NULL;
 	}
@@ -2680,6 +2684,8 @@ static struct variable3 vrrp_vars[] = {
 	 vrrp_snmp_instance, 3, {3, 1, 35}},
 	{VRRP_SNMP_INSTANCE_V3_CHECKSUM_AS_V2, ASN_INTEGER, RONLY,
 	 vrrp_snmp_instance, 3, {3, 1, 36}},
+	{VRRP_SNMP_INSTANCE_FAULTINITEXITDELAY, ASN_UNSIGNED, RONLY,
+	 vrrp_snmp_instance, 3, {3, 1, 37}},
 
 	/* vrrpTrackedInterfaceTable */
 	{VRRP_SNMP_TRACKEDINTERFACE_NAME, ASN_OCTET_STR, RONLY,