diff --git a/doc/KEEPALIVED-MIB.txt b/doc/KEEPALIVED-MIB.txt index f86f491cb..5db007a40 100644 --- a/doc/KEEPALIVED-MIB.txt +++ b/doc/KEEPALIVED-MIB.txt @@ -946,6 +946,7 @@ VrrpInstanceEntry ::= SEQUENCE { vrrpInstanceMulticastAddressType InetAddressType, vrrpInstanceMulticastAddress InetAddress, vrrpInstanceV3ChecksumAsV2 INTEGER + vrrpInstanceFaultInitExitDelay Unsigned32 } vrrpInstanceIndex OBJECT-TYPE @@ -1251,6 +1252,15 @@ vrrpInstanceV3ChecksumAsV2 OBJECT-TYPE "True if VRRPv3 IPv4 checksum excludes pseudo-header." ::= { vrrpInstanceEntry 36 } +vrrpInstanceFaultInitExitDelay OBJECT-TYPE + SYNTAX Unsigned32 + UNITS "seconds" + MAX-ACCESS read-only + STATUS current + DESCRIPTION + "Delay after a fault after which the instance participates in master election. 0 means that there is no delay." + ::= { vrrpInstanceEntry 37 } + vrrpTrackedInterfaceTable OBJECT-TYPE SYNTAX SEQUENCE OF VrrpTrackedInterfaceEntry MAX-ACCESS not-accessible @@ -5102,6 +5112,7 @@ vrrpInstanceGroup OBJECT-GROUP vrrpInstanceMulticastAddressType, vrrpInstanceMulticastAddress, vrrpInstanceV3ChecksumAsV2, + vrrpInstanceFaultInitExitDelay, vrrpTrackedInterfaceName, vrrpTrackedInterfaceWeight, vrrpTrackedInterfaceWgtRev, diff --git a/keepalived/include/vrrp.h b/keepalived/include/vrrp.h index d96dfa8f3..158162eed 100644 --- a/keepalived/include/vrrp.h +++ b/keepalived/include/vrrp.h @@ -386,6 +386,22 @@ typedef struct _vrrp_t { * prio is allowed. 0 means no delay. */ timeval_t preempt_time; /* Time after which preemption can happen */ + unsigned long fault_init_exit_delay; /* Additional seconds*TIMER_HZ that the instance + * remains in BACKUP state after moving out of + * FAULT on INIT state before transitioning to + * MASTER. 0 means no delay. + */ + timeval_t fault_init_exit_time; /* Timestamp when the instance can become MASTER, + * after fault_init_exit_delay is applied. + */ + timeval_t block_socket_time; /* Ignore the messages received on the socket until + * this timestamp, to implement the + * fault_init_exit_delay (+ possibly startup_delay) + */ + bool fault_init_delay_needed;/* Indicates that we need to apply + * fault_init_exit_delay when sands_timer is + * initialized. + */ int state; /* internal state (init/backup/master/fault) */ #ifdef _WITH_SNMP_VRRP_ int configured_state; /* the configured state of the instance */ diff --git a/keepalived/vrrp/vrrp.c b/keepalived/vrrp/vrrp.c index 28abad2f7..a883df78a 100644 --- a/keepalived/vrrp/vrrp.c +++ b/keepalived/vrrp/vrrp.c @@ -1973,6 +1973,13 @@ vrrp_state_leave_fault(vrrp_t * vrrp) /* Set the down timer */ vrrp->master_adver_int = vrrp->adver_int; vrrp->ms_down_timer = VRRP_MS_DOWN_TIMER(vrrp); + if (vrrp->state == VRRP_STATE_BACK && vrrp->fault_init_exit_delay > 0) + vrrp->fault_init_delay_needed = true; + else if (vrrp->state == VRRP_STATE_FAULT) { + vrrp->fault_init_delay_needed = false; + vrrp->fault_init_exit_time = time_now; + vrrp->block_socket_time = time_now; + } vrrp_init_instance_sands(vrrp); vrrp->last_transition = timer_now(); } @@ -3378,6 +3385,10 @@ vrrp_complete_instance(vrrp_t * vrrp) , vrrp->iname); vrrp->preempt_delay = false; } + if (vrrp->fault_init_exit_delay) { + report_config_error(CONFIG_GENERAL_ERROR, "(%s) Warning - fault init exit delay will not work with initial state MASTER - clearing", vrrp->iname); + vrrp->fault_init_exit_delay = false; + } } if (vrrp->preempt_delay) { if (vrrp->strict_mode) { @@ -3393,6 +3404,12 @@ vrrp_complete_instance(vrrp_t * vrrp) vrrp->preempt_delay = 0; } } + if (vrrp->fault_init_exit_delay) { + if (vrrp->strict_mode) { + report_config_error(CONFIG_GENERAL_ERROR, "(%s) fault_init_exit_delay is incompatible with strict mode - resetting", vrrp->iname); + vrrp->fault_init_exit_delay = 0; + } + } if (vrrp->down_timer_adverts != VRRP_DOWN_TIMER_ADVERTS && vrrp->strict_mode) { report_config_error(CONFIG_GENERAL_ERROR, "(%s) down_timer_adverts is incompatible with" @@ -5046,12 +5063,40 @@ vrrp_complete_init(void) vrrp = vrrp_exist(old_vrrp, &vrrp_data->vrrp); if (vrrp) { + timeval_t time_now; + /* If we have detected a fault, don't override it */ if (vrrp->state == VRRP_STATE_FAULT || vrrp->num_script_init) continue; vrrp->state = old_vrrp->state; vrrp->wantstate = old_vrrp->state; + vrrp->fault_init_exit_time = old_vrrp->fault_init_exit_time; + vrrp->fault_init_delay_needed = old_vrrp->fault_init_delay_needed; + vrrp->block_socket_time = old_vrrp->block_socket_time; + + /* If there is a fault_init_exit_time in progress (less than the current time), + * we update the fault_init_exit_time, to reflect the updated fault_init_exit_delay + */ + time_now = timer_now(); + if (timercmp(&time_now, &old_vrrp->fault_init_exit_time, >=)) + continue; + if (vrrp->fault_init_exit_delay == old_vrrp->fault_init_exit_delay) + continue; + + if (vrrp->fault_init_exit_delay > old_vrrp->fault_init_exit_delay) { + vrrp->fault_init_exit_time = timer_add_long(vrrp->fault_init_exit_time, + vrrp->fault_init_exit_delay - old_vrrp->fault_init_exit_delay); + vrrp->block_socket_time = timer_add_long(vrrp->block_socket_time, + vrrp->fault_init_exit_delay - old_vrrp->fault_init_exit_delay); + } else { + vrrp->fault_init_exit_time = timer_sub_long(vrrp->fault_init_exit_time, + old_vrrp->fault_init_exit_delay - vrrp->fault_init_exit_delay); + vrrp->block_socket_time = timer_sub_long(vrrp->block_socket_time, + old_vrrp->fault_init_exit_delay - vrrp->fault_init_exit_delay); + } + log_message(LOG_INFO, "(%s) changing fault_init_exit_time as propagation delay is changed from %f seconds to %f seconds", vrrp->iname, + (float)old_vrrp->fault_init_exit_delay/TIMER_HZ, (float)vrrp->fault_init_exit_delay/TIMER_HZ); } } diff --git a/keepalived/vrrp/vrrp_data.c b/keepalived/vrrp/vrrp_data.c index a6cd176d1..322701d96 100644 --- a/keepalived/vrrp/vrrp_data.c +++ b/keepalived/vrrp/vrrp_data.c @@ -787,6 +787,9 @@ dump_vrrp(FILE *fp, const vrrp_t *vrrp) if (vrrp->preempt_delay) conf_write(fp, " Preempt delay = %g secs", vrrp->preempt_delay / TIMER_HZ_DOUBLE); + if (vrrp->fault_init_exit_delay) + conf_write(fp, " Fault Init Exit delay = %g secs", + vrrp->fault_init_exit_delay / TIMER_HZ_DOUBLE); conf_write(fp, " Promote_secondaries = %s", __test_bit(VRRP_FLAG_PROMOTE_SECONDARIES, &vrrp->flags) ? "enabled" : "disabled"); #if defined _WITH_VRRP_AUTH_ if (vrrp->auth_type) { diff --git a/keepalived/vrrp/vrrp_json.c b/keepalived/vrrp/vrrp_json.c index b5062ae2f..0912f24e9 100644 --- a/keepalived/vrrp/vrrp_json.c +++ b/keepalived/vrrp/vrrp_json.c @@ -195,6 +195,7 @@ vrrp_json_data_dump(json_writer_t *wr, vrrp_t *vrrp) #endif jsonw_bool_field(wr, "nopreempt", __test_bit(VRRP_FLAG_NOPREEMPT, &vrrp->flags)); jsonw_uint_field(wr, "preempt_delay", vrrp->preempt_delay / TIMER_HZ); + jsonw_uint_field(wr, "fault_init_exit_delay", vrrp->fault_init_exit_delay / TIMER_HZ); jsonw_uint_field(wr, "state", vrrp->state); jsonw_uint_field(wr, "wantstate", vrrp->wantstate); jsonw_uint_field(wr, "version", vrrp->version); diff --git a/keepalived/vrrp/vrrp_parser.c b/keepalived/vrrp/vrrp_parser.c index 424e376fa..83a52cba4 100644 --- a/keepalived/vrrp/vrrp_parser.c +++ b/keepalived/vrrp/vrrp_parser.c @@ -1144,6 +1144,16 @@ vrrp_preempt_delay_handler(const vector_t *strvec) current_vrrp->preempt_delay = preempt_delay; } static void +vrrp_fault_init_exit_delay_handler(const vector_t *strvec) +{ + unsigned fault_init_exit_delay; + + if (!read_decimal_unsigned_strvec(strvec, 1, &fault_init_exit_delay, 0, TIMER_MAX_SEC * TIMER_HZ, TIMER_HZ_DIGITS, true)) + report_config_error(CONFIG_GENERAL_ERROR, "(%s) fault_init_exit_delay not valid! must be between 0-%u", current_vrrp->iname, TIMER_MAX_SEC); + else + current_vrrp->fault_init_exit_delay = fault_init_exit_delay; +} +static void vrrp_notify_backup_handler(const vector_t *strvec) { if (current_vrrp->script_backup) { @@ -2211,6 +2221,7 @@ init_vrrp_keywords(bool active) install_keyword("preempt", &vrrp_preempt_handler); install_keyword("nopreempt", &vrrp_nopreempt_handler); install_keyword("preempt_delay", &vrrp_preempt_delay_handler); + install_keyword("fault_init_exit_delay", &vrrp_fault_init_exit_delay_handler); install_keyword("debug", &vrrp_debug_handler); install_keyword_quoted("notify_backup", &vrrp_notify_backup_handler); install_keyword_quoted("notify_master", &vrrp_notify_master_handler); diff --git a/keepalived/vrrp/vrrp_scheduler.c b/keepalived/vrrp/vrrp_scheduler.c index c08dc03aa..bcf61954c 100644 --- a/keepalived/vrrp/vrrp_scheduler.c +++ b/keepalived/vrrp/vrrp_scheduler.c @@ -241,6 +241,15 @@ vrrp_init_state(list_head_t *l) * very quickly (1usec) */ vrrp->state = VRRP_STATE_BACK; vrrp->ms_down_timer = 1; + if (vrrp->fault_init_exit_delay > 0) + { + vrrp->fault_init_delay_needed = true; + vrrp->block_socket_time = timer_add_long( + vrrp_delayed_start_time.tv_sec ? vrrp_delayed_start_time: time_now, + vrrp->fault_init_exit_delay); + vrrp->fault_init_exit_time = timer_add_long(vrrp->block_socket_time, vrrp->ms_down_timer); + } + } // TODO Do we need -> vrrp_restore_interface(vrrp, false, false); @@ -252,6 +261,15 @@ vrrp_init_state(list_head_t *l) } else vrrp->ms_down_timer = VRRP_MS_DOWN_TIMER(vrrp); + if (!reload && vrrp->fault_init_exit_delay > 0) + { + vrrp->fault_init_delay_needed = true; + vrrp->block_socket_time = timer_add_long( + vrrp_delayed_start_time.tv_sec ? vrrp_delayed_start_time: time_now, + vrrp->fault_init_exit_delay); + vrrp->fault_init_exit_time = timer_add_long(vrrp->block_socket_time, vrrp->ms_down_timer); + } + #ifdef _WITH_SNMP_RFCV3_ vrrp->stats->next_master_reason = VRRPV3_MASTER_REASON_MASTER_NO_RESPONSE; #endif @@ -278,6 +296,7 @@ vrrp_init_state(list_head_t *l) if (vrrp->state != VRRP_STATE_BACK) { log_message(LOG_INFO, "(%s) Entering BACKUP STATE (init)", vrrp->iname); vrrp->state = VRRP_STATE_BACK; + vrrp->fault_init_delay_needed = true; } } else { /* Note: if we have alpha mode scripts, we enter fault state, but don't want @@ -325,6 +344,18 @@ vrrp_init_instance_sands(vrrp_t *vrrp) vrrp->sands = timer_add_long(vrrp_delayed_start_time, vrrp->ms_down_timer); else vrrp->sands = timer_add_long(time_now, vrrp->ms_down_timer); + if (vrrp->fault_init_delay_needed) + { + log_message(LOG_INFO, "Applied vrrp fault init exit delay of %f seconds on instance (%s)", + (float)vrrp->fault_init_exit_delay/TIMER_HZ, vrrp->iname); + vrrp->sands = timer_add_long(vrrp->sands, vrrp->fault_init_exit_delay); + vrrp->fault_init_delay_needed = false; + vrrp->fault_init_exit_time = timer_add_long(time_now, vrrp->fault_init_exit_delay); + if (vrrp_delayed_start_time.tv_sec) + vrrp->block_socket_time = timer_add_long(vrrp_delayed_start_time, vrrp->fault_init_exit_delay); + else + vrrp->block_socket_time = timer_add_long(time_now, vrrp->fault_init_exit_delay); + } } else if (vrrp->state == VRRP_STATE_FAULT || vrrp->state == VRRP_STATE_INIT) vrrp->sands.tv_sec = TIMER_DISABLED; @@ -897,6 +928,16 @@ vrrp_dispatcher_read_timeout(sock_t *sock) prev_state = vrrp->state; if (vrrp->state == VRRP_STATE_BACK) { + if (vrrp->fault_init_exit_delay > 0) + { + timeval_t now = timer_now(); + /* Do not transition to master before fault_init_exit_time */ + if (timercmp(&vrrp->fault_init_exit_time, &now, >)) { + vrrp_init_instance_sands(vrrp); + break; + } + } + if (__test_bit(LOG_DETAIL_BIT, &debug)) log_message(LOG_INFO, "(%s) Receive advertisement timeout", vrrp->iname); vrrp_goto_master(vrrp); @@ -1109,6 +1150,16 @@ vrrp_dispatcher_read(sock_t *sock) continue; } + if (vrrp->fault_init_exit_delay > 0) + { + /* Ignore the messages received on this socket till + * fault_init_exit_delay (+ possible startup_delay) have expired. */ + set_time_now(); + if (timercmp(&time_now, &vrrp->block_socket_time, <)) { + continue; + } + } + /* Save non packet data */ vrrp->pkt_saddr = src_addr; vrrp->rx_ttl_hl = -1; /* Default to not received */ diff --git a/keepalived/vrrp/vrrp_snmp.c b/keepalived/vrrp/vrrp_snmp.c index d8f859168..bd7682702 100644 --- a/keepalived/vrrp/vrrp_snmp.c +++ b/keepalived/vrrp/vrrp_snmp.c @@ -226,6 +226,7 @@ enum snmp_vrrp_magic { VRRP_SNMP_INSTANCE_MULTICAST_ADDRESSTYPE, VRRP_SNMP_INSTANCE_MULTICAST_ADDRESS, VRRP_SNMP_INSTANCE_V3_CHECKSUM_AS_V2, + VRRP_SNMP_INSTANCE_FAULTINITEXITDELAY, VRRP_SNMP_TRACKEDINTERFACE_NAME, VRRP_SNMP_TRACKEDINTERFACE_WEIGHT, VRRP_SNMP_TRACKEDINTERFACE_WEIGHT_REVERSE, @@ -2236,6 +2237,9 @@ vrrp_snmp_instance(struct variable *vp, oid *name, size_t *length, case VRRP_SNMP_INSTANCE_V3_CHECKSUM_AS_V2: long_ret.u = (__test_bit(VRRP_FLAG_V3_CHECKSUM_AS_V2, &rt->flags)) ? 1 : 2; return PTR_CAST(u_char, &long_ret); + case VRRP_SNMP_INSTANCE_FAULTINITEXITDELAY: + long_ret.u = rt->fault_init_exit_delay / TIMER_HZ; + return PTR_CAST(u_char, &long_ret); default: return NULL; } @@ -2680,6 +2684,8 @@ static struct variable3 vrrp_vars[] = { vrrp_snmp_instance, 3, {3, 1, 35}}, {VRRP_SNMP_INSTANCE_V3_CHECKSUM_AS_V2, ASN_INTEGER, RONLY, vrrp_snmp_instance, 3, {3, 1, 36}}, + {VRRP_SNMP_INSTANCE_FAULTINITEXITDELAY, ASN_UNSIGNED, RONLY, + vrrp_snmp_instance, 3, {3, 1, 37}}, /* vrrpTrackedInterfaceTable */ {VRRP_SNMP_TRACKEDINTERFACE_NAME, ASN_OCTET_STR, RONLY,