hw-mgmt: thermal: Fan state validation after re-insertion

A customer (case #4486661) was reported regarding
the fan start issues after re-insertion. After re-inserting
fans in some systems, it may fail to start rotating.

A stuck fan can start rotating by either of the following actions:
  1) Pulling out and reinserting an additional fans.
  2) Lowering the PWM of all fans for a few seconds.

The commit ff627 introduced FR #4521169 to Thermal
control 2.5. This patch enables the feature for Thermal
control 2.0.

Bugs:  #4360106
NVBug: #5317658

Signed-off-by: Ciju Rajan K <[email protected]>
Signed-off-by: Oleksandr Shamray <[email protected]>

Author: ciju-nvidia

usr/usr/bin/hw_management_thermal_control.py

@@ -95,6 +95,10 @@ class CONST(object):
     SYS_CONF_SENSOR_LIST_PARAM = "sensor_list"
     SYS_CONF_ERR_MASK = "error_mask"
     SYS_CONF_REDUNDANCY_PARAM = "redundancy"
+    SYS_CONF_GENERAL_CONFIG_PARAM = "general_config"
+    SYS_CONF_FAN_STEADY_STATE_DELAY = "fan_steady_state_delay"
+    SYS_CONF_FAN_STEADY_STATE_PWM = "fan_steady_state_pwm"
+    SYS_CONF_FAN_STEADY_ATTENTION_ITEMS = "attention_fans"
 
     # *************************
     # Folders definition
@@ -169,6 +173,10 @@ class CONST(object):
     # FAN RPM tolerance in percent
     FAN_RPM_TOLERANCE = 30
 
+    # attention fan insertion recovery defaults
+    FAN_STEADY_STATE_DELAY_DEF = 0
+    FAN_STEADY_STATE_PWM_DEF = 50
+
     # default system devices
     PSU_COUNT_DEF = 2
     FAN_DRWR_COUNT_DEF = 6
@@ -2035,6 +2043,11 @@ def __init__(self, cmd_arg, sys_config, name, tc_logger):
 
         self.rpm_valid_state = True
 
+        self.insert_status = 0
+        self.insert_event_ts = 0
+        self.insert_failed = False
+        self.insert_event = False
+
     # ----------------------------------------------------------------------
     def sensor_configure(self):
         """
@@ -2053,6 +2066,11 @@ def sensor_configure(self):
         self.fan_shutdown(False)
         self.pwm_set = self.read_pwm(CONST.PWM_MIN)
 
+        self.insert_status = 0
+        self.insert_event_ts = 0
+        self.insert_failed = False
+        self.insert_event = False
+
     # ----------------------------------------------------------------------
     def refresh_attr(self):
         """
@@ -2116,6 +2134,42 @@ def _get_status(self):
                 self.log.error("Value reading from file: {}".format(status_filename))
         return status
 
+    # ----------------------------------------------------------------------
+    def update_insert_state(self):
+        """
+        @summary: Update insert state
+        """
+        status = self._get_status()
+        if status:
+            if not self.insert_status:
+                self.insert_event_ts = self.get_timestump()
+                self.insert_event = True
+        else:
+            self.insert_event = 0
+            self.insert_failed = False
+        self.insert_status = status
+
+    # ----------------------------------------------------------------------
+    def is_insert_failed(self):
+        """
+        @summary: Check if insert failed
+        """
+        if self.insert_event:
+            if self.insert_event_ts + CONST.FAN_RELAX_TIME * 1000 <= self.get_timestump():
+                fan_fault_list = self._get_fault()
+                if any(x == 1 for x in fan_fault_list):
+                    self.insert_failed = True
+                self.insert_event = False
+
+        return self.insert_failed
+
+    # ----------------------------------------------------------------------
+    def reset_insert_failed_state(self):
+        """
+        @summary: Reset insert failed state
+        """
+        self.insert_failed = False
+
     # ----------------------------------------------------------------------
     def _get_fault(self):
         """
@@ -2687,6 +2741,18 @@ def __init__(self, cmd_arg, tc_logger):
                 self.exit.wait(10)
             self.log.notice("PWM control activated", 1)
 
+        self.attention_fans_lst = get_dict_val_by_path(self.sys_config, [CONST.SYS_CONF_GENERAL_CONFIG_PARAM, CONST.SYS_CONF_FAN_STEADY_ATTENTION_ITEMS])
+        if self.attention_fans_lst:
+            self.fan_steady_state_delay = get_dict_val_by_path(self.sys_config, [CONST.SYS_CONF_GENERAL_CONFIG_PARAM, CONST.SYS_CONF_FAN_STEADY_STATE_DELAY])
+            if not self.fan_steady_state_delay:
+                self.fan_steady_state_delay = CONST.FAN_STEADY_STATE_DELAY_DEF
+            self.fan_steady_state_pwm = get_dict_val_by_path(self.sys_config, [CONST.SYS_CONF_GENERAL_CONFIG_PARAM, CONST.SYS_CONF_FAN_STEADY_STATE_PWM])
+            if not self.fan_steady_state_pwm:
+                self.fan_steady_state_delay = CONST.FAN_STEADY_STATE_PWM_DEF
+            self.log.info("Fan {} insertion recovery enabled: delay {}s, pwm {}%".format(self.attention_fans_lst,
+                                                                                         self.fan_steady_state_delay,
+                                                                                         self.fan_steady_state_pwm))
+
         # Set PWM to the default state while we are waiting for system configuration
         self.log.notice("Set FAN PWM {}".format(self.pwm_target), 1)
         if not self.write_pwm(self.pwm_target, validate=True):
@@ -2936,6 +3002,30 @@ def _get_chassis_fan_dir(self):
 
         return pref_dir
 
+    # ---------------------------------------------------------------------
+    def _is_attention_fan_insertion_fail(self):
+        fan_insert_failed = False
+        for fan_obj in self.attention_fans:
+            fan_obj.update_insert_state()
+            if fan_obj.is_insert_failed():
+                self.log.notice("{} fan not started after insertion".format(fan_obj.name))
+                fan_obj.reset_insert_failed_state()
+                fan_insert_failed = True
+                break
+        return fan_insert_failed
+
+    # ---------------------------------------------------------------------
+    def _attention_fan_insertion_recovery(self):
+        pwm = self.read_pwm(100)
+        self.log.notice("Attention fan not started after insertion: Setting pwm to {}% from {}%".format(self.fan_steady_state_pwm, pwm), 1)
+        self._update_chassis_fan_speed(self.fan_steady_state_pwm, force=True)
+        self.log.info("Waiting {}s for newly inserted fan to stabilize".format(self.fan_steady_state_delay))
+        timeout = current_milli_time() + 1000 * self.fan_steady_state_delay
+        while timeout > current_milli_time():
+            self.exit.wait(1)
+        self.log.info("Resuming normal operation: Setting pwm back to {}%".format(pwm))
+        self._update_chassis_fan_speed(pwm, force=True)
+
     # ----------------------------------------------------------------------
     def _update_psu_fan_speed(self, pwm):
         """
@@ -3313,6 +3403,9 @@ def load_configuration(self):
         if CONST.SYS_CONF_REDUNDANCY_PARAM not in sys_config:
             sys_config[CONST.SYS_CONF_REDUNDANCY_PARAM] = {}
 
+        if CONST.SYS_CONF_GENERAL_CONFIG_PARAM not in sys_config:
+            sys_config[CONST.SYS_CONF_GENERAL_CONFIG_PARAM] = {}
+
         self.sys_config = sys_config
 
     # ----------------------------------------------------------------------
@@ -3522,6 +3615,16 @@ def init(self):
         self.dev_obj_list.sort(key=lambda x: x.name)
         self.write_file(CONST.PERIODIC_REPORT_FILE, self.periodic_report_time)
 
+        self.attention_fans = []
+        if self.attention_fans_lst:
+            for fan_drwr_name in self.attention_fans_lst:
+                fan_drwr_obj = self._get_dev_obj(fan_drwr_name)
+                if not fan_drwr_obj:
+                    self.log.warn("Dev name {} missing in system_config".format(fan_drwr_name))
+                    continue
+                self.attention_fans.append(fan_drwr_obj)
+                self.log.info("{} added to attention_fans".format(fan_drwr_name))
+
     # ----------------------------------------------------------------------
     def start(self, reason=""):
         """
@@ -3627,6 +3730,11 @@ def run(self):
                 self.exit.wait(30)
                 continue
 
+            if self._is_attention_fan_insertion_fail():
+                self.log.info("Attention fan insertion failed, trying to recover")
+                self._attention_fan_insertion_recovery()
+                continue
+
             if self._is_suspend():
                 self.stop(reason="suspend")
                 self.exit.wait(5)