Avoid deadlock issues caused by logging in a loop.

Signed-off-by: wangfakang <[email protected]>

Author: wangfakang

src/misc/param.cc

@@ -61,21 +61,28 @@ void initEnv() {
 
 void ncclLoadParam(char const* env, int64_t deftVal, int64_t uninitialized, int64_t* cache) {
   static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+  if (__builtin_expect(__atomic_load_n(cache, __ATOMIC_RELAXED) != uninitialized, true)) {
+    return;
+  }
+
   pthread_mutex_lock(&mutex);
   if (__atomic_load_n(cache, __ATOMIC_RELAXED) == uninitialized) {
     const char* str = ncclGetEnv(env);
     int64_t value = deftVal;
     if (str && strlen(str) > 0) {
       errno = 0;
       value = strtoll(str, nullptr, 0);
-      if (errno) {
-        value = deftVal;
-        INFO(NCCL_ALL,"Invalid value %s for %s, using default %lld.", str, env, (long long)deftVal);
-      } else {
-        INFO(NCCL_ENV,"%s set by environment to %lld.", env, (long long)value);
-      }
+      value = errno ? deftVal : value;
     }
+    // To prevent deadlock issues caused by logging in a loop,
+    // so cache the value before the log operation.
     __atomic_store_n(cache, value, __ATOMIC_RELAXED);
+
+    if (errno) {
+      INFO(NCCL_ALL,"Invalid value %s for %s, using default %lld.", str, env, (long long)deftVal);
+    } else {
+      INFO(NCCL_ENV,"%s set by environment to %lld.", env, (long long)value);
+    }
   }
   pthread_mutex_unlock(&mutex);
 }
@@ -84,4 +91,4 @@ const char *ncclGetEnv(const char *name) {
   static pthread_once_t once = PTHREAD_ONCE_INIT;
   pthread_once(&once, initEnv);
   return getenv(name);
-}
+}