More logging in socketPollConnect and let some error cases retry

Recently we noticed a hang case during NCCL bootstrap.

Some signature in job error log:
rank_604.s12gn3064.4031852.log:[0604/1024][2025-02-16 22:31:34,422] [misc/socket.cc:586] [s12gn3064:pid=3942408] NCCL WARN socketPollConnect poll() returned 1, no POLLOUT events
rank_607.s12gn3064.4031852.log:[0607/1024][2025-02-16 22:31:34,418] [misc/socket.cc:586] [s12gn3064:pid=3942411] NCCL WARN socketPollConnect poll() returned 1, no POLLOUT events

For our case, the issue is caused by the destination node dropping TCP SYN packets, which triggers TCP connection timeout on src side. However, this is general case. When poll returns 1 and there is no POLLOUT event, it could be POLLERR or POLLHUP. For such cases, we would want to go to socketConnectCheck for retry, instead of returning error.

Testing:
we reproduced the issue. The job ran into this case and retry worked. The job completed successfully.

[03/16][2025-02-25 15:46:48,389] [misc/socket.cc:602] [s11gn1268:pid=3987720] NCCL WARN socketPollConnect poll() failed, ret 1, error Connection timed out, revents 8, connect to 172.16.30.229:39953
[03/16][2025-02-25 15:46:48,390] [misc/socket.cc:551] [s11gn1268:pid=3987720] NCCL INFO socketPollConnect: connect returned Connection timed out, retrying (1/34) after sleep for 100 msec

Author: Yinglin Sun

src/misc/socket.cc

@@ -565,8 +565,12 @@ static ncclResult_t socketStartConnect(struct ncclSocket* sock) {
 
 static ncclResult_t socketPollConnect(struct ncclSocket* sock) {
   struct pollfd pfd;
-  int timeout = 1, ret;
-  socklen_t rlen = sizeof(int);
+  int timeout = 1, ret, sockErrorCode;
+  socklen_t errCodeLen = sizeof(sockErrorCode);
+  char sockstr[SOCKET_NAME_MAXLEN+1];
+  const char *warnStr = "socketPollConnect poll() failed, ret %d, error %s, revents %hd, connect to %s";
+
+  ncclSocketToString(&sock->addr, sockstr);
 
   memset(&pfd, 0, sizeof(struct pollfd));
   pfd.fd = sock->fd;
@@ -576,16 +580,25 @@ static ncclResult_t socketPollConnect(struct ncclSocket* sock) {
   if (ret == 0 || (ret < 0 && errno == EINTR)) {
     return ncclSuccess;
   } else if (ret < 0) {
-    WARN("socketPollConnect poll() failed with error %s", strerror(errno));
+    WARN(warnStr, ret, strerror(errno), pfd.revents, sockstr);
     return ncclRemoteError;
-  } else if (ret != 1 || (pfd.revents & POLLOUT) == 0) {
-    WARN("socketPollConnect poll() returned %d%s", ret, (pfd.revents & POLLOUT) ? "" : ", no POLLOUT events");
+  } else if (ret != 1) {
+    WARN(warnStr, ret, strerror(errno), pfd.revents, sockstr);
     return ncclSystemError;
   }
 
   /* check socket status */
-  SYSCHECK(getsockopt(sock->fd, SOL_SOCKET, SO_ERROR, (void*)&ret, &rlen), "getsockopt");
-  return socketConnectCheck(sock, ret, __func__);
+  SYSCHECK(getsockopt(sock->fd, SOL_SOCKET, SO_ERROR, (void*)&sockErrorCode, &errCodeLen), "getsockopt");
+  if ((pfd.revents & POLLOUT) == 0) {
+    /*
+     * When poll returns 1 however no POLLOUT, it could be POLLERR or POLLHUP.
+     * Those cases could be triggered by transient networking issue, so we would
+     * want to go to socketConnectCheck for retry, instead of returning error directly.
+     */
+    WARN(warnStr, ret, strerror(sockErrorCode), pfd.revents, sockstr);
+  }
+
+  return socketConnectCheck(sock, sockErrorCode, __func__);
 }
 
 ncclResult_t ncclSocketPollConnect(struct ncclSocket* sock) {