issue: 3788369 Optimize is_rts() checks in data path

While a socket is locked, it's state cannot be changed and the only
scenario to change the state is to call polling or TCP timers.

In the TX path we don't call timers and polling is called when the send
buffer is full. Therefore, it's enough to check the connection state
under the lock once for non-blocking sockets. For blocking sockets the
check can be moved after the polling, because the connection state
cannot be changed before.

Check for g_b_exit is relevant only for blocking operations, so check it
just before polling.

Signed-off-by: Dmytro Podgornyi <[email protected]>

Author: pasis

src/core/sock/sockinfo_tcp.cpp

@@ -1052,9 +1052,6 @@ ssize_t sockinfo_tcp::tcp_tx(xlio_tx_call_attr_t &tx_arg)
         return ret;
     }
 
-    if (unlikely(!is_connected_and_ready_to_send())) {
-        return -1;
-    }
     si_tcp_logfunc("tx: iov=%p niovs=%d", p_iov, sz_iov);
 
     if (m_sysvar_rx_poll_on_tx_tcp) {
@@ -1079,6 +1076,9 @@ ssize_t sockinfo_tcp::tcp_tx(xlio_tx_call_attr_t &tx_arg)
                         [](size_t sum, const iovec &curr) { return sum + curr.iov_len; });
     lock_tcp_con();
 
+    if (unlikely(!is_connected_and_ready_to_send())) {
+        return tcp_tx_handle_errno_and_unlock(errno);
+    }
     if (TCP_WND_UNAVALABLE(m_pcb, total_iov_len)) {
         return tcp_tx_handle_errno_and_unlock(EAGAIN);
     }
@@ -1099,13 +1099,8 @@ ssize_t sockinfo_tcp::tcp_tx(xlio_tx_call_attr_t &tx_arg)
             unsigned tx_size = sndbuf_available();
 
             if (tx_size == 0) {
-                if (unlikely(!is_rts())) {
-                    si_tcp_logdbg("TX on disconnected socket");
-                    return tcp_tx_handle_errno_and_unlock(ECONNRESET);
-                }
                 // force out TCP data before going on wait()
                 tcp_output(&m_pcb);
-
                 return tcp_tx_handle_sndbuf_unavailable(total_tx, is_dummy, is_non_file_zerocopy,
                                                         errno_tmp);
             }
@@ -1134,15 +1129,6 @@ ssize_t sockinfo_tcp::tcp_tx(xlio_tx_call_attr_t &tx_arg)
                 tx_size = std::min(remainder, tx_size);
             }
 
-            if (unlikely(!is_rts())) {
-                si_tcp_logdbg("TX on disconnected socket");
-                return tcp_tx_handle_errno_and_unlock(ECONNRESET);
-            }
-            if (unlikely(g_b_exit)) {
-                return tcp_tx_handle_partial_send_and_unlock(total_tx, EINTR, is_dummy,
-                                                             is_non_file_zerocopy, errno_tmp);
-            }
-
             const struct iovec iov = {.iov_base = tx_ptr, .iov_len = tx_size};
             err = tcp_write_express(&m_pcb, &iov, 1, &tx_arg.priv);
             if (unlikely(err != ERR_OK)) {
@@ -1211,6 +1197,9 @@ ssize_t sockinfo_tcp::tcp_tx_slow_path(xlio_tx_call_attr_t &tx_arg)
 
     lock_tcp_con();
 
+    if (unlikely(!is_connected_and_ready_to_send())) {
+        return tcp_tx_handle_errno_and_unlock(errno);
+    }
     if (cannot_do_requested_dummy_send(m_pcb, tx_arg)) {
         return tcp_tx_handle_errno_and_unlock(EAGAIN);
     }
@@ -1246,10 +1235,6 @@ ssize_t sockinfo_tcp::tcp_tx_slow_path(xlio_tx_call_attr_t &tx_arg)
              *    - block until space is available
              */
             if (tx_size == 0) {
-                if (unlikely(!is_rts())) {
-                    si_tcp_logdbg("TX on disconnected socket");
-                    return tcp_tx_handle_errno_and_unlock(ECONNRESET);
-                }
                 // force out TCP data before going on wait()
                 tcp_output(&m_pcb);
 
@@ -1259,7 +1244,16 @@ ssize_t sockinfo_tcp::tcp_tx_slow_path(xlio_tx_call_attr_t &tx_arg)
                                                             errno_tmp);
                 }
 
+                if (unlikely(g_b_exit)) {
+                    return tcp_tx_handle_partial_send_and_unlock(total_tx, EINTR, is_dummy,
+                                                                 is_send_zerocopy, errno_tmp);
+                }
+
                 tx_size = tx_wait(block_this_run);
+                if (unlikely(!is_rts())) {
+                    si_tcp_logdbg("TX on disconnected socket");
+                    return tcp_tx_handle_errno_and_unlock(ECONNRESET);
+                }
             }
 
             tx_size = std::min<size_t>(p_iov[i].iov_len - pos, tx_size);
@@ -1276,15 +1270,6 @@ ssize_t sockinfo_tcp::tcp_tx_slow_path(xlio_tx_call_attr_t &tx_arg)
                 tx_size = std::min(remainder, tx_size);
             }
             do {
-                if (unlikely(!is_rts())) {
-                    si_tcp_logdbg("TX on disconnected socket");
-                    return tcp_tx_handle_errno_and_unlock(ECONNRESET);
-                }
-                if (unlikely(g_b_exit)) {
-                    return tcp_tx_handle_partial_send_and_unlock(total_tx, EINTR, is_dummy,
-                                                                 is_send_zerocopy, errno_tmp);
-                }
-
                 err_t err;
                 if (apiflags & XLIO_TX_PACKET_ZEROCOPY) {
                     const struct iovec iov = {.iov_base = tx_ptr, .iov_len = tx_size};
@@ -1315,7 +1300,16 @@ ssize_t sockinfo_tcp::tcp_tx_slow_path(xlio_tx_call_attr_t &tx_arg)
                         }
                     }
 
+                    if (unlikely(g_b_exit)) {
+                        return tcp_tx_handle_partial_send_and_unlock(total_tx, EINTR, is_dummy,
+                                                                     is_send_zerocopy, errno_tmp);
+                    }
+
                     rx_wait(poll_count, true);
+                    if (unlikely(!is_rts())) {
+                        si_tcp_logdbg("TX on disconnected socket");
+                        return tcp_tx_handle_errno_and_unlock(ECONNRESET);
+                    }
 
                     // AlexV:Avoid from going to sleep, for the blocked socket of course, since
                     // progress engine may consume an arrived credit and it will not wakeup the
@@ -6168,10 +6162,6 @@ inline bool sockinfo_tcp::handle_bind_no_port(int &bind_ret, in_port_t in_port,
 int sockinfo_tcp::tcp_tx_express(const struct iovec *iov, unsigned iov_len, uint32_t mkey,
                                  unsigned flags, void *opaque_op)
 {
-    if (unlikely(!is_connected_and_ready_to_send())) {
-        return -1;
-    }
-
     pbuf_desc mdesc;
 
     switch (flags & XLIO_EXPRESS_OP_TYPE_MASK) {
@@ -6194,6 +6184,10 @@ int sockinfo_tcp::tcp_tx_express(const struct iovec *iov, unsigned iov_len, uint
 
     lock_tcp_con();
 
+    if (unlikely(!is_connected_and_ready_to_send())) {
+        return tcp_tx_handle_errno_and_unlock(errno);
+    }
+
     err_t err = tcp_write_express(&m_pcb, iov, iov_len, &mdesc);
     if (unlikely(err != ERR_OK)) {
         // The only error in tcp_write_express() is a memory error.
@@ -6216,10 +6210,6 @@ int sockinfo_tcp::tcp_tx_express(const struct iovec *iov, unsigned iov_len, uint
 
 int sockinfo_tcp::tcp_tx_express_inline(const struct iovec *iov, unsigned iov_len, unsigned flags)
 {
-    if (unlikely(!is_connected_and_ready_to_send())) {
-        return -1;
-    }
-
     pbuf_desc mdesc;
     int bytes_written = 0;
 
@@ -6228,6 +6218,10 @@ int sockinfo_tcp::tcp_tx_express_inline(const struct iovec *iov, unsigned iov_le
 
     lock_tcp_con();
 
+    if (unlikely(!is_connected_and_ready_to_send())) {
+        return tcp_tx_handle_errno_and_unlock(errno);
+    }
+
     for (unsigned i = 0; i < iov_len; ++i) {
         bytes_written += iov[i].iov_len;
         err_t err = tcp_write(&m_pcb, iov[i].iov_base, iov[i].iov_len, 0, &mdesc);
@@ -6323,7 +6317,6 @@ ssize_t sockinfo_tcp::tcp_tx_handle_partial_send_and_unlock(ssize_t total_tx, in
 
 bool sockinfo_tcp::is_connected_and_ready_to_send()
 {
-    /* TODO should we add !g_b_exit here? */
     if (unlikely(!is_rts())) {
         if (m_conn_state == TCP_CONN_TIMEOUT) {
             si_tcp_logdbg("TX timed out");