issue: 3697720 Option to postpone socket close

Add VMA option VMA_DEFERRED_CLOSE to postpone shadow socket close.
Disabled by default (no changes vs previous code).

Signed-off-by: Iftah Levi <[email protected]>

Author: iftahl

src/vma/main.cpp

@@ -573,6 +573,7 @@ void print_vma_global_settings()
 		VLOG_PARAM_NUMBER("MSS", safe_mce_sys().lwip_mss, MCE_DEFAULT_MSS, SYS_VAR_MSS);	break;
 	}
 	VLOG_PARAM_NUMSTR("TCP CC Algorithm", safe_mce_sys().lwip_cc_algo_mod, MCE_DEFAULT_LWIP_CC_ALGO_MOD, SYS_VAR_TCP_CC_ALGO, lwip_cc_algo_str(safe_mce_sys().lwip_cc_algo_mod));
+	VLOG_PARAM_STRING("Deferred close", safe_mce_sys().deferred_close, MCE_DEFAULT_DEFERRED_CLOSE, SYS_VAR_DEFERRED_CLOSE, safe_mce_sys().deferred_close ? "Enabled " : "Disabled");
 	VLOG_PARAM_STRING("Polling Rx on Tx TCP", safe_mce_sys().rx_poll_on_tx_tcp, MCE_DEFAULT_RX_POLL_ON_TX_TCP, SYS_VAR_VMA_RX_POLL_ON_TX_TCP, safe_mce_sys().rx_poll_on_tx_tcp ? "Enabled " : "Disabled");
 	VLOG_PARAM_STRING("Trig dummy send getsockname()", safe_mce_sys().trigger_dummy_send_getsockname, MCE_DEFAULT_TRIGGER_DUMMY_SEND_GETSOCKNAME, SYS_VAR_VMA_TRIGGER_DUMMY_SEND_GETSOCKNAME, safe_mce_sys().trigger_dummy_send_getsockname ? "Enabled " : "Disabled");
 

src/vma/sock/sock-redirect.cpp

@@ -310,9 +310,9 @@ void dbg_check_if_need_to_send_mcpkt()
 	dbg_check_if_need_to_send_mcpkt_prevent_nested_calls--;
 }
 
-void handle_close(int fd, bool cleanup, bool passthrough)
+bool handle_close(int fd, bool cleanup, bool passthrough)
 {
-	
+	bool to_close_now = true;
 	srdr_logfunc("Cleanup fd=%d", fd);
 
 	if (g_p_fd_collection) {
@@ -321,12 +321,17 @@ void handle_close(int fd, bool cleanup, bool passthrough)
 
 		if (fd_collection_get_sockfd(fd)) {
 			g_p_fd_collection->del_sockfd(fd, cleanup);
+			if (safe_mce_sys().deferred_close) {
+				to_close_now = false;
+			}
 		}
 		if (fd_collection_get_epfd(fd)) {
 			g_p_fd_collection->del_epfd(fd, cleanup);
 		}
 
 	}
+
+	return to_close_now;
 }
 
 
@@ -912,9 +917,9 @@ int close(int __fd)
 
 	srdr_logdbg_entry("fd=%d", __fd);
 
-	handle_close(__fd);
+	bool close_now = handle_close(__fd);
 
-	return orig_os_api.close(__fd);
+	return close_now ? orig_os_api.close(__fd) : 0;
 }
 
 extern "C"

src/vma/sock/sock-redirect.h

@@ -177,7 +177,7 @@ extern iomux_stats_t* g_p_epoll_stats;
 
 int do_global_ctors();
 void reset_globals();
-void handle_close(int fd, bool cleanup = false, bool passthrough = false);
+bool handle_close(int fd, bool cleanup = false, bool passthrough = false);
 
 // allow calling our socket(...) implementation safely from within libvma.so
 // this is critical in case VMA was loaded using dlopen and not using LD_PRELOAD

src/vma/sock/socket_fd_api.cpp

@@ -56,6 +56,9 @@ socket_fd_api::socket_fd_api(int fd) : m_epoll_event_flags(0), m_fd(fd), m_n_sys
 
 socket_fd_api::~socket_fd_api()
 {
+	if (safe_mce_sys().deferred_close && (m_fd >= 0)) {
+		orig_os_api.close(m_fd);
+	}
 }
 
 

src/vma/util/sys_vars.cpp

@@ -619,6 +619,7 @@ void mce_sys_var::get_env_params()
 	neigh_wait_till_send_arp_msec = MCE_DEFAULT_NEIGH_UC_ARP_DELAY_MSEC;
 	timer_netlink_update_msec = MCE_DEFAULT_NETLINK_TIMER_MSEC;
 
+	deferred_close = MCE_DEFAULT_DEFERRED_CLOSE;
 	rx_poll_on_tx_tcp	= MCE_DEFAULT_RX_POLL_ON_TX_TCP;
 	trigger_dummy_send_getsockname = MCE_DEFAULT_TRIGGER_DUMMY_SEND_GETSOCKNAME;
 
@@ -1267,6 +1268,10 @@ void mce_sys_var::get_env_params()
 	if ((env_ptr = getenv(SYS_VAR_TCP_CC_ALGO)) != NULL)
 		lwip_cc_algo_mod = (uint32_t)atoi(env_ptr);
 
+	if ((env_ptr = getenv(SYS_VAR_DEFERRED_CLOSE)) != NULL) {
+		deferred_close = atoi(env_ptr) ? true : false;
+	}
+
 	if ((env_ptr = getenv(SYS_VAR_VMA_RX_POLL_ON_TX_TCP)) != NULL)
 		rx_poll_on_tx_tcp = atoi(env_ptr) ? true : false;
 

src/vma/util/sys_vars.h

@@ -424,6 +424,9 @@ struct mce_sys_var {
 	uint32_t 	vma_time_measure_num_samples;
 	char 		vma_time_measure_filename[PATH_MAX];
 	sysctl_reader_t & sysctl_reader;
+	// Workaround for #3440429: postpone close(2) to the socket destructor, so the sockfd is closed
+	// after the rfs rule is destroyed. Otherwise, flow_tag or TCP port can be reused too early.
+	bool deferred_close;
 	bool		rx_poll_on_tx_tcp;
 	hyper_t		hypervisor;
 	bool		trigger_dummy_send_getsockname;
@@ -567,6 +570,7 @@ extern mce_sys_var & safe_mce_sys();
 
 #define SYS_VAR_VMA_TIME_MEASURE_NUM_SAMPLES		"VMA_TIME_MEASURE_NUM_SAMPLES"
 #define SYS_VAR_VMA_TIME_MEASURE_DUMP_FILE		"VMA_TIME_MEASURE_DUMP_FILE"
+#define SYS_VAR_DEFERRED_CLOSE "VMA_DEFERRED_CLOSE"
 #define SYS_VAR_VMA_RX_POLL_ON_TX_TCP			"VMA_RX_POLL_ON_TX_TCP"
 #define SYS_VAR_VMA_TRIGGER_DUMMY_SEND_GETSOCKNAME	"VMA_TRIGGER_DUMMY_SEND_GETSOCKNAME"
 
@@ -701,6 +705,7 @@ extern mce_sys_var & safe_mce_sys();
 #endif /* DEFINED_TSO */
 #define MCE_DEFAULT_RX_POLL_ON_TX_TCP			(false)
 #define MCE_DEFAULT_TRIGGER_DUMMY_SEND_GETSOCKNAME	(false)
+#define MCE_DEFAULT_DEFERRED_CLOSE (false)
 
 #define MCE_ALIGNMENT					((unsigned long)63)
 #define RX_BUF_SIZE(mtu)				((mtu) + IPOIB_HDR_LEN + GRH_HDR_LEN) // RX buffers are larger in IB