issue: 1792164 Force tx completion for zerocopy buffers

ZCOPY packets should notify application as soon as possible to
confirm one that user buffers are free to reuse. So force completion
signal for such work requests.

Signed-off-by: Igor Ivanov <[email protected]>

Author: igor-ivanov

src/vma/dev/qp_mgr.cpp

@@ -100,7 +100,6 @@ qp_mgr::qp_mgr(const ring_simple* p_ring, const ib_ctx_handler* p_context,
 	m_ibv_rx_sg_array = new ibv_sge[m_n_sysvar_rx_num_wr_to_post_recv];
 	m_ibv_rx_wr_array = new ibv_recv_wr[m_n_sysvar_rx_num_wr_to_post_recv];
 
-	set_unsignaled_count();
 	memset(&m_rate_limit, 0, sizeof(struct vma_rate_limit_t));
 
 	qp_logfunc("");
@@ -336,7 +335,6 @@ void qp_mgr::up()
 	release_tx_buffers();
 
 	/* clean any link to completions with error we might have */
-	set_unsignaled_count();
 	m_p_last_tx_mem_buf_desc = NULL;
 
 	modify_qp_to_ready_state();
@@ -497,7 +495,6 @@ void qp_mgr::trigger_completion_for_all_sent_packets()
 
 		// Close the Tx unsignaled send list
 		set_unsignaled_count();
-		m_p_last_tx_mem_buf_desc = NULL;
 
 		if (!m_p_ring->m_tx_num_wr_free) {
 			qp_logdbg("failed to trigger completion for all packets due to no available wr");
@@ -599,9 +596,19 @@ inline int qp_mgr::send_to_wire(vma_ibv_send_wr* p_send_wqe, vma_wr_tx_packet_at
 int qp_mgr::send(vma_ibv_send_wr* p_send_wqe, vma_wr_tx_packet_attr attr)
 {
 	mem_buf_desc_t* p_mem_buf_desc = (mem_buf_desc_t *)p_send_wqe->wr_id;
+	/* Control tx completions:
+	 * - VMA_TX_WRE_BATCHING - The number of Tx Work Request Elements used
+	 *   until a completion signal is requested.
+	 * - ZCOPY packets should notify application as soon as possible to
+	 *   confirm one that user buffers are free to reuse. So force completion
+	 *   signal for such work requests.
+	 * - First call of send() should do completion. It means that
+	 *   m_n_unsignaled_count must be zero for this time.
+	 */
+	bool request_comp = (is_completion_need() ||
+			(p_mem_buf_desc->m_flags & mem_buf_desc_t::ZCOPY));
 
 	qp_logfunc("VERBS send, unsignaled_count: %d", m_n_unsignaled_count);
-	bool request_comp = is_completion_need();
 
 #ifdef VMA_TIME_MEASURE
 	TAKE_T_TX_POST_SEND_START;
@@ -636,7 +643,6 @@ int qp_mgr::send(vma_ibv_send_wr* p_send_wqe, vma_wr_tx_packet_attr attr)
 		int ret;
 
 		set_unsignaled_count();
-		m_p_last_tx_mem_buf_desc = NULL;
 
 		// Poll the Tx CQ
 		uint64_t dummy_poll_sn = 0;

src/vma/dev/qp_mgr.h

@@ -172,7 +172,10 @@ friend class cq_mgr_mp;
 
 	int             configure(struct ibv_comp_channel* p_rx_comp_event_channel);
 	virtual int     prepare_ibv_qp(vma_ibv_qp_init_attr& qp_init_attr) = 0;
-	inline void     set_unsignaled_count(void) { m_n_unsignaled_count = m_n_sysvar_tx_num_wr_to_signal - 1;	}
+	inline void     set_unsignaled_count(void) {
+		m_n_unsignaled_count = m_n_sysvar_tx_num_wr_to_signal - 1;
+		m_p_last_tx_mem_buf_desc = NULL;
+	}
 
 	virtual cq_mgr* init_rx_cq_mgr(struct ibv_comp_channel* p_rx_comp_event_channel);
 	virtual cq_mgr* init_tx_cq_mgr(void);

src/vma/dev/qp_mgr_eth_mlx5.cpp

@@ -887,7 +887,6 @@ void qp_mgr_eth_mlx5::trigger_completion_for_all_sent_packets()
 
 		// Close the Tx unsignaled send list
 		set_unsignaled_count();
-		m_p_last_tx_mem_buf_desc = NULL;
 
 		if (!m_p_ring->m_tx_num_wr_free) {
 			qp_logdbg("failed to trigger completion for all packets due to no available wr");