issue: 4656811 Removing TSO

Removing TSO (TCP Segmentation offload) compilation flavor.

Signed-off-by: Alexander Grissik <[email protected]>

Author: AlexanderGrissik

.ci/matrix_job.yaml

@@ -63,7 +63,6 @@ matrix:
   axes:
     flags:
       - default
-      - extra
     arch:
       - x86_64
       - aarch64

README

@@ -115,7 +115,6 @@ Example:
  VMA DETAILS: TCP max syn rate               0 (no limit)               [VMA_TCP_MAX_SYN_RATE]
  VMA DETAILS: Tx Mem Segs TCP                1000000                    [VMA_TX_SEGS_TCP]
  VMA DETAILS: Tx Mem Bufs                    200000                     [VMA_TX_BUFS]
- VMA DETAILS: Tx Mem Buf size                0                          [VMA_TX_BUF_SIZE]
  VMA DETAILS: Tx QP WRE                      2048                       [VMA_TX_WRE]
  VMA DETAILS: Tx QP WRE Batching             64                         [VMA_TX_WRE_BATCHING]
  VMA DETAILS: Tx Max QP INLINE               204                        [VMA_TX_MAX_INLINE]
@@ -264,7 +263,6 @@ latency
      VMA_RX_WRE = 256                         (default: 16000)
      VMA_RX_WRE_BATCHING = 4                  (default: 64)
      VMA_RX_POLL = -1                         (default: 100000)
-     VMA_TSO = Disable                        (default: Enable)
      VMA_RX_PREFETCH_BYTES_BEFORE_POLL = 256  (default: 0)
      VMA_GRO_STREAMS_MAX = 0                  (default: 32)
      VMA_SELECT_POLL = -1                     (default: 100000)
@@ -354,11 +352,6 @@ VMA_TX_BUFS
 Number of global Tx data buffer elements allocation.
 Default value is 200000
 
-VMA_TX_BUF_SIZE
-Size of Tx data buffer elements allocation.
-Can not be less then MTU (Maximum Transfer Unit) and greater than 0xFF00.
-Default value is calculated basing on VMA_MTU and VMA_MSS.
-
 VMA_TX_WRE
 Number of Work Request Elements allocated in all transmit QP's.
 The number of QP's can change according to the number of network offloaded
@@ -491,16 +484,6 @@ Value range is -1, 0 to 100,000,000
 Where value of -1 is used for infinite polling
 Default value is 100000
 
-VMA_TSO
-With Segmentation Offload, or TCP Large Send, TCP can pass a buffer to be
-transmitted that is bigger than the maximum transmission unit (MTU) supported
-by the medium. Intelligent adapters implement large sends by using the
-prototype TCP and IP headers of the incoming send buffer to carve out segments
-of required size. Copying the prototype header and options, then calculating
-the sequence number and checksum fields creates TCP segment headers.
-Expected benefits: Throughput increase and CPU unload.
-Default value is 1 (Enable)
-
 VMA_RX_POLL_INIT
 VMA maps all UDP sockets as potential offloaded capable. Only after the
 ADD_MEMBERSHIP does the offload start to work and the CQ polling kicks in VMA.

config/m4/verbs.m4

@@ -150,7 +150,6 @@ if test "x$vma_cv_verbs" == x2; then
     CHECK_VERBS_ATTRIBUTE([IBV_EXP_DEVICE_ATTR_MAX_DM_SIZE], [infiniband/verbs_exp.h], [IBV_DM])
     CHECK_VERBS_ATTRIBUTE([IBV_EXP_QP_RATE_LIMIT], [infiniband/verbs_exp.h], [IBV_PACKET_PACING_CAPS])
     CHECK_VERBS_ATTRIBUTE([IBV_EXP_QP_SUPPORT_BURST], [infiniband/verbs_exp.h], [IBV_QP_SUPPORT_BURST])
-    CHECK_VERBS_ATTRIBUTE([IBV_EXP_WR_TSO], [infiniband/verbs_exp.h], [OPCODE_TSO])
     CHECK_VERBS_ATTRIBUTE([IBV_EXP_DEVICE_CROSS_CHANNEL], [infiniband/verbs_exp.h], [IBV_DEVICE_CROSS_CHANNEL])
 
     #
@@ -180,8 +179,6 @@ fi
 # Check Upstream
 #
 if test "x$vma_cv_verbs" == x3; then
-    CHECK_VERBS_ATTRIBUTE([IBV_WR_TSO], [infiniband/verbs.h], [OPCODE_TSO])
-
     if test "x$vma_cv_directverbs" == x3; then
         CHECK_VERBS_ATTRIBUTE([MLX5_OPCODE_NOP], [infiniband/mlx5dv.h], [IBV_WR_NOP])
         CHECK_VERBS_MEMBER([struct mlx5dv_clock_info.last_cycles], [infiniband/mlx5dv.h], [IBV_CLOCK_INFO])

configure.ac

@@ -255,20 +255,6 @@ else
     AC_MSG_RESULT([no])
 fi
 
-# Control TSO usage
-#
-AC_ARG_ENABLE([tso],
-    AS_HELP_STRING([--enable-tso],
-        [Enable TSO availability (default=no)]))
-AC_MSG_CHECKING(
-    [for tso support])
-if test "x$enable_tso" = xyes -a "x$vma_cv_attribute_ex_OPCODE_TSO" = xyes; then
-    AC_DEFINE_UNQUOTED([DEFINED_TSO], [1], [Define to 1 to use TSO])
-    AC_MSG_RESULT([yes])
-else
-    AC_MSG_RESULT([no])
-fi
-
 AC_MSG_CHECKING([for md5 version of VMA statistics is])
 STATS_PROTOCOL_VER=`md5sum ${srcdir}/src/vma/util/vma_stats.h | awk '{ print $1}'`
 AC_DEFINE_UNQUOTED(STATS_PROTOCOL_VER, "${STATS_PROTOCOL_VER}", [Stats Protocol Version])

contrib/build_pkg.sh

@@ -54,9 +54,9 @@ contrib/build_pkg.sh -s
  * Prepare source package directly from github using tag or branch
 contrib/build_pkg.sh -s -i /tmp/libvma-9.0.2 -c 9.0.2
  * Pass none default configuration options
-contrib/build_pkg.sh -b -s -a "configure_options=--enable-tso"
+contrib/build_pkg.sh -b -s -a "configure_options=--enable-debug"
  * Create release
-env PRJ_RELEASE=1 contrib/build_pkg.sh -b -s -a "configure_options=--enable-tso"
+env PRJ_RELEASE=1 contrib/build_pkg.sh -b -s -a "configure_options=--enable-debug"
 EOF
             exit 1
             ;;

contrib/jenkins_tests/rpm.sh

@@ -84,7 +84,10 @@ if [ $opt_binrpm -eq 1 ]; then
 fi
 
 if [ $opt_checkpkg -eq 1 ]; then
-    test_exec="env RPM_BUILD_NCPUS=${NPROC} PRJ_RELEASE=1 ${WORKSPACE}/contrib/build_pkg.sh -b -s -a \"configure_options=$jenkins_test_custom_configure\" -i ${WORKSPACE} -o ${rpm_dir}/dist-pkg"
+    if [ -n "$jenkins_test_custom_configure" ]; then
+        custom_configure_option="-a \"configure_options=$jenkins_test_custom_configure\""
+    fi
+    test_exec="env RPM_BUILD_NCPUS=${NPROC} PRJ_RELEASE=1 ${WORKSPACE}/contrib/build_pkg.sh -b -s $custom_configure_option -i ${WORKSPACE} -o ${rpm_dir}/dist-pkg"
     do_check_result "$test_exec" "$test_id" "checkpkg" "$rpm_tap" "${rpm_dir}/rpm-${test_id}"
     test_id=$((test_id+1))
 fi

contrib/test_jenkins.sh

@@ -97,11 +97,7 @@ do_check_env
 TARGET=${TARGET:=all}
 i=0
 if [ "$TARGET" == "all" -o "$TARGET" == "default" ]; then
-    target_list[$i]="default: --disable-tso"
-    i=$((i+1))
-fi
-if [ "$TARGET" == "all" -o "$TARGET" == "extra" ]; then
-    target_list[$i]="extra: --enable-tso"
+    target_list[$i]="default: "
     i=$((i+1))
 fi
 

src/vma/dev/ib_ctx_handler.h

@@ -48,9 +48,6 @@ class ib_ctx_handler : public event_handler_ibverbs
 	inline char*            get_ibname() { return (m_p_ibv_device ? m_p_ibv_device->name : (char *)""); }
 	struct ibv_context*     get_ibv_context() { return m_p_ibv_context; }
 	vma_ibv_device_attr*    get_ibv_device_attr() { return vma_get_device_orig_attr(m_p_ibv_device_attr); }
-#ifdef DEFINED_TSO
-	vma_ibv_device_attr_ex* get_ibv_device_attr_ex() { return m_p_ibv_device_attr; }
-#endif /* DEFINED_TSO */
 	uint32_t                mem_reg(void *addr, size_t length, uint64_t access);
 	void                    mem_dereg(uint32_t lkey);
 	struct ibv_mr*          get_mem_reg(uint32_t lkey);

src/vma/dev/qp_mgr.cpp

@@ -60,16 +60,7 @@ qp_mgr::qp_mgr(struct qp_mgr_desc *desc, const uint32_t tx_num_wr):
 	,m_n_ip_id_base(0)
 	,m_n_ip_id_offset(0)
 {
-#ifdef DEFINED_TSO
-	memset(&m_qp_cap, 0, sizeof(m_qp_cap));
-	m_qp_cap.max_inline_data = safe_mce_sys().tx_max_inline;
-	m_qp_cap.max_send_sge = (m_p_ring->is_tso() ?
-		m_p_ib_ctx_handler->get_ibv_device_attr()->max_sge : MCE_DEFAULT_TX_NUM_SGE);
-        m_qp_cap.max_recv_sge = (m_p_ring->is_socketxtreme()) ? 1 : MCE_DEFAULT_RX_NUM_SGE;
-#else
 	m_max_inline_data = 0;
-#endif /* DEFINED_TSO */
-
 	m_ibv_rx_sg_array = new ibv_sge[m_n_sysvar_rx_num_wr_to_post_recv];
 	m_ibv_rx_wr_array = new ibv_recv_wr[m_n_sysvar_rx_num_wr_to_post_recv];
 
@@ -188,78 +179,6 @@ int qp_mgr::configure(struct qp_mgr_desc *desc)
 	vma_ibv_qp_init_attr qp_init_attr;
 	memset(&qp_init_attr, 0, sizeof(qp_init_attr));
 
-#ifdef DEFINED_TSO
-	// TODO: m_tx_num_wr and m_rx_num_wr should be part of m_qp_cap
-	// and assigned as a result of ibv_query_qp()
-	m_qp_cap.max_send_wr = m_tx_num_wr;
-	m_qp_cap.max_recv_wr = m_rx_num_wr;
-
-	memcpy(&qp_init_attr.cap, &m_qp_cap, sizeof(qp_init_attr.cap));
-	qp_init_attr.recv_cq = m_p_cq_mgr_rx->get_ibv_cq_hndl();
-	qp_init_attr.send_cq = m_p_cq_mgr_tx->get_ibv_cq_hndl();
-	qp_init_attr.sq_sig_all = 0;
-
-	// In case of enabled TSO we need to take into account amount of SGE together with header inline
-	// Per PRM maximum of CTRL + ETH + ETH_HEADER_INLINE+DATA_PTR*NUM_SGE+MAX_INLINE+INLINE_SIZE
-	// MLX5 return 32678 WQEBBs at max so minimal number 
-	int max_wqe_sz = 16+14+m_p_ring->m_tso.max_header_sz+16*qp_init_attr.cap.max_send_sge+qp_init_attr.cap.max_inline_data+4;
-	int num_wr = 32678*64/max_wqe_sz;
-	qp_logdbg("calculated max_wqe_sz=%d num_wr=%d", max_wqe_sz, num_wr);
-	if (num_wr < (signed)m_tx_num_wr) {
-		qp_init_attr.cap.max_send_wr = num_wr; // force min for create_qp or you will have error of memory allocation
-	}
-
-	qp_logdbg("Requested QP parameters: "
-			"wre: tx = %d rx = %d "
-			"sge: tx = %d rx = %d "
-			"inline: %d",
-			qp_init_attr.cap.max_send_wr, qp_init_attr.cap.max_recv_wr,
-			qp_init_attr.cap.max_send_sge, qp_init_attr.cap.max_recv_sge,
-			qp_init_attr.cap.max_inline_data);
-
-	// Create the QP
-	if (prepare_ibv_qp(qp_init_attr)) {
-		return -1;
-	}
-
-	qp_logdbg("Configured QP parameters: "
-			"wre: tx = %d rx = %d "
-			"sge: tx = %d rx = %d "
-			"inline: %d",
-			qp_init_attr.cap.max_send_wr, qp_init_attr.cap.max_recv_wr,
-			qp_init_attr.cap.max_send_sge, qp_init_attr.cap.max_recv_sge,
-			qp_init_attr.cap.max_inline_data);
-
-	/* Check initial parameters with actual */
-	enum ibv_qp_attr_mask attr_mask = IBV_QP_CAP;
-	struct ibv_qp_attr tmp_ibv_qp_attr;
-	struct ibv_qp_init_attr tmp_ibv_qp_init_attr;
-	IF_VERBS_FAILURE(ibv_query_qp(m_qp, &tmp_ibv_qp_attr, attr_mask,
-			 &tmp_ibv_qp_init_attr)) {
-			qp_logerr("ibv_query_qp failed (errno=%d %m)", errno);
-			return -1;
-	} ENDIF_VERBS_FAILURE;
-	m_qp_cap.max_send_wr = min(tmp_ibv_qp_attr.cap.max_send_wr, m_qp_cap.max_send_wr);
-	m_qp_cap.max_recv_wr = min(tmp_ibv_qp_attr.cap.max_recv_wr, m_qp_cap.max_recv_wr);
-	m_qp_cap.max_send_sge = min(tmp_ibv_qp_attr.cap.max_send_sge, m_qp_cap.max_send_sge);
-	m_qp_cap.max_recv_sge = min(tmp_ibv_qp_attr.cap.max_recv_sge, m_qp_cap.max_recv_sge);
-	m_qp_cap.max_inline_data = min(tmp_ibv_qp_attr.cap.max_inline_data, m_qp_cap.max_inline_data);
-
-	if ( m_qp_cap.max_send_wr < m_tx_num_wr ) {
-		qp_logwarn("Amount of requested TX_WRE %d lowered to %d", m_tx_num_wr, m_qp_cap.max_send_wr);
-		m_tx_num_wr = m_qp_cap.max_send_wr;
-		m_p_ring->set_tx_num_wr( m_tx_num_wr );
-	}
-
-	qp_logdbg("Used QP (num=%d) "
-			"wre: tx = %d rx = %d "
-			"sge: tx = %d rx = %d "
-			"inline: %d",
-			m_qp->qp_num,
-			m_qp_cap.max_send_wr, m_qp_cap.max_recv_wr,
-			m_qp_cap.max_send_sge, m_qp_cap.max_recv_sge,
-			m_qp_cap.max_inline_data);
-#else
 	// Check device capabilities for max SG elements
 	uint32_t tx_max_inline = safe_mce_sys().tx_max_inline;
 	uint32_t rx_num_sge = (m_p_ring->is_socketxtreme() ? 1 : MCE_DEFAULT_RX_NUM_SGE);
@@ -282,7 +201,6 @@ int qp_mgr::configure(struct qp_mgr_desc *desc)
 	qp_logdbg("Created QP (num=%d) with %d tx wre and inline=%d and %d rx "
 		"wre and %d sge", m_qp->qp_num, m_tx_num_wr, m_max_inline_data,
 		m_rx_num_wr, rx_num_sge);
-#endif /* DEFINED_TSO */
 
 #if defined(DEFINED_ROCE_LAG)
 	if (desc->slave && desc->slave->lag_tx_port_affinity > 0) {
@@ -680,13 +598,6 @@ int qp_mgr_eth::prepare_ibv_qp(vma_ibv_qp_init_attr& qp_init_attr)
 	qp_init_attr.qp_type = IBV_QPT_RAW_PACKET;
 	vma_ibv_qp_init_attr_comp_mask(m_p_ib_ctx_handler->get_ibv_pd(), qp_init_attr);
 
-#ifdef DEFINED_TSO
-	if (m_p_ring->is_tso()) {
-		vma_ibv_qp_init_attr_tso(qp_init_attr, m_p_ring->get_max_header_sz());
-		qp_logdbg("create qp with max_tso_header = %d", m_p_ring->get_max_header_sz());
-	}
-#endif /* DEFINED_TSO */
-
 	m_qp = vma_ibv_create_qp(m_p_ib_ctx_handler->get_ibv_pd(), &qp_init_attr);
 
 	BULLSEYE_EXCLUDE_BLOCK_START
@@ -701,8 +612,6 @@ int qp_mgr_eth::prepare_ibv_qp(vma_ibv_qp_init_attr& qp_init_attr)
 	}
 	BULLSEYE_EXCLUDE_BLOCK_END
 
-#ifdef DEFINED_TSO
-#else
 	enum ibv_qp_attr_mask attr_mask = IBV_QP_CAP;
 	struct ibv_qp_attr tmp_ibv_qp_attr;
 	struct ibv_qp_init_attr tmp_ibv_qp_init_attr;
@@ -711,16 +620,18 @@ int qp_mgr_eth::prepare_ibv_qp(vma_ibv_qp_init_attr& qp_init_attr)
 			qp_logerr("ibv_query_qp failed (errno=%d %m)", errno);
 			return -1;
 	} ENDIF_VERBS_FAILURE;
+
 	uint32_t tx_max_inline = safe_mce_sys().tx_max_inline;
 	m_max_inline_data = min(tmp_ibv_qp_attr.cap.max_inline_data, tx_max_inline);
+
 	qp_logdbg("requested max inline = %d QP, actual max inline = %d, "
 		"VMA max inline set to %d, max_send_wr=%d, max_recv_wr=%d, "
 		"max_recv_sge=%d, max_send_sge=%d",
 		tx_max_inline, tmp_ibv_qp_init_attr.cap.max_inline_data,
 		m_max_inline_data, tmp_ibv_qp_attr.cap.max_send_wr,
 		tmp_ibv_qp_attr.cap.max_recv_wr, tmp_ibv_qp_attr.cap.max_recv_sge,
 		tmp_ibv_qp_attr.cap.max_send_sge);
-#endif /* DEFINED_TSO */
+
 	return 0;
 }
 
@@ -750,13 +661,6 @@ int qp_mgr_ib::prepare_ibv_qp(vma_ibv_qp_init_attr& qp_init_attr)
 	qp_init_attr.qp_type = IBV_QPT_UD;
 	vma_ibv_qp_init_attr_comp_mask(m_p_ib_ctx_handler->get_ibv_pd(), qp_init_attr);
 
-#ifdef DEFINED_TSO
-	if (m_p_ring->is_tso()) {
-		vma_ibv_qp_init_attr_tso(qp_init_attr, m_p_ring->get_max_header_sz());
-		qp_logdbg("create qp with max_tso_header = %d", m_p_ring->get_max_header_sz());
-	}
-#endif /* DEFINED_TSO */
-
 	if (m_underly_qpn) {
 		ibv_source_qpn_set(qp_init_attr, m_underly_qpn);
 		qp_logdbg("create qp using underly qpn = 0x%X", m_underly_qpn);
@@ -781,8 +685,6 @@ int qp_mgr_ib::prepare_ibv_qp(vma_ibv_qp_init_attr& qp_init_attr)
 	}
 	BULLSEYE_EXCLUDE_BLOCK_END
 
-#ifdef DEFINED_TSO
-#else
 	enum ibv_qp_attr_mask attr_mask = IBV_QP_CAP;
 	struct ibv_qp_attr tmp_ibv_qp_attr;
 	struct ibv_qp_init_attr tmp_ibv_qp_init_attr;
@@ -791,16 +693,17 @@ int qp_mgr_ib::prepare_ibv_qp(vma_ibv_qp_init_attr& qp_init_attr)
 			qp_logerr("ibv_query_qp failed (errno=%d %m)", errno);
 			return -1;
 	} ENDIF_VERBS_FAILURE;
+
 	uint32_t tx_max_inline = safe_mce_sys().tx_max_inline;
 	m_max_inline_data = min(tmp_ibv_qp_attr.cap.max_inline_data, tx_max_inline);
+
 	qp_logdbg("requested max inline = %d QP, actual max inline = %d, "
 		"VMA max inline set to %d, max_send_wr=%d, max_recv_wr=%d, "
 		"max_recv_sge=%d, max_send_sge=%d",
 		tx_max_inline, tmp_ibv_qp_init_attr.cap.max_inline_data,
 		m_max_inline_data, tmp_ibv_qp_attr.cap.max_send_wr,
 		tmp_ibv_qp_attr.cap.max_recv_wr, tmp_ibv_qp_attr.cap.max_recv_sge,
 		tmp_ibv_qp_attr.cap.max_send_sge);
-#endif /* DEFINED_TSO */
 
 	return 0;
 }

src/vma/dev/qp_mgr.h

@@ -74,17 +74,7 @@ friend class cq_mgr_mp;
 	virtual void        post_recv_buffer(mem_buf_desc_t* p_mem_buf_desc); // Post for receive single mem_buf_desc
 	void                post_recv_buffers(descq_t* p_buffers, size_t count); // Post for receive a list of mem_buf_desc
 	int                 send(vma_ibv_send_wr* p_send_wqe, vma_wr_tx_packet_attr attr);
-
-#ifdef DEFINED_TSO
-	inline uint32_t     get_max_inline_data() const {
-		return m_qp_cap.max_inline_data;
-	}
-	inline uint32_t     get_max_send_sge() const {
-		return m_qp_cap.max_send_sge;
-	}
-#else
 	uint32_t            get_max_inline_data() const {return m_max_inline_data; }
-#endif /* DEFINED_TSO */
 	int                 get_port_num() const { return m_port_num; }
 	virtual uint16_t    get_partiton() const { return 0; };
 	virtual uint32_t    get_underly_qpn() const { return 0; };
@@ -107,29 +97,20 @@ friend class cq_mgr_mp;
 	static inline bool  is_lib_mlx5(const char* device_name) {return strstr(device_name, "mlx5");}
 	virtual void        dm_release_data(mem_buf_desc_t* buff) { NOT_IN_USE(buff); }
 	virtual bool        fill_hw_descriptors(vma_mlx_hw_device_data &data) {NOT_IN_USE(data);return false;};
+
 protected:
 	struct ibv_qp*      m_qp;
 	uint64_t*           m_rq_wqe_idx_to_wrid;
-
 	ring_simple*        m_p_ring;
 	uint8_t             m_port_num;
 	ib_ctx_handler*     m_p_ib_ctx_handler;
-
-#ifdef DEFINED_TSO
-	struct ibv_qp_cap   m_qp_cap;
-#else
 	uint32_t            m_max_inline_data;
-#endif /* DEFINED_TSO */
 	uint32_t            m_max_qp_wr;
-
 	cq_mgr*             m_p_cq_mgr_rx;
 	cq_mgr*             m_p_cq_mgr_tx;
-
 	uint32_t            m_rx_num_wr;
 	uint32_t            m_tx_num_wr;
-
 	bool                m_hw_dummy_send_support;
-
 	uint32_t            m_n_sysvar_rx_num_wr_to_post_recv;
 	const uint32_t      m_n_sysvar_tx_num_wr_to_signal;
 	const uint32_t      m_n_sysvar_rx_prefetch_bytes_before_poll;
@@ -143,7 +124,6 @@ friend class cq_mgr_mp;
 	// send wr
 	uint32_t            m_n_unsignaled_count;
 	mem_buf_desc_t*     m_p_last_tx_mem_buf_desc; // Remembered so we can list several mem_buf_desc_t on a single notification request
-
 	mem_buf_desc_t*     m_p_prev_rx_desc_pushed;
 
 	// generating packet IDs