prov/efa: Introduce ep option FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV

include/rdma/fi_ext.h

@@ -77,6 +77,7 @@ enum {
 	FI_OPT_EFA_SENDRECV_IN_ORDER_ALIGNED_128_BYTES, /* bool */
 	FI_OPT_EFA_WRITE_IN_ORDER_ALIGNED_128_BYTES, /* bool */
 	FI_OPT_EFA_HOMOGENEOUS_PEERS,   /* bool */
+	FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV,     /* bool */
 };
 
 struct fi_fid_export {

man/fi_efa.7.md

@@ -179,6 +179,13 @@ provider for AWS Neuron or Habana SynapseAI.
   is kicked off, due to a current device limitation.
   The default value is false.
 
+*FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV - bool*
+: This option only applies to the fi_setopt() call.
+  It is used to disable unsolicited write recv for this endpoint, which can reduce
+  the likelihood of CQ overflow. The default value is true.
+  For efa-direct, FI_RX_CQ_DATA is required when FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV
+  is false, or it will return -FI_EOPNOTSUPP for the call to fi_setopt().
+
 # PROVIDER SPECIFIC DOMAIN OPS
 The efa provider exports extensions for operations
 that are not provided by the standard libfabric interface. These extensions

prov/efa/src/efa_base_ep.c

@@ -341,8 +341,17 @@ static int efa_base_ep_create_qp(struct efa_base_ep *base_ep,
 
 	assert(tx_cq->unsolicited_write_recv_enabled == rx_cq->unsolicited_write_recv_enabled);
 
-	/* If user intend to post rx buffer for cq data, we shouldn't enable unsolicited write recv */
-	use_unsolicited_write_recv = tx_cq->unsolicited_write_recv_enabled && !(base_ep->info->mode & FI_RX_CQ_DATA);
+	if (EFA_INFO_TYPE_IS_DIRECT(base_ep->info)) {
+		/* If user intend to post rx buffer for cq data, we shouldn't
+		 * enable unsolicited write recv */
+		use_unsolicited_write_recv =
+			tx_cq->unsolicited_write_recv_enabled && !(base_ep->info->mode & FI_RX_CQ_DATA);
+	} else {
+		/* RDM full protocol doesn't support FI_RX_CQ_DATA. 
+		 * Set FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV to false to disable unsolicited write recv. */
+		use_unsolicited_write_recv =
+			tx_cq->unsolicited_write_recv_enabled && base_ep->use_unsolicited_write_recv;
+	}
 	EFA_INFO(FI_LOG_EP_CTRL, "creating QP with unsolicited write recv status: %d\n", use_unsolicited_write_recv);
 	ret = efa_qp_create(&base_ep->qp, &attr_ex, base_ep->info->tx_attr->tclass,
 			    use_unsolicited_write_recv);
@@ -517,6 +526,7 @@ int efa_base_ep_construct(struct efa_base_ep *base_ep,
 	/* TODO: update inject_rma_size to inline size after firmware
 	 * supports inline rdma write */
 	base_ep->inject_rma_size = 0;
+	base_ep->use_unsolicited_write_recv = true;
 	return 0;
 }
 

prov/efa/src/efa_base_ep.h

@@ -93,6 +93,7 @@ struct efa_base_ep {
 	/* Only used by RDM ep type */
 	struct efa_qp *user_recv_qp; /* Separate qp to receive pkts posted by users */
 	struct efa_recv_wr *user_recv_wr_vec;
+	bool use_unsolicited_write_recv;
 };
 
 int efa_base_ep_bind_av(struct efa_base_ep *base_ep, struct efa_av *av);

prov/efa/src/efa_ep.c

@@ -171,6 +171,17 @@ static int efa_ep_setopt(fid_t fid, int level, int optname, const void *optval,
 	/* no op as efa direct ep will not handshake with peers */
 	case FI_OPT_EFA_HOMOGENEOUS_PEERS:
 		break;
+	case FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV:
+		if (optlen != sizeof(bool))
+			return -FI_EINVAL;
+		if (!(ep->info->mode & FI_RX_CQ_DATA) && !*(bool *)optval) {
+			EFA_WARN(FI_LOG_EP_CTRL,
+				 "FI_RX_CQ_DATA is required when unsolicited "
+				 "write recv is disabled.\n");
+			return -FI_EOPNOTSUPP;
+		}
+		ep->use_unsolicited_write_recv = *(bool *)optval;
+		break;
 	default:
 		EFA_INFO(FI_LOG_EP_CTRL, "Unknown / unsupported endpoint option\n");
 		return -FI_ENOPROTOOPT;

prov/efa/src/rdm/efa_rdm_ep_fiops.c

@@ -1748,6 +1748,11 @@ static int efa_rdm_ep_setopt(fid_t fid, int level, int optname,
 			return -FI_EINVAL;
 		efa_rdm_ep->homogeneous_peers = *(bool *)optval;
 		break;
+	case FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV:
+		if (optlen != sizeof(bool))
+			return -FI_EINVAL;
+		efa_rdm_ep->base_ep.use_unsolicited_write_recv = *(bool *)optval;
+		break;
 	default:
 		EFA_INFO(FI_LOG_EP_CTRL, "Unknown endpoint option\n");
 		return -FI_ENOPROTOOPT;

prov/efa/test/efa_unit_test_ep.c

@@ -1877,3 +1877,72 @@ void test_efa_base_ep_disable_unsolicited_write_recv_with_rx_cq_data(struct efa_
 	/* When FI_RX_CQ_DATA is set, unsolicited write recv should be disabled */
 	assert_false(efa_base_ep->qp->unsolicited_write_recv_enabled);
 }
+
+/**
+ * @brief Test that unsolicited write recv is disabled when FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV is false
+ */
+void test_efa_rdm_ep_setopt_cq_flow_control(struct efa_resource **state)
+{
+	struct efa_resource *resource = *state;
+	struct efa_rdm_ep *ep;
+	bool optval = false;
+
+	efa_unit_test_resource_construct_ep_not_enabled(resource, FI_EP_RDM, EFA_FABRIC_NAME);
+
+	ep = container_of(resource->ep, struct efa_rdm_ep,
+		base_ep.util_ep.ep_fid);
+	assert_true(ep->base_ep.use_unsolicited_write_recv);
+	assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT,
+				   FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV, &optval,
+				   sizeof(optval)),
+			 FI_SUCCESS);
+	assert_false(ep->base_ep.use_unsolicited_write_recv);
+	assert_int_equal(fi_enable(resource->ep), 0);
+	assert_false(ep->base_ep.qp->unsolicited_write_recv_enabled);
+}
+
+/**
+ * @brief Test disabling FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV will fail without FI_RX_CQ_DATA in efa direct
+ */
+void test_efa_direct_ep_setopt_cq_flow_control_no_rx_cq_data(struct efa_resource **state)
+{
+	struct efa_resource *resource = *state;
+	struct efa_base_ep *efa_base_ep;
+	bool optval = false;
+
+	efa_unit_test_resource_construct_ep_not_enabled(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
+	assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT,
+				   FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV, &optval,
+				   sizeof(optval)),
+			 -FI_EOPNOTSUPP);
+	efa_base_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
+	assert_true(efa_base_ep->use_unsolicited_write_recv);
+}
+
+/**
+ * @brief Test setting FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV with FI_RX_CQ_DATA will disable unsolicited write recv
+ */
+void test_efa_direct_ep_setopt_cq_flow_control_with_rx_cq_data(struct efa_resource **state)
+{
+	struct efa_resource *resource = *state;
+	struct efa_base_ep *efa_base_ep;
+	bool optval = false;
+
+	resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
+	assert_non_null(resource->hints);
+
+	resource->hints->mode |= FI_RX_CQ_DATA;
+
+	efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 18),
+	                                            resource->hints, false, true);
+
+	efa_base_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
+	assert_true(efa_base_ep->use_unsolicited_write_recv);
+	assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT,
+				   FI_OPT_EFA_USE_UNSOLICITED_WRITE_RECV, &optval,
+				   sizeof(optval)),
+			 FI_SUCCESS);
+	assert_false(efa_base_ep->use_unsolicited_write_recv);
+	assert_int_equal(fi_enable(resource->ep), 0);
+	assert_false(efa_base_ep->qp->unsolicited_write_recv_enabled);
+}

prov/efa/test/efa_unit_tests.c

@@ -198,6 +198,9 @@ int main(void)
 		cmocka_unit_test_setup_teardown(test_efa_ep_lock_type_mutex, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_shm_ep_different_info, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_base_ep_disable_unsolicited_write_recv_with_rx_cq_data, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_setopt_cq_flow_control, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_efa_direct_ep_setopt_cq_flow_control_no_rx_cq_data, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_efa_direct_ep_setopt_cq_flow_control_with_rx_cq_data, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		/* end efa_unit_test_ep.c */
 
 		/* begin efa_unit_test_cq.c */

prov/efa/test/efa_unit_tests.h

@@ -409,6 +409,9 @@ void test_efa_ep_lock_type_no_op();
 void test_efa_ep_lock_type_mutex();
 void test_efa_rdm_ep_shm_ep_different_info();
 void test_efa_base_ep_disable_unsolicited_write_recv_with_rx_cq_data();
+void test_efa_rdm_ep_setopt_cq_flow_control();
+void test_efa_direct_ep_setopt_cq_flow_control_no_rx_cq_data();
+void test_efa_direct_ep_setopt_cq_flow_control_with_rx_cq_data();
 
 /* begin efa_unit_test_data_path_direct.c */
 void test_efa_data_path_direct_rdma_read_multiple_sge_fail();