Add RCCL_P2P_BATCH_ENABLE parameter to multinode

rccl tests.

- Added nccl_p2p_batch_enable test parameter with values ["0", "1"] to rccl_multinode_cvs.py
- Updated rccl_lib.py to include the parameter in MPI command execution
- Added parameter to rccl_config.json configuration file
- Doubles test combinations to cover P2P batching enable/disable cases.

Signed-off-by: Ignatious Johnson <[email protected]>

Author: cijohnson

input/config_file/rccl/rccl_config.json

@@ -41,6 +41,7 @@
        "nccl_ib_tc": "0",
        "nccl_ib_split_data_on_qps": "0",
        "nccl_pxn_disable": [ "0", "1" ],
+       "nccl_p2p_batch_enable": [ "0", "1" ],
        "nccl_net_plugin": "none",
        "verify_bus_bw": "False",
        "verify_bw_dip": "True",

lib/rccl_lib.py

@@ -312,7 +312,7 @@ def rccl_cluster_test( phdl, shdl, test_name, cluster_node_list, vpc_node_list,
         ib_rx_queue_len=8192, ucx_tls='tcp', hcoll_enable_mcast_all=0, \
         nccl_cumem_enable=0, nccl_ib_timeout=30, nccl_ib_sl=0, \
         nccl_ib_tc=41, nccl_ib_split_data_on_qps=0, nccl_pxn_disable=1, \
-        nccl_net_plugin=None, user_password=None, \
+        nccl_p2p_batch_enable=1, nccl_net_plugin=None, user_password=None, \
         min_channels=64, max_channels=64, \
         data_type="float", \
         user_key_file=None, verify_bus_bw=False, \
@@ -418,6 +418,7 @@ def rccl_cluster_test( phdl, shdl, test_name, cluster_node_list, vpc_node_list,
         -x NCCL_IB_TC={nccl_ib_tc} \
         -x NCCL_IB_SPLIT_DATA_ON_QPS={nccl_ib_split_data_on_qps} \
         -x NCCL_PXN_DISABLE={nccl_pxn_disable} \
+        -x RCCL_P2P_BATCH_ENABLE={nccl_p2p_batch_enable} \
         -x NCCL_NET_PLUGIN={nccl_net_plugin} \
         {RCCL_TESTS_INSTALL_DIR}/{test_name} -b {start_msg_size} -e {end_msg_size} -f {step_function} \
         -g {threads_per_gpu} -c {check_iteration_count} -w {warmup_iterations} \
@@ -482,7 +483,7 @@ def rccl_cluster_test_default( phdl, shdl, test_name, cluster_node_list, vpc_nod
         ib_rx_queue_len=8192, ucx_tls='tcp', hcoll_enable_mcast_all=0, \
         nccl_cumem_enable=0, nccl_ib_timeout=30, nccl_ib_sl=0, \
         nccl_ib_tc=41, nccl_ib_split_data_on_qps=0, nccl_pxn_disable=1, \
-        nccl_net_plugin=None, user_password=None, \
+        nccl_p2p_batch_enable=1, nccl_net_plugin=None, user_password=None, \
         min_channels=64, max_channels=64, \
         user_key_file=None, verify_bus_bw=False, \
         verify_bw_dip=True, verify_lat_dip=True, exp_results_dict=None ):
@@ -584,6 +585,7 @@ def rccl_cluster_test_default( phdl, shdl, test_name, cluster_node_list, vpc_nod
         --mca btl_tcp_if_exclude lo,docker0,usb0 \
         -x UCX_NET_DEVICES={net_dev_list} \
         -x UCX_TLS={ucx_tls} \
+        -x RCCL_P2P_BATCH_ENABLE={nccl_p2p_batch_enable} \
         -x NCCL_NET_PLUGIN={nccl_net_plugin} \
         {RCCL_TESTS_INSTALL_DIR}/{test_name} -b {start_msg_size} -e {end_msg_size} -f {step_function} \
         -g {threads_per_gpu} -c {check_iteration_count} -w {warmup_iterations} \

tests/rccl/rccl_multinode_cvs.py

@@ -242,9 +242,10 @@ def pytest_generate_tests(metafunc):
     rccl_protocol_list = rccl.get("rccl_protocol", ["simple", "LL128", "LL"])
     qp_scale_list = rccl.get("qp_scale", ["1", "2"])
     nccl_pxn_disable_list = rccl.get("nccl_pxn_disable", [ "1", "0" ])
+    nccl_p2p_batch_enable_list = rccl.get("nccl_p2p_batch_enable", [ "1", "0" ])
 
     # Only parametrize fixtures used by this test
-    all_keys = ("rccl_collective", "rccl_algo", "rccl_protocol", "qp_scale", "nccl_pxn_disable")
+    all_keys = ("rccl_collective", "rccl_algo", "rccl_protocol", "qp_scale", "nccl_pxn_disable", "nccl_p2p_batch_enable")
 
     active = [k for k in all_keys if k in metafunc.fixturenames]
     if not active:
@@ -256,6 +257,7 @@ def pytest_generate_tests(metafunc):
         "rccl_protocol": rccl_protocol_list,
         "qp_scale": qp_scale_list,
         "nccl_pxn_disable": nccl_pxn_disable_list,
+        "nccl_p2p_batch_enable": nccl_p2p_batch_enable_list,
     }
     domains = [domain_by_key[k] for k in active]
 
@@ -329,7 +331,7 @@ def test_disable_firewall( phdl ):
 
 
 def test_rccl_perf(phdl, shdl, cluster_dict, config_dict, rccl_collective, rccl_algo, \
-       rccl_protocol, qp_scale, nccl_pxn_disable ):
+       rccl_protocol, qp_scale, nccl_pxn_disable, nccl_p2p_batch_enable ):
 
     """
     Execute RCCL performance test across the cluster with given parameters.
@@ -422,6 +424,7 @@ def test_rccl_perf(phdl, shdl, cluster_dict, config_dict, rccl_collective, rccl_
        nccl_ib_tc              = config_dict['nccl_ib_tc'], \
        nccl_ib_split_data_on_qps  = config_dict['nccl_ib_split_data_on_qps'], \
        nccl_pxn_disable        = nccl_pxn_disable, \
+       nccl_p2p_batch_enable   = nccl_p2p_batch_enable, \
        nccl_net_plugin         = config_dict['nccl_net_plugin'], \
        user_key_file           = cluster_dict['priv_key_file'], \
        verify_bus_bw           = config_dict['verify_bus_bw'], \
@@ -432,7 +435,7 @@ def test_rccl_perf(phdl, shdl, cluster_dict, config_dict, rccl_collective, rccl_
 
 
     print(result_dict)
-    key_name = f'{rccl_collective}-{rccl_algo}-{rccl_protocol}-{qp_scale}-{nccl_pxn_disable}'
+    key_name = f'{rccl_collective}-{rccl_algo}-{rccl_protocol}-{qp_scale}-{nccl_pxn_disable}-{nccl_p2p_batch_enable}'
     rccl_res_dict[key_name] = result_dict
 
     # Scan dmesg between start and end times cluster wide ..