From 231cf78b3b2092a02eef03b0c74f92a8ce290749 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Wed, 24 Jun 2026 20:04:20 +0000 Subject: [PATCH] sync_daemon_mpi: give daemon a distinct PMIX_NAMESPACE The MPI sync daemon shares the traced application's inherited PMIx identity (PMIX_NAMESPACE). Two MPI instances claiming the same (namespace, rank) collide in the node-local SHM bootstrap and deadlock (n=128 --ppn 64) or abort with a truncation error (n=4). Append a fixed suffix to PMIX_NAMESPACE before any MPI call so the daemon's session gets its own WORLD. Verified on 2 nodes with THAPI_SYNC_DAEMON=mpi: fix -> exit 0, full tally; control (fix disabled) -> hang. Default remains fs pending MPICH feedback. Co-Authored-By: Claude Opus 4.8 --- xprof/sync_daemon_mpi.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/xprof/sync_daemon_mpi.cpp b/xprof/sync_daemon_mpi.cpp index 05a33060..18a06c17 100644 --- a/xprof/sync_daemon_mpi.cpp +++ b/xprof/sync_daemon_mpi.cpp @@ -2,6 +2,7 @@ #include "mpi.h" #include #include +#include using namespace daemon_proto; @@ -125,6 +126,10 @@ int main(int argc, char **argv) { } const int fd = atoi(argv[1]); + // WA for MPI_Session bug when running concurrently with MPI apps. + if (const char *ns = getenv("PMIX_NAMESPACE")) + setenv("PMIX_NAMESPACE", (std::string(ns) + "_thapi_sync_daemon").c_str(), 1); + CHECK_MPI(MPIX_Init_Session(&lib_shandle, &MPI_COMM_WORLD_THAPI)); CHECK_MPI(MPI_Comm_split_type(MPI_COMM_WORLD_THAPI, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &MPI_COMM_NODE));