Merge pull request #1515 from ROCm/hotfix-swdev-494236

limits OPENMP threading in clients (#2753)

Author: vamovsik

clients/benchmarks/client.cpp

@@ -1360,6 +1360,8 @@ void fix_batch(int argc, char* argv[])
 int main(int argc, char* argv[])
 try
 {
+    rocblas_client_init();
+
     fix_batch(argc, argv);
     Arguments   arg;
     std::string function;
@@ -1844,6 +1846,8 @@ try
     int status = 0;
     // TODO: query for any failed tests
 
+    rocblas_client_shutdown();
+
     return status;
 }
 catch(const std::invalid_argument& exp)

clients/common/client_utility.cpp

@@ -35,6 +35,10 @@
 #include <new>
 #include <stdexcept>
 #include <stdlib.h>
+#include <thread>
+#ifdef _OPENMP
+#include <omp.h>
+#endif
 
 #ifdef WIN32
 #define strcasecmp(A, B) _stricmp(A, B)
@@ -54,6 +58,33 @@ namespace fs = std::experimental::filesystem;
 #include <fcntl.h>
 #endif
 
+void rocblas_client_init()
+{
+    // limit OMP usage as deadlock issues seen in reference library
+#ifdef _OPENMP
+    const int processor_count = std::thread::hardware_concurrency();
+    if(processor_count > 0)
+    {
+        const int omp_current_threads = omp_get_max_threads();
+        if(omp_current_threads >= processor_count)
+        {
+            int limiter           = processor_count > 4 ? processor_count - 2 : processor_count;
+            int omp_limit_threads = std::max(1, limiter);
+
+            if(omp_limit_threads != omp_current_threads)
+            {
+                omp_set_num_threads(omp_limit_threads);
+
+                rocblas_cerr << "rocBLAS info: client (OPENMP) reduced omp_set_num_threads to "
+                             << omp_limit_threads << std::endl;
+            }
+        }
+    }
+#endif
+}
+
+void rocblas_client_shutdown() {}
+
 /* ============================================================================================ */
 // Return path of this executable
 std::string rocblas_exepath()

clients/gtest/rocblas_gtest_main.cpp

@@ -284,6 +284,8 @@ static void rocblas_set_test_device()
  *****************/
 int main(int argc, char** argv)
 {
+    rocblas_client_init();
+
     std::string args = rocblas_capture_args(argc, argv);
 
     auto* no_signal_handling = getenv("ROCBLAS_TEST_NO_SIGACTION");
@@ -333,5 +335,7 @@ int main(int argc, char** argv)
 
     //rocblas_shutdown();
 
+    rocblas_client_shutdown();
+
     return status;
 }

clients/include/client_utility.hpp

@@ -88,6 +88,10 @@
 
 #define NOOP (void)0
 
+// general global initializations
+void rocblas_client_init();
+void rocblas_client_shutdown();
+
 /*!
  * Initialize rocBLAS for the requested number of  HIP devices
  * and report the time taken to complete the initialization.