issue: Start event handler thread during XLIO init

Event handler manager is a global object with a single internal thread.
Once XLIO is initialized, there is no option not to start the internal
thread and it's used starting from the initialization method (to create
a timer event for netlink).

Therefore, there is no point in trying to start the thread with each
post action to the event handler. This can save CPU in CPS scenario.

Signed-off-by: Dmytro Podgornyi <[email protected]>

Author: pasis

src/core/event/event_handler_manager.cpp

@@ -344,56 +344,53 @@ int event_handler_manager::start_thread()
 {
     cpu_set_t cpu_set;
     pthread_attr_t tattr;
+    int ret;
+    bool affinity_requested = false;
 
     if (!m_b_continue_running) {
+        errno = ECANCELED;
         return -1;
     }
-
     if (m_event_handler_tid != 0) {
         return 0;
     }
 
-    // m_reg_action_q.reserve(); //todo change to vector and reserve
-
-    BULLSEYE_EXCLUDE_BLOCK_START
-    if (pthread_attr_init(&tattr)) {
-        evh_logpanic("Failed to initialize thread attributes");
+    ret = pthread_attr_init(&tattr);
+    if (ret != 0) {
+        return -1;
     }
-    BULLSEYE_EXCLUDE_BLOCK_END
 
     cpu_set = safe_mce_sys().internal_thread_affinity;
     if (strcmp(safe_mce_sys().internal_thread_affinity_str, "-1") &&
-        !strcmp(safe_mce_sys().internal_thread_cpuset,
-                MCE_DEFAULT_INTERNAL_THREAD_CPUSET)) { // no affinity
-        BULLSEYE_EXCLUDE_BLOCK_START
-        if (pthread_attr_setaffinity_np(&tattr, sizeof(cpu_set), &cpu_set)) {
-            evh_logpanic("Failed to set CPU affinity");
+        !strcmp(safe_mce_sys().internal_thread_cpuset, MCE_DEFAULT_INTERNAL_THREAD_CPUSET)) {
+        ret = pthread_attr_setaffinity_np(&tattr, sizeof(cpu_set), &cpu_set);
+        if (ret != 0) {
+            evh_logwarn("Failed to set event handler thread affinity. (errno=%d)", errno);
         }
-        BULLSEYE_EXCLUDE_BLOCK_END
+        affinity_requested = (ret == 0);
     } else {
         evh_logdbg("Internal thread affinity not set.");
     }
 
-    int ret = pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this);
-    if (ret) {
-        // maybe it's the cset issue? try without affinity
-        evh_logwarn("Failed to start event handler thread with thread affinity - trying without. "
-                    "[errno=%d %s]",
-                    ret, strerror(ret));
-        BULLSEYE_EXCLUDE_BLOCK_START
-        if (pthread_attr_init(&tattr)) {
-            evh_logpanic("Failed to initialize thread attributes");
-        }
-        if (pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this)) {
-            evh_logpanic("Failed to start event handler thread");
-        }
-        BULLSEYE_EXCLUDE_BLOCK_END
+    ret = pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this);
+    if (ret != 0 && affinity_requested) {
+        // Try without affinity in case this is a cset issue.
+        evh_logwarn("Failed to start event handler thread with a thread affinity. Trying default "
+                    "thread affinity. (errno=%d)",
+                    errno);
+        pthread_attr_destroy(&tattr);
+        ret = pthread_attr_init(&tattr)
+            ?: pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this);
     }
-
+    // Destroy will either succeed or return EINVAL if the init fails in the above block.
     pthread_attr_destroy(&tattr);
 
-    evh_logdbg("Started event handler thread");
-    return 0;
+    if (ret == 0) {
+        evh_logdbg("Started event handler thread.");
+    } else {
+        evh_logerr("Failed to start event handler thread. (errno=%d)", errno);
+    }
+    return ret;
 }
 
 void event_handler_manager::stop_thread()
@@ -483,15 +480,11 @@ void event_handler_manager::post_new_reg_action(reg_action_t &reg_action)
         return;
     }
 
-    start_thread();
-
     evh_logfunc("add event action %s (%d)", reg_action_str(reg_action.type), reg_action.type);
 
-    bool is_empty = false;
+    bool is_empty;
     m_reg_action_q_lock.lock();
-    if (m_p_reg_action_q_to_push_to->empty()) {
-        is_empty = true;
-    }
+    is_empty = m_p_reg_action_q_to_push_to->empty();
     m_p_reg_action_q_to_push_to->push_back(reg_action);
     m_reg_action_q_lock.unlock();
     if (is_empty) {

src/core/event/event_handler_manager.h

@@ -208,6 +208,7 @@ class event_handler_manager : public wakeup_pipe {
     void register_command_event(int fd, command *cmd);
 
     void *thread_loop();
+    int start_thread();
     void stop_thread();
     bool is_running() { return m_b_continue_running; };
 
@@ -251,7 +252,6 @@ class event_handler_manager : public wakeup_pipe {
     void handle_registration_action(reg_action_t &reg_action);
     void process_ibverbs_event(event_handler_map_t::iterator &i);
     void process_rdma_cm_event(event_handler_map_t::iterator &i);
-    int start_thread();
 
     void event_channel_post_process_for_rdma_events(void *p_event);
     void *event_channel_pre_process_for_rdma_events(void *p_event_channel_handle, void **p_event);

src/core/main.cpp

@@ -983,6 +983,7 @@ static void do_global_ctors_helper()
 {
     static lock_spin_recursive g_globals_lock;
     std::lock_guard<decltype(g_globals_lock)> lock(g_globals_lock);
+    int rc;
 
     if (g_init_global_ctors_done) {
         return;
@@ -1018,6 +1019,12 @@ static void do_global_ctors_helper()
 
     // Create all global management objects
     NEW_CTOR(g_p_event_handler_manager, event_handler_manager());
+    rc = g_p_event_handler_manager->start_thread();
+    if (rc != 0) {
+        BULLSEYE_EXCLUDE_BLOCK_START
+        throw_xlio_exception("Failed to start event handler thread.\n");
+        BULLSEYE_EXCLUDE_BLOCK_END
+    }
 
     xlio_shmem_stats_open(&g_p_vlogger_level, &g_p_vlogger_details);
     *g_p_vlogger_level = g_vlogger_level;