launchmon: Add IBM Spectrum and PowerLE support

Adjust for OpenPower ABI functional call convension
change: The ABI adds two different entry points, one
for intramodule call and the other intermodule call
(being done through TOC).

Port for IBM Spectrum jsrun. Add new spectrum definition
and also adjust back end deamon code as to how it should
synchronize itself with Spectrum MPI target.

Adjust how to handle new thread creation. When a new thread
is created, we will get the SIGTRAP notification via waitpid
on the parent thread that spawns a new thread.
But on a recent Linux kernal, we have to check the high
order bits of waitpid returned status as:

leftshift status by 8 bits and then to see if
it is equal to (SIGTRAP | (LINUX_TRACER_EVENT_CLONE << 8).

Add a work around for jsrun's broken MPIR_attach_fifo.
It is expecting ASCII 1 to be sent to the FIFO when
it should expect a numeric 1. Need to drop it when
IBM ultimately fixes the issue. (Bug filed to IBM).

Author: Dong H. Ahn

launchmon/src/linux/lmon_api/lmon_be_sync_mpi.cxx

@@ -116,6 +116,7 @@ lmon_rc_e LMON_be_procctl_init(rm_catalogue_e rmtype, MPIR_PROCDESC_EXT *ptab,
     case RC_cray:
     case RC_gupc:
     case RC_mpiexec_hydra:
+    case RC_ibm_spectrum:
       //
       // Call generic Linux init
       //
@@ -170,6 +171,7 @@ lmon_rc_e LMON_be_procctl_stop(rm_catalogue_e rmtype, MPIR_PROCDESC_EXT *ptab,
     case RC_cray:
     case RC_gupc:
     case RC_mpiexec_hydra:
+    case RC_ibm_spectrum:
       //
       // Call generic Linux stop
       //
@@ -230,6 +232,7 @@ lmon_rc_e LMON_be_procctl_run(rm_catalogue_e rmtype, int signum,
     case RC_cray:
     case RC_gupc:
     case RC_mpiexec_hydra:
+    case RC_ibm_spectrum:
       //
       // Call generic Linux run
       //
@@ -292,6 +295,7 @@ lmon_rc_e LMON_be_procctl_initdone(rm_catalogue_e rmtype,
     case RC_cray:
     case RC_gupc:
     case RC_mpiexec_hydra:
+    case RC_ibm_spectrum:
       //
       // Call generic Linux initdone
       //
@@ -346,6 +350,7 @@ lmon_rc_e LMON_be_procctl_done(rm_catalogue_e rmtype, MPIR_PROCDESC_EXT *ptab,
     case RC_cray:
     case RC_gupc:
     case RC_mpiexec_hydra:
+    case RC_ibm_spectrum:
       //
       // You need to do nothing for these resource managers
       //
@@ -402,6 +407,7 @@ lmon_rc_e LMON_be_procctl_perf(rm_catalogue_e rmtype, MPIR_PROCDESC_EXT *ptab,
     case RC_cray:
     case RC_gupc:
     case RC_mpiexec_hydra:
+    case RC_ibm_spectrum:
       //
       // You need to do nothing for these resource managers
       //

launchmon/src/linux/main.cxx

@@ -54,7 +54,7 @@ int main(int argc, char* argv[]) {
   try {
     int rc = EXIT_FAILURE;
 #if X86_ARCHITECTURE || X86_64_ARCHITECTURE || PPC_ARCHITECTURE || \
-    AARCH64_ARCHITECTURE
+    AARCH64_ARCHITECTURE || POWERLE_ARCHITECTURE
     //
     // driver instantiation for the linux platform.
     //

launchmon/src/linux/sdbg_linux_driver_impl.hxx

@@ -91,7 +91,7 @@ linux_driver_t<LINUX_DRIVER_TEMPLPARAM>::create_process(pid_t pid,
 
 #if X86_ARCHITECTURE || X86_64_ARCHITECTURE
   return_proc = new linux_x86_process_t(pid, mi, md, mt, mc);
-#elif PPC_ARCHITECTURE
+#elif PPC_ARCHITECTURE || POWERLE_ARCHITECTURE
   return_proc = new linux_ppc_process_t(pid, mi, md, mt, mc);
 #elif IA64_ARCHITECTURE
   return_proc = new linux_ia64_process_t(pid, mi, md, mt, mc);
@@ -122,7 +122,7 @@ linux_driver_t<LINUX_DRIVER_TEMPLPARAM>::create_process(pid_t pid,
 //
 #if X86_ARCHITECTURE || X86_64_ARCHITECTURE
   return_proc = new linux_x86_process_t(pid, mi);
-#elif PPC_ARCHITECTURE
+#elif PPC_ARCHITECTURE || POWERLE_ARCHITECTURE
   return_proc = new linux_ppc_process_t(pid, mi);
 #elif IA64_ARCHITECTURE
   return_proc = new linux_ia64_process_t(pid, mi);

launchmon/src/linux/sdbg_linux_launchmon.cxx

@@ -1055,16 +1055,16 @@ bool linux_launchmon_t::handle_mpir_variables(
         p.set_launch_hidden_bp(NULL);
       }
       la_bp = new linux_breakpoint_t();
-      la_bp->set_address_at(launch_bp_sym.get_relocated_address());
+      la_bp->set_address_at(launch_bp_sym.get_relocated_lowest_address());
 
 #if PPC_ARCHITECTURE
       //
       // DHA Mar 05 2009
       // PowerPC Linux has begun to change the linking convention
       // such that binaries no longer export direct function
       // symbols. (e.g., .MPIR_Breakpoint). But rather, undotted
-      // global data symbols (e.g., MPIR_Breakpoint) contains the
-      // address for the corresponding function.
+      // global data symbols (e.g., MPIR_Breakpoint) is the function
+      // descriptor
       //
       // Added indirect breakpoint support for that and use this
       // method on all PPC systems across the board including
@@ -1486,16 +1486,15 @@ launchmon_event_e linux_launchmon_t::decipher_an_event(
         //
         // Parent gets SIGTRAP when a new thread is created
         // Used to be: return_ev = LM_STOP_NOT_INTERESTED;
-        int upper16;
-        upper16 = event.get_rawstatus() >> 16;
-        if (upper16 == LINUX_TRACER_EVENT_CLONE) {
+        int high = event.get_rawstatus() >> 8;
+        if (high == (SIGTRAP | (LINUX_TRACER_EVENT_CLONE << 8))) {
           return_ev = LM_STOP_AT_THREAD_CREATION;
-        } else {
-          //
-          // SIGTRAP due to fork for example
-          //
-          return_ev = LM_STOP_NOT_INTERESTED;
-        }
+         } else {
+           //
+           // SIGTRAP due to fork for example
+           //
+           return_ev = LM_STOP_NOT_INTERESTED;
+         }
       } else if (event.get_signum() == SIGSTOP) {
         return_ev = LM_RELAY_SIGNAL;
 
@@ -1578,12 +1577,13 @@ launchmon_rc_e linux_launchmon_t::handle_trap_after_attach_event(
 
 #if MEASURE_TRACING_COST
     beginTS = gettimeofdayD();
+#endif
+
     {
-      self_trace_t::trace(true,  // print always
+      self_trace_t::trace(LEVELCHK(level2),
                           MODULENAME, 0,
                           "The RM process has just been trapped due to attach");
     }
-#endif
 
     bool use_cxt = true;
     image_base_t<T_VA, elf_wrapper> *dynloader_im = NULL;
@@ -1649,7 +1649,7 @@ launchmon_rc_e linux_launchmon_t::handle_trap_after_attach_event(
         dynloader_im->get_a_symbol(p.get_loader_breakpoint_sym());
 
     lo_bp = new linux_breakpoint_t();
-    addr_dl_bp = dynload_sym.get_relocated_address();
+    addr_dl_bp = dynload_sym.get_relocated_lowest_address();
     lo_bp->set_address_at(addr_dl_bp);
 #if PPC_ARCHITECTURE
     lo_bp->set_use_indirection();
@@ -1682,7 +1682,14 @@ launchmon_rc_e linux_launchmon_t::handle_trap_after_attach_event(
           get_tracer()->tracer_continue(p, use_cxt);
           int fifofd = 0;
           if ((fifofd = open(fifopathbuf, O_WRONLY)) >= 0) {
+#if POWERLE_ARCHITECTURE
+            char wakeup = '1';
+            self_trace_t::trace(
+                  true, MODULENAME, 0,
+                  "Warning: Sending ASCII 1 to FIFO to work around a jsrun bug");
+#else
             char wakeup = (char)1;
+#endif
             if (lmon_write_raw(fifofd, &wakeup, 1) != 1) {
               self_trace_t::trace(
                   true, MODULENAME, 0,
@@ -1781,9 +1788,9 @@ launchmon_rc_e linux_launchmon_t::handle_trap_after_exec_event(
 #endif
 
     {
-      self_trace_t::trace(true,  // print always
-                          MODULENAME, 0,
-                          "The RM process has just been forked and exec'ed.");
+      self_trace_t::trace(LEVELCHK(level2),
+                          MODULENAME, 0, "The RM process (%d) has "
+                          "just been forked and exec'ed.", p.get_pid (true));
     }
 
     bool use_cxt = true;
@@ -1830,7 +1837,7 @@ launchmon_rc_e linux_launchmon_t::handle_trap_after_exec_event(
 //
 // Corner case; we deal with a heuristics
 //
-#if PPC_ARCHITECTURE
+#if PPC_ARCHITECTURE || POWERLE_ARCHITECTURE
       //
       // DHA Mar 05 2009
       // There're systems that do not directly
@@ -1858,7 +1865,7 @@ launchmon_rc_e linux_launchmon_t::handle_trap_after_exec_event(
 #else
           = p.get_gprset(use_cxt)->get_pc() & 0xffff0000;
 #endif
-#else /* PPC_ARCHITECTURE */
+#else /* PPC_ARCHITECTURE || POWERLE_ARCHITECTURE */
       //
       // This requires the actual page size to compute this loader load
       // address implictly. Just using the following bits for now.
@@ -1915,13 +1922,13 @@ launchmon_rc_e linux_launchmon_t::handle_trap_after_exec_event(
     countHandler++;
     // accum and countHandler now contain the cost of this handler which
     // is invoked just once per job
-#endif
-
     {
       self_trace_t::trace(
           true,  // print always
           MODULENAME, 0, "Just continued the RM process out of the first trap");
     }
+#endif
+
 
     set_last_seen(gettimeofdayD());
     return LAUNCHMON_OK;
@@ -2652,12 +2659,19 @@ launchmon_rc_e linux_launchmon_t::handle_thrcreate_request(
     memset(&tinfo, '\0', sizeof(tinfo));
     tinfo.ti_lid = (lwpid_t)newlwpid;
 
+    {
+       self_trace_t::trace(LEVELCHK(level2), MODULENAME,0,
+                           "thread creation request event handler "
+                           "invoked for thread (%d)", newlwpid);
+    }
+
+
     if (p.get_thrlist().find(tinfo.ti_lid) == p.get_thrlist().end()) {
 // this thread has not been seen
 #if X86_ARCHITECTURE || X86_64_ARCHITECTURE
       thread_base_t<SDBG_LINUX_DFLT_INSTANTIATION> *thrinfo =
           new linux_x86_thread_t();
-#elif PPC_ARCHITECTURE
+#elif PPC_ARCHITECTURE || POWERLE_ARCHITECTURE
       thread_base_t<SDBG_LINUX_DFLT_INSTANTIATION> *thrinfo =
           new linux_ppc_thread_t();
 #elif AARCH64_ARCHITECTURE

launchmon/src/linux/sdbg_linux_mach.cxx

@@ -520,7 +520,7 @@ bool linux_x86_process_t::basic_init(const std::string& mi,
   return true;
 }
 
-#elif PPC_ARCHITECTURE
+#elif PPC_ARCHITECTURE || POWERLE_ARCHITECTURE
 
 ////////////////////////////////////////////////////////////////////
 //

launchmon/src/linux/sdbg_linux_mach.hxx

@@ -516,7 +516,7 @@ struct ps_prochandle {
   process_base_t<SDBG_LINUX_DFLT_INSTANTIATION>* p;
 };
 
-#elif PPC_ARCHITECTURE
+#elif PPC_ARCHITECTURE || POWERLE_ARCHITECTURE
 
 //! linux_ppc_gpr_set_t:
 /*!

launchmon/src/linux/sdbg_linux_std.hxx

@@ -356,7 +356,7 @@ const T_VA T_UNINIT_HEX = 0xdeadbeef;
 #define SDBG_LINUX_DFLT_INSTANTIATION \
   T_VA, T_WT, T_IT, T_GRS, T_FRS, my_thrinfo_t, elf_wrapper
 
-#elif PPC_ARCHITECTURE
+#elif PPC_ARCHITECTURE || POWERLE_ARCHITECTURE
 
 //
 //

launchmon/src/linux/sdbg_linux_symtab.hxx

@@ -203,6 +203,8 @@ class linux_image_t : public image_base_t<VA, elf_wrapper> {
             level);
   }
 
+  VA get_local_entry_point (const unsigned char o);
+
   // For self tracing
   //
   std::string MODULENAME;

launchmon/src/linux/sdbg_linux_symtab_impl.hxx

@@ -28,6 +28,7 @@
  *
  *
  *  Update Log:
+ *        May 19 2018 DHA: Added OpenPower ABI's dual entry points.
  *        Oct 27 2010 DHA: Added is_defined, is_globally_visible,
  *                         is_locally_visible virtual methods.
  *        Dec 20 2009 DHA: Fixed a bug that arose when Mark's patch
@@ -482,6 +483,10 @@ linux_image_t<LINUX_IMAGE_TEMPLPARAM>::read_linkage_symbols() throw(
       decode_type(first_sym->st_info, tmp);
       a_linksym->set_type(tmp);
       a_linksym->set_defined((first_sym->st_shndx != SHN_UNDEF) ? true : false);
+      a_linksym->set_info(first_sym->st_info);
+      a_linksym->set_other(first_sym->st_other);
+      a_linksym->set_local_entry_offset(
+          get_local_entry_point(first_sym->st_other));
 
       string keystr(symname);
 
@@ -864,6 +869,47 @@ void linux_image_t<LINUX_IMAGE_TEMPLPARAM>::decode_visibility(
   }
 }
 
+
+//! PRIVATE: linux_image_t<VA>::get_local_entry_point --
+/*!
+    Calculate the entry point used by an intramodule function call
+*/
+template <LINUX_IMAGE_TEMPLATELIST>
+VA
+linux_image_t<LINUX_IMAGE_TEMPLPARAM>::get_local_entry_point (const unsigned char o)
+{
+  VA rc = 0;
+#if POWERLE_ARCHITECTURE
+  /* The "OpenPOWER ABI for Linux Supplement, Power Architecture 64-Bit ELF V2
+   * ABI, Advance":
+   * "The OpenPOWER ABI uses the three most-significant bits
+   * in the symbol st_other field to specify the number of instructions between a
+   * function's global entry point and local entry point. The global entry point
+   * is used when it is necessary to set up the TOC pointer (r2) for the
+   * function. The local entry point is used when r2 is known to already be valid
+   * for the function. A value of zero in these bits asserts that the function
+   * does not use r2."
+   */
+  const int code = (o >> 5) & 0x7;
+  switch (code)
+    {
+    case 2: /* 1 instruction */
+    case 3: /* 2 instructions */
+    case 4: /* 4 instructions */
+    case 5: /* 8 instructions */
+    case 6: /* 16 instructions */
+      rc = (1 << (code - 2)) * 4; /* TODO: augment template param to includ IT */
+      break;
+    case 0: /* local == global */
+    case 1: /* local == global */
+    case 7: /* Reserved */
+      break;
+    }  /* switch */
+#endif
+  return rc;
+}
+
+
 template <LINUX_IMAGE_TEMPLATELIST>
 void linux_image_t<LINUX_IMAGE_TEMPLPARAM>::set_image_base_address(VA ba) {
   image_base_t<VA, elf_wrapper>::set_image_base_address(ba);

launchmon/src/linux/sdbg_proc_service.cxx

@@ -62,7 +62,7 @@ extern "C" {
 #include <limits.h>
 }
 
-#if X86_ARCHITECTURE || PPC_ARCHITECTURE
+#if X86_ARCHITECTURE || PPC_ARCHITECTURE || POWERLE_ARCHITECTURE
 
 #ifndef PTRACE_GET_THREAD_AREA
 #define PTRACE_GET_THREAD_AREA 25
@@ -243,7 +243,7 @@ extern "C" ps_err_e ps_get_thread_area(const struct ps_prochandle *ph,
                                        lwpid_t lpid, int x, psaddr_t *addr) {
   bool use_cxt = true;
 
-#if X86_ARCHITECTURE || PPC_ARCHITECTURE
+#if X86_ARCHITECTURE || PPC_ARCHITECTURE || POWERLE_ARCHITECTURE
   /*
    * How to fetch thread-specific area for x86/linux and powerPC/linux
    *