Fix memusage reporting

components/homme/src/preqx/share/prim_state_mod.F90

@@ -649,7 +649,8 @@ subroutine print_software_statistics(hybrid, nets, nete)
 
     type(hybrid_t), intent(in) :: hybrid
     integer, intent(in) :: nets, nete
-    integer :: ok, size, rss_int, share, text, datastack, ie
+    integer :: ok, ie
+    integer(8) :: size, rss_int, share, text, datastack
     real(kind=real_kind) :: rss, rss_min, rss_max, rss_mean
 
     ok = GPTLget_memusage(size, rss_int, share, text, datastack)

components/homme/src/share/compose_test_mod.F90

@@ -157,7 +157,8 @@ subroutine print_software_statistics(hybrid, nets, nete)
 
     type(hybrid_t), intent(in) :: hybrid
     integer, intent(in) :: nets, nete
-    integer :: ok, size, rss_int, share, text, datastack, ie
+    integer :: ok, ie
+    integer(8) :: size, rss_int, share, text, datastack
     real(kind=real_kind) :: rss, rss_min, rss_max, rss_mean
 
     ok = GPTLget_memusage(size, rss_int, share, text, datastack)

share/timing/CMakeLists.txt

@@ -57,6 +57,9 @@ string(APPEND TIMING_CPPDEFS " -D${UPVAR}_INTERFACE")
 if (DEBUG)
   string(APPEND TIMING_CPPDEFS " -DDEBUG")
 endif()
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  string(APPEND TIMING_CPPDEFS " -DHAVE_SLASHPROC")
+endif()
 
 if (CPRE)
   # Not sure what to do here or if this is really needed

share/timing/GPTLget_memusage.c

@@ -7,7 +7,7 @@
 ** get_memusage:
 **
 **   Designed to be called from Fortran, returns information about memory
-**   usage in each of 5 input int* args.  On Linux read from the /proc
+**   usage in each of 5 input long long* args.  On Linux read from the /proc
 **   filesystem because getrusage() returns placebos (zeros).  Return -1 for
 **   values which are unavailable or ambiguous on a particular architecture.
 **   Reported numbers are in kilobytes.
@@ -62,7 +62,7 @@
 
 #define PRINT_MEMUSAGE 0
 
-int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack)
+int GPTLget_memusage (long long *size, long long *rss, long long *share, long long *text, long long *datastack)
 {
 #if defined (BGP)
   long long alloc, total;
@@ -108,13 +108,14 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack
 #elif (defined HAVE_SLASHPROC)
   FILE *fd;                       /* file descriptor for fopen */
   int pid;                        /* process id */
-  char file[24];                  /* full path to file in /proc */
-  int dum;                        /* placeholder for unused return arguments */
-  int ret;                        /* function return value */
-  static int pg_sz = -1;          /* page size */
+  char file[32];                  /* full path to file in /proc */
+  char line[128];                 /* line buffer for reading /proc/status */
 
   /*
-  ** The file we want to open is /proc/<pid>/statm
+  ** Read from /proc/<pid>/status which reports labeled fields in KB.
+  ** Use VmHWM (peak RSS) for size instead of VmSize (virtual address space)
+  ** because VmSize is inflated by GPU BAR mappings and other non-physical
+  ** reservations, while VmHWM is the true memory high-water mark.
   */
 
   pid = (int) getpid ();
@@ -123,35 +124,30 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack
     return -1;
   }
 
-  sprintf (file, "/proc/%d/statm", pid);
-  if ((fd = fopen (file, "r")) < 0) {
+  *size      = -1;
+  *rss       = -1;
+  *share     = -1;
+  *text      = -1;
+  *datastack = -1;
+
+  sprintf (file, "/proc/%d/status", pid);
+  if ((fd = fopen (file, "r")) == NULL) {
     fprintf (stderr, "get_memusage: bad attempt to open %s\n", file);
     return -1;
   }
 
-  /*
-  ** Read the desired data from the /proc filesystem directly into the output
-  ** arguments, close the file and return.
-  */
-
-  ret = fscanf (fd, "%d %d %d %d %d %d %d",
-		size, rss, share, text, datastack, &dum, &dum);
-  ret = fclose (fd);
-
-  // read page size once
-  if (pg_sz == -1) {
-    pg_sz = sysconf(_SC_PAGESIZE) / 1024;
+  while (fgets (line, sizeof(line), fd)) {
+    if      (sscanf (line, "VmHWM: %lld", size)      == 1) ;
+    else if (sscanf (line, "VmRSS: %lld", rss)       == 1) ;
+    else if (sscanf (line, "VmData: %lld", datastack) == 1) ;
+    else if (sscanf (line, "VmExe: %lld", text)       == 1) ;
+    else if (sscanf (line, "RssFile: %lld", share)    == 1) ;
   }
+  fclose (fd);
 
-  // convert from pages to KBs
-  *size      = *size      * pg_sz;
-  *rss       = *rss       * pg_sz;
-  *share     = *share     * pg_sz;
-  *text      = *text      * pg_sz;
-  *datastack = *datastack * pg_sz;
 #if PRINT_MEMUSAGE
-  fprintf (stderr, "get_memusage: size=%d KB, rss=%d KB, share=%d KB, text=%d KB, datastack=%d KB, page_size=%d KB\n",
-           *size, *rss, *share, *text, *datastack, pg_sz);
+  fprintf (stderr, "get_memusage: size=%lld KB, rss=%lld KB, share=%lld KB, text=%lld KB, datastack=%lld KB\n",
+           *size, *rss, *share, *text, *datastack);
 #endif
 
   return 0;
@@ -167,7 +163,7 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack
   fd = popen (cmd, "r");
 
   if (fd) {
-    fscanf (fd, "%d %d %d", size, rss, text);
+    fscanf (fd, "%lld %lld %lld", size, rss, text);
     *share     = -1;
     *datastack = -1;
     (void) pclose (fd);
@@ -183,12 +179,12 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack
     return -1;
 
   *size      = -1;
-  *rss       = usage.ru_maxrss; // in KBs
+  *rss       = (long long) usage.ru_maxrss; // in KBs
   *share     = -1;
   *text      = -1;
   *datastack = -1;
 #ifdef IRIX64
-  *datastack = usage.ru_idrss + usage.ru_isrss;
+  *datastack = (long long) (usage.ru_idrss + usage.ru_isrss);
 #endif
   return 0;
 

share/timing/GPTLprint_memusage.c

@@ -20,7 +20,6 @@
 #include <spi/include/kernel/memory.h>
 #endif
 
-static int nearest_powerof2 (const int);
 static int convert_to_mb = 1;   /* true */
 
 int GPTLprint_memusage (const char *str)
@@ -47,109 +46,27 @@ int GPTLprint_memusage (const char *str)
   return 0;
 
 #else
-  int size, size2;                        /* process size (returned from OS) */
-  int rss, rss2;                          /* resident set size (returned from OS) */
-  int share, share2;                      /* shared data segment size (returned from OS) */
-  int text, text2;                        /* text segment size (returned from OS) */
-  int datastack, datastack2;              /* data/stack size (returned from OS) */
-  static int kbytesperblock = -1;         /* convert to Kbytes (init to invalid) */
-  static const int nbytes =1024*1024*1024;/* allocate 1 GB */
-  void *space;                            /* allocated space */
+  long long size;                           /* peak RSS in KB (VmHWM on Linux) */
+  long long rss;                            /* resident set size in KB */
+  long long share;                          /* shared pages in KB */
+  long long text;                           /* text segment in KB */
+  long long datastack;                      /* data segment in KB */
 
   setbuf(stdout, NULL); // don't buffer stdout, flush
   if (GPTLget_memusage (&size, &rss, &share, &text, &datastack) < 0) {
     printf ("GPTLprint_memusage: GPTLget_memusage failed.\n");
     return -1;
   }
 
-#if (defined HAVE_SLASHPROC || defined __APPLE__)
-  if (kbytesperblock == -1) {
-    kbytesperblock = sysconf(_SC_PAGESIZE) / 1024;
-    printf ("GPTLprint_memusage: Using Kbytesperpage=%d\n", kbytesperblock);
-  }
-
-  /*
-  ** Determine size in bytes of memory usage info presented by the OS. Method: allocate a
-  ** known amount of memory and see how much bigger the process becomes.
-  */
-
-  if (convert_to_mb && kbytesperblock == -1 && (space = malloc (nbytes))) {
-    memset (space, 0, nbytes);  /* ensure the space is really allocated */
-    if (GPTLget_memusage (&size2, &rss2, &share2, &text2, &datastack2) == 0) {
-      if (size2 > size) {
-	/*
-	** Estimate bytes per block, then refine to nearest power of 2.
-	** The assumption is that the OS presents memory usage info in
-	** units that are a power of 2.
-	*/
-        kbytesperblock = (int) ((nbytes / (double) (size2 - size)) + 0.5);
-        kbytesperblock = nearest_powerof2 (kbytesperblock);
-        printf ("GPTLprint_memusage: Using Kbytesperblock=%d\n", kbytesperblock);
-      } else {
-        printf ("GPTLprint_memusage: highwater did not increase.\n");
-      }
-    } else {
-      printf ("GPTLprint_memusage: call GPTLget_memusage failed.\n");
-    }
-    free (space);
-  }
-
-  if (kbytesperblock > 0) {
+  /* GPTLget_memusage returns values in KB on all platforms */
+  if (convert_to_mb)
     printf ("%s sysmem size=%.1f MB rss=%.1f MB share=%.1f MB text=%.1f MB datastack=%.1f MB\n",
             str, size/1024., rss/1024., share/1024., text/1024., datastack/1024.);
-  } else {
-    printf ("%s sysmem size=%d rss=%d share=%d text=%d datastack=%d\n",
-            str, size, rss, share, text, datastack);
-  }
-
-#else
-
-  /*
-  ** Use max rss as returned by getrusage. If someone knows how to
-  ** get the process size under AIX please tell me.
-  */
-
-  if (convert_to_mb)
-    printf ("%s max rss=%.1f MB\n", str, rss*1024.);
   else
-    printf ("%s max rss=%d\n", str, rss);
-#endif
+    printf ("%s sysmem size=%lld rss=%lld share=%lld text=%lld datastack=%lld\n",
+            str, size, rss, share, text, datastack);
 
   return 0;
 #endif
 }
 
-/*
-** nearest_powerof2:
-**   Determine nearest integer which is a power of 2.
-**   Note: algorithm can't use anything that requires -lm because this is a library,
-**   and we don't want to burden the user with having to add extra libraries to the
-**   link line.
-**
-** Input arguments:
-**   val: input integer
-**
-** Return value: nearest integer to val which is a power of 2
-*/
-
-static int nearest_powerof2 (const int val)
-{
-  int lower;  /* power of 2 which is just less than val */
-  int higher; /* power of 2 which is just more than val */
-  int delta1; /* difference between val and lower */
-  int delta2; /* difference between val and higher */
-
-  if (val < 2)
-    return 0;
-
-  for (higher = 1; higher < val; higher *= 2)
-    lower = higher;
-
-  delta1 = val - lower;
-  delta2 = higher - val;
-
-  if (delta1 < delta2)
-    return lower;
-  else
-    return higher;
-}

share/timing/f_wrappers.c

@@ -192,7 +192,7 @@ int gptlget_eventvalue (const char *timername, const char *eventname, int *t, do
 			int nc1, int nc2);
 int gptlget_nregions (int *t, int *nregions);
 int gptlget_regionname (int *t, int *region, char *name, int nc);
-int gptlget_memusage (int *size, int *rss, int *share, int *text, int *datastack);
+int gptlget_memusage (long long *size, long long *rss, long long *share, long long *text, long long *datastack);
 int gptlprint_memusage (const char *str, int nc);
 #ifdef HAVE_PAPI
 int gptl_papilibraryinit (void);
@@ -534,7 +534,7 @@ int gptlget_regionname (int *t, int *region, char *name, int nc)
   return ret;
 }
 
-int gptlget_memusage (int *size, int *rss, int *share, int *text, int *datastack)
+int gptlget_memusage (long long *size, long long *rss, long long *share, long long *text, long long *datastack)
 {
   return GPTLget_memusage (size, rss, share, text, datastack);
 }

share/timing/gptl.h

@@ -161,7 +161,7 @@ extern int GPTLbarrier (int, const char *);
 
 extern int GPTLreset (void);
 extern int GPTLfinalize (void);
-extern int GPTLget_memusage (int *, int *, int *, int *, int *);
+extern int GPTLget_memusage (long long *, long long *, long long *, long long *, long long *);
 extern int GPTLprint_memusage (const char *);
 extern int GPTLenable (void);
 extern int GPTLdisable (void);

share/util/shr_mem_mod.F90

@@ -31,9 +31,9 @@ subroutine shr_mem_init(prt, strbuf)
     !----- local -----
 
     ! --- Memory stats ---
-    integer :: msize                   ! memory size (high water)
-    integer :: mrss0,mrss1,mrss2       ! temporary rss
-    integer :: mshare,mtext,mdatastack
+    integer(8) :: msize                ! memory size (high water)
+    integer(8) :: mrss0,mrss1,mrss2    ! temporary rss
+    integer(8) :: mshare,mtext,mdatastack
     logical :: lprt
     integer :: ierr
 
@@ -95,8 +95,8 @@ subroutine shr_mem_getusage(r_msize,r_mrss,prt)
     logical, optional :: prt
 
     !----- local ---
-    integer :: msize,mrss
-    integer :: mshare,mtext,mdatastack
+    integer(8) :: msize,mrss
+    integer(8) :: mshare,mtext,mdatastack
     integer :: ierr
     integer :: GPTLget_memusage, GPTLprint_memusage