From 7981fc70b12cc21b63dfba99b6b751596df0f49e Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Thu, 25 Jul 2019 19:18:52 -0600
Subject: [PATCH 01/13] Changed the behavior of shift in mdtest to mimic that
 in IOR so that it shifts by _entire nodes_ not by ranks.

---
 src/mdtest.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/mdtest.c b/src/mdtest.c
index 8613e3b..2d15c9f 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -1362,7 +1362,7 @@ void print_help (void) {
         "\t-p: pre-iteration delay (in seconds)\n"
         "\t-r: only remove files or directories left behind by previous runs\n"
         "\t-R: randomly stat files (optional argument for random seed)\n"
-        "\t-s: stride between the number of tasks for each test\n"
+        "\t-s: stride between the number of nodes for each test\n"
         "\t-S: shared file access (file only, no directories)\n"
         "\t-t: time unique working directory overhead\n"
         "\t-T: only stat files/dirs\n"
@@ -2191,7 +2191,8 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
     pid = getpid();
     uid = getuid();
 
-    nodeCount = size / CountTasksPerNode(testComm);
+    tasksPerNode = CountTasksPerNode(testComm);
+    nodeCount = size / tasksPerNode;
 
     if (rank == 0) {
         fprintf(out_logfile, "-- started at %s --\n\n", PrintTimestamp());
@@ -2267,6 +2268,11 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
         fflush( out_logfile );
     }
 
+    /* set the shift to mimic IOR and shift by procs per node */
+    if (nstride > 0) {
+        nstride *= tasksPerNode;
+    }
+
     /* setup total number of items and number of items per dir */
     if (depth <= 0) {
         num_dirs_in_tree = 1;

From 524d053be1dd789cd4ba241011f9039def364d96 Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Fri, 26 Jul 2019 08:55:24 -0600
Subject: [PATCH 02/13] Making shift work in mdtest as well as it works in IOR
 and on a per-node basis. Also added printing the nodemap so we can check the
 allocation.

---
 src/mdtest.c    | 16 +++++++++++-----
 src/utilities.c | 38 ++++++++++++++++++++++++++++++++++++++
 src/utilities.h |  1 +
 3 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/src/mdtest.c b/src/mdtest.c
index 2d15c9f..cbb0de8 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -2268,11 +2268,6 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
         fflush( out_logfile );
     }
 
-    /* set the shift to mimic IOR and shift by procs per node */
-    if (nstride > 0) {
-        nstride *= tasksPerNode;
-    }
-
     /* setup total number of items and number of items per dir */
     if (depth <= 0) {
         num_dirs_in_tree = 1;
@@ -2381,6 +2376,17 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
     }
 
     if (rank == 0) display_freespace(testdirpath);
+    int packedByNode = QueryNodeMapping(testComm);
+
+    /* set the shift to mimic IOR and shift by procs per node */
+    if (nstride > 0) {
+        if ( packedByNode ) {
+            nstride *= tasksPerNode;
+        }
+        if (rank == 0) {
+            fprintf(out_logfile, "Shifting ranks by %d for each phase.\n", nstride);
+        }
+    }
 
     if (verbose >= 3 && rank == 0) {
         fprintf(out_logfile,  "V-3: main (after display_freespace): testdirpath is \"%s\"\n", testdirpath );
diff --git a/src/utilities.c b/src/utilities.c
index c0c7c61..493eb89 100755
--- a/src/utilities.c
+++ b/src/utilities.c
@@ -213,12 +213,50 @@ void DumpBuffer(void *buffer,
         return;
 }                               /* DumpBuffer() */
 
+/* a function that prints an int array where each index corresponds to a rank
+   and the value is whether that rank is on the same host as root.
+   Also returns 1 if rank 1 is on same host and 0 otherwise
+*/
+int QueryNodeMapping(MPI_Comm comm) {
+    char localhost[MAX_PATHLEN], roothost[MAX_PATHLEN];
+    int num_ranks;
+    MPI_Comm_size(comm, &num_ranks);
+    int *node_map = (int*)malloc(sizeof(int) * num_ranks);
+    if ( ! node_map ) {
+        FAIL("malloc");
+    }
+    if (gethostname(localhost, MAX_PATHLEN) != 0) {
+        FAIL("gethostname()");
+    }
+    if (rank==0) {
+        strncpy(roothost,localhost,MAX_PATHLEN);
+    }
+
+    /* have rank 0 broadcast out its hostname */
+    MPI_Bcast(roothost, MAX_PATHLEN, MPI_CHAR, 0, comm);
+    //printf("Rank %d received root host as %s\n", rank, roothost);
+    /* then every rank figures out whether it is same host as root and then gathers that */
+    int same_as_root = strcmp(roothost,localhost) == 0;
+    MPI_Gather( &same_as_root, 1, MPI_INT, node_map, 1, MPI_INT, 0, comm);
+    if (rank==0) {
+        fprintf( out_logfile, "Nodemap: " );
+        for ( int i = 0; i < num_ranks; i++ ) {
+            fprintf( out_logfile, "%d", node_map[i] );
+        }
+        fprintf( out_logfile, "\n" );
+    }
+    int ret = node_map[1] == 1;
+    free(node_map);
+    return ret;
+}
+
 #if MPI_VERSION >= 3
 int CountTasksPerNode(MPI_Comm comm) {
     /* modern MPI provides a simple way to get the local process count */
     MPI_Comm shared_comm;
     int count;
 
+
     MPI_Comm_split_type (comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shared_comm);
     MPI_Comm_size (shared_comm, &count);
     MPI_Comm_free (&shared_comm);
diff --git a/src/utilities.h b/src/utilities.h
index b646174..d811261 100755
--- a/src/utilities.h
+++ b/src/utilities.h
@@ -65,6 +65,7 @@ void SetHints (MPI_Info *, char *);
 void ShowHints (MPI_Info *);
 char *HumanReadable(IOR_offset_t value, int base);
 int CountTasksPerNode(MPI_Comm comm);
+int QueryNodeMapping(MPI_Comm comm);
 void DelaySecs(int delay);
 void updateParsedOptions(IOR_param_t * options, options_all_t * global_options);
 size_t NodeMemoryStringToBytes(char *size_str);

From 945487f7437f56a6e85c9b726a744af1be83c789 Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Sat, 27 Jul 2019 09:20:20 -0600
Subject: [PATCH 03/13] Better debug message when stat fails

---
 src/mdtest.c | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/src/mdtest.c b/src/mdtest.c
index cbb0de8..d9f28db 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -635,19 +635,13 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch
         }
 
         if (-1 == backend->stat (item, &buf, &param)) {
-            if (dirs) {
-                if ( verbose >= 3 ) {
-                    fprintf( out_logfile, "V-3: Stat'ing directory \"%s\"\n", item );
-                    fflush( out_logfile );
-                }
-                FAIL("unable to stat directory");
-            } else {
-                if ( verbose >= 3 ) {
-                    fprintf( out_logfile, "V-3: Stat'ing file \"%s\"\n", item );
-                    fflush( out_logfile );
-                }
-                FAIL("unable to stat file");
+            char msg_buf[4096];
+            if ( verbose >=3 ) {
+                fprintf( out_logfile, "V-3: Stat'ing %s \"%s\"\n", dirs ? "directory" : "file", item );
+                fflush( out_logfile );
             }
+            snprintf(msg_buf, 4096, "unable to stat %s %s", dirs ? "directory" : "file", item);
+            FAIL("unable to stat directory");
         }
     }
 }

From 9d6480d46e5658a269b6bf7e7022f5ad55fba203 Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Sat, 27 Jul 2019 11:30:07 -0600
Subject: [PATCH 04/13] Fixed bug in the nstride calculation where only 0 was
 computing it correctly.  Had to bcast it out

---
 src/mdtest.c    | 16 +++++++++++++++-
 src/utilities.c |  1 +
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/mdtest.c b/src/mdtest.c
index d9f28db..072dc29 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -494,6 +494,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const
             if (collective) {
                 collective_helper(dirs, create, temp_path, 0, progress);
             } else {
+                printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path);
                 create_remove_items_helper(dirs, create, temp_path, 0, progress);
             }
         }
@@ -522,6 +523,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const
                 if (collective) {
                     collective_helper(dirs, create, temp_path, currDir*items_per_dir, progress);
                 } else {
+                    printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path);
                     create_remove_items_helper(dirs, create, temp_path, currDir*items_per_dir, progress);
                 }
             }
@@ -641,7 +643,7 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch
                 fflush( out_logfile );
             }
             snprintf(msg_buf, 4096, "unable to stat %s %s", dirs ? "directory" : "file", item);
-            FAIL("unable to stat directory");
+            FAIL(msg_buf);
         }
     }
 }
@@ -792,6 +794,7 @@ void collective_create_remove(const int create, const int dirs, const int ntasks
             sprintf(rm_name, "mdtest.%d.", (i+(3*nstride))%ntasks);
         }
         if (unique_dir_per_task) {
+            printf("DEBUG %5d Rank %d i %d nstride %d ntasks %d", __LINE__, rank, i, nstride, ntasks);
             sprintf(unique_mk_dir, "%s/mdtest_tree.%d.0", testdir,
                     (i+(0*nstride))%ntasks);
             sprintf(unique_chdir_dir, "%s/mdtest_tree.%d.0", testdir,
@@ -1085,6 +1088,8 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
     char temp_path[MAX_PATHLEN];
     MPI_Comm_size(testComm, &size);
 
+    printf("DEBUG %5d Rank %d file_test on %s\n", __LINE__, rank, path);
+
     if (( rank == 0 ) && ( verbose >= 1 )) {
         fprintf( out_logfile, "V-1: Entering file_test...\n" );
         fflush( out_logfile );
@@ -1099,7 +1104,9 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
         prep_testdir(iteration, dir_iter);
 
         if (unique_dir_per_task) {
+            printf("DEBUG %5d Rank %d operating on %s\n", __LINE__, rank, temp_path);
             unique_dir_access(MK_UNI_DIR, temp_path);
+            printf("DEBUG %5d Rank %d operating on %s\n", __LINE__, rank, temp_path);
             if (!time_unique_dir_overhead) {
                 offset_timers(t, 0);
             }
@@ -1107,6 +1114,8 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
+
+
         if (verbose >= 3 && rank == 0) {
             fprintf(out_logfile,  "V-3: file_test: create path is \"%s\"\n", temp_path );
             fflush( out_logfile );
@@ -1250,6 +1259,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
                 collective_create_remove(0, 0, ntasks, temp_path, progress);
             }
         } else {
+            printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path);
             create_remove_items(0, 0, 0, 0, temp_path, 0, progress);
         }
       }
@@ -1942,12 +1952,15 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
           sprintf(rm_name, "mdtest.%d.", (rank+(3*nstride))%i);
       }
       if (unique_dir_per_task) {
+          printf("DEBUG %5d Rank %d i %d nstride %d\n", __LINE__, rank, i, nstride);
           sprintf(unique_mk_dir, "mdtest_tree.%d.0",  (rank+(0*nstride))%i);
           sprintf(unique_chdir_dir, "mdtest_tree.%d.0", (rank+(1*nstride))%i);
           sprintf(unique_stat_dir, "mdtest_tree.%d.0", (rank+(2*nstride))%i);
           sprintf(unique_read_dir, "mdtest_tree.%d.0", (rank+(3*nstride))%i);
           sprintf(unique_rm_dir, "mdtest_tree.%d.0", (rank+(4*nstride))%i);
           unique_rm_uni_dir[0] = 0;
+          printf("DEBUG %5d Rank %d mk_dir %s chdir %s stat_dir %s read_dir %s rm_dir %s\n", __LINE__, 
+                  rank, unique_mk_dir,unique_chdir_dir,unique_stat_dir,unique_read_dir,unique_rm_dir);
       }
 
       if (verbose >= 3 && rank == 0) {
@@ -1965,6 +1978,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
           if (pre_delay) {
               DelaySecs(pre_delay);
           }
+          printf("DEBUG %5d Rank %d gonna file_test on %s\n", __LINE__, rank, unique_mk_dir);
           file_test(j, i, unique_mk_dir, progress);
       }
   }
diff --git a/src/utilities.c b/src/utilities.c
index 493eb89..5a3168c 100755
--- a/src/utilities.c
+++ b/src/utilities.c
@@ -246,6 +246,7 @@ int QueryNodeMapping(MPI_Comm comm) {
         fprintf( out_logfile, "\n" );
     }
     int ret = node_map[1] == 1;
+    MPI_Bcast(&ret, 1, MPI_INT, 0, comm);
     free(node_map);
     return ret;
 }

From f6491fcd37d9365fd0b2d502d4d3a79b5af755a0 Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Sat, 27 Jul 2019 13:22:15 -0600
Subject: [PATCH 05/13] Cleaned up the verbose messages by creating a macro and
 a function

---
 src/mdtest.c | 546 ++++++++++++++-------------------------------------
 1 file changed, 147 insertions(+), 399 deletions(-)

diff --git a/src/mdtest.c b/src/mdtest.c
index 072dc29..bfeeda1 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -36,6 +36,7 @@
 #include <inttypes.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <stdarg.h>
 
 #include "option.h"
 #include "utilities.h"
@@ -173,16 +174,30 @@ typedef struct{
 /* for making/removing unique directory && stating/deleting subdirectory */
 enum {MK_UNI_DIR, STAT_SUB_DIR, READ_SUB_DIR, RM_SUB_DIR, RM_UNI_DIR};
 
+/* a helper function for passing debug and verbose messages.
+   use the MACRO as it will insert __LINE__ for you.
+   Pass the verbose level for root to print, then the verbose level for anyone to print.
+   Pass -1 to suppress the print for anyone.
+   Then do the standard printf stuff.  This function adds the newline for you.
+*/
+#define VERBOSE(root,any,...) VerboseMessage(root,any,__LINE__,__VA_ARGS__)
+void VerboseMessage (int root_level, int any_level, int line, char * format, ...) {
+    if ((rank==0 && verbose >= root_level) || (any_level > 0 && verbose >= any_level)) {
+        char buffer[1024];
+        va_list args;
+        va_start (args, format);
+        vsnprintf (buffer, 1024, format, args);
+        va_end (args);
+        fprintf( out_logfile, "V-%d: Rank %3d Line %5d %s\n", root_level, rank, line, buffer );
+    }
+}
 
 void offset_timers(double * t, int tcount) {
     double toffset;
     int i;
 
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering offset_timers...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"V-1: Entering offset_timers..." );
 
     toffset = GetTimeStamp() - t[tcount];
     for (i = 0; i < tcount+1; i++) {
@@ -196,10 +211,7 @@ void parse_dirpath(char *dirpath_arg) {
     int i = 0;
 
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering parse_dirpath...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1, "Entering parse_dirpath..." );
 
     tmp = dirpath_arg;
 
@@ -240,10 +252,7 @@ static void prep_testdir(int j, int dir_iter){
  */
 
 void unique_dir_access(int opt, char *to) {
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering unique_dir_access...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"Entering unique_dir_access..." );
 
     if (opt == MK_UNI_DIR) {
         MPI_Barrier(testComm);
@@ -263,20 +272,13 @@ static void create_remove_dirs (const char *path, bool create, uint64_t itemNum)
     char curr_item[MAX_PATHLEN];
     const char *operation = create ? "create" : "remove";
 
-    if (( rank == 0 )                                         &&
-        ( verbose >= 3 )                                      &&
-        (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
-
-        fprintf(out_logfile, "V-3: %s dir: "LLU"\n", operation, itemNum);
-        fflush(out_logfile);
+    if ( (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
+        VERBOSE(3,5,"dir: "LLU"", operation, itemNum);
     }
 
     //create dirs
     sprintf(curr_item, "%s/dir.%s%" PRIu64, path, create ? mk_name : rm_name, itemNum);
-    if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-        fprintf(out_logfile, "V-3: create_remove_items_helper (dirs %s): curr_item is \"%s\"\n", operation, curr_item);
-        fflush(out_logfile);
-    }
+    VERBOSE(3,5,"create_remove_items_helper (dirs %s): curr_item is \"%s\"", operation, curr_item);
 
     if (create) {
         if (backend->mkdir(curr_item, DIRMODE, &param) == -1) {
@@ -292,20 +294,13 @@ static void create_remove_dirs (const char *path, bool create, uint64_t itemNum)
 static void remove_file (const char *path, uint64_t itemNum) {
     char curr_item[MAX_PATHLEN];
 
-    if (( rank == 0 )                                       &&
-        ( verbose >= 3 )                                    &&
-        (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
-
-        fprintf(out_logfile, "V-3: remove file: "LLU"\n", itemNum);
-        fflush(out_logfile);
+    if ( (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
+        VERBOSE(3,5,"remove file: "LLU"\n", itemNum);
     }
 
     //remove files
     sprintf(curr_item, "%s/file.%s"LLU"", path, rm_name, itemNum);
-    if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-        fprintf(out_logfile, "V-3: create_remove_items_helper (non-dirs remove): curr_item is \"%s\"\n", curr_item);
-        fflush(out_logfile);
-    }
+    VERBOSE(3,5,"create_remove_items_helper (non-dirs remove): curr_item is \"%s\"", curr_item);
     if (!(shared_file && rank != 0)) {
         backend->delete (curr_item, &param);
     }
@@ -315,28 +310,18 @@ static void create_file (const char *path, uint64_t itemNum) {
     char curr_item[MAX_PATHLEN];
     void *aiori_fh;
 
-    if (( rank == 0 )                                             &&
-        ( verbose >= 3 )                                          &&
-        (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
-
-        fprintf(out_logfile, "V-3: create file: "LLU"\n", itemNum);
-        fflush(out_logfile);
+    if ( (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
+        VERBOSE(3,5,"create file: "LLU"", itemNum);
     }
 
     //create files
     sprintf(curr_item, "%s/file.%s"LLU"", path, mk_name, itemNum);
-    if ((rank == 0 && verbose >= 3) || verbose >= 5) {
-        fprintf(out_logfile, "V-3: create_remove_items_helper (non-dirs create): curr_item is \"%s\"\n", curr_item);
-        fflush(out_logfile);
-    }
+    VERBOSE(3,5,"create_remove_items_helper (non-dirs create): curr_item is \"%s\"", curr_item);
 
     if (collective_creates) {
         param.openFlags = IOR_WRONLY;
 
-        if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-            fprintf(out_logfile,  "V-3: create_remove_items_helper (collective): open...\n" );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"create_remove_items_helper (collective): open..." );
 
         if (make_node)
             aiori_fh = backend->mknod (curr_item);
@@ -353,10 +338,7 @@ static void create_file (const char *path, uint64_t itemNum) {
         param.openFlags = IOR_CREAT | IOR_WRONLY;
         param.filePerProc = !shared_file;
 
-        if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-            fprintf(out_logfile,  "V-3: create_remove_items_helper (non-collective, shared): open...\n" );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"create_remove_items_helper (non-collective, shared): open..." );
 
         if (make_node)
             aiori_fh = backend->mknod (curr_item);
@@ -368,10 +350,7 @@ static void create_file (const char *path, uint64_t itemNum) {
     }
 
     if (write_bytes > 0) {
-        if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-            fprintf(out_logfile,  "V-3: create_remove_items_helper: write...\n" );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"create_remove_items_helper: write..." );
 
         /*
          * According to Bill Loewe, writes are only done one time, so they are always at
@@ -384,10 +363,7 @@ static void create_file (const char *path, uint64_t itemNum) {
         }
     }
 
-    if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-        fprintf(out_logfile,  "V-3: create_remove_items_helper: close...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(3,5,"create_remove_items_helper: close..." );
 
     if (!make_node)
         backend->close (aiori_fh, &param);
@@ -397,10 +373,7 @@ static void create_file (const char *path, uint64_t itemNum) {
 void create_remove_items_helper(const int dirs, const int create, const char *path,
                                 uint64_t itemNum, rank_progress_t * progress) {
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering create_remove_items_helper...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"Entering create_remove_items_helper..." );
 
     for (uint64_t i = progress->items_start; i < progress->items_per_dir ; ++i) {
         if (!dirs) {
@@ -426,10 +399,7 @@ void create_remove_items_helper(const int dirs, const int create, const char *pa
 void collective_helper(const int dirs, const int create, const char* path, uint64_t itemNum, rank_progress_t * progress) {
     char curr_item[MAX_PATHLEN];
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering collective_helper...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"Entering collective_helper..." );
     for (uint64_t i = progress->items_start ; i < progress->items_per_dir ; ++i) {
         if (dirs) {
             create_remove_dirs (path, create, itemNum + i);
@@ -437,10 +407,7 @@ void collective_helper(const int dirs, const int create, const char* path, uint6
         }
 
         sprintf(curr_item, "%s/file.%s"LLU"", path, create ? mk_name : rm_name, itemNum+i);
-        if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-            fprintf(out_logfile, "V-3: create file: %s\n", curr_item);
-            fflush(out_logfile);
-        }
+        VERBOSE(3,5,"create file: %s", curr_item);
 
         if (create) {
             void *aiori_fh;
@@ -474,19 +441,13 @@ void create_remove_items(int currDepth, const int dirs, const int create, const
     unsigned long long currDir = dirNum;
 
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering create_remove_items, currDepth = %d...\n", currDepth );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"Entering create_remove_items, currDepth = %d...", currDepth );
 
 
     memset(dir, 0, MAX_PATHLEN);
     strcpy(temp_path, path);
 
-    if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-        fprintf(out_logfile,  "V-3: create_remove_items (start): temp_path is \"%s\"\n", temp_path );
-        fflush(out_logfile);
-    }
+    VERBOSE(3,5,"create_remove_items (start): temp_path is \"%s\"", temp_path );
 
     if (currDepth == 0) {
         /* create items at this depth */
@@ -513,10 +474,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const
             strcat(temp_path, "/");
             strcat(temp_path, dir);
 
-            if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-                fprintf(out_logfile,  "V-3: create_remove_items (for loop): temp_path is \"%s\"\n", temp_path );
-                fflush(out_logfile);
-            }
+            VERBOSE(3,5,"create_remove_items (for loop): temp_path is \"%s\"", temp_path );
 
             /* create the items in this branch */
             if (!leaf_only || (leaf_only && currDepth == depth)) {
@@ -553,10 +511,7 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch
     uint64_t parent_dir, item_num = 0;
     char item[MAX_PATHLEN], temp[MAX_PATHLEN];
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering mdtest_stat...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"Entering mdtest_stat..." );
 
     uint64_t stop_items = items;
 
@@ -592,15 +547,13 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch
 
         /* create name of file/dir to stat */
         if (dirs) {
-            if ((verbose >= 5 || (rank == 0 && verbose >= 3)) && (i%ITEM_COUNT == 0) && (i != 0)) {
-                fprintf(out_logfile, "V-3: stat dir: "LLU"\n", i);
-                fflush(out_logfile);
+            if ((i%ITEM_COUNT == 0) && (i != 0)) {
+                VERBOSE(3,5,"stat dir: "LLU"", i);
             }
             sprintf(item, "dir.%s"LLU"", stat_name, item_num);
         } else {
-            if ((verbose >= 5 || (rank == 0 && verbose >= 3)) && (i%ITEM_COUNT == 0) && (i != 0)) {
-                fprintf(out_logfile, "V-3: stat file: "LLU"\n", i);
-                fflush(out_logfile);
+            if ((i%ITEM_COUNT == 0) && (i != 0)) {
+                VERBOSE(3,5,"stat file: "LLU"", i);
             }
             sprintf(item, "file.%s"LLU"", stat_name, item_num);
         }
@@ -627,21 +580,9 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch
         strcpy( item, temp );
 
         /* below temp used to be hiername */
-        if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-            if (dirs) {
-                fprintf(out_logfile, "V-3: mdtest_stat dir : %s\n", item);
-            } else {
-                fprintf(out_logfile, "V-3: mdtest_stat file: %s\n", item);
-            }
-            fflush(out_logfile);
-        }
-
+        VERBOSE(3,5,"mdtest_stat %4s: %s", (dirs ? "dir" : "file"), item);
         if (-1 == backend->stat (item, &buf, &param)) {
             char msg_buf[4096];
-            if ( verbose >=3 ) {
-                fprintf( out_logfile, "V-3: Stat'ing %s \"%s\"\n", dirs ? "directory" : "file", item );
-                fflush( out_logfile );
-            }
             snprintf(msg_buf, 4096, "unable to stat %s %s", dirs ? "directory" : "file", item);
             FAIL(msg_buf);
         }
@@ -655,10 +596,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) {
     char item[MAX_PATHLEN], temp[MAX_PATHLEN];
     void *aiori_fh;
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering mdtest_read...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"Entering mdtest_read..." );
 
     /* allocate read buffer */
     if (read_bytes > 0) {
@@ -703,9 +641,8 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) {
 
         /* create name of file to read */
         if (!dirs) {
-            if ((verbose >= 5 || (rank == 0 && verbose >= 3)) && (i%ITEM_COUNT == 0) && (i != 0)) {
-                fprintf(out_logfile, "V-3: read file: "LLU"\n", i);
-                fflush(out_logfile);
+            if ((i%ITEM_COUNT == 0) && (i != 0)) {
+                VERBOSE(3,5,"read file: "LLU"", i);
             }
             sprintf(item, "file.%s"LLU"", read_name, item_num);
         }
@@ -732,12 +669,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) {
         strcpy( item, temp );
 
         /* below temp used to be hiername */
-        if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-            if (!dirs) {
-                fprintf(out_logfile, "V-3: mdtest_read file: %s\n", item);
-            }
-            fflush(out_logfile);
-        }
+        VERBOSE(3,5,"mdtest_read file: %s", item);
 
         /* open file for reading */
         param.openFlags = O_RDONLY;
@@ -763,10 +695,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) {
 void collective_create_remove(const int create, const int dirs, const int ntasks, const char *path, rank_progress_t * progress) {
     char temp[MAX_PATHLEN];
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering collective_create_remove...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"Entering collective_create_remove on %s", path );
 
     /* rank 0 does all of the creates and removes for all of the ranks */
     for (int i = 0 ; i < ntasks ; ++i) {
@@ -794,7 +723,7 @@ void collective_create_remove(const int create, const int dirs, const int ntasks
             sprintf(rm_name, "mdtest.%d.", (i+(3*nstride))%ntasks);
         }
         if (unique_dir_per_task) {
-            printf("DEBUG %5d Rank %d i %d nstride %d ntasks %d", __LINE__, rank, i, nstride, ntasks);
+            VERBOSE(3,5,"i %d nstride %d ntasks %d", i, nstride, ntasks);
             sprintf(unique_mk_dir, "%s/mdtest_tree.%d.0", testdir,
                     (i+(0*nstride))%ntasks);
             sprintf(unique_chdir_dir, "%s/mdtest_tree.%d.0", testdir,
@@ -809,10 +738,7 @@ void collective_create_remove(const int create, const int dirs, const int ntasks
         }
 
         /* Now that everything is set up as it should be, do the create or remove */
-        if (verbose >= 5 || (rank == 0 && verbose >= 3)) {
-            fprintf(out_logfile, "V-3: collective_create_remove (create_remove_items): temp is \"%s\"\n", temp);
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"collective_create_remove (create_remove_items): temp is \"%s\"", temp);
 
         create_remove_items(0, dirs, create, 1, temp, 0, progress);
     }
@@ -851,10 +777,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
 
     MPI_Comm_size(testComm, &size);
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering directory_test...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"Entering directory_test on %s", path );
 
     MPI_Barrier(testComm);
     t[0] = GetTimeStamp();
@@ -872,10 +795,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        if (verbose >= 3 && rank == 0) {
-            fprintf(out_logfile,  "V-3: directory_test: create path is \"%s\"\n", temp_path );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,-1,"directory_test: create path is \"%s\"", temp_path );
 
         /* "touch" the files */
         if (collective_creates) {
@@ -907,10 +827,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        if (verbose >= 3 && rank == 0) {
-            fprintf(out_logfile,  "V-3: directory_test: stat path is \"%s\"\n", temp_path );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"stat path is \"%s\"", temp_path );
 
         /* stat directories */
         if (random_seed > 0) {
@@ -939,10 +856,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        if (verbose >= 3 && rank == 0) {
-            fprintf(out_logfile,  "V-3: directory_test: read path is \"%s\"\n", temp_path );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"directory_test: read path is \"%s\"", temp_path );
 
         /* read directories */
         if (random_seed > 0) {
@@ -970,10 +884,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        if (verbose >= 3 && rank == 0) {
-            fprintf(out_logfile,  "V-3: directory_test: remove directories path is \"%s\"\n", temp_path );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"directory_test: remove directories path is \"%s\"", temp_path );
 
         /* remove directories */
         if (collective_creates) {
@@ -998,10 +909,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        if (verbose >= 3 && rank == 0) {
-            fprintf(out_logfile,  "V-3: directory_test: remove unique directories path is \"%s\"\n", temp_path );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"directory_test: remove unique directories path is \"%s\"\n", temp_path );
     }
 
     if (unique_dir_per_task && !time_unique_dir_overhead) {
@@ -1034,30 +942,21 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
         summary_table[iteration].stonewall_last_item[3] = items;
     }
 
-    if (verbose >= 1 && rank == 0) {
-        fprintf(out_logfile, "V-1:   Directory creation: %14.3f sec, %14.3f ops/sec\n",
-               t[1] - t[0], summary_table[iteration].rate[0]);
-        fprintf(out_logfile, "V-1:   Directory stat    : %14.3f sec, %14.3f ops/sec\n",
-               t[2] - t[1], summary_table[iteration].rate[1]);
-/* N/A
-   fprintf(out_logfile, "V-1:   Directory read    : %14.3f sec, %14.3f ops/sec\n",
-   t[3] - t[2], summary_table[iteration].rate[2]);
-*/
-        fprintf(out_logfile, "V-1:   Directory removal : %14.3f sec, %14.3f ops/sec\n",
-               t[4] - t[3], summary_table[iteration].rate[3]);
-        fflush(out_logfile);
-    }
+    VERBOSE(1,-1,"   Directory creation: %14.3f sec, %14.3f ops/sec\n", t[1] - t[0], summary_table[iteration].rate[0]);
+    VERBOSE(1,-1,"   Directory stat    : %14.3f sec, %14.3f ops/sec\n", t[2] - t[1], summary_table[iteration].rate[1]);
+    /* N/A
+    VERBOSE(1,-1,"   Directory read    : %14.3f sec, %14.3f ops/sec\n", t[3] - t[2], summary_table[iteration].rate[2]);
+    */
+    VERBOSE(1,-1,"   Directory removal : %14.3f sec, %14.3f ops/sec\n", t[4] - t[3], summary_table[iteration].rate[3]);
 }
 
 /* Returns if the stonewall was hit */
 int updateStoneWallIterations(int iteration, rank_progress_t * progress, double tstart){
   int hit = 0;
-  if (verbose >= 1 ) {
-    fprintf( out_logfile, "V-1: rank %d stonewall hit with %lld items\n", rank, (long long) progress->items_done );
-    fflush( out_logfile );
-  }
   uint64_t done = progress->items_done;
   long long unsigned max_iter = 0;
+
+  VERBOSE(1,1,"stonewall hit with %lld items", (long long) progress->items_done );
   MPI_Allreduce(& progress->items_done, & max_iter, 1, MPI_LONG_LONG_INT, MPI_MAX, testComm);
   summary_table[iteration].stonewall_time[MDTEST_FILE_CREATE_NUM] = GetTimeStamp() - tstart;
 
@@ -1088,12 +987,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
     char temp_path[MAX_PATHLEN];
     MPI_Comm_size(testComm, &size);
 
-    printf("DEBUG %5d Rank %d file_test on %s\n", __LINE__, rank, path);
-
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering file_test...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(3,5,"Entering file_test on %s", path);
 
     MPI_Barrier(testComm);
     t[0] = GetTimeStamp();
@@ -1104,9 +998,8 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
         prep_testdir(iteration, dir_iter);
 
         if (unique_dir_per_task) {
-            printf("DEBUG %5d Rank %d operating on %s\n", __LINE__, rank, temp_path);
             unique_dir_access(MK_UNI_DIR, temp_path);
-            printf("DEBUG %5d Rank %d operating on %s\n", __LINE__, rank, temp_path);
+            VERBOSE(5,5,"operating on %s", temp_path);
             if (!time_unique_dir_overhead) {
                 offset_timers(t, 0);
             }
@@ -1116,10 +1009,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
 
 
 
-        if (verbose >= 3 && rank == 0) {
-            fprintf(out_logfile,  "V-3: file_test: create path is \"%s\"\n", temp_path );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,-1,"file_test: create path is \"%s\"", temp_path );
 
         /* "touch" the files */
         if (collective_creates) {
@@ -1136,9 +1026,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
 
           if (hit){
             progress->stone_wall_timer_seconds = 0;
-            if (verbose > 1){
-              printf("stonewall rank %d: %lld of %lld \n", rank, (long long) progress->items_start, (long long) progress->items_per_dir);
-            }
+            VERBOSE(1,1,"stonewall: %lld of %lld", (long long) progress->items_start, (long long) progress->items_per_dir);
             create_remove_items(0, 0, 1, 0, temp_path, 0, progress);
             // now reset the values
             progress->stone_wall_timer_seconds = stone_wall_timer_seconds;
@@ -1163,8 +1051,8 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
         if (rank == 0) {
           if(expected_items == -1){
             fprintf(out_logfile, "WARNING: could not read stonewall status file\n");
-          }else if(verbose >= 1){
-            fprintf(out_logfile, "Read stonewall status; items: "LLU"\n", items);
+          }else {
+            VERBOSE(1,1, "Read stonewall status; items: "LLU"\n", items);
           }
         }
       }
@@ -1188,10 +1076,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        if (verbose >= 3 && rank == 0) {
-            fprintf(out_logfile,  "V-3: file_test: stat path is \"%s\"\n", temp_path );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"file_test: stat path is \"%s\"", temp_path );
 
         /* stat files */
         mdtest_stat((random_seed > 0 ? 1 : 0), 0, dir_iter, temp_path, progress);
@@ -1216,10 +1101,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        if (verbose >= 3 && rank == 0) {
-            fprintf(out_logfile,  "V-3: file_test: read path is \"%s\"\n", temp_path );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"file_test: read path is \"%s\"", temp_path );
 
         /* read files */
         if (random_seed > 0) {
@@ -1249,17 +1131,14 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        if (verbose >= 3 && rank == 0) {
-            fprintf(out_logfile,  "V-3: file_test: rm directories path is \"%s\"\n", temp_path );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"file_test: rm directories path is \"%s\"", temp_path );
 
         if (collective_creates) {
             if (rank == 0) {
                 collective_create_remove(0, 0, ntasks, temp_path, progress);
             }
         } else {
-            printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path);
+            VERBOSE(3,5,"gonna create %s", temp_path);
             create_remove_items(0, 0, 0, 0, temp_path, 0, progress);
         }
       }
@@ -1276,10 +1155,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
             strcpy( temp_path, path );
         }
 
-        if (verbose >= 3 && rank == 0) {
-            fprintf(out_logfile,  "V-3: file_test: rm unique directories path is \"%s\"\n", temp_path );
-            fflush( out_logfile );
-        }
+        VERBOSE(3,5,"file_test: rm unique directories path is \"%s\"", temp_path );
     }
 
     if (unique_dir_per_task && !time_unique_dir_overhead) {
@@ -1316,17 +1192,10 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
         summary_table[iteration].stonewall_last_item[7] = items;
     }
 
-    if (verbose >= 1 && rank == 0) {
-        fprintf(out_logfile, "V-1:   File creation     : %14.3f sec, %14.3f ops/sec\n",
-               t[1] - t[0], summary_table[iteration].rate[4]);
-        fprintf(out_logfile, "V-1:   File stat         : %14.3f sec, %14.3f ops/sec\n",
-               t[2] - t[1], summary_table[iteration].rate[5]);
-        fprintf(out_logfile, "V-1:   File read         : %14.3f sec, %14.3f ops/sec\n",
-               t[3] - t[2], summary_table[iteration].rate[6]);
-        fprintf(out_logfile, "V-1:   File removal      : %14.3f sec, %14.3f ops/sec\n",
-               t[4] - t[3], summary_table[iteration].rate[7]);
-        fflush(out_logfile);
-    }
+    VERBOSE(1,-1,"  File creation     : %14.3f sec, %14.3f ops/sec", t[1] - t[0], summary_table[iteration].rate[4]);
+    VERBOSE(1,-1,"  File stat         : %14.3f sec, %14.3f ops/sec", t[2] - t[1], summary_table[iteration].rate[5]);
+    VERBOSE(1,-1,"  File read         : %14.3f sec, %14.3f ops/sec", t[3] - t[2], summary_table[iteration].rate[6]);
+    VERBOSE(1,-1,"  File removal      : %14.3f sec, %14.3f ops/sec", t[4] - t[3], summary_table[iteration].rate[7]);
 }
 
 void print_help (void) {
@@ -1398,10 +1267,7 @@ void summarize_results(int iterations) {
     double all[iterations * size * tableSize];
 
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering summarize_results...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"Entering summarize_results..." );
 
     MPI_Barrier(testComm);
     for(int i=0; i < iterations; i++){
@@ -1544,16 +1410,10 @@ void valid_tests() {
 
     if (!create_only && !stat_only && !read_only && !remove_only) {
         create_only = stat_only = read_only = remove_only = 1;
-        if (( rank == 0 ) && ( verbose >= 1 )) {
-            fprintf( out_logfile, "V-1: main: Setting create/stat/read/remove_only to True\n" );
-            fflush( out_logfile );
-        }
+        VERBOSE(1,-1,"main: Setting create/stat/read/remove_only to True" );
     }
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering valid_tests...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"Entering valid_tests..." );
 
     /* if dirs_only and files_only were both left unset, set both now */
     if (!dirs_only && !files_only) {
@@ -1645,10 +1505,7 @@ void show_file_system_size(char *file_system) {
     ior_aiori_statfs_t stat_buf;
     int ret;
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering show_file_system_size...\n" );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,-1,"Entering show_file_system_size..." );
 
     ret = backend->statfs (file_system, &stat_buf, &param);
     if (0 != ret) {
@@ -1699,15 +1556,7 @@ void display_freespace(char *testdirpath)
     int  directoryFound   = 0;
 
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering display_freespace...\n" );
-        fflush( out_logfile );
-    }
-
-    if (verbose >= 3 && rank == 0) {
-        fprintf(out_logfile,  "V-3: testdirpath is \"%s\"\n", testdirpath );
-        fflush( out_logfile );
-    }
+    VERBOSE(3,5,"Entering display_freespace on %s...", testdirpath );
 
     strcpy(dirpath, testdirpath);
 
@@ -1726,17 +1575,9 @@ void display_freespace(char *testdirpath)
         strcpy(dirpath, ".");
     }
 
-    if (verbose >= 3 && rank == 0) {
-        fprintf(out_logfile,  "V-3: Before show_file_system_size, dirpath is \"%s\"\n", dirpath );
-        fflush( out_logfile );
-    }
-
+    VERBOSE(3,5,"Before show_file_system_size, dirpath is \"%s\"", dirpath );
     show_file_system_size(dirpath);
-
-    if (verbose >= 3 && rank == 0) {
-        fprintf(out_logfile,  "V-3: After show_file_system_size, dirpath is \"%s\"\n", dirpath );
-        fflush( out_logfile );
-    }
+    VERBOSE(3,5, "After show_file_system_size, dirpath is \"%s\"\n", dirpath );
 
     return;
 }
@@ -1748,20 +1589,13 @@ void create_remove_directory_tree(int create,
     char dir[MAX_PATHLEN];
 
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        fprintf( out_logfile, "V-1: Entering create_remove_directory_tree, currDepth = %d...\n", currDepth );
-        fflush( out_logfile );
-    }
+    VERBOSE(1,5,"Entering create_remove_directory_tree on %s, currDepth = %d...", path, currDepth );
 
     if (currDepth == 0) {
         sprintf(dir, "%s/%s.%d/", path, base_tree_name, dirNum);
 
         if (create) {
-            if (rank == 0 && verbose >= 2) {
-                fprintf(out_logfile, "V-2: Making directory \"%s\"\n", dir);
-                fflush(out_logfile);
-            }
-
+            VERBOSE(2,5,"Making directory \"%s\"", dir);
             if (-1 == backend->mkdir (dir, DIRMODE, &param)) {
                 fprintf(out_logfile, "error could not create directory \"%s\"\n", dir);
             }
@@ -1770,11 +1604,7 @@ void create_remove_directory_tree(int create,
         create_remove_directory_tree(create, ++currDepth, dir, ++dirNum, progress);
 
         if (!create) {
-            if (rank == 0 && verbose >= 2) {
-                fprintf(out_logfile, "V-2: Remove directory \"%s\"\n", dir);
-                fflush(out_logfile);
-            }
-
+            VERBOSE(2,5,"Remove directory \"%s\"", dir);
             if (-1 == backend->rmdir(dir, &param)) {
                 FAIL("Unable to remove directory");
             }
@@ -1790,11 +1620,7 @@ void create_remove_directory_tree(int create,
             strcat(temp_path, dir);
 
             if (create) {
-                if (rank == 0 && verbose >= 2) {
-                    fprintf(out_logfile, "V-2: Making directory \"%s\"\n", temp_path);
-                    fflush(out_logfile);
-                }
-
+                VERBOSE(2,5,"Making directory \"%s\"", temp_path);
                 if (-1 == backend->mkdir(temp_path, DIRMODE, &param)) {
                     FAIL("Unable to create directory");
                 }
@@ -1805,11 +1631,7 @@ void create_remove_directory_tree(int create,
             currDepth--;
 
             if (!create) {
-                if (rank == 0 && verbose >= 2) {
-                    fprintf(out_logfile, "V-2: Remove directory \"%s\"\n", temp_path);
-                    fflush(out_logfile);
-                }
-
+                VERBOSE(2,5,"Remove directory \"%s\"", temp_path);
                 if (-1 == backend->rmdir(temp_path, &param)) {
                     FAIL("Unable to remove directory");
                 }
@@ -1833,18 +1655,12 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
   double startCreate, endCreate;
   int k;
 
-  if (rank == 0 && verbose >= 1) {
-      fprintf(out_logfile, "V-1: main: * iteration %d *\n", j+1);
-      fflush(out_logfile);
-  }
+  VERBOSE(1,-1,"main: * iteration %d *", j+1);
 
   for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
     prep_testdir(j, dir_iter);
 
-    if (verbose >= 2 && rank == 0) {
-        fprintf(out_logfile,  "V-2: main (for j loop): making testdir, \"%s\"\n", testdir );
-        fflush( out_logfile );
-    }
+    VERBOSE(2,5,"main (for j loop): making testdir, \"%s\"", testdir );
     if ((rank < path_count) && backend->access(testdir, F_OK, &param) != 0) {
         if (backend->mkdir(testdir, DIRMODE, &param) != 0) {
             FAIL("Unable to create test directory");
@@ -1869,13 +1685,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
                 for (k=0; k<size; k++) {
                     sprintf(base_tree_name, "mdtest_tree.%d", k);
 
-                    if (verbose >= 3 && rank == 0) {
-                        fprintf(out_logfile,
-                            "V-3: main (create hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"\n",
-                            testdir );
-                        fflush( out_logfile );
-                    }
-
+                    VERBOSE(3,5,"main (create hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"", testdir );
                     /*
                      * Let's pass in the path to the directory we most recently made so that we can use
                      * full paths in the other calls.
@@ -1887,13 +1697,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
                     }
                 }
             } else if (!collective_creates) {
-                if (verbose >= 3 && rank == 0) {
-                    fprintf(out_logfile,
-                        "V-3: main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with \"%s\"\n",
-                        testdir );
-                    fflush( out_logfile );
-                }
-
+                VERBOSE(3,5,"main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with \"%s\"", testdir );
                 /*
                  * Let's pass in the path to the directory we most recently made so that we can use
                  * full paths in the other calls.
@@ -1902,12 +1706,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
             }
         } else {
             if (rank == 0) {
-                if (verbose >= 3 && rank == 0) {
-                    fprintf(out_logfile,
-                        "V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with \"%s\"\n",
-                        testdir );
-                    fflush( out_logfile );
-                }
+                VERBOSE(3,5,"main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with \"%s\"", testdir );
 
                 /*
                  * Let's pass in the path to the directory we most recently made so that we can use
@@ -1924,11 +1723,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
       summary_table->time[8] = (endCreate - startCreate);
       summary_table->items[8] = num_dirs_in_tree;
       summary_table->stonewall_last_item[8] = num_dirs_in_tree;
-      if (verbose >= 1 && rank == 0) {
-          fprintf(out_logfile, "V-1: main:   Tree creation     : %14.3f sec, %14.3f ops/sec\n",
-                 (endCreate - startCreate), summary_table->rate[8]);
-          fflush(out_logfile);
-      }
+      VERBOSE(1,-1,"V-1: main:   Tree creation     : %14.3f sec, %14.3f ops/sec", (endCreate - startCreate), summary_table->rate[8]);
   }
   sprintf(unique_mk_dir, "%s.0", base_tree_name);
   sprintf(unique_chdir_dir, "%s.0", base_tree_name);
@@ -1938,10 +1733,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
   unique_rm_uni_dir[0] = 0;
 
   if (!unique_dir_per_task) {
-      if (verbose >= 3 && rank == 0) {
-          fprintf(out_logfile,  "V-3: main: Using unique_mk_dir, \"%s\"\n", unique_mk_dir );
-          fflush( out_logfile );
-      }
+    VERBOSE(3,-1,"V-3: main: Using unique_mk_dir, \"%s\"", unique_mk_dir );
   }
 
   if (rank < i) {
@@ -1963,10 +1755,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
                   rank, unique_mk_dir,unique_chdir_dir,unique_stat_dir,unique_read_dir,unique_rm_dir);
       }
 
-      if (verbose >= 3 && rank == 0) {
-          fprintf(out_logfile,  "V-3: main: Copied unique_mk_dir, \"%s\", to topdir\n", unique_mk_dir );
-          fflush( out_logfile );
-      }
+      VERBOSE(3,-1,"V-3: main: Copied unique_mk_dir, \"%s\", to topdir", unique_mk_dir );
 
       if (dirs_only && !shared_file) {
           if (pre_delay) {
@@ -1985,10 +1774,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
 
   /* remove directory structure */
   if (!unique_dir_per_task) {
-      if (verbose >= 3 && rank == 0) {
-          fprintf(out_logfile,  "V-3: main: Using testdir, \"%s\"\n", testdir );
-          fflush( out_logfile );
-      }
+      VERBOSE(3,-1,"main: Using testdir, \"%s\"", testdir );
   }
 
   MPI_Barrier(testComm);
@@ -2006,12 +1792,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
                 for (k=0; k<size; k++) {
                     sprintf(base_tree_name, "mdtest_tree.%d", k);
 
-                    if (verbose >= 3 && rank == 0) {
-                        fprintf(out_logfile,
-                            "V-3: main (remove hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"\n",
-                            testdir );
-                        fflush( out_logfile );
-                    }
+                    VERBOSE(3,-1,"main (remove hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"", testdir );
 
                     /*
                      * Let's pass in the path to the directory we most recently made so that we can use
@@ -2024,12 +1805,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
                     }
                 }
             } else if (!collective_creates) {
-                if (verbose >= 3 && rank == 0) {
-                    fprintf(out_logfile,
-                        "V-3: main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with \"%s\"\n",
-                        testdir );
-                    fflush( out_logfile );
-                }
+                VERBOSE(3,-1,"main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with \"%s\"", testdir );
 
                 /*
                  * Let's pass in the path to the directory we most recently made so that we can use
@@ -2039,12 +1815,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
             }
         } else {
             if (rank == 0) {
-                if (verbose >= 3 && rank == 0) {
-                    fprintf(out_logfile,
-                        "V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with \"%s\"\n",
-                        testdir );
-                    fflush( out_logfile );
-                }
+                VERBOSE(3,-1,"V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with \"%s\"", testdir );
 
                 /*
                  * Let's pass in the path to the directory we most recently made so that we can use
@@ -2061,16 +1832,8 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
       summary_table->time[9] = endCreate - startCreate;
       summary_table->items[9] = num_dirs_in_tree;
       summary_table->stonewall_last_item[8] = num_dirs_in_tree;
-      if (verbose >= 1 && rank == 0) {
-          fprintf(out_logfile, "V-1: main   Tree removal      : %14.3f sec, %14.3f ops/sec\n",
-                 (endCreate - startCreate), summary_table->rate[9]);
-          fflush(out_logfile);
-      }
-
-      if (( rank == 0 ) && ( verbose >=2 )) {
-          fprintf( out_logfile, "V-2: main (at end of for j loop): Removing testdir of \"%s\"\n", testdir );
-          fflush( out_logfile );
-      }
+      VERBOSE(1,-1,"main   Tree removal      : %14.3f sec, %14.3f ops/sec", (endCreate - startCreate), summary_table->rate[9]);
+      VERBOSE(2,-1,"main (at end of for j loop): Removing testdir of \"%s\"\n", testdir );
 
       for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
         prep_testdir(j, dir_iter);
@@ -2204,8 +1967,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
 
     if (rank == 0) {
         fprintf(out_logfile, "-- started at %s --\n\n", PrintTimestamp());
-        fprintf(out_logfile, "mdtest-%s was launched with %d total task(s) on %d node(s)\n",
-               RELEASE_VERS, size, nodeCount);
+        fprintf(out_logfile, "mdtest-%s was launched with %d total task(s) on %d node(s)\n", RELEASE_VERS, size, nodeCount);
         fflush(out_logfile);
     }
 
@@ -2239,42 +2001,39 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
     }
     valid_tests();
 
-    if (( rank == 0 ) && ( verbose >= 1 )) {
-        // option_print_current(options);
-        fprintf (out_logfile, "api                     : %s\n", param.api);
-        fprintf( out_logfile, "barriers                : %s\n", ( barriers ? "True" : "False" ));
-        fprintf( out_logfile, "collective_creates      : %s\n", ( collective_creates ? "True" : "False" ));
-        fprintf( out_logfile, "create_only             : %s\n", ( create_only ? "True" : "False" ));
-        fprintf( out_logfile, "dirpath(s):\n" );
-        for ( i = 0; i < path_count; i++ ) {
-            fprintf( out_logfile, "\t%s\n", filenames[i] );
-        }
-        fprintf( out_logfile, "dirs_only               : %s\n", ( dirs_only ? "True" : "False" ));
-        fprintf( out_logfile, "read_bytes              : "LLU"\n", read_bytes );
-        fprintf( out_logfile, "read_only               : %s\n", ( read_only ? "True" : "False" ));
-        fprintf( out_logfile, "first                   : %d\n", first );
-        fprintf( out_logfile, "files_only              : %s\n", ( files_only ? "True" : "False" ));
-        fprintf( out_logfile, "iterations              : %d\n", iterations );
-        fprintf( out_logfile, "items_per_dir           : "LLU"\n", items_per_dir );
-        fprintf( out_logfile, "last                    : %d\n", last );
-        fprintf( out_logfile, "leaf_only               : %s\n", ( leaf_only ? "True" : "False" ));
-        fprintf( out_logfile, "items                   : "LLU"\n", items );
-        fprintf( out_logfile, "nstride                 : %d\n", nstride );
-        fprintf( out_logfile, "pre_delay               : %d\n", pre_delay );
-        fprintf( out_logfile, "remove_only             : %s\n", ( leaf_only ? "True" : "False" ));
-        fprintf( out_logfile, "random_seed             : %d\n", random_seed );
-        fprintf( out_logfile, "stride                  : %d\n", stride );
-        fprintf( out_logfile, "shared_file             : %s\n", ( shared_file ? "True" : "False" ));
-        fprintf( out_logfile, "time_unique_dir_overhead: %s\n", ( time_unique_dir_overhead ? "True" : "False" ));
-        fprintf( out_logfile, "stone_wall_timer_seconds: %d\n", stone_wall_timer_seconds);
-        fprintf( out_logfile, "stat_only               : %s\n", ( stat_only ? "True" : "False" ));
-        fprintf( out_logfile, "unique_dir_per_task     : %s\n", ( unique_dir_per_task ? "True" : "False" ));
-        fprintf( out_logfile, "write_bytes             : "LLU"\n", write_bytes );
-        fprintf( out_logfile, "sync_file               : %s\n", ( sync_file ? "True" : "False" ));
-        fprintf( out_logfile, "depth                   : %d\n", depth );
-        fprintf( out_logfile, "make_node               : %d\n", make_node );
-        fflush( out_logfile );
+    // option_print_current(options);
+    VERBOSE(1,-1, "api                     : %s", param.api);
+    VERBOSE(1,-1, "barriers                : %s", ( barriers ? "True" : "False" ));
+    VERBOSE(1,-1, "collective_creates      : %s", ( collective_creates ? "True" : "False" ));
+    VERBOSE(1,-1, "create_only             : %s", ( create_only ? "True" : "False" ));
+    VERBOSE(1,-1, "dirpath(s):" );
+    for ( i = 0; i < path_count; i++ ) {
+        VERBOSE(1,-1, "\t%s", filenames[i] );
     }
+    VERBOSE(1,-1, "dirs_only               : %s", ( dirs_only ? "True" : "False" ));
+    VERBOSE(1,-1, "read_bytes              : "LLU"", read_bytes );
+    VERBOSE(1,-1, "read_only               : %s", ( read_only ? "True" : "False" ));
+    VERBOSE(1,-1, "first                   : %d", first );
+    VERBOSE(1,-1, "files_only              : %s", ( files_only ? "True" : "False" ));
+    VERBOSE(1,-1, "iterations              : %d", iterations );
+    VERBOSE(1,-1, "items_per_dir           : "LLU"", items_per_dir );
+    VERBOSE(1,-1, "last                    : %d", last );
+    VERBOSE(1,-1, "leaf_only               : %s", ( leaf_only ? "True" : "False" ));
+    VERBOSE(1,-1, "items                   : "LLU"", items );
+    VERBOSE(1,-1, "nstride                 : %d", nstride );
+    VERBOSE(1,-1, "pre_delay               : %d", pre_delay );
+    VERBOSE(1,-1, "remove_only             : %s", ( leaf_only ? "True" : "False" ));
+    VERBOSE(1,-1, "random_seed             : %d", random_seed );
+    VERBOSE(1,-1, "stride                  : %d", stride );
+    VERBOSE(1,-1, "shared_file             : %s", ( shared_file ? "True" : "False" ));
+    VERBOSE(1,-1, "time_unique_dir_overhead: %s", ( time_unique_dir_overhead ? "True" : "False" ));
+    VERBOSE(1,-1, "stone_wall_timer_seconds: %d", stone_wall_timer_seconds);
+    VERBOSE(1,-1, "stat_only               : %s", ( stat_only ? "True" : "False" ));
+    VERBOSE(1,-1, "unique_dir_per_task     : %s", ( unique_dir_per_task ? "True" : "False" ));
+    VERBOSE(1,-1, "write_bytes             : "LLU"", write_bytes );
+    VERBOSE(1,-1, "sync_file               : %s", ( sync_file ? "True" : "False" ));
+    VERBOSE(1,-1, "depth                   : %d", depth );
+    VERBOSE(1,-1, "make_node               : %d", make_node );
 
     /* setup total number of items and number of items per dir */
     if (depth <= 0) {
@@ -2378,10 +2137,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
     }
 
     /* display disk usage */
-    if (verbose >= 3 && rank == 0) {
-        fprintf(out_logfile,  "V-3: main (before display_freespace): testdirpath is \"%s\"\n", testdirpath );
-        fflush( out_logfile );
-    }
+    VERBOSE(3,-1,"main (before display_freespace): testdirpath is \"%s\"", testdirpath );
 
     if (rank == 0) display_freespace(testdirpath);
     int packedByNode = QueryNodeMapping(testComm);
@@ -2396,10 +2152,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
         }
     }
 
-    if (verbose >= 3 && rank == 0) {
-        fprintf(out_logfile,  "V-3: main (after display_freespace): testdirpath is \"%s\"\n", testdirpath );
-        fflush( out_logfile );
-    }
+    VERBOSE(3,-1,"main (after display_freespace): testdirpath is \"%s\"", testdirpath );
 
     if (rank == 0) {
         if (random_seed > 0) {
@@ -2468,11 +2221,9 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
                 fprintf(out_logfile, "\n%d tasks, "LLU" directories\n", i, items_all);
             }
         }
-        if (rank == 0 && verbose >= 1) {
-            fprintf(out_logfile, "\n");
-            fprintf(out_logfile, "   Operation               Duration              Rate\n");
-            fprintf(out_logfile, "   ---------               --------              ----\n");
-        }
+        VERBOSE(1,-1,"");
+        VERBOSE(1,-1,"   Operation               Duration              Rate");
+        VERBOSE(1,-1,"   ---------               --------              ----");
 
         for (j = 0; j < iterations; j++) {
             // keep track of the current status for stonewalling
@@ -2484,10 +2235,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
         }
     }
 
-    if (rank == 0) {
-        fprintf(out_logfile, "\n-- finished at %s --\n", PrintTimestamp());
-        fflush(out_logfile);
-    }
+    VERBOSE(0,-1,"-- finished at %s --\n", PrintTimestamp());
 
     if (random_seed > 0) {
         free(rand_array);

From e767ef3de980698677450452bd4933a621d4c1c0 Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Sat, 27 Jul 2019 13:26:39 -0600
Subject: [PATCH 06/13] Remove extraneous print_help function that was causing
 people to have to edit the same string in two different locations

---
 src/mdtest.c | 62 ----------------------------------------------------
 1 file changed, 62 deletions(-)

diff --git a/src/mdtest.c b/src/mdtest.c
index bfeeda1..368acf1 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -455,7 +455,6 @@ void create_remove_items(int currDepth, const int dirs, const int create, const
             if (collective) {
                 collective_helper(dirs, create, temp_path, 0, progress);
             } else {
-                printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path);
                 create_remove_items_helper(dirs, create, temp_path, 0, progress);
             }
         }
@@ -481,7 +480,6 @@ void create_remove_items(int currDepth, const int dirs, const int create, const
                 if (collective) {
                     collective_helper(dirs, create, temp_path, currDir*items_per_dir, progress);
                 } else {
-                    printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path);
                     create_remove_items_helper(dirs, create, temp_path, currDir*items_per_dir, progress);
                 }
             }
@@ -1198,66 +1196,6 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
     VERBOSE(1,-1,"  File removal      : %14.3f sec, %14.3f ops/sec", t[4] - t[3], summary_table[iteration].rate[7]);
 }
 
-void print_help (void) {
-    int j;
-
-    char APIs[1024];
-    char APIs_legacy[1024];
-    aiori_supported_apis(APIs, APIs_legacy, MDTEST);
-    char apiStr[1024];
-    sprintf(apiStr, "API for I/O [%s]", APIs);
-
-    fprintf(out_logfile,
-        "Usage: mdtest [-b branching_factor] [-B] [-c] [-C] [-d testdir] [-D] [-e number_of_bytes_to_read]\n"
-        "              [-E] [-f first] [-F] [-h] [-i iterations] [-I items_per_dir] [-k] [-l last] [-L]\n"
-        "              [-n number_of_items] [-N stride_length] [-p seconds] [-r]\n"
-        "              [-R[seed]] [-s stride] [-S] [-t] [-T] [-u] [-v] [-a API]\n"
-        "              [-V verbosity_value] [-w number_of_bytes_to_write] [-W seconds] [-y] [-z depth] -Z\n"
-        "\t-a: %s\n"
-        "\t-b: branching factor of hierarchical directory structure\n"
-        "\t-B: no barriers between phases\n"
-        "\t-c: collective creates: task 0 does all creates\n"
-        "\t-C: only create files/dirs\n"
-        "\t-d: the directory in which the tests will run\n"
-        "\t-D: perform test on directories only (no files)\n"
-        "\t-e: bytes to read from each file\n"
-        "\t-E: only read files/dir\n"
-        "\t-f: first number of tasks on which the test will run\n"
-        "\t-F: perform test on files only (no directories)\n"
-        "\t-h: prints this help message\n"
-        "\t-i: number of iterations the test will run\n"
-        "\t-I: number of items per directory in tree\n"
-        "\t-k: use mknod\n"
-        "\t-l: last number of tasks on which the test will run\n"
-        "\t-L: files only at leaf level of tree\n"
-        "\t-n: every process will creat/stat/read/remove # directories and files\n"
-        "\t-N: stride # between neighbor tasks for file/dir operation (local=0)\n"
-        "\t-p: pre-iteration delay (in seconds)\n"
-        "\t-r: only remove files or directories left behind by previous runs\n"
-        "\t-R: randomly stat files (optional argument for random seed)\n"
-        "\t-s: stride between the number of nodes for each test\n"
-        "\t-S: shared file access (file only, no directories)\n"
-        "\t-t: time unique working directory overhead\n"
-        "\t-T: only stat files/dirs\n"
-        "\t-u: unique working directory for each task\n"
-        "\t-v: verbosity (each instance of option increments by one)\n"
-        "\t-V: verbosity value\n"
-        "\t-w: bytes to write to each file after it is created\n"
-        "\t-W: number in seconds; stonewall timer, write as many seconds and ensure all processes did the same number of operations (currently only stops during create phase)\n"
-        "\t-x: StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs\n"
-        "\t-y: sync file after writing\n"
-        "\t-z: depth of hierarchical directory structure\n"
-        "\t-Z: print time instead of rate\n",
-        apiStr
-        );
-
-    MPI_Initialized(&j);
-    if (j) {
-        MPI_Finalize();
-    }
-    exit(0);
-}
-
 void summarize_results(int iterations) {
     char access[MAX_PATHLEN];
     int i, j, k;

From d69957e55b52137d62692872596c755d7fbe727f Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Sat, 27 Jul 2019 14:31:49 -0600
Subject: [PATCH 07/13] Final changes cleaning up the output messages

---
 src/mdtest.c | 105 ++++++++++++++++++++++-----------------------------
 1 file changed, 46 insertions(+), 59 deletions(-)

diff --git a/src/mdtest.c b/src/mdtest.c
index 368acf1..75558c1 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -188,7 +188,14 @@ void VerboseMessage (int root_level, int any_level, int line, char * format, ...
         va_start (args, format);
         vsnprintf (buffer, 1024, format, args);
         va_end (args);
-        fprintf( out_logfile, "V-%d: Rank %3d Line %5d %s\n", root_level, rank, line, buffer );
+        if (root_level == 0 && any_level == -1) {
+            /* No header when it is just the standard output */
+            fprintf( out_logfile, "%s\n", buffer );
+        } else {
+            /* add a header when the verbose is greater than 0 */
+            fprintf( out_logfile, "V-%d: Rank %3d Line %5d %s\n", root_level, rank, line, buffer );
+        }
+        fflush(out_logfile);
     }
 }
 
@@ -211,7 +218,7 @@ void parse_dirpath(char *dirpath_arg) {
     int i = 0;
 
 
-    VERBOSE(1,-1, "Entering parse_dirpath..." );
+    VERBOSE(1,-1, "Entering parse_dirpath on %s...", dirpath_arg );
 
     tmp = dirpath_arg;
 
@@ -252,8 +259,6 @@ static void prep_testdir(int j, int dir_iter){
  */
 
 void unique_dir_access(int opt, char *to) {
-    VERBOSE(1,-1,"Entering unique_dir_access..." );
-
     if (opt == MK_UNI_DIR) {
         MPI_Barrier(testComm);
         sprintf( to, "%s/%s", testdir, unique_chdir_dir );
@@ -266,6 +271,7 @@ void unique_dir_access(int opt, char *to) {
     } else if (opt == RM_UNI_DIR) {
         sprintf( to, "%s/%s", testdir, unique_rm_uni_dir );
     }
+    VERBOSE(1,-1,"Entering unique_dir_access, set it to %s", to );
 }
 
 static void create_remove_dirs (const char *path, bool create, uint64_t itemNum) {
@@ -373,7 +379,7 @@ static void create_file (const char *path, uint64_t itemNum) {
 void create_remove_items_helper(const int dirs, const int create, const char *path,
                                 uint64_t itemNum, rank_progress_t * progress) {
 
-    VERBOSE(1,-1,"Entering create_remove_items_helper..." );
+    VERBOSE(1,-1,"Entering create_remove_items_helper on %s", path );
 
     for (uint64_t i = progress->items_start; i < progress->items_per_dir ; ++i) {
         if (!dirs) {
@@ -399,7 +405,7 @@ void create_remove_items_helper(const int dirs, const int create, const char *pa
 void collective_helper(const int dirs, const int create, const char* path, uint64_t itemNum, rank_progress_t * progress) {
     char curr_item[MAX_PATHLEN];
 
-    VERBOSE(1,-1,"Entering collective_helper..." );
+    VERBOSE(1,-1,"Entering collective_helper on %s", path );
     for (uint64_t i = progress->items_start ; i < progress->items_per_dir ; ++i) {
         if (dirs) {
             create_remove_dirs (path, create, itemNum + i);
@@ -441,7 +447,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const
     unsigned long long currDir = dirNum;
 
 
-    VERBOSE(1,-1,"Entering create_remove_items, currDepth = %d...", currDepth );
+    VERBOSE(1,-1,"Entering create_remove_items on %s, currDepth = %d...", path, currDepth );
 
 
     memset(dir, 0, MAX_PATHLEN);
@@ -509,7 +515,7 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch
     uint64_t parent_dir, item_num = 0;
     char item[MAX_PATHLEN], temp[MAX_PATHLEN];
 
-    VERBOSE(1,-1,"Entering mdtest_stat..." );
+    VERBOSE(1,-1,"Entering mdtest_stat on %s", path );
 
     uint64_t stop_items = items;
 
@@ -594,7 +600,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) {
     char item[MAX_PATHLEN], temp[MAX_PATHLEN];
     void *aiori_fh;
 
-    VERBOSE(1,-1,"Entering mdtest_read..." );
+    VERBOSE(1,-1,"Entering mdtest_read on %s", path );
 
     /* allocate read buffer */
     if (read_bytes > 0) {
@@ -940,12 +946,12 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
         summary_table[iteration].stonewall_last_item[3] = items;
     }
 
-    VERBOSE(1,-1,"   Directory creation: %14.3f sec, %14.3f ops/sec\n", t[1] - t[0], summary_table[iteration].rate[0]);
-    VERBOSE(1,-1,"   Directory stat    : %14.3f sec, %14.3f ops/sec\n", t[2] - t[1], summary_table[iteration].rate[1]);
+    VERBOSE(1,-1,"   Directory creation: %14.3f sec, %14.3f ops/sec", t[1] - t[0], summary_table[iteration].rate[0]);
+    VERBOSE(1,-1,"   Directory stat    : %14.3f sec, %14.3f ops/sec", t[2] - t[1], summary_table[iteration].rate[1]);
     /* N/A
-    VERBOSE(1,-1,"   Directory read    : %14.3f sec, %14.3f ops/sec\n", t[3] - t[2], summary_table[iteration].rate[2]);
+    VERBOSE(1,-1,"   Directory read    : %14.3f sec, %14.3f ops/sec", t[3] - t[2], summary_table[iteration].rate[2]);
     */
-    VERBOSE(1,-1,"   Directory removal : %14.3f sec, %14.3f ops/sec\n", t[4] - t[3], summary_table[iteration].rate[3]);
+    VERBOSE(1,-1,"   Directory removal : %14.3f sec, %14.3f ops/sec", t[4] - t[3], summary_table[iteration].rate[3]);
 }
 
 /* Returns if the stonewall was hit */
@@ -967,10 +973,7 @@ int updateStoneWallIterations(int iteration, rank_progress_t * progress, double
   if(items != (sum_accessed / size)){
     summary_table[iteration].stonewall_item_sum[MDTEST_FILE_CREATE_NUM] = sum_accessed;
     summary_table[iteration].stonewall_item_min[MDTEST_FILE_CREATE_NUM] = min_accessed * size;
-    if (rank == 0){
-      fprintf( out_logfile, "Continue stonewall hit min: %lld max: %lld avg: %.1f \n", min_accessed, max_iter, ((double) sum_accessed) / size);
-      fflush( out_logfile );
-    }
+    VERBOSE(0,-1, "Continue stonewall hit min: %lld max: %lld avg: %.1f \n", min_accessed, max_iter, ((double) sum_accessed) / size);
     hit = 1;
   }
   progress->items_start = done;
@@ -1218,12 +1221,9 @@ void summarize_results(int iterations) {
 
     if (rank == 0) {
 
-        fprintf(out_logfile, "\nSUMMARY %s: (of %d iterations)\n", print_time ? "time": "rate", iterations);
-        fprintf(out_logfile,
-            "   Operation                      Max            Min           Mean        Std Dev\n");
-        fprintf(out_logfile,
-            "   ---------                      ---            ---           ----        -------\n");
-        fflush(out_logfile);
+        VERBOSE(0,-1,"\nSUMMARY %s: (of %d iterations)", print_time ? "time": "rate", iterations);
+        VERBOSE(0,-1,"   Operation                      Max            Min           Mean        Std Dev");
+        VERBOSE(0,-1,"   ---------                      ---            ---           ----        -------");
 
         /* if files only access, skip entries 0-3 (the dir tests) */
         if (files_only && !dirs_only) {
@@ -1342,8 +1342,7 @@ void summarize_results(int iterations) {
 void valid_tests() {
 
     if (((stone_wall_timer_seconds > 0) && (branch_factor > 1)) || ! barriers) {
-        fprintf(out_logfile, "Error, stone wall timer does only work with a branch factor <= 1 and with barriers\n");
-        MPI_Abort(testComm, 1);
+        FAIL( "Error, stone wall timer does only work with a branch factor <= 1 and with barriers\n");
     }
 
     if (!create_only && !stat_only && !read_only && !remove_only) {
@@ -1443,7 +1442,7 @@ void show_file_system_size(char *file_system) {
     ior_aiori_statfs_t stat_buf;
     int ret;
 
-    VERBOSE(1,-1,"Entering show_file_system_size..." );
+    VERBOSE(1,-1,"Entering show_file_system_size on %s", file_system );
 
     ret = backend->statfs (file_system, &stat_buf, &param);
     if (0 != ret) {
@@ -1475,14 +1474,10 @@ void show_file_system_size(char *file_system) {
 
 
     /* show results */
-    fprintf(out_logfile, "Path: %s\n", real_path);
-    fprintf(out_logfile, "FS: %.1f %s   Used FS: %2.1f%%   ",
-            total_file_system_size_hr, file_system_unit_str,
-            used_file_system_percentage);
-    fprintf(out_logfile, "Inodes: %.1f %s   Used Inodes: %2.1f%%\n",
-            (double)total_inodes / (double)inode_unit_val,
-            inode_unit_str, used_inode_percentage);
-    fflush(out_logfile);
+    VERBOSE(0,-1,"Path: %s", real_path);
+    VERBOSE(0,-1,"FS: %.1f %s   Used FS: %2.1f%%   Inodes: %.1f %s   Used Inodes: %2.1f%%\n", 
+            total_file_system_size_hr, file_system_unit_str, used_file_system_percentage,
+            (double)total_inodes / (double)inode_unit_val, inode_unit_str, used_inode_percentage);
 
     return;
 }
@@ -1682,15 +1677,14 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
           sprintf(rm_name, "mdtest.%d.", (rank+(3*nstride))%i);
       }
       if (unique_dir_per_task) {
-          printf("DEBUG %5d Rank %d i %d nstride %d\n", __LINE__, rank, i, nstride);
+          VERBOSE(3,5,"i %d nstride %d", i, nstride);
           sprintf(unique_mk_dir, "mdtest_tree.%d.0",  (rank+(0*nstride))%i);
           sprintf(unique_chdir_dir, "mdtest_tree.%d.0", (rank+(1*nstride))%i);
           sprintf(unique_stat_dir, "mdtest_tree.%d.0", (rank+(2*nstride))%i);
           sprintf(unique_read_dir, "mdtest_tree.%d.0", (rank+(3*nstride))%i);
           sprintf(unique_rm_dir, "mdtest_tree.%d.0", (rank+(4*nstride))%i);
           unique_rm_uni_dir[0] = 0;
-          printf("DEBUG %5d Rank %d mk_dir %s chdir %s stat_dir %s read_dir %s rm_dir %s\n", __LINE__, 
-                  rank, unique_mk_dir,unique_chdir_dir,unique_stat_dir,unique_read_dir,unique_rm_dir);
+          VERBOSE(5,5,"mk_dir %s chdir %s stat_dir %s read_dir %s rm_dir %s\n", unique_mk_dir,unique_chdir_dir,unique_stat_dir,unique_read_dir,unique_rm_dir);
       }
 
       VERBOSE(3,-1,"V-3: main: Copied unique_mk_dir, \"%s\", to topdir", unique_mk_dir );
@@ -1705,7 +1699,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
           if (pre_delay) {
               DelaySecs(pre_delay);
           }
-          printf("DEBUG %5d Rank %d gonna file_test on %s\n", __LINE__, rank, unique_mk_dir);
+          VERBOSE(3,5,"will file_test on %s", unique_mk_dir);
           file_test(j, i, unique_mk_dir, progress);
       }
   }
@@ -1903,20 +1897,15 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
     tasksPerNode = CountTasksPerNode(testComm);
     nodeCount = size / tasksPerNode;
 
-    if (rank == 0) {
-        fprintf(out_logfile, "-- started at %s --\n\n", PrintTimestamp());
-        fprintf(out_logfile, "mdtest-%s was launched with %d total task(s) on %d node(s)\n", RELEASE_VERS, size, nodeCount);
-        fflush(out_logfile);
+    char cmd_buffer[4096];
+    strncpy(cmd_buffer, argv[0], 4096);
+    for (i = 1; i < argc; i++) {
+        snprintf(&cmd_buffer[strlen(cmd_buffer)], 4096-strlen(cmd_buffer), " \"%s\"", argv[i]);
     }
 
-    if (rank == 0) {
-        fprintf(out_logfile, "Command line used: %s", argv[0]);
-        for (i = 1; i < argc; i++) {
-            fprintf(out_logfile, " \"%s\"", argv[i]);
-        }
-        fprintf(out_logfile, "\n");
-        fflush(out_logfile);
-    }
+    VERBOSE(0,-1,"-- started at %s --\n", PrintTimestamp());
+    VERBOSE(0,-1,"mdtest-%s was launched with %d total task(s) on %d node(s)", RELEASE_VERS, size, nodeCount);
+    VERBOSE(0,-1,"Command line used: %s", cmd_buffer);
 
     /* adjust special variables */
     barriers = ! no_barriers;
@@ -2082,19 +2071,17 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
 
     /* set the shift to mimic IOR and shift by procs per node */
     if (nstride > 0) {
-        if ( packedByNode ) {
+        if ( nodeCount > 1 && packedByNode ) {
             nstride *= tasksPerNode;
         }
-        if (rank == 0) {
-            fprintf(out_logfile, "Shifting ranks by %d for each phase.\n", nstride);
-        }
+        VERBOSE(0,5,"Shifting ranks by %d for each phase.", nstride);
     }
 
     VERBOSE(3,-1,"main (after display_freespace): testdirpath is \"%s\"", testdirpath );
 
     if (rank == 0) {
         if (random_seed > 0) {
-            fprintf(out_logfile, "random seed: %d\n", random_seed);
+            VERBOSE(0,-1,"random seed: %d", random_seed);
         }
     }
 
@@ -2147,16 +2134,16 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
               items_all *= num_dirs_in_tree_calc;
             }
             if (files_only && dirs_only) {
-                fprintf(out_logfile, "\n%d tasks, "LLU" files/directories\n", i, items_all);
+                VERBOSE(0,-1,"%d tasks, "LLU" files/directories", i, items_all);
             } else if (files_only) {
                 if (!shared_file) {
-                    fprintf(out_logfile, "\n%d tasks, "LLU" files\n", i, items_all);
+                    VERBOSE(0,-1,"%d tasks, "LLU" files", i, items_all);
                 }
                 else {
-                    fprintf(out_logfile, "\n%d tasks, 1 file\n", i);
+                    VERBOSE(0,-1,"%d tasks, 1 file", i);
                 }
             } else if (dirs_only) {
-                fprintf(out_logfile, "\n%d tasks, "LLU" directories\n", i, items_all);
+                VERBOSE(0,-1,"%d tasks, "LLU" directories", i, items_all);
             }
         }
         VERBOSE(1,-1,"");

From 629ff810b70cd98f8eaec51167154af70dcc3386 Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Sat, 27 Jul 2019 15:27:20 -0600
Subject: [PATCH 08/13] Got IOR shifting to work regardless of whether
 node/task mapping is round-robin or contiguous

---
 src/ior.c | 20 ++++++++++++++++----
 src/ior.h |  1 +
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/ior.c b/src/ior.c
index 3ef411c..bb2b4c2 100755
--- a/src/ior.c
+++ b/src/ior.c
@@ -939,6 +939,7 @@ static void InitTests(IOR_test_t *tests, MPI_Comm com)
                 params->testComm = com;
                 params->nodes = params->numTasks / tasksPerNode;
                 params->tasksPerNode = tasksPerNode;
+                params->packedTasks = QueryNodeMapping(com);
                 if (params->numTasks == 0) {
                   params->numTasks = size;
                 }
@@ -1360,7 +1361,13 @@ static void TestIoSys(IOR_test_t *test)
                         }
                         if (params->reorderTasks) {
                                 /* move two nodes away from writing node */
-                                rankOffset = (2 * params->tasksPerNode) % params->numTasks;
+                                int shift = 1;
+                                if (params->packedTasks) {
+                                    shift = params->tasksPerNode;
+                                } else {
+                                    shift = 1;
+                                }
+                                rankOffset = (2 * shift) % params->numTasks;
                         }
 
                         // update the check buffer
@@ -1395,9 +1402,14 @@ static void TestIoSys(IOR_test_t *test)
                         /* Get rankOffset [file offset] for this process to read, based on -C,-Z,-Q,-X options */
                         /* Constant process offset reading */
                         if (params->reorderTasks) {
-                                /* move taskPerNodeOffset nodes[1==default] away from writing node */
-                                rankOffset = (params->taskPerNodeOffset *
-                                          params->tasksPerNode) % params->numTasks;
+                                /* move one node away from writing node */ 
+                                int shift = 1;
+                                if (params->packedTasks) {
+                                    shift=params->tasksPerNode;
+                                } else {
+                                    shift=1;
+                                }
+                                rankOffset = (params->taskPerNodeOffset * shift) % params->numTasks;
                         }
                         /* random process offset reading */
                         if (params->reorderTasksRandom) {
diff --git a/src/ior.h b/src/ior.h
index 5c46efe..2ea6c05 100755
--- a/src/ior.h
+++ b/src/ior.h
@@ -100,6 +100,7 @@ typedef struct
     int numTasks;                    /* number of tasks for test */
     int nodes;                       /* number of nodes for test */
     int tasksPerNode;                /* number of tasks per node */
+    int packedTasks;                 /* are the tasks round-robin across nodes or are they packed tightly*/ 
     int repetitions;                 /* number of repetitions of test */
     int repCounter;                  /* rep counter */
     int multiFile;                   /* multiple files */

From 168a4077938e29cd46f627612e13b6dc6363ddeb Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Sun, 28 Jul 2019 09:55:00 -0600
Subject: [PATCH 09/13] Fixed inconsistent spacing that Andreas commented upon

---
 src/mdtest.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mdtest.c b/src/mdtest.c
index 75558c1..074463e 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -551,12 +551,12 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch
 
         /* create name of file/dir to stat */
         if (dirs) {
-            if ((i%ITEM_COUNT == 0) && (i != 0)) {
+            if ( (i % ITEM_COUNT == 0) && (i != 0)) {
                 VERBOSE(3,5,"stat dir: "LLU"", i);
             }
             sprintf(item, "dir.%s"LLU"", stat_name, item_num);
         } else {
-            if ((i%ITEM_COUNT == 0) && (i != 0)) {
+            if ( (i % ITEM_COUNT == 0) && (i != 0)) {
                 VERBOSE(3,5,"stat file: "LLU"", i);
             }
             sprintf(item, "file.%s"LLU"", stat_name, item_num);

From b2d486f749c16fe3375dfc4389732bfae08db5f9 Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Sun, 28 Jul 2019 10:07:03 -0600
Subject: [PATCH 10/13] Followed Andreas suggestion to replace escape double
 quotes within printf's with single quotes

---
 src/mdtest.c | 74 ++++++++++++++++++++++++++--------------------------
 1 file changed, 37 insertions(+), 37 deletions(-)

diff --git a/src/mdtest.c b/src/mdtest.c
index 074463e..e35a20e 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -284,7 +284,7 @@ static void create_remove_dirs (const char *path, bool create, uint64_t itemNum)
 
     //create dirs
     sprintf(curr_item, "%s/dir.%s%" PRIu64, path, create ? mk_name : rm_name, itemNum);
-    VERBOSE(3,5,"create_remove_items_helper (dirs %s): curr_item is \"%s\"", operation, curr_item);
+    VERBOSE(3,5,"create_remove_items_helper (dirs %s): curr_item is '%s'", operation, curr_item);
 
     if (create) {
         if (backend->mkdir(curr_item, DIRMODE, &param) == -1) {
@@ -306,7 +306,7 @@ static void remove_file (const char *path, uint64_t itemNum) {
 
     //remove files
     sprintf(curr_item, "%s/file.%s"LLU"", path, rm_name, itemNum);
-    VERBOSE(3,5,"create_remove_items_helper (non-dirs remove): curr_item is \"%s\"", curr_item);
+    VERBOSE(3,5,"create_remove_items_helper (non-dirs remove): curr_item is '%s'", curr_item);
     if (!(shared_file && rank != 0)) {
         backend->delete (curr_item, &param);
     }
@@ -322,7 +322,7 @@ static void create_file (const char *path, uint64_t itemNum) {
 
     //create files
     sprintf(curr_item, "%s/file.%s"LLU"", path, mk_name, itemNum);
-    VERBOSE(3,5,"create_remove_items_helper (non-dirs create): curr_item is \"%s\"", curr_item);
+    VERBOSE(3,5,"create_remove_items_helper (non-dirs create): curr_item is '%s'", curr_item);
 
     if (collective_creates) {
         param.openFlags = IOR_WRONLY;
@@ -453,7 +453,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const
     memset(dir, 0, MAX_PATHLEN);
     strcpy(temp_path, path);
 
-    VERBOSE(3,5,"create_remove_items (start): temp_path is \"%s\"", temp_path );
+    VERBOSE(3,5,"create_remove_items (start): temp_path is '%s'", temp_path );
 
     if (currDepth == 0) {
         /* create items at this depth */
@@ -479,7 +479,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const
             strcat(temp_path, "/");
             strcat(temp_path, dir);
 
-            VERBOSE(3,5,"create_remove_items (for loop): temp_path is \"%s\"", temp_path );
+            VERBOSE(3,5,"create_remove_items (for loop): temp_path is '%s'", temp_path );
 
             /* create the items in this branch */
             if (!leaf_only || (leaf_only && currDepth == depth)) {
@@ -742,7 +742,7 @@ void collective_create_remove(const int create, const int dirs, const int ntasks
         }
 
         /* Now that everything is set up as it should be, do the create or remove */
-        VERBOSE(3,5,"collective_create_remove (create_remove_items): temp is \"%s\"", temp);
+        VERBOSE(3,5,"collective_create_remove (create_remove_items): temp is '%s'", temp);
 
         create_remove_items(0, dirs, create, 1, temp, 0, progress);
     }
@@ -799,7 +799,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        VERBOSE(3,-1,"directory_test: create path is \"%s\"", temp_path );
+        VERBOSE(3,-1,"directory_test: create path is '%s'", temp_path );
 
         /* "touch" the files */
         if (collective_creates) {
@@ -831,7 +831,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        VERBOSE(3,5,"stat path is \"%s\"", temp_path );
+        VERBOSE(3,5,"stat path is '%s'", temp_path );
 
         /* stat directories */
         if (random_seed > 0) {
@@ -860,7 +860,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        VERBOSE(3,5,"directory_test: read path is \"%s\"", temp_path );
+        VERBOSE(3,5,"directory_test: read path is '%s'", temp_path );
 
         /* read directories */
         if (random_seed > 0) {
@@ -888,7 +888,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        VERBOSE(3,5,"directory_test: remove directories path is \"%s\"", temp_path );
+        VERBOSE(3,5,"directory_test: remove directories path is '%s'", temp_path );
 
         /* remove directories */
         if (collective_creates) {
@@ -913,7 +913,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        VERBOSE(3,5,"directory_test: remove unique directories path is \"%s\"\n", temp_path );
+        VERBOSE(3,5,"directory_test: remove unique directories path is '%s'\n", temp_path );
     }
 
     if (unique_dir_per_task && !time_unique_dir_overhead) {
@@ -1010,7 +1010,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
 
 
 
-        VERBOSE(3,-1,"file_test: create path is \"%s\"", temp_path );
+        VERBOSE(3,-1,"file_test: create path is '%s'", temp_path );
 
         /* "touch" the files */
         if (collective_creates) {
@@ -1077,7 +1077,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        VERBOSE(3,5,"file_test: stat path is \"%s\"", temp_path );
+        VERBOSE(3,5,"file_test: stat path is '%s'", temp_path );
 
         /* stat files */
         mdtest_stat((random_seed > 0 ? 1 : 0), 0, dir_iter, temp_path, progress);
@@ -1102,7 +1102,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        VERBOSE(3,5,"file_test: read path is \"%s\"", temp_path );
+        VERBOSE(3,5,"file_test: read path is '%s'", temp_path );
 
         /* read files */
         if (random_seed > 0) {
@@ -1132,7 +1132,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
             sprintf( temp_path, "%s/%s", testdir, path );
         }
 
-        VERBOSE(3,5,"file_test: rm directories path is \"%s\"", temp_path );
+        VERBOSE(3,5,"file_test: rm directories path is '%s'", temp_path );
 
         if (collective_creates) {
             if (rank == 0) {
@@ -1156,7 +1156,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
             strcpy( temp_path, path );
         }
 
-        VERBOSE(3,5,"file_test: rm unique directories path is \"%s\"", temp_path );
+        VERBOSE(3,5,"file_test: rm unique directories path is '%s'", temp_path );
     }
 
     if (unique_dir_per_task && !time_unique_dir_overhead) {
@@ -1508,9 +1508,9 @@ void display_freespace(char *testdirpath)
         strcpy(dirpath, ".");
     }
 
-    VERBOSE(3,5,"Before show_file_system_size, dirpath is \"%s\"", dirpath );
+    VERBOSE(3,5,"Before show_file_system_size, dirpath is '%s'", dirpath );
     show_file_system_size(dirpath);
-    VERBOSE(3,5, "After show_file_system_size, dirpath is \"%s\"\n", dirpath );
+    VERBOSE(3,5, "After show_file_system_size, dirpath is '%s'\n", dirpath );
 
     return;
 }
@@ -1528,16 +1528,16 @@ void create_remove_directory_tree(int create,
         sprintf(dir, "%s/%s.%d/", path, base_tree_name, dirNum);
 
         if (create) {
-            VERBOSE(2,5,"Making directory \"%s\"", dir);
+            VERBOSE(2,5,"Making directory '%s'", dir);
             if (-1 == backend->mkdir (dir, DIRMODE, &param)) {
-                fprintf(out_logfile, "error could not create directory \"%s\"\n", dir);
+                fprintf(out_logfile, "error could not create directory '%s'\n", dir);
             }
         }
 
         create_remove_directory_tree(create, ++currDepth, dir, ++dirNum, progress);
 
         if (!create) {
-            VERBOSE(2,5,"Remove directory \"%s\"", dir);
+            VERBOSE(2,5,"Remove directory '%s'", dir);
             if (-1 == backend->rmdir(dir, &param)) {
                 FAIL("Unable to remove directory");
             }
@@ -1553,7 +1553,7 @@ void create_remove_directory_tree(int create,
             strcat(temp_path, dir);
 
             if (create) {
-                VERBOSE(2,5,"Making directory \"%s\"", temp_path);
+                VERBOSE(2,5,"Making directory '%s'", temp_path);
                 if (-1 == backend->mkdir(temp_path, DIRMODE, &param)) {
                     FAIL("Unable to create directory");
                 }
@@ -1564,7 +1564,7 @@ void create_remove_directory_tree(int create,
             currDepth--;
 
             if (!create) {
-                VERBOSE(2,5,"Remove directory \"%s\"", temp_path);
+                VERBOSE(2,5,"Remove directory '%s'", temp_path);
                 if (-1 == backend->rmdir(temp_path, &param)) {
                     FAIL("Unable to remove directory");
                 }
@@ -1593,7 +1593,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
   for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
     prep_testdir(j, dir_iter);
 
-    VERBOSE(2,5,"main (for j loop): making testdir, \"%s\"", testdir );
+    VERBOSE(2,5,"main (for j loop): making testdir, '%s'", testdir );
     if ((rank < path_count) && backend->access(testdir, F_OK, &param) != 0) {
         if (backend->mkdir(testdir, DIRMODE, &param) != 0) {
             FAIL("Unable to create test directory");
@@ -1618,7 +1618,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
                 for (k=0; k<size; k++) {
                     sprintf(base_tree_name, "mdtest_tree.%d", k);
 
-                    VERBOSE(3,5,"main (create hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"", testdir );
+                    VERBOSE(3,5,"main (create hierarchical directory loop-collective): Calling create_remove_directory_tree with '%s'", testdir );
                     /*
                      * Let's pass in the path to the directory we most recently made so that we can use
                      * full paths in the other calls.
@@ -1630,7 +1630,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
                     }
                 }
             } else if (!collective_creates) {
-                VERBOSE(3,5,"main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with \"%s\"", testdir );
+                VERBOSE(3,5,"main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with '%s'", testdir );
                 /*
                  * Let's pass in the path to the directory we most recently made so that we can use
                  * full paths in the other calls.
@@ -1639,7 +1639,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
             }
         } else {
             if (rank == 0) {
-                VERBOSE(3,5,"main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with \"%s\"", testdir );
+                VERBOSE(3,5,"main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with '%s'", testdir );
 
                 /*
                  * Let's pass in the path to the directory we most recently made so that we can use
@@ -1666,7 +1666,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
   unique_rm_uni_dir[0] = 0;
 
   if (!unique_dir_per_task) {
-    VERBOSE(3,-1,"V-3: main: Using unique_mk_dir, \"%s\"", unique_mk_dir );
+    VERBOSE(3,-1,"V-3: main: Using unique_mk_dir, '%s'", unique_mk_dir );
   }
 
   if (rank < i) {
@@ -1687,7 +1687,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
           VERBOSE(5,5,"mk_dir %s chdir %s stat_dir %s read_dir %s rm_dir %s\n", unique_mk_dir,unique_chdir_dir,unique_stat_dir,unique_read_dir,unique_rm_dir);
       }
 
-      VERBOSE(3,-1,"V-3: main: Copied unique_mk_dir, \"%s\", to topdir", unique_mk_dir );
+      VERBOSE(3,-1,"V-3: main: Copied unique_mk_dir, '%s', to topdir", unique_mk_dir );
 
       if (dirs_only && !shared_file) {
           if (pre_delay) {
@@ -1706,7 +1706,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
 
   /* remove directory structure */
   if (!unique_dir_per_task) {
-      VERBOSE(3,-1,"main: Using testdir, \"%s\"", testdir );
+      VERBOSE(3,-1,"main: Using testdir, '%s'", testdir );
   }
 
   MPI_Barrier(testComm);
@@ -1724,7 +1724,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
                 for (k=0; k<size; k++) {
                     sprintf(base_tree_name, "mdtest_tree.%d", k);
 
-                    VERBOSE(3,-1,"main (remove hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"", testdir );
+                    VERBOSE(3,-1,"main (remove hierarchical directory loop-collective): Calling create_remove_directory_tree with '%s'", testdir );
 
                     /*
                      * Let's pass in the path to the directory we most recently made so that we can use
@@ -1737,7 +1737,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
                     }
                 }
             } else if (!collective_creates) {
-                VERBOSE(3,-1,"main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with \"%s\"", testdir );
+                VERBOSE(3,-1,"main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with '%s'", testdir );
 
                 /*
                  * Let's pass in the path to the directory we most recently made so that we can use
@@ -1747,7 +1747,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
             }
         } else {
             if (rank == 0) {
-                VERBOSE(3,-1,"V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with \"%s\"", testdir );
+                VERBOSE(3,-1,"V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with '%s'", testdir );
 
                 /*
                  * Let's pass in the path to the directory we most recently made so that we can use
@@ -1765,7 +1765,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
       summary_table->items[9] = num_dirs_in_tree;
       summary_table->stonewall_last_item[8] = num_dirs_in_tree;
       VERBOSE(1,-1,"main   Tree removal      : %14.3f sec, %14.3f ops/sec", (endCreate - startCreate), summary_table->rate[9]);
-      VERBOSE(2,-1,"main (at end of for j loop): Removing testdir of \"%s\"\n", testdir );
+      VERBOSE(2,-1,"main (at end of for j loop): Removing testdir of '%s'\n", testdir );
 
       for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
         prep_testdir(j, dir_iter);
@@ -1900,7 +1900,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
     char cmd_buffer[4096];
     strncpy(cmd_buffer, argv[0], 4096);
     for (i = 1; i < argc; i++) {
-        snprintf(&cmd_buffer[strlen(cmd_buffer)], 4096-strlen(cmd_buffer), " \"%s\"", argv[i]);
+        snprintf(&cmd_buffer[strlen(cmd_buffer)], 4096-strlen(cmd_buffer), " '%s'", argv[i]);
     }
 
     VERBOSE(0,-1,"-- started at %s --\n", PrintTimestamp());
@@ -2064,7 +2064,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
     }
 
     /* display disk usage */
-    VERBOSE(3,-1,"main (before display_freespace): testdirpath is \"%s\"", testdirpath );
+    VERBOSE(3,-1,"main (before display_freespace): testdirpath is '%s'", testdirpath );
 
     if (rank == 0) display_freespace(testdirpath);
     int packedByNode = QueryNodeMapping(testComm);
@@ -2077,7 +2077,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
         VERBOSE(0,5,"Shifting ranks by %d for each phase.", nstride);
     }
 
-    VERBOSE(3,-1,"main (after display_freespace): testdirpath is \"%s\"", testdirpath );
+    VERBOSE(3,-1,"main (after display_freespace): testdirpath is '%s'", testdirpath );
 
     if (rank == 0) {
         if (random_seed > 0) {

From 0ffec67d2b35f60cd99cfb8307a2f342414c3bcc Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Sun, 28 Jul 2019 10:25:42 -0600
Subject: [PATCH 11/13] Following Julian's suggestion about better naming

---
 src/ior.c    | 18 +++++++-----------
 src/ior.h    |  2 +-
 src/mdtest.c |  6 ++++--
 3 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/src/ior.c b/src/ior.c
index bb2b4c2..64530fd 100755
--- a/src/ior.c
+++ b/src/ior.c
@@ -939,7 +939,7 @@ static void InitTests(IOR_test_t *tests, MPI_Comm com)
                 params->testComm = com;
                 params->nodes = params->numTasks / tasksPerNode;
                 params->tasksPerNode = tasksPerNode;
-                params->packedTasks = QueryNodeMapping(com);
+                params->tasksBlockMapping = QueryNodeMapping(com);
                 if (params->numTasks == 0) {
                   params->numTasks = size;
                 }
@@ -1361,11 +1361,9 @@ static void TestIoSys(IOR_test_t *test)
                         }
                         if (params->reorderTasks) {
                                 /* move two nodes away from writing node */
-                                int shift = 1;
-                                if (params->packedTasks) {
-                                    shift = params->tasksPerNode;
-                                } else {
-                                    shift = 1;
+                                int shift = 1; /* assume a by-node (round-robin) mapping of tasks to nodes */
+                                if (params->tasksBlockMapping) {
+                                    shift = params->tasksPerNode; /* switch to by-slot (contiguous block) mapping */
                                 }
                                 rankOffset = (2 * shift) % params->numTasks;
                         }
@@ -1403,11 +1401,9 @@ static void TestIoSys(IOR_test_t *test)
                         /* Constant process offset reading */
                         if (params->reorderTasks) {
                                 /* move one node away from writing node */ 
-                                int shift = 1;
-                                if (params->packedTasks) {
-                                    shift=params->tasksPerNode;
-                                } else {
-                                    shift=1;
+                                int shift = 1; /* assume a by-node (round-robin) mapping of tasks to nodes */
+                                if (params->tasksBlockMapping) {
+                                    shift=params->tasksPerNode; /* switch to a by-slot (contiguous block) mapping */
                                 }
                                 rankOffset = (params->taskPerNodeOffset * shift) % params->numTasks;
                         }
diff --git a/src/ior.h b/src/ior.h
index 2ea6c05..e245b08 100755
--- a/src/ior.h
+++ b/src/ior.h
@@ -100,7 +100,7 @@ typedef struct
     int numTasks;                    /* number of tasks for test */
     int nodes;                       /* number of nodes for test */
     int tasksPerNode;                /* number of tasks per node */
-    int packedTasks;                 /* are the tasks round-robin across nodes or are they packed tightly*/ 
+    int tasksBlockMapping;           /* are the tasks in contiguous blocks across nodes or round-robin */ 
     int repetitions;                 /* number of repetitions of test */
     int repCounter;                  /* rep counter */
     int multiFile;                   /* multiple files */
diff --git a/src/mdtest.c b/src/mdtest.c
index e35a20e..0005c0a 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -2067,11 +2067,13 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
     VERBOSE(3,-1,"main (before display_freespace): testdirpath is '%s'", testdirpath );
 
     if (rank == 0) display_freespace(testdirpath);
-    int packedByNode = QueryNodeMapping(testComm);
+    int tasksBlockMapping = QueryNodeMapping(testComm);
 
     /* set the shift to mimic IOR and shift by procs per node */
     if (nstride > 0) {
-        if ( nodeCount > 1 && packedByNode ) {
+        if ( nodeCount > 1 && tasksBlockMapping ) {
+            /* the user set the stride presumably to get the consumer tasks on a different node than the producer tasks
+               however, if the mpirun scheduler placed the tasks by-slot (in a contiguous block) then we need to adjust the shift by ppn */
             nstride *= tasksPerNode;
         }
         VERBOSE(0,5,"Shifting ranks by %d for each phase.", nstride);

From a3c37808dadebc12c21ed4bad1ac55de5d25fa31 Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Sun, 28 Jul 2019 11:17:11 -0600
Subject: [PATCH 12/13] Made FAIL take variable args so we can pass printf like
 args to it

---
 src/mdtest.c    | 40 +++++++++++++++++++---------------------
 src/utilities.c | 12 ++++++++++++
 src/utilities.h | 19 +++++--------------
 3 files changed, 36 insertions(+), 35 deletions(-)

diff --git a/src/mdtest.c b/src/mdtest.c
index 0005c0a..127aa5f 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -288,11 +288,11 @@ static void create_remove_dirs (const char *path, bool create, uint64_t itemNum)
 
     if (create) {
         if (backend->mkdir(curr_item, DIRMODE, &param) == -1) {
-            FAIL("unable to create directory");
+            FAIL("unable to create directory %s", curr_item);
         }
     } else {
         if (backend->rmdir(curr_item, &param) == -1) {
-            FAIL("unable to remove directory");
+            FAIL("unable to remove directory %s", curr_item);
         }
     }
 }
@@ -334,7 +334,7 @@ static void create_file (const char *path, uint64_t itemNum) {
 	else
             aiori_fh = backend->open (curr_item, &param);
         if (NULL == aiori_fh) {
-            FAIL("unable to open file");
+            FAIL("unable to open file %s", curr_item);
         }
 
         /*
@@ -351,7 +351,7 @@ static void create_file (const char *path, uint64_t itemNum) {
 	else
             aiori_fh = backend->create (curr_item, &param);
         if (NULL == aiori_fh) {
-            FAIL("unable to create file");
+            FAIL("unable to create file %s", curr_item);
         }
     }
 
@@ -365,7 +365,7 @@ static void create_file (const char *path, uint64_t itemNum) {
         param.offset = 0;
         param.fsyncPerWrite = sync_file;
         if ( write_bytes != (size_t) backend->xfer (WRITE, aiori_fh, (IOR_size_t *) write_buffer, write_bytes, &param)) {
-            FAIL("unable to write file");
+            FAIL("unable to write file %s", curr_item);
         }
     }
 
@@ -422,7 +422,7 @@ void collective_helper(const int dirs, const int create, const char* path, uint6
             param.openFlags = IOR_WRONLY | IOR_CREAT;
             aiori_fh = backend->create (curr_item, &param);
             if (NULL == aiori_fh) {
-                FAIL("unable to create file");
+                FAIL("unable to create file %s", curr_item);
             }
 
             backend->close (aiori_fh, &param);
@@ -586,9 +586,7 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch
         /* below temp used to be hiername */
         VERBOSE(3,5,"mdtest_stat %4s: %s", (dirs ? "dir" : "file"), item);
         if (-1 == backend->stat (item, &buf, &param)) {
-            char msg_buf[4096];
-            snprintf(msg_buf, 4096, "unable to stat %s %s", dirs ? "directory" : "file", item);
-            FAIL(msg_buf);
+            FAIL("unable to stat %s %s", dirs ? "directory" : "file", item);
         }
     }
 }
@@ -679,13 +677,13 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) {
         param.openFlags = O_RDONLY;
         aiori_fh = backend->open (item, &param);
         if (NULL == aiori_fh) {
-            FAIL("unable to open file");
+            FAIL("unable to open file %s", item);
         }
 
         /* read file */
         if (read_bytes > 0) {
             if (read_bytes != (size_t) backend->xfer (READ, aiori_fh, (IOR_size_t *) read_buffer, read_bytes, &param)) {
-                FAIL("unable to read file");
+                FAIL("unable to read file %s", item);
             }
         }
 
@@ -1342,7 +1340,7 @@ void summarize_results(int iterations) {
 void valid_tests() {
 
     if (((stone_wall_timer_seconds > 0) && (branch_factor > 1)) || ! barriers) {
-        FAIL( "Error, stone wall timer does only work with a branch factor <= 1 and with barriers\n");
+        FAIL( "Error, stone wall timer does only work with a branch factor <= 1 (current is %d) and with barriers\n", branch_factor);
     }
 
     if (!create_only && !stat_only && !read_only && !remove_only) {
@@ -1446,7 +1444,7 @@ void show_file_system_size(char *file_system) {
 
     ret = backend->statfs (file_system, &stat_buf, &param);
     if (0 != ret) {
-        FAIL("unable to stat file system");
+        FAIL("unable to stat file system %s", file_system);
     }
 
     total_file_system_size = stat_buf.f_blocks * stat_buf.f_bsize;
@@ -1469,7 +1467,7 @@ void show_file_system_size(char *file_system) {
         * 100;
 
     if (realpath(file_system, real_path) == NULL) {
-        FAIL("unable to use realpath()");
+        FAIL("unable to use realpath() on file system %s", file_system);
     }
 
 
@@ -1539,7 +1537,7 @@ void create_remove_directory_tree(int create,
         if (!create) {
             VERBOSE(2,5,"Remove directory '%s'", dir);
             if (-1 == backend->rmdir(dir, &param)) {
-                FAIL("Unable to remove directory");
+                FAIL("Unable to remove directory %s", dir);
             }
         }
     } else if (currDepth <= depth) {
@@ -1555,7 +1553,7 @@ void create_remove_directory_tree(int create,
             if (create) {
                 VERBOSE(2,5,"Making directory '%s'", temp_path);
                 if (-1 == backend->mkdir(temp_path, DIRMODE, &param)) {
-                    FAIL("Unable to create directory");
+                    FAIL("Unable to create directory %s", temp_path);
                 }
             }
 
@@ -1566,7 +1564,7 @@ void create_remove_directory_tree(int create,
             if (!create) {
                 VERBOSE(2,5,"Remove directory '%s'", temp_path);
                 if (-1 == backend->rmdir(temp_path, &param)) {
-                    FAIL("Unable to remove directory");
+                    FAIL("Unable to remove directory %s", temp_path);
                 }
             }
 
@@ -1596,7 +1594,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
     VERBOSE(2,5,"main (for j loop): making testdir, '%s'", testdir );
     if ((rank < path_count) && backend->access(testdir, F_OK, &param) != 0) {
         if (backend->mkdir(testdir, DIRMODE, &param) != 0) {
-            FAIL("Unable to create test directory");
+            FAIL("Unable to create test directory %s", testdir);
         }
     }
   }
@@ -1772,7 +1770,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t
         if ((rank < path_count) && backend->access(testdir, F_OK, &param) == 0) {
             //if (( rank == 0 ) && access(testdir, F_OK) == 0) {
             if (backend->rmdir(testdir, &param) == -1) {
-                FAIL("unable to remove directory");
+                FAIL("unable to remove directory %s", testdir);
             }
         }
       }
@@ -2049,7 +2047,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
     if (path_count == 0) { /* special case where no directory path provided with '-d' option */
         char *ret = getcwd(testdirpath, MAX_PATHLEN);
         if (ret == NULL) {
-            FAIL("Unable to get current working directory");
+            FAIL("Unable to get current working directory on %s", testdirpath);
         }
         path_count = 1;
     } else {
@@ -2059,7 +2057,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
     /*   if directory does not exist, create it */
     if ((rank < path_count) && backend->access(testdirpath, F_OK, &param) != 0) {
         if (backend->mkdir(testdirpath, DIRMODE, &param) != 0) {
-            FAIL("Unable to create test directory path");
+            FAIL("Unable to create test directory path %s", testdirpath);
         }
     }
 
diff --git a/src/utilities.c b/src/utilities.c
index 5a3168c..8eb4485 100755
--- a/src/utilities.c
+++ b/src/utilities.c
@@ -20,6 +20,7 @@
 #  define _GNU_SOURCE            /* Needed for O_DIRECT in fcntl */
 #endif                           /* __linux__ */
 
+#include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
@@ -75,6 +76,17 @@ void* safeMalloc(uint64_t size){
   return d;
 }
 
+void FailMessage(int rank, const char *location, char *format, ...) {
+    char msg[4096];                                                  
+    va_list args;
+    va_start(args, format);
+    vsnprintf(msg, 4096, format, args);
+    va_end(args);
+    fprintf(out_logfile, "%s: Process %d: FAILED in %s, %s: %s\n", 
+                PrintTimestamp(), rank, location, msg, strerror(errno));                               
+    fflush(out_logfile);                                        
+    MPI_Abort(testComm, 1);                                    
+}
 
 size_t NodeMemoryStringToBytes(char *size_str)
 {
diff --git a/src/utilities.h b/src/utilities.h
index d811261..0964169 100755
--- a/src/utilities.h
+++ b/src/utilities.h
@@ -36,23 +36,14 @@ extern enum OutputFormat_t outputFormat;  /* format of the output */
 
 
 #ifdef __linux__
-#define FAIL(msg) do {                                                   \
-        fprintf(out_logfile, "%s: Process %d: FAILED in %s, %s: %s\n",   \
-                PrintTimestamp(), rank, __func__,                       \
-                msg, strerror(errno));                                   \
-        fflush(out_logfile);                                             \
-        MPI_Abort(testComm, 1);                                          \
-    } while(0)
+#define ERROR_LOCATION __func__
 #else
-#define FAIL(msg) do {                                                   \
-        fprintf(out_logfile, "%s: Process %d: FAILED at %d, %s: %s\n",   \
-                PrintTimestamp(), rank, __LINE__,                       \
-                msg, strerror(errno));                                   \
-        fflush(out_logfile);                                             \
-        MPI_Abort(testComm, 1);                                          \
-    } while(0)
+#define ERROR_LOCATION __LINE__
 #endif
 
+#define FAIL(...) FailMessage(rank, ERROR_LOCATION, __VA_ARGS__)
+void FailMessage(int rank, const char *location, char *format, ...);
+
 void* safeMalloc(uint64_t size);
 void set_o_direct_flag(int *fd);
 

From 3890b71b54f6bc3b8d931173bb6fb27090ddc652 Mon Sep 17 00:00:00 2001
From: John Bent <johnbent@AMR-U535110L001.ad.seagate.com>
Date: Thu, 1 Aug 2019 09:42:03 +0900
Subject: [PATCH 13/13] Fixed issues and followed suggestions from Glenn's
 review of the PR

---
 src/ior.c           | 4 ++--
 src/mdtest.c        | 4 ++--
 src/parse_options.c | 2 +-
 src/utilities.c     | 4 ++--
 src/utilities.h     | 2 +-
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/ior.c b/src/ior.c
index 64530fd..15b1d07 100755
--- a/src/ior.c
+++ b/src/ior.c
@@ -939,7 +939,7 @@ static void InitTests(IOR_test_t *tests, MPI_Comm com)
                 params->testComm = com;
                 params->nodes = params->numTasks / tasksPerNode;
                 params->tasksPerNode = tasksPerNode;
-                params->tasksBlockMapping = QueryNodeMapping(com);
+                params->tasksBlockMapping = QueryNodeMapping(com,false);
                 if (params->numTasks == 0) {
                   params->numTasks = size;
                 }
@@ -1223,7 +1223,7 @@ static void TestIoSys(IOR_test_t *test)
         }
         if (rank == 0 && params->reorderTasks == TRUE && verbose >= VERBOSE_1) {
                 fprintf(out_logfile,
-                        "Using reorderTasks '-C' (expecting block, not cyclic, task assignment)\n");
+                        "Using reorderTasks '-C' (useful to avoid read cache in client)\n");
                 fflush(out_logfile);
         }
         params->tasksPerNode = CountTasksPerNode(testComm);
diff --git a/src/mdtest.c b/src/mdtest.c
index 127aa5f..f65ea7f 100644
--- a/src/mdtest.c
+++ b/src/mdtest.c
@@ -1862,7 +1862,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
       {'l', NULL,        "last number of tasks on which the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & last},
       {'L', NULL,        "files only at leaf level of tree", OPTION_FLAG, 'd', & leaf_only},
       {'n', NULL,        "every process will creat/stat/read/remove # directories and files", OPTION_OPTIONAL_ARGUMENT, 'l', & items},
-      {'N', NULL,        "stride # between neighbor tasks for file/dir operation (local=0)", OPTION_OPTIONAL_ARGUMENT, 'd', & nstride},
+      {'N', NULL,        "stride # between tasks for file/dir operation (local=0; set to 1 to avoid client cache)", OPTION_OPTIONAL_ARGUMENT, 'd', & nstride},
       {'p', NULL,        "pre-iteration delay (in seconds)", OPTION_OPTIONAL_ARGUMENT, 'd', & pre_delay},
       {'R', NULL,        "random access to files (only for stat)", OPTION_FLAG, 'd', & randomize},
       {0, "random-seed", "random seed for -R", OPTION_OPTIONAL_ARGUMENT, 'd', & random_seed},
@@ -2065,7 +2065,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
     VERBOSE(3,-1,"main (before display_freespace): testdirpath is '%s'", testdirpath );
 
     if (rank == 0) display_freespace(testdirpath);
-    int tasksBlockMapping = QueryNodeMapping(testComm);
+    int tasksBlockMapping = QueryNodeMapping(testComm, true);
 
     /* set the shift to mimic IOR and shift by procs per node */
     if (nstride > 0) {
diff --git a/src/parse_options.c b/src/parse_options.c
index ba72413..af30c36 100755
--- a/src/parse_options.c
+++ b/src/parse_options.c
@@ -471,7 +471,7 @@ option_help * createGlobalOptions(IOR_param_t * params){
     {'A', NULL,        "refNum -- user supplied reference number to include in the summary", OPTION_OPTIONAL_ARGUMENT, 'd', & params->referenceNumber},
     {'b', NULL,        "blockSize -- contiguous bytes to write per task  (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & params->blockSize},
     {'c', NULL,        "collective -- collective I/O", OPTION_FLAG, 'd', & params->collective},
-    {'C', NULL,        "reorderTasks -- changes task ordering to n+1 ordering for readback", OPTION_FLAG, 'd', & params->reorderTasks},
+    {'C', NULL,        "reorderTasks -- changes task ordering for readback (useful to avoid client cache)", OPTION_FLAG, 'd', & params->reorderTasks},
     {'d', NULL,        "interTestDelay -- delay between reps in seconds", OPTION_OPTIONAL_ARGUMENT, 'd', & params->interTestDelay},
     {'D', NULL,        "deadlineForStonewalling -- seconds before stopping write or read phase", OPTION_OPTIONAL_ARGUMENT, 'd', & params->deadlineForStonewalling},
     {.help="  -O stoneWallingWearOut=1           -- once the stonewalling timout is over, all process finish to access the amount of data", .arg = OPTION_OPTIONAL_ARGUMENT},
diff --git a/src/utilities.c b/src/utilities.c
index 8eb4485..99d26e7 100755
--- a/src/utilities.c
+++ b/src/utilities.c
@@ -229,7 +229,7 @@ void DumpBuffer(void *buffer,
    and the value is whether that rank is on the same host as root.
    Also returns 1 if rank 1 is on same host and 0 otherwise
 */
-int QueryNodeMapping(MPI_Comm comm) {
+int QueryNodeMapping(MPI_Comm comm, int print_nodemap) {
     char localhost[MAX_PATHLEN], roothost[MAX_PATHLEN];
     int num_ranks;
     MPI_Comm_size(comm, &num_ranks);
@@ -250,7 +250,7 @@ int QueryNodeMapping(MPI_Comm comm) {
     /* then every rank figures out whether it is same host as root and then gathers that */
     int same_as_root = strcmp(roothost,localhost) == 0;
     MPI_Gather( &same_as_root, 1, MPI_INT, node_map, 1, MPI_INT, 0, comm);
-    if (rank==0) {
+    if ( print_nodemap && rank==0) {
         fprintf( out_logfile, "Nodemap: " );
         for ( int i = 0; i < num_ranks; i++ ) {
             fprintf( out_logfile, "%d", node_map[i] );
diff --git a/src/utilities.h b/src/utilities.h
index 0964169..d2c9962 100755
--- a/src/utilities.h
+++ b/src/utilities.h
@@ -56,7 +56,7 @@ void SetHints (MPI_Info *, char *);
 void ShowHints (MPI_Info *);
 char *HumanReadable(IOR_offset_t value, int base);
 int CountTasksPerNode(MPI_Comm comm);
-int QueryNodeMapping(MPI_Comm comm);
+int QueryNodeMapping(MPI_Comm comm, int print_nodemap);
 void DelaySecs(int delay);
 void updateParsedOptions(IOR_param_t * options, options_all_t * global_options);
 size_t NodeMemoryStringToBytes(char *size_str);