Teach IOR about GPFS hints (gpfs_fcntl)

GPFS supports a "gpfs_fcntl" method for hinting various things,
including "i'm about to write this block of data".  Let's see if, for
the cost of a few system calls, we can wrangle the GPFS locking system
into allowing concurrent access with less overhead. (new IOR parameter
gpfsHintAccess)

Also, drop all locks on a file immediately after open/creation in the
shared file case, since we know all processes will touch unique regions
of the file.  It may or may not be a good idea to release all file locks
after opening.  Processes will then have to re-acquire locks already
held.   (new IOR parameter gpfsReleaseToken)

configure.ac

@@ -20,7 +20,7 @@ AX_PROG_CC_MPI
 # Checks for libraries.
 
 # Checks for header files.
-AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h])
+AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h gpfs.h gpfs_fcntl.h])
 
 # Checks for typedefs, structures, and compiler characteristics.
 AC_TYPE_SIZE_T

doc/USER_GUIDE

@@ -341,6 +341,13 @@ LUSTRE-SPECIFIC:
 
   * lustreIgnoreLocks    - disable lustre range locking [0]
 
+GPFS-SPECIFIC:
+================
+  * gpfsHintAccess       - use gpfs_fcntl hints to pre-declare accesses
+
+  * gpfsReleaseToken     - immediately after opening or creating file, release
+			   all locks.  Might help mitigate lock-revocation
+			   traffic when many proceses write/read to same file.
 
 ***********************
 * 5. VERBOSITY LEVELS *

src/aiori-POSIX.c

@@ -32,6 +32,13 @@
 #include <lustre/lustre_user.h>
 #endif
 
+#ifdef HAVE_GPFS_H
+#include <gpfs.h>
+#endif
+#ifdef HAVE_GPFS_FCNTL_H
+#include <gpfs_fcntl.h>
+#endif
+
 #include "ior.h"
 #include "aiori.h"
 #include "iordef.h"
@@ -91,6 +98,78 @@ void set_o_direct_flag(int *fd)
         *fd |= O_DIRECT;
 }
 
+#ifdef HAVE_GPFS_FCNTL_H
+void gpfs_free_all_locks(int fd)
+{
+        int rc;
+        struct {
+                gpfsFcntlHeader_t header;
+                gpfsFreeRange_t release;
+        } release_all;
+        release_all.header.totalLength = sizeof(release_all);
+        release_all.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
+        release_all.header.fcntlReserved = 0;
+
+        release_all.release.structLen = sizeof(release_all.release);
+        release_all.release.structType = GPFS_FREE_RANGE;
+        release_all.release.start = 0;
+        release_all.release.length = 0;
+
+        rc = gpfs_fcntl(fd, &release_all);
+        if (verbose >= VERBOSE_0 && rc != 0) {
+                EWARN("gpfs_fcntl release all locks hint failed.");
+        }
+}
+void gpfs_access_start(int fd, IOR_offset_t length, IOR_param_t *param, int access)
+{
+        int rc;
+        struct {
+                gpfsFcntlHeader_t header;
+                gpfsAccessRange_t access;
+        } take_locks;
+
+        take_locks.header.totalLength = sizeof(take_locks);
+        take_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
+        take_locks.header.fcntlReserved = 0;
+
+        take_locks.access.structLen = sizeof(take_locks.access);
+        take_locks.access.structType = GPFS_ACCESS_RANGE;
+        take_locks.access.start = param->offset;
+        take_locks.access.length = length;
+        take_locks.access.isWrite = (access == WRITE);
+
+        rc = gpfs_fcntl(fd, &take_locks);
+        if (verbose >= VERBOSE_2 && rc != 0) {
+                EWARN("gpfs_fcntl access range hint failed.");
+        }
+}
+
+void gpfs_access_end(int fd, IOR_offset_t length, IOR_param_t *param, int access)
+{
+        int rc;
+        struct {
+                gpfsFcntlHeader_t header;
+                gpfsFreeRange_t free;
+        } free_locks;
+
+
+        free_locks.header.totalLength = sizeof(free_locks);
+        free_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
+        free_locks.header.fcntlReserved = 0;
+
+        free_locks.free.structLen = sizeof(free_locks.free);
+        free_locks.free.structType = GPFS_FREE_RANGE;
+        free_locks.free.start = param->offset;
+        free_locks.free.length = length;
+
+        rc = gpfs_fcntl(fd, &free_locks);
+        if (verbose >= VERBOSE_2 && rc != 0) {
+                EWARN("gpfs_fcntl free range hint failed.");
+        }
+}
+
+#endif
+
 /*
  * Creat and open a file through the POSIX interface.
  */
@@ -166,6 +245,14 @@ static void *POSIX_Create(char *testFileName, IOR_param_t * param)
         }
 #endif                          /* HAVE_LUSTRE_LUSTRE_USER_H */
 
+#ifdef HAVE_GPFS_FCNTL_H
+        /* in the single shared file case, immediately release all locks, with
+         * the intent that we can avoid some byte range lock revocation:
+         * everyone will be writing/reading from individual regions */
+        if (param->gpfs_release_token ) {
+                gpfs_free_all_locks(*fd);
+        }
+#endif
         return ((void *)fd);
 }
 
@@ -201,6 +288,11 @@ static void *POSIX_Open(char *testFileName, IOR_param_t * param)
         }
 #endif                          /* HAVE_LUSTRE_LUSTRE_USER_H */
 
+#ifdef HAVE_GPFS_FCNTL_H
+        if(param->gpfs_release_token) {
+                gpfs_free_all_locks(*fd);
+        }
+#endif
         return ((void *)fd);
 }
 
@@ -218,6 +310,13 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer,
 
         fd = *(int *)file;
 
+#ifdef HAVE_GPFS_FCNTL_H
+        if (param->gpfs_hint_access) {
+                gpfs_access_start(fd, length, param, access);
+        }
+#endif
+
+
         /* seek to offset */
         if (lseek64(fd, param->offset, SEEK_SET) == -1)
                 ERR("lseek64() failed");
@@ -268,6 +367,11 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer,
                 ptr += rc;
                 xferRetries++;
         }
+#ifdef HAVE_GPFS_FCNTL_H
+        if (param->gpfs_hint_access) {
+                gpfs_access_end(fd, length, param, access);
+        }
+#endif
         return (length);
 }
 

src/ior.c

@@ -1601,6 +1601,10 @@ static void ShowTest(IOR_param_t * test)
                 test->setTimeStampSignature);
         fprintf(stdout, "\t%s=%d\n", "collective", test->collective);
         fprintf(stdout, "\t%s=%lld", "segmentCount", test->segmentCount);
+#ifdef HAVE_GPFS_FCNTL_H
+        fprintf(stdout, "\t%s=%d\n", "gpfsHintAccess", test->gpfs_hint_access);
+        fprintf(stdout, "\t%s=%d\n", "gpfsReleaseToken", test->gpfs_release_token);
+#endif
         if (strcmp(test->api, "HDF5") == 0) {
                 fprintf(stdout, " (datasets)");
         }

src/ior.h

@@ -126,6 +126,11 @@ typedef struct
     int lustre_set_striping;         /* flag that we need to set lustre striping */
     int lustre_ignore_locks;
 
+    /* gpfs variables */
+    int gpfs_hint_access;          /* use gpfs "access range" hint */
+    int gpfs_release_token;        /* immediately release GPFS tokens after
+                                      creating or opening a file */
+
     int id;                          /* test's unique ID */
     int intraTestBarriers;           /* barriers between open/op and op/close */
 } IOR_param_t;

src/parse_options.c

@@ -263,6 +263,16 @@ void DecodeDirective(char *line, IOR_param_t *params)
                 ERR("ior was not compiled with Lustre support");
 #endif
                 params->lustre_ignore_locks = atoi(value);
+        } else if (strcasecmp(option, "gpfshintaccess") == 0) {
+#ifndef HAVE_GPFS_FCNTL_H
+                ERR("ior was not compiled with GPFS hint support");
+#endif
+                params->gpfs_hint_access = atoi(value);
+        } else if (strcasecmp(option, "gpfsreleasetoken") == 0) {
+#ifndef HAVE_GPFS_FCNTL_H
+                ERR("ior was not compiled with GPFS hint support");
+#endif
+                params->gpfs_release_token = atoi(value);
         } else if (strcasecmp(option, "numtasks") == 0) {
                 params->numTasks = atoi(value);
 		RecalculateExpectedFileSize(params);