From aa604c1d38de803aa0db3f3abb5515e0ed1857ea Mon Sep 17 00:00:00 2001 From: Rob Latham Date: Thu, 26 Sep 2013 13:48:50 +0000 Subject: [PATCH] Teach IOR about GPFS hints (gpfs_fcntl) GPFS supports a "gpfs_fcntl" method for hinting various things, including "i'm about to write this block of data". Let's see if, for the cost of a few system calls, we can wrangle the GPFS locking system into allowing concurrent access with less overhead. (new IOR parameter gpfsHintAccess) Also, drop all locks on a file immediately after open/creation in the shared file case, since we know all processes will touch unique regions of the file. It may or may not be a good idea to release all file locks after opening. Processes will then have to re-acquire locks already held. (new IOR parameter gpfsReleaseToken) --- configure.ac | 2 +- doc/USER_GUIDE | 7 +++ src/aiori-POSIX.c | 104 ++++++++++++++++++++++++++++++++++++++++++++ src/ior.c | 4 ++ src/ior.h | 5 +++ src/parse_options.c | 10 +++++ 6 files changed, 131 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 47fee3d..ed5a8aa 100644 --- a/configure.ac +++ b/configure.ac @@ -20,7 +20,7 @@ AX_PROG_CC_MPI # Checks for libraries. # Checks for header files. -AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h]) +AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h gpfs.h gpfs_fcntl.h]) # Checks for typedefs, structures, and compiler characteristics. AC_TYPE_SIZE_T diff --git a/doc/USER_GUIDE b/doc/USER_GUIDE index 5ef5027..97e3342 100644 --- a/doc/USER_GUIDE +++ b/doc/USER_GUIDE @@ -341,6 +341,13 @@ LUSTRE-SPECIFIC: * lustreIgnoreLocks - disable lustre range locking [0] +GPFS-SPECIFIC: +================ + * gpfsHintAccess - use gpfs_fcntl hints to pre-declare accesses + + * gpfsReleaseToken - immediately after opening or creating file, release + all locks. Might help mitigate lock-revocation + traffic when many proceses write/read to same file. *********************** * 5. VERBOSITY LEVELS * diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 63791d9..ca6065e 100644 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -32,6 +32,13 @@ #include #endif +#ifdef HAVE_GPFS_H +#include +#endif +#ifdef HAVE_GPFS_FCNTL_H +#include +#endif + #include "ior.h" #include "aiori.h" #include "iordef.h" @@ -91,6 +98,78 @@ void set_o_direct_flag(int *fd) *fd |= O_DIRECT; } +#ifdef HAVE_GPFS_FCNTL_H +void gpfs_free_all_locks(int fd) +{ + int rc; + struct { + gpfsFcntlHeader_t header; + gpfsFreeRange_t release; + } release_all; + release_all.header.totalLength = sizeof(release_all); + release_all.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION; + release_all.header.fcntlReserved = 0; + + release_all.release.structLen = sizeof(release_all.release); + release_all.release.structType = GPFS_FREE_RANGE; + release_all.release.start = 0; + release_all.release.length = 0; + + rc = gpfs_fcntl(fd, &release_all); + if (verbose >= VERBOSE_0 && rc != 0) { + EWARN("gpfs_fcntl release all locks hint failed."); + } +} +void gpfs_access_start(int fd, IOR_offset_t length, IOR_param_t *param, int access) +{ + int rc; + struct { + gpfsFcntlHeader_t header; + gpfsAccessRange_t access; + } take_locks; + + take_locks.header.totalLength = sizeof(take_locks); + take_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION; + take_locks.header.fcntlReserved = 0; + + take_locks.access.structLen = sizeof(take_locks.access); + take_locks.access.structType = GPFS_ACCESS_RANGE; + take_locks.access.start = param->offset; + take_locks.access.length = length; + take_locks.access.isWrite = (access == WRITE); + + rc = gpfs_fcntl(fd, &take_locks); + if (verbose >= VERBOSE_2 && rc != 0) { + EWARN("gpfs_fcntl access range hint failed."); + } +} + +void gpfs_access_end(int fd, IOR_offset_t length, IOR_param_t *param, int access) +{ + int rc; + struct { + gpfsFcntlHeader_t header; + gpfsFreeRange_t free; + } free_locks; + + + free_locks.header.totalLength = sizeof(free_locks); + free_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION; + free_locks.header.fcntlReserved = 0; + + free_locks.free.structLen = sizeof(free_locks.free); + free_locks.free.structType = GPFS_FREE_RANGE; + free_locks.free.start = param->offset; + free_locks.free.length = length; + + rc = gpfs_fcntl(fd, &free_locks); + if (verbose >= VERBOSE_2 && rc != 0) { + EWARN("gpfs_fcntl free range hint failed."); + } +} + +#endif + /* * Creat and open a file through the POSIX interface. */ @@ -166,6 +245,14 @@ static void *POSIX_Create(char *testFileName, IOR_param_t * param) } #endif /* HAVE_LUSTRE_LUSTRE_USER_H */ +#ifdef HAVE_GPFS_FCNTL_H + /* in the single shared file case, immediately release all locks, with + * the intent that we can avoid some byte range lock revocation: + * everyone will be writing/reading from individual regions */ + if (param->gpfs_release_token ) { + gpfs_free_all_locks(*fd); + } +#endif return ((void *)fd); } @@ -201,6 +288,11 @@ static void *POSIX_Open(char *testFileName, IOR_param_t * param) } #endif /* HAVE_LUSTRE_LUSTRE_USER_H */ +#ifdef HAVE_GPFS_FCNTL_H + if(param->gpfs_release_token) { + gpfs_free_all_locks(*fd); + } +#endif return ((void *)fd); } @@ -218,6 +310,13 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer, fd = *(int *)file; +#ifdef HAVE_GPFS_FCNTL_H + if (param->gpfs_hint_access) { + gpfs_access_start(fd, length, param, access); + } +#endif + + /* seek to offset */ if (lseek64(fd, param->offset, SEEK_SET) == -1) ERR("lseek64() failed"); @@ -268,6 +367,11 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer, ptr += rc; xferRetries++; } +#ifdef HAVE_GPFS_FCNTL_H + if (param->gpfs_hint_access) { + gpfs_access_end(fd, length, param, access); + } +#endif return (length); } diff --git a/src/ior.c b/src/ior.c index e9d6c20..163a59c 100644 --- a/src/ior.c +++ b/src/ior.c @@ -1601,6 +1601,10 @@ static void ShowTest(IOR_param_t * test) test->setTimeStampSignature); fprintf(stdout, "\t%s=%d\n", "collective", test->collective); fprintf(stdout, "\t%s=%lld", "segmentCount", test->segmentCount); +#ifdef HAVE_GPFS_FCNTL_H + fprintf(stdout, "\t%s=%d\n", "gpfsHintAccess", test->gpfs_hint_access); + fprintf(stdout, "\t%s=%d\n", "gpfsReleaseToken", test->gpfs_release_token); +#endif if (strcmp(test->api, "HDF5") == 0) { fprintf(stdout, " (datasets)"); } diff --git a/src/ior.h b/src/ior.h index 60ae944..b02c264 100644 --- a/src/ior.h +++ b/src/ior.h @@ -126,6 +126,11 @@ typedef struct int lustre_set_striping; /* flag that we need to set lustre striping */ int lustre_ignore_locks; + /* gpfs variables */ + int gpfs_hint_access; /* use gpfs "access range" hint */ + int gpfs_release_token; /* immediately release GPFS tokens after + creating or opening a file */ + int id; /* test's unique ID */ int intraTestBarriers; /* barriers between open/op and op/close */ } IOR_param_t; diff --git a/src/parse_options.c b/src/parse_options.c index 452bcdb..9f66f1d 100644 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -263,6 +263,16 @@ void DecodeDirective(char *line, IOR_param_t *params) ERR("ior was not compiled with Lustre support"); #endif params->lustre_ignore_locks = atoi(value); + } else if (strcasecmp(option, "gpfshintaccess") == 0) { +#ifndef HAVE_GPFS_FCNTL_H + ERR("ior was not compiled with GPFS hint support"); +#endif + params->gpfs_hint_access = atoi(value); + } else if (strcasecmp(option, "gpfsreleasetoken") == 0) { +#ifndef HAVE_GPFS_FCNTL_H + ERR("ior was not compiled with GPFS hint support"); +#endif + params->gpfs_release_token = atoi(value); } else if (strcasecmp(option, "numtasks") == 0) { params->numTasks = atoi(value); RecalculateExpectedFileSize(params);