Teach IOR about GPFS hints (gpfs_fcntl)
GPFS supports a "gpfs_fcntl" method for hinting various things, including "i'm about to write this block of data". Let's see if, for the cost of a few system calls, we can wrangle the GPFS locking system into allowing concurrent access with less overhead. (new IOR parameter gpfsHintAccess) Also, drop all locks on a file immediately after open/creation in the shared file case, since we know all processes will touch unique regions of the file. It may or may not be a good idea to release all file locks after opening. Processes will then have to re-acquire locks already held. (new IOR parameter gpfsReleaseToken)master
parent
6a7ddfe60c
commit
aa604c1d38
|
@ -20,7 +20,7 @@ AX_PROG_CC_MPI
|
||||||
# Checks for libraries.
|
# Checks for libraries.
|
||||||
|
|
||||||
# Checks for header files.
|
# Checks for header files.
|
||||||
AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h])
|
AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h gpfs.h gpfs_fcntl.h])
|
||||||
|
|
||||||
# Checks for typedefs, structures, and compiler characteristics.
|
# Checks for typedefs, structures, and compiler characteristics.
|
||||||
AC_TYPE_SIZE_T
|
AC_TYPE_SIZE_T
|
||||||
|
|
|
@ -341,6 +341,13 @@ LUSTRE-SPECIFIC:
|
||||||
|
|
||||||
* lustreIgnoreLocks - disable lustre range locking [0]
|
* lustreIgnoreLocks - disable lustre range locking [0]
|
||||||
|
|
||||||
|
GPFS-SPECIFIC:
|
||||||
|
================
|
||||||
|
* gpfsHintAccess - use gpfs_fcntl hints to pre-declare accesses
|
||||||
|
|
||||||
|
* gpfsReleaseToken - immediately after opening or creating file, release
|
||||||
|
all locks. Might help mitigate lock-revocation
|
||||||
|
traffic when many proceses write/read to same file.
|
||||||
|
|
||||||
***********************
|
***********************
|
||||||
* 5. VERBOSITY LEVELS *
|
* 5. VERBOSITY LEVELS *
|
||||||
|
|
|
@ -32,6 +32,13 @@
|
||||||
#include <lustre/lustre_user.h>
|
#include <lustre/lustre_user.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifdef HAVE_GPFS_H
|
||||||
|
#include <gpfs.h>
|
||||||
|
#endif
|
||||||
|
#ifdef HAVE_GPFS_FCNTL_H
|
||||||
|
#include <gpfs_fcntl.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
#include "ior.h"
|
#include "ior.h"
|
||||||
#include "aiori.h"
|
#include "aiori.h"
|
||||||
#include "iordef.h"
|
#include "iordef.h"
|
||||||
|
@ -91,6 +98,78 @@ void set_o_direct_flag(int *fd)
|
||||||
*fd |= O_DIRECT;
|
*fd |= O_DIRECT;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef HAVE_GPFS_FCNTL_H
|
||||||
|
void gpfs_free_all_locks(int fd)
|
||||||
|
{
|
||||||
|
int rc;
|
||||||
|
struct {
|
||||||
|
gpfsFcntlHeader_t header;
|
||||||
|
gpfsFreeRange_t release;
|
||||||
|
} release_all;
|
||||||
|
release_all.header.totalLength = sizeof(release_all);
|
||||||
|
release_all.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
|
||||||
|
release_all.header.fcntlReserved = 0;
|
||||||
|
|
||||||
|
release_all.release.structLen = sizeof(release_all.release);
|
||||||
|
release_all.release.structType = GPFS_FREE_RANGE;
|
||||||
|
release_all.release.start = 0;
|
||||||
|
release_all.release.length = 0;
|
||||||
|
|
||||||
|
rc = gpfs_fcntl(fd, &release_all);
|
||||||
|
if (verbose >= VERBOSE_0 && rc != 0) {
|
||||||
|
EWARN("gpfs_fcntl release all locks hint failed.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
void gpfs_access_start(int fd, IOR_offset_t length, IOR_param_t *param, int access)
|
||||||
|
{
|
||||||
|
int rc;
|
||||||
|
struct {
|
||||||
|
gpfsFcntlHeader_t header;
|
||||||
|
gpfsAccessRange_t access;
|
||||||
|
} take_locks;
|
||||||
|
|
||||||
|
take_locks.header.totalLength = sizeof(take_locks);
|
||||||
|
take_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
|
||||||
|
take_locks.header.fcntlReserved = 0;
|
||||||
|
|
||||||
|
take_locks.access.structLen = sizeof(take_locks.access);
|
||||||
|
take_locks.access.structType = GPFS_ACCESS_RANGE;
|
||||||
|
take_locks.access.start = param->offset;
|
||||||
|
take_locks.access.length = length;
|
||||||
|
take_locks.access.isWrite = (access == WRITE);
|
||||||
|
|
||||||
|
rc = gpfs_fcntl(fd, &take_locks);
|
||||||
|
if (verbose >= VERBOSE_2 && rc != 0) {
|
||||||
|
EWARN("gpfs_fcntl access range hint failed.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void gpfs_access_end(int fd, IOR_offset_t length, IOR_param_t *param, int access)
|
||||||
|
{
|
||||||
|
int rc;
|
||||||
|
struct {
|
||||||
|
gpfsFcntlHeader_t header;
|
||||||
|
gpfsFreeRange_t free;
|
||||||
|
} free_locks;
|
||||||
|
|
||||||
|
|
||||||
|
free_locks.header.totalLength = sizeof(free_locks);
|
||||||
|
free_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
|
||||||
|
free_locks.header.fcntlReserved = 0;
|
||||||
|
|
||||||
|
free_locks.free.structLen = sizeof(free_locks.free);
|
||||||
|
free_locks.free.structType = GPFS_FREE_RANGE;
|
||||||
|
free_locks.free.start = param->offset;
|
||||||
|
free_locks.free.length = length;
|
||||||
|
|
||||||
|
rc = gpfs_fcntl(fd, &free_locks);
|
||||||
|
if (verbose >= VERBOSE_2 && rc != 0) {
|
||||||
|
EWARN("gpfs_fcntl free range hint failed.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Creat and open a file through the POSIX interface.
|
* Creat and open a file through the POSIX interface.
|
||||||
*/
|
*/
|
||||||
|
@ -166,6 +245,14 @@ static void *POSIX_Create(char *testFileName, IOR_param_t * param)
|
||||||
}
|
}
|
||||||
#endif /* HAVE_LUSTRE_LUSTRE_USER_H */
|
#endif /* HAVE_LUSTRE_LUSTRE_USER_H */
|
||||||
|
|
||||||
|
#ifdef HAVE_GPFS_FCNTL_H
|
||||||
|
/* in the single shared file case, immediately release all locks, with
|
||||||
|
* the intent that we can avoid some byte range lock revocation:
|
||||||
|
* everyone will be writing/reading from individual regions */
|
||||||
|
if (param->gpfs_release_token ) {
|
||||||
|
gpfs_free_all_locks(*fd);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
return ((void *)fd);
|
return ((void *)fd);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -201,6 +288,11 @@ static void *POSIX_Open(char *testFileName, IOR_param_t * param)
|
||||||
}
|
}
|
||||||
#endif /* HAVE_LUSTRE_LUSTRE_USER_H */
|
#endif /* HAVE_LUSTRE_LUSTRE_USER_H */
|
||||||
|
|
||||||
|
#ifdef HAVE_GPFS_FCNTL_H
|
||||||
|
if(param->gpfs_release_token) {
|
||||||
|
gpfs_free_all_locks(*fd);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
return ((void *)fd);
|
return ((void *)fd);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -218,6 +310,13 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer,
|
||||||
|
|
||||||
fd = *(int *)file;
|
fd = *(int *)file;
|
||||||
|
|
||||||
|
#ifdef HAVE_GPFS_FCNTL_H
|
||||||
|
if (param->gpfs_hint_access) {
|
||||||
|
gpfs_access_start(fd, length, param, access);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
/* seek to offset */
|
/* seek to offset */
|
||||||
if (lseek64(fd, param->offset, SEEK_SET) == -1)
|
if (lseek64(fd, param->offset, SEEK_SET) == -1)
|
||||||
ERR("lseek64() failed");
|
ERR("lseek64() failed");
|
||||||
|
@ -268,6 +367,11 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer,
|
||||||
ptr += rc;
|
ptr += rc;
|
||||||
xferRetries++;
|
xferRetries++;
|
||||||
}
|
}
|
||||||
|
#ifdef HAVE_GPFS_FCNTL_H
|
||||||
|
if (param->gpfs_hint_access) {
|
||||||
|
gpfs_access_end(fd, length, param, access);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
return (length);
|
return (length);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1601,6 +1601,10 @@ static void ShowTest(IOR_param_t * test)
|
||||||
test->setTimeStampSignature);
|
test->setTimeStampSignature);
|
||||||
fprintf(stdout, "\t%s=%d\n", "collective", test->collective);
|
fprintf(stdout, "\t%s=%d\n", "collective", test->collective);
|
||||||
fprintf(stdout, "\t%s=%lld", "segmentCount", test->segmentCount);
|
fprintf(stdout, "\t%s=%lld", "segmentCount", test->segmentCount);
|
||||||
|
#ifdef HAVE_GPFS_FCNTL_H
|
||||||
|
fprintf(stdout, "\t%s=%d\n", "gpfsHintAccess", test->gpfs_hint_access);
|
||||||
|
fprintf(stdout, "\t%s=%d\n", "gpfsReleaseToken", test->gpfs_release_token);
|
||||||
|
#endif
|
||||||
if (strcmp(test->api, "HDF5") == 0) {
|
if (strcmp(test->api, "HDF5") == 0) {
|
||||||
fprintf(stdout, " (datasets)");
|
fprintf(stdout, " (datasets)");
|
||||||
}
|
}
|
||||||
|
|
|
@ -126,6 +126,11 @@ typedef struct
|
||||||
int lustre_set_striping; /* flag that we need to set lustre striping */
|
int lustre_set_striping; /* flag that we need to set lustre striping */
|
||||||
int lustre_ignore_locks;
|
int lustre_ignore_locks;
|
||||||
|
|
||||||
|
/* gpfs variables */
|
||||||
|
int gpfs_hint_access; /* use gpfs "access range" hint */
|
||||||
|
int gpfs_release_token; /* immediately release GPFS tokens after
|
||||||
|
creating or opening a file */
|
||||||
|
|
||||||
int id; /* test's unique ID */
|
int id; /* test's unique ID */
|
||||||
int intraTestBarriers; /* barriers between open/op and op/close */
|
int intraTestBarriers; /* barriers between open/op and op/close */
|
||||||
} IOR_param_t;
|
} IOR_param_t;
|
||||||
|
|
|
@ -263,6 +263,16 @@ void DecodeDirective(char *line, IOR_param_t *params)
|
||||||
ERR("ior was not compiled with Lustre support");
|
ERR("ior was not compiled with Lustre support");
|
||||||
#endif
|
#endif
|
||||||
params->lustre_ignore_locks = atoi(value);
|
params->lustre_ignore_locks = atoi(value);
|
||||||
|
} else if (strcasecmp(option, "gpfshintaccess") == 0) {
|
||||||
|
#ifndef HAVE_GPFS_FCNTL_H
|
||||||
|
ERR("ior was not compiled with GPFS hint support");
|
||||||
|
#endif
|
||||||
|
params->gpfs_hint_access = atoi(value);
|
||||||
|
} else if (strcasecmp(option, "gpfsreleasetoken") == 0) {
|
||||||
|
#ifndef HAVE_GPFS_FCNTL_H
|
||||||
|
ERR("ior was not compiled with GPFS hint support");
|
||||||
|
#endif
|
||||||
|
params->gpfs_release_token = atoi(value);
|
||||||
} else if (strcasecmp(option, "numtasks") == 0) {
|
} else if (strcasecmp(option, "numtasks") == 0) {
|
||||||
params->numTasks = atoi(value);
|
params->numTasks = atoi(value);
|
||||||
RecalculateExpectedFileSize(params);
|
RecalculateExpectedFileSize(params);
|
||||||
|
|
Loading…
Reference in New Issue