From 738190bd79a7877b6d634637d35ca28ad7ab354e Mon Sep 17 00:00:00 2001 From: Oliver Steffen Date: Thu, 21 Sep 2017 17:12:31 +0200 Subject: [PATCH] add support for tuning BeeGFS parameters --- configure.ac | 2 +- doc/USER_GUIDE | 8 +++ src/aiori-POSIX.c | 119 ++++++++++++++++++++++++++++++++++++++++++++ src/ior.c | 3 ++ src/ior.h | 3 ++ src/parse_options.c | 16 ++++++ 6 files changed, 150 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 54f4a37..159cb4c 100755 --- a/configure.ac +++ b/configure.ac @@ -20,7 +20,7 @@ AX_PROG_CC_MPI # Checks for libraries. # Checks for header files. -AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h plfs.h hdfs.h]) +AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h plfs.h hdfs.h beegfs/beegfs.h]) # Checks for typedefs, structures, and compiler characteristics. AC_TYPE_SIZE_T diff --git a/doc/USER_GUIDE b/doc/USER_GUIDE index 20f8ecb..9957fec 100755 --- a/doc/USER_GUIDE +++ b/doc/USER_GUIDE @@ -349,6 +349,14 @@ GPFS-SPECIFIC: all locks. Might help mitigate lock-revocation traffic when many proceses write/read to same file. +BeeGFS-SPECIFIC (POSIX only): +================ + * beegfsNumTargets - set the number of storage targets to use + + * beegfsChunkSize - set the striping chunk size. Must be a power of two, + and greater than 64kiB, (e.g.: 256k, 1M, ...) + + *********************** * 5. VERBOSITY LEVELS * *********************** diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 403ff45..9b06d6b 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -41,6 +41,12 @@ # include #endif +#ifdef HAVE_BEEGFS_BEEGFS_H +#include +#include +#include +#endif + #include "ior.h" #include "aiori.h" #include "iordef.h" @@ -158,6 +164,101 @@ void gpfs_access_end(int fd, IOR_offset_t length, IOR_param_t *param, int access #endif +#ifdef HAVE_BEEGFS_BEEGFS_H + +int mkTempInDir(char* dirPath) +{ + unsigned long len = strlen(dirPath) + 8; + char* tmpfilename = (char*)malloc(sizeof (char)*len+1); + snprintf(tmpfilename, len, "%s/XXXXXX", dirPath); + + int fd = mkstemp(tmpfilename); + unlink(tmpfilename); + free(tmpfilename); + + return fd; +} + +bool beegfs_getStriping(char* dirPath, u_int16_t* numTargetsOut, unsigned* chunkSizeOut) +{ + bool retVal = false; + + int fd = mkTempInDir(dirPath); + if (fd) { + unsigned stripePattern = 0; + retVal = beegfs_getStripeInfo(fd, &stripePattern, chunkSizeOut, numTargetsOut); + close(fd); + } + + return retVal; +} + +bool beegfs_isOptionSet(int opt) { + return opt != -1; +} + +/* + * Create a file on a BeeGFS file system with striping parameters + */ +bool beegfs_createFilePath(char* filepath, mode_t mode, int numTargets, int chunkSize) +{ + bool retVal = false; + char* dirTmp = strdup(filepath); + char* dir = dirname(dirTmp); + DIR* parentDirS = opendir(dir); + if (!parentDirS) { + ERR("Failed to get directory"); + } + else + { + int parentDirFd = dirfd(parentDirS); + if (parentDirFd < 0) + { + ERR("Failed to get directory descriptor"); + } + else + { + bool isBeegfs = beegfs_testIsBeeGFS(parentDirFd); + if (!isBeegfs) + { + WARN("Not a BeeGFS file system"); + } + else + { + if ( !beegfs_isOptionSet(numTargets) + || !beegfs_isOptionSet(chunkSize)) { + u_int16_t defaultNumTargets = 0; + unsigned defaultChunkSize = 0; + bool haveDefaults = beegfs_getStriping(dir, + &defaultNumTargets, + &defaultChunkSize); + if (!haveDefaults) + ERR("Failed to get default BeeGFS striping values"); + + numTargets = beegfs_isOptionSet(numTargets) ? + numTargets : defaultNumTargets; + chunkSize = beegfs_isOptionSet(chunkSize) ? + chunkSize : defaultChunkSize; + } + + char* filenameTmp = strdup(filepath); + char* filename = basename(filepath); + bool isFileCreated = beegfs_createFile(parentDirFd, filename, + mode, numTargets, chunkSize); + if (!isFileCreated) + ERR("Could not create file"); + retVal = true; + free(filenameTmp); + } + } + closedir(parentDirS); + } + free(dirTmp); + return retVal; +} +#endif /* HAVE_BEEGFS_BEEGFS_H */ + + /* * Creat and open a file through the POSIX interface. */ @@ -219,10 +320,28 @@ static void *POSIX_Create(char *testFileName, IOR_param_t * param) } } else { #endif /* HAVE_LUSTRE_LUSTRE_USER_H */ + fd_oflag |= O_CREAT | O_RDWR; + +#ifdef HAVE_BEEGFS_BEEGFS_H + if (beegfs_isOptionSet(param->beegfs_chunkSize) + || beegfs_isOptionSet(param->beegfs_numTargets)) { + bool result = beegfs_createFilePath(testFileName, + 0664, + param->beegfs_numTargets, + param->beegfs_chunkSize); + if (result) { + fd_oflag &= ~O_CREAT; + } else { + EWARN("BeeGFS tuning failed"); + } + } +#endif /* HAVE_BEEGFS_BEEGFS_H */ + *fd = open64(testFileName, fd_oflag, 0664); if (*fd < 0) ERR("open64() failed"); + #ifdef HAVE_LUSTRE_LUSTRE_USER_H } diff --git a/src/ior.c b/src/ior.c index 52d9384..341492e 100755 --- a/src/ior.c +++ b/src/ior.c @@ -242,6 +242,9 @@ void init_IOR_Param_t(IOR_param_t * p) p->io_buf = NULL; p->etags = NULL; p->part_number = 0; + + p->beegfs_numTargets = -1; + p->beegfs_chunkSize = -1; } /* diff --git a/src/ior.h b/src/ior.h index c9c1292..c97382c 100755 --- a/src/ior.h +++ b/src/ior.h @@ -205,6 +205,9 @@ typedef struct int gpfs_hint_access; /* use gpfs "access range" hint */ int gpfs_release_token; /* immediately release GPFS tokens after creating or opening a file */ + /* beegfs variables */ + int beegfs_numTargets; /* number storage targets to use */ + int beegfs_chunkSize; /* srtipe pattern for new files */ int id; /* test's unique ID */ int intraTestBarriers; /* barriers between open/op and op/close */ diff --git a/src/parse_options.c b/src/parse_options.c index 5377cb5..0abc1be 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -25,6 +25,8 @@ #include "aiori.h" #include "parse_options.h" +#define ISPOWEROFTWO(x) ((x != 0) && !(x & (x - 1))) + IOR_param_t initialTestParams; /* @@ -298,6 +300,20 @@ void DecodeDirective(char *line, IOR_param_t *params) ERR("ior was not compiled with GPFS hint support"); #endif params->gpfs_release_token = atoi(value); + } else if (strcasecmp(option, "beegfsNumTargets") == 0) { +#ifndef HAVE_BEEGFS_BEEGFS_H + ERR("ior was not compiled with BeeGFS support"); +#endif + params->beegfs_numTargets = atoi(value); + if (params->beegfs_numTargets < 1) + ERR("beegfsNumTargets must be >= 1"); + } else if (strcasecmp(option, "beegfsChunkSize") == 0) { + #ifndef HAVE_BEEGFS_BEEGFS_H + ERR("ior was not compiled with BeeGFS support"); + #endif + params->beegfs_chunkSize = StringToBytes(value); + if (!ISPOWEROFTWO(params->beegfs_chunkSize) || params->beegfs_chunkSize < (1<<16)) + ERR("beegfsChunkSize must be a power of two and >64k"); } else if (strcasecmp(option, "numtasks") == 0) { params->numTasks = atoi(value); RecalculateExpectedFileSize(params);