From aa0b8c92d22befe7c06e6d43fe010c6364f21c50 Mon Sep 17 00:00:00 2001 From: Julian Kunkel Date: Fri, 20 Oct 2017 18:02:24 +0200 Subject: [PATCH 1/4] New stonewalling option to create a wear out phase. Once a process hits the stonewall (timelimit), they all figure out the maximum pair read/written. Each proces continues to read/write until the maximum number of pairs is reached, this simulates the wear out. --- src/ior.c | 139 +++++++++++++++++++++++++++----------------- src/ior.h | 6 +- src/parse_options.c | 11 +++- 3 files changed, 98 insertions(+), 58 deletions(-) diff --git a/src/ior.c b/src/ior.c index 89eec11..47542ce 100755 --- a/src/ior.c +++ b/src/ior.c @@ -93,7 +93,7 @@ static void ShowTest(IOR_param_t *); static void PrintLongSummaryAllTests(IOR_test_t *tests_head); static void TestIoSys(IOR_test_t *); static void ValidateTests(IOR_param_t *); -static IOR_offset_t WriteOrRead(IOR_param_t * test, void *fd, int access, IOR_io_buffers* ioBuffers); +static IOR_offset_t WriteOrRead(IOR_param_t * test, IOR_results_t * results, void *fd, int access, IOR_io_buffers* ioBuffers); static void WriteTimes(IOR_param_t *, double **, int, int); /********************************** M A I N ***********************************/ @@ -169,6 +169,9 @@ int main(int argc, char **argv) sleep(5); printf("\trank %d: awake.\n", rank); } + if(rank == 0 && tptr->params.stoneWallingWearOut){ + fprintf(stdout, "Pairs deadlineForStonewallingaccessed: %lld\n", (long long) tptr->results->pairs_accessed); + } TestIoSys(tptr); } @@ -366,6 +369,10 @@ static void CheckFileSize(IOR_test_t *test, IOR_offset_t dataMoved, int rep) fprintf(stdout, "WARNING: Using actual aggregate bytes moved = %lld.\n", (long long) results->aggFileSizeFromXfer[rep]); + if(params->deadlineForStonewalling){ + fprintf(stdout, + "WARNING: maybe caused by deadlineForStonewalling\n"); + } } } } @@ -785,6 +792,7 @@ static void DisplayUsage(char **argv) " -W checkWrite -- check read after write", " -x singleXferAttempt -- do not retry transfer if incomplete", " -X N reorderTasksRandomSeed -- random seed for -Z option", + " -y stoneWallingWearOut -- once the stonewalling timout is over, all process finish to access the amount of data", " -Y fsyncPerWrite -- perform fsync after each POSIX write", " -z randomOffset -- access is to random, not sequential, offsets within a file", " -Z reorderTasksRandom -- changes task ordering to random ordering for readback", @@ -1556,8 +1564,8 @@ static void ShowSetup(IOR_param_t *params) } #endif /* HAVE_LUSTRE_LUSTRE_USER_H */ if (params->deadlineForStonewalling > 0) { - printf("\tUsing stonewalling = %d second(s)\n", - params->deadlineForStonewalling); + printf("\tUsing stonewalling = %d second(s)%s\n", + params->deadlineForStonewalling, params->stoneWallingWearOut ? " with phase out" : ""); } fflush(stdout); } @@ -1577,6 +1585,7 @@ static void ShowTest(IOR_param_t * test) fprintf(stdout, "\t%s=%s\n", "hintsFileName", test->hintsFileName); fprintf(stdout, "\t%s=%d\n", "deadlineForStonewall", test->deadlineForStonewalling); + fprintf(stdout, "\t%s=%d\n", "stoneWallingWearOut", test->stoneWallingWearOut); fprintf(stdout, "\t%s=%d\n", "maxTimeDuration", test->maxTimeDuration); fprintf(stdout, "\t%s=%d\n", "outlierThreshold", test->outlierThreshold); @@ -2078,7 +2087,7 @@ static void TestIoSys(IOR_test_t *test) CurrentTimeString()); } timer[2][rep] = GetTimeStamp(); - dataMoved = WriteOrRead(params, fd, WRITE, &ioBuffers); + dataMoved = WriteOrRead(params, results, fd, WRITE, &ioBuffers); if (params->verbose >= VERBOSE_4) { printf("* data moved = %llu\n", dataMoved); fflush(stdout); @@ -2131,7 +2140,7 @@ static void TestIoSys(IOR_test_t *test) GetTestFileName(testFileName, params); params->open = WRITECHECK; fd = backend->open(testFileName, params); - dataMoved = WriteOrRead(params, fd, WRITECHECK, &ioBuffers); + dataMoved = WriteOrRead(params, results, fd, WRITECHECK, &ioBuffers); backend->close(fd, params); rankOffset = 0; } @@ -2203,7 +2212,7 @@ static void TestIoSys(IOR_test_t *test) CurrentTimeString()); } timer[8][rep] = GetTimeStamp(); - dataMoved = WriteOrRead(params, fd, operation_flag, &ioBuffers); + dataMoved = WriteOrRead(params, results, fd, operation_flag, &ioBuffers); timer[9][rep] = GetTimeStamp(); if (params->intraTestBarriers) MPI_CHECK(MPI_Barrier(testComm), @@ -2540,22 +2549,71 @@ static IOR_offset_t *GetOffsetArrayRandom(IOR_param_t * test, int pretendRank, return (offsetArray); } +static IOR_offset_t WriteOrReadSingle(IOR_offset_t pairCnt, IOR_offset_t *offsetArray, int pretendRank, + IOR_offset_t * transferCount, int * errors, IOR_param_t * test, int * fd, IOR_io_buffers* ioBuffers, int access){ + IOR_offset_t amtXferred; + IOR_offset_t transfer; + + void *buffer = ioBuffers->buffer; + void *checkBuffer = ioBuffers->checkBuffer; + void *readCheckBuffer = ioBuffers->readCheckBuffer; + + test->offset = offsetArray[pairCnt]; + + transfer = test->transferSize; + if (access == WRITE) { + /* + * fills each transfer with a unique pattern + * containing the offset into the file + */ + if (test->storeFileOffset == TRUE) { + FillBuffer(buffer, test, test->offset, pretendRank); + } + amtXferred = + backend->xfer(access, fd, buffer, transfer, test); + if (amtXferred != transfer) + ERR("cannot write to file"); + } else if (access == READ) { + amtXferred = + backend->xfer(access, fd, buffer, transfer, test); + if (amtXferred != transfer) + ERR("cannot read from file"); + } else if (access == WRITECHECK) { + memset(checkBuffer, 'a', transfer); + amtXferred = + backend->xfer(access, fd, checkBuffer, transfer, + test); + if (amtXferred != transfer) + ERR("cannot read from file write check"); + (*transferCount)++; + *errors += CompareBuffers(buffer, checkBuffer, transfer, + *transferCount, test, + WRITECHECK); + } else if (access == READCHECK) { + amtXferred = backend->xfer(access, fd, buffer, transfer, test); + if (amtXferred != transfer){ + ERR("cannot read from file"); + } + if (test->storeFileOffset == TRUE) { + FillBuffer(readCheckBuffer, test, test->offset, pretendRank); + } + *errors += CompareBuffers(readCheckBuffer, buffer, transfer, *transferCount, test, READCHECK); + } + return amtXferred; +} + /* * Write or Read data to file(s). This loops through the strides, writing * out the data to each block in transfer sizes, until the remainder left is 0. */ -static IOR_offset_t WriteOrRead(IOR_param_t * test, void *fd, int access, IOR_io_buffers* ioBuffers) +static IOR_offset_t WriteOrRead(IOR_param_t * test, IOR_results_t * results, void *fd, int access, IOR_io_buffers* ioBuffers) { int errors = 0; IOR_offset_t amtXferred; - IOR_offset_t transfer; IOR_offset_t transferCount = 0; IOR_offset_t pairCnt = 0; IOR_offset_t *offsetArray; int pretendRank; - void *buffer = ioBuffers->buffer; - void *checkBuffer = ioBuffers->checkBuffer; - void *readCheckBuffer = ioBuffers->readCheckBuffer; IOR_offset_t dataMoved = 0; /* for data rate calculation */ double startForStonewall; int hitStonewall; @@ -2578,54 +2636,29 @@ static IOR_offset_t WriteOrRead(IOR_param_t * test, void *fd, int access, IOR_io /* loop over offsets to access */ while ((offsetArray[pairCnt] != -1) && !hitStonewall) { - test->offset = offsetArray[pairCnt]; - - transfer = test->transferSize; - if (access == WRITE) { - /* - * fills each transfer with a unique pattern - * containing the offset into the file - */ - if (test->storeFileOffset == TRUE) { - FillBuffer(buffer, test, test->offset, pretendRank); - } - amtXferred = - backend->xfer(access, fd, buffer, transfer, test); - if (amtXferred != transfer) - ERR("cannot write to file"); - } else if (access == READ) { - amtXferred = - backend->xfer(access, fd, buffer, transfer, test); - if (amtXferred != transfer) - ERR("cannot read from file"); - } else if (access == WRITECHECK) { - memset(checkBuffer, 'a', transfer); - amtXferred = - backend->xfer(access, fd, checkBuffer, transfer, - test); - if (amtXferred != transfer) - ERR("cannot read from file write check"); - transferCount++; - errors += CompareBuffers(buffer, checkBuffer, transfer, - transferCount, test, - WRITECHECK); - } else if (access == READCHECK) { - amtXferred = backend->xfer(access, fd, buffer, transfer, test); - if (amtXferred != transfer){ - ERR("cannot read from file"); - } - if (test->storeFileOffset == TRUE) { - FillBuffer(readCheckBuffer, test, test->offset, pretendRank); - } - errors += CompareBuffers(readCheckBuffer, buffer, transfer, transferCount, test, READCHECK); - } - dataMoved += amtXferred; + dataMoved += WriteOrReadSingle(pairCnt, offsetArray, pretendRank, & transferCount, & errors, test, fd, ioBuffers, access); pairCnt++; hitStonewall = ((test->deadlineForStonewalling != 0) && ((GetTimeStamp() - startForStonewall) > test->deadlineForStonewalling)); } + if (test->stoneWallingWearOut){ + MPI_CHECK(MPI_Allreduce(& pairCnt, &results->pairs_accessed, + 1, MPI_LONG_LONG_INT, MPI_MAX, testComm), "cannot reduce pairs moved"); + if (verbose >= VERBOSE_1){ + printf("%d: stonewalling pairs accessed globally: %lld this rank: %lld\n", rank, (long long) results->pairs_accessed, (long long) pairCnt); + } + if(pairCnt != results->pairs_accessed){ + // some work needs still to be done ! + for(; pairCnt < results->pairs_accessed; pairCnt++ ) { + dataMoved += WriteOrReadSingle(pairCnt, offsetArray, pretendRank, & transferCount, & errors, test, fd, ioBuffers, access); + } + } + }else{ + results->pairs_accessed = pairCnt; + } + totalErrorCount += CountErrors(test, access, errors); diff --git a/src/ior.h b/src/ior.h index c9c1292..2d81abe 100755 --- a/src/ior.h +++ b/src/ior.h @@ -5,7 +5,7 @@ * 8-chars of indenting is ridiculous. If you really want 8-spaces, * then change the mode-line to use tabs, and configure your personal * editor environment to use 8-space tab-stops. - * + * */ /******************************************************************************\ * * @@ -67,7 +67,7 @@ enum PACKET_TYPE /* A struct to hold the buffers so we can pass 1 pointer around instead of 3 */ -typedef struct IO_BUFFERS +typedef struct IO_BUFFERS { void* buffer; void* checkBuffer; @@ -141,6 +141,7 @@ typedef struct int useExistingTestFile; /* do not delete test file before access */ int storeFileOffset; /* use file offset as stored signature */ int deadlineForStonewalling; /* max time in seconds to run any test phase */ + int stoneWallingWearOut; /* wear out the stonewalling, once the timout is over, each process has to write the same amount */ int maxTimeDuration; /* max time in minutes to run each test */ int outlierThreshold; /* warn on outlier N seconds from mean */ int verbose; /* verbosity */ @@ -215,6 +216,7 @@ typedef struct typedef struct { double *writeTime; double *readTime; + size_t pairs_accessed; // number of I/Os done, useful for deadlineForStonewalling IOR_offset_t *aggFileSizeFromStat; IOR_offset_t *aggFileSizeFromXfer; IOR_offset_t *aggFileSizeForBW; diff --git a/src/parse_options.c b/src/parse_options.c index 5377cb5..8db715a 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -169,6 +169,8 @@ void DecodeDirective(char *line, IOR_param_t *params) strcpy(params->hintsFileName, value); } else if (strcasecmp(option, "deadlineforstonewalling") == 0) { params->deadlineForStonewalling = atoi(value); + } else if (strcasecmp(option, "stoneWallingWearOut") == 0) { + params->stoneWallingWearOut = atoi(value); } else if (strcasecmp(option, "maxtimeduration") == 0) { params->maxTimeDuration = atoi(value); } else if (strcasecmp(option, "outlierthreshold") == 0) { @@ -430,7 +432,7 @@ IOR_test_t *ReadConfigScript(char *scriptName) IOR_test_t *ParseCommandLine(int argc, char **argv) { static const char *opts = - "a:A:b:BcCd:D:eEf:FgG:hHi:Ij:J:kKl:mM:nN:o:O:pPqQ:rRs:St:T:uU:vVwWxX:YzZ"; + "a:A:b:BcCd:D:eEf:FgG:hHi:Ij:J:kKl:mM:nN:o:O:pPqQ:rRs:St:T:uU:vVwWxX:YzZy"; int c, i; static IOR_test_t *tests = NULL; @@ -523,13 +525,13 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) initialTestParams.dataPacketType = incompressible; break; case 't': /* timestamp */ - initialTestParams.dataPacketType = timestamp; + initialTestParams.dataPacketType = timestamp; break; case 'o': /* offset packet */ initialTestParams.storeFileOffset = TRUE; initialTestParams.dataPacketType = offset; break; - default: + default: fprintf(stdout, "Unknown arguement for -l %s generic assumed\n", optarg); break; @@ -610,6 +612,9 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) case 'X': initialTestParams.reorderTasksRandomSeed = atoi(optarg); break; + case 'y': + initialTestParams.stoneWallingWearOut = TRUE; + break; case 'Y': initialTestParams.fsyncPerWrite = TRUE; break; From 96e75cc83220280c1207a5988c7bd0b7c87a6c07 Mon Sep 17 00:00:00 2001 From: Julian Kunkel Date: Fri, 20 Oct 2017 18:13:29 +0200 Subject: [PATCH 2/4] stoneWallingWearOutIterations allows to read data back using the same number of iterations as used for the write with stoneWallingWearOut. --- src/ior.c | 5 +++-- src/ior.h | 1 + src/parse_options.c | 5 ++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/ior.c b/src/ior.c index 47542ce..7e71c19 100755 --- a/src/ior.c +++ b/src/ior.c @@ -793,6 +793,7 @@ static void DisplayUsage(char **argv) " -x singleXferAttempt -- do not retry transfer if incomplete", " -X N reorderTasksRandomSeed -- random seed for -Z option", " -y stoneWallingWearOut -- once the stonewalling timout is over, all process finish to access the amount of data", + " -1 stoneWallingWearOutIterations stop after processing this number of iterations, needed for reading data back written with -y", " -Y fsyncPerWrite -- perform fsync after each POSIX write", " -z randomOffset -- access is to random, not sequential, offsets within a file", " -Z reorderTasksRandom -- changes task ordering to random ordering for readback", @@ -2635,13 +2636,13 @@ static IOR_offset_t WriteOrRead(IOR_param_t * test, IOR_results_t * results, voi > test->deadlineForStonewalling)); /* loop over offsets to access */ - while ((offsetArray[pairCnt] != -1) && !hitStonewall) { + while ((offsetArray[pairCnt] != -1) && !hitStonewall ) { dataMoved += WriteOrReadSingle(pairCnt, offsetArray, pretendRank, & transferCount, & errors, test, fd, ioBuffers, access); pairCnt++; hitStonewall = ((test->deadlineForStonewalling != 0) && ((GetTimeStamp() - startForStonewall) - > test->deadlineForStonewalling)); + > test->deadlineForStonewalling)) || (test->stoneWallingWearOutIterations != 0 && pairCnt == test->stoneWallingWearOutIterations) ; } if (test->stoneWallingWearOut){ MPI_CHECK(MPI_Allreduce(& pairCnt, &results->pairs_accessed, diff --git a/src/ior.h b/src/ior.h index 2d81abe..1159883 100755 --- a/src/ior.h +++ b/src/ior.h @@ -142,6 +142,7 @@ typedef struct int storeFileOffset; /* use file offset as stored signature */ int deadlineForStonewalling; /* max time in seconds to run any test phase */ int stoneWallingWearOut; /* wear out the stonewalling, once the timout is over, each process has to write the same amount */ + int stoneWallingWearOutIterations; /* the number of iterations for the stonewallingWearOut, needed for readBack */ int maxTimeDuration; /* max time in minutes to run each test */ int outlierThreshold; /* warn on outlier N seconds from mean */ int verbose; /* verbosity */ diff --git a/src/parse_options.c b/src/parse_options.c index 8db715a..26fbef0 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -432,7 +432,7 @@ IOR_test_t *ReadConfigScript(char *scriptName) IOR_test_t *ParseCommandLine(int argc, char **argv) { static const char *opts = - "a:A:b:BcCd:D:eEf:FgG:hHi:Ij:J:kKl:mM:nN:o:O:pPqQ:rRs:St:T:uU:vVwWxX:YzZy"; + "a:A:b:BcCd:D:eEf:FgG:hHi:Ij:J:kKl:mM:nN:o:O:pPqQ:rRs:St:T:uU:vVwWxX:YzZy1:"; int c, i; static IOR_test_t *tests = NULL; @@ -612,6 +612,9 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) case 'X': initialTestParams.reorderTasksRandomSeed = atoi(optarg); break; + case '1': + initialTestParams.stoneWallingWearOutIterations = atoi(optarg); + break; case 'y': initialTestParams.stoneWallingWearOut = TRUE; break; From 072eb452fd800087a2f9efaa0e2000dc59b35b27 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 21 Oct 2017 11:59:09 +0200 Subject: [PATCH 3/4] Stonewalling wear-out only with long options. --- src/ior.c | 8 ++++---- src/parse_options.c | 8 ++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/ior.c b/src/ior.c index 7e71c19..53b6fe3 100755 --- a/src/ior.c +++ b/src/ior.c @@ -169,11 +169,11 @@ int main(int argc, char **argv) sleep(5); printf("\trank %d: awake.\n", rank); } + TestIoSys(tptr); + if(rank == 0 && tptr->params.stoneWallingWearOut){ fprintf(stdout, "Pairs deadlineForStonewallingaccessed: %lld\n", (long long) tptr->results->pairs_accessed); } - - TestIoSys(tptr); } if (verbose < 0) @@ -753,6 +753,8 @@ static void DisplayUsage(char **argv) " -C reorderTasks -- changes task ordering to n+1 ordering for readback", " -d N interTestDelay -- delay between reps in seconds", " -D N deadlineForStonewalling -- seconds before stopping write or read phase", + " -O stoneWallingWearOut=1 -- once the stonewalling timout is over, all process finish to access the amount of data", + " -O stoneWallingWearOutIterations=N -- stop after processing this number of iterations, needed for reading data back written with stoneWallingWearOut", " -e fsync -- perform fsync upon POSIX write close", " -E useExistingTestFile -- do not remove test file before write access", " -f S scriptFile -- test script name", @@ -792,8 +794,6 @@ static void DisplayUsage(char **argv) " -W checkWrite -- check read after write", " -x singleXferAttempt -- do not retry transfer if incomplete", " -X N reorderTasksRandomSeed -- random seed for -Z option", - " -y stoneWallingWearOut -- once the stonewalling timout is over, all process finish to access the amount of data", - " -1 stoneWallingWearOutIterations stop after processing this number of iterations, needed for reading data back written with -y", " -Y fsyncPerWrite -- perform fsync after each POSIX write", " -z randomOffset -- access is to random, not sequential, offsets within a file", " -Z reorderTasksRandom -- changes task ordering to random ordering for readback", diff --git a/src/parse_options.c b/src/parse_options.c index 26fbef0..9520930 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -171,6 +171,8 @@ void DecodeDirective(char *line, IOR_param_t *params) params->deadlineForStonewalling = atoi(value); } else if (strcasecmp(option, "stoneWallingWearOut") == 0) { params->stoneWallingWearOut = atoi(value); + } else if (strcasecmp(option, "stoneWallingWearOutIterations") == 0) { + params->stoneWallingWearOutIterations = atoi(value); } else if (strcasecmp(option, "maxtimeduration") == 0) { params->maxTimeDuration = atoi(value); } else if (strcasecmp(option, "outlierthreshold") == 0) { @@ -612,12 +614,6 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) case 'X': initialTestParams.reorderTasksRandomSeed = atoi(optarg); break; - case '1': - initialTestParams.stoneWallingWearOutIterations = atoi(optarg); - break; - case 'y': - initialTestParams.stoneWallingWearOut = TRUE; - break; case 'Y': initialTestParams.fsyncPerWrite = TRUE; break; From 0916d432cd3d92f8981e25bcb93abd684c0b7b50 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 21 Oct 2017 12:02:03 +0200 Subject: [PATCH 4/4] Removed short option parsing. --- src/parse_options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parse_options.c b/src/parse_options.c index 9520930..e2cd653 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -434,7 +434,7 @@ IOR_test_t *ReadConfigScript(char *scriptName) IOR_test_t *ParseCommandLine(int argc, char **argv) { static const char *opts = - "a:A:b:BcCd:D:eEf:FgG:hHi:Ij:J:kKl:mM:nN:o:O:pPqQ:rRs:St:T:uU:vVwWxX:YzZy1:"; + "a:A:b:BcCd:D:eEf:FgG:hHi:Ij:J:kKl:mM:nN:o:O:pPqQ:rRs:St:T:uU:vVwWxX:YzZ"; int c, i; static IOR_test_t *tests = NULL;