From b5963380ae9ee4939c2c0e632371b64f0b796461 Mon Sep 17 00:00:00 2001 From: Julian Kunkel Date: Tue, 9 Feb 2021 17:54:14 +0000 Subject: [PATCH] Feature: IOR rank details in CSV file (#334) * IOR: Store individual rank results into a CSV file #333 Example usage: ior -O saveRankPerformanceDetailsCSV=test.csv --- src/ior.c | 63 ++++++++++++++++++++++++++++++++++++--------- src/ior.h | 1 + src/parse_options.c | 16 ++++++++++++ 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/src/ior.c b/src/ior.c index 986f3de..dd0f048 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1243,6 +1243,55 @@ WriteTimes(IOR_param_t *test, const double *timer, const int iteration, timerName); } } + +static void StoreRankInformation(IOR_test_t *test, double *timer, const int rep, const int access){ + IOR_param_t *params = &test->params; + double totalTime = timer[5] - timer[0]; + double accessTime = timer[3] - timer[2]; + double times[] = {totalTime, accessTime}; + + if(rank == 0){ + FILE* fd = fopen(params->saveRankDetailsCSV, "a"); + if (fd == NULL){ + FAIL("Cannot open saveRankPerformanceDetailsCSV file for writes!"); + } + int size; + MPI_Comm_size(params->testComm, & size); + double *all_times = malloc(2* size * sizeof(double)); + MPI_Gather(times, 2, MPI_DOUBLE, all_times, 2, MPI_DOUBLE, 0, params->testComm); + IOR_point_t *point = (access == WRITE) ? &test->results[rep].write : &test->results[rep].read; + double file_size = ((double) point->aggFileSizeForBW) / size; + + for(int i=0; i < size; i++){ + char buff[1024]; + sprintf(buff, "%s,%d,%.10e,%.10e,%.10e,%.10e\n", access==WRITE ? "write" : "read", i, all_times[i*2], all_times[i*2+1], file_size/all_times[i*2], file_size/all_times[i*2+1] ); + int ret = fwrite(buff, strlen(buff), 1, fd); + if(ret != 1){ + WARN("Couln't append to saveRankPerformanceDetailsCSV file\n"); + break; + } + } + fclose(fd); + }else{ + MPI_Gather(& times, 2, MPI_DOUBLE, NULL, 2, MPI_DOUBLE, 0, testComm); + } +} + +static void ProcessIterResults(IOR_test_t *test, double *timer, const int rep, const int access){ + IOR_param_t *params = &test->params; + + if (verbose >= VERBOSE_3) + WriteTimes(params, timer, rep, access); + ReduceIterResults(test, timer, rep, access); + if (params->outlierThreshold) { + CheckForOutliers(params, timer, access); + } + + if(params->saveRankDetailsCSV){ + StoreRankInformation(test, timer, rep, access); + } +} + /* * Using the test parameters, run iteration(s) of single test. */ @@ -1383,12 +1432,7 @@ static void TestIoSys(IOR_test_t *test) use actual amount of byte moved */ CheckFileSize(test, testFileName, dataMoved, rep, WRITE); - if (verbose >= VERBOSE_3) - WriteTimes(params, timer, rep, WRITE); - ReduceIterResults(test, timer, rep, WRITE); - if (params->outlierThreshold) { - CheckForOutliers(params, timer, WRITE); - } + ProcessIterResults(test, timer, rep, WRITE); /* check if in this round we run write with stonewalling */ if(params->deadlineForStonewalling > 0){ @@ -1513,12 +1557,7 @@ static void TestIoSys(IOR_test_t *test) use actual amount of byte moved */ CheckFileSize(test, testFileName, dataMoved, rep, READ); - if (verbose >= VERBOSE_3) - WriteTimes(params, timer, rep, READ); - ReduceIterResults(test, timer, rep, READ); - if (params->outlierThreshold) { - CheckForOutliers(params, timer, READ); - } + ProcessIterResults(test, timer, rep, READ); } if (!params->keepFile diff --git a/src/ior.h b/src/ior.h index 6252f78..e4663db 100755 --- a/src/ior.h +++ b/src/ior.h @@ -130,6 +130,7 @@ typedef struct IOR_offset_t expectedAggFileSize; /* calculated aggregate file size */ IOR_offset_t randomPrefillBlocksize; /* prefill option for random IO, the amount of data used for prefill */ + char * saveRankDetailsCSV; /* save the details about the performance to a file */ int summary_every_test; /* flag to print summary every test, not just at end */ int uniqueDir; /* use unique directory for each fpp */ int useExistingTestFile; /* do not delete test file before access */ diff --git a/src/parse_options.c b/src/parse_options.c index 05fa78f..82fab98 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -103,6 +103,21 @@ void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_opt } printf("Writing output to %s\n", value); } + } else if (strcasecmp(option, "saveRankPerformanceDetailsCSV") == 0){ + if (rank == 0){ + // check that the file is writeable, truncate it and add header + FILE* fd = fopen(value, "w"); + if (fd == NULL){ + FAIL("Cannot open saveRankPerformanceDetailsCSV file for write!"); + } + char buff[] = "access,rank,runtime-with-openclose,runtime,throughput-withopenclose,throughput\n"; + int ret = fwrite(buff, strlen(buff), 1, fd); + if(ret != 1){ + FAIL("Cannot write header to saveRankPerformanceDetailsCSV file"); + } + fclose(fd); + } + params->saveRankDetailsCSV = strdup(value); } else if (strcasecmp(option, "summaryFormat") == 0) { if(strcasecmp(value, "default") == 0){ outputFormat = OUTPUT_DEFAULT; @@ -439,6 +454,7 @@ option_help * createGlobalOptions(IOR_param_t * params){ {0, "warningAsErrors", "Any warning should lead to an error.", OPTION_FLAG, 'd', & params->warningAsErrors}, {.help=" -O summaryFile=FILE -- store result data into this file", .arg = OPTION_OPTIONAL_ARGUMENT}, {.help=" -O summaryFormat=[default,JSON,CSV] -- use the format for outputting the summary", .arg = OPTION_OPTIONAL_ARGUMENT}, + {.help=" -O saveRankPerformanceDetailsCSV= -- store the performance of each rank into the named CSV file.", .arg = OPTION_OPTIONAL_ARGUMENT}, {0, "dryRun", "do not perform any I/Os just run evtl. inputs print dummy output", OPTION_FLAG, 'd', & params->dryRun}, LAST_OPTION, };