2012-01-09 00:51:04 +04:00
|
|
|
/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
|
|
|
|
* vim:expandtab:shiftwidth=8:tabstop=8:
|
|
|
|
*/
|
2011-06-17 23:20:43 +04:00
|
|
|
/******************************************************************************\
|
|
|
|
* *
|
|
|
|
* Copyright (c) 2003, The Regents of the University of California *
|
|
|
|
* See the file COPYRIGHT for a complete copyright notice and license. *
|
|
|
|
* *
|
|
|
|
\******************************************************************************/
|
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
#ifdef HAVE_CONFIG_H
|
2014-07-31 03:17:21 +04:00
|
|
|
# include "config.h"
|
2011-11-12 04:40:45 +04:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#include <stdio.h>
|
|
|
|
#include <stdlib.h>
|
2012-01-07 05:29:45 +04:00
|
|
|
#include <unistd.h>
|
2011-11-12 02:22:17 +04:00
|
|
|
#include <ctype.h> /* tolower() */
|
2011-11-12 04:40:45 +04:00
|
|
|
#include <errno.h>
|
2011-06-17 23:20:43 +04:00
|
|
|
#include <math.h>
|
|
|
|
#include <mpi.h>
|
|
|
|
#include <string.h>
|
2018-10-08 23:47:28 +03:00
|
|
|
|
|
|
|
#if defined(HAVE_STRINGS_H)
|
|
|
|
#include <strings.h>
|
|
|
|
#endif
|
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
#include <sys/stat.h> /* struct stat */
|
2011-06-17 23:20:43 +04:00
|
|
|
#include <time.h>
|
2014-07-31 03:17:21 +04:00
|
|
|
|
2011-06-17 23:20:43 +04:00
|
|
|
#ifndef _WIN32
|
2014-07-31 03:17:21 +04:00
|
|
|
# include <sys/time.h> /* gettimeofday() */
|
|
|
|
# include <sys/utsname.h> /* uname() */
|
2011-06-17 23:20:43 +04:00
|
|
|
#endif
|
2014-07-31 03:17:21 +04:00
|
|
|
|
2021-02-18 13:40:42 +03:00
|
|
|
#ifdef HAVE_CUDA
|
|
|
|
#include <cuda_runtime.h>
|
|
|
|
#endif
|
|
|
|
|
2011-11-10 04:34:16 +04:00
|
|
|
#include <assert.h>
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
#include "ior.h"
|
2018-07-08 15:38:05 +03:00
|
|
|
#include "ior-internal.h"
|
2011-11-12 04:40:45 +04:00
|
|
|
#include "aiori.h"
|
|
|
|
#include "utilities.h"
|
|
|
|
#include "parse_options.h"
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2018-09-19 20:06:05 +03:00
|
|
|
#define IOR_NB_TIMERS 6
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
/* file scope globals */
|
|
|
|
extern char **environ;
|
2018-07-07 13:42:21 +03:00
|
|
|
static int totalErrorCount;
|
|
|
|
static const ior_aiori_t *backend;
|
2011-10-28 03:50:05 +04:00
|
|
|
|
2011-12-13 09:00:18 +04:00
|
|
|
static void DestroyTests(IOR_test_t *tests_head);
|
2011-11-12 04:40:45 +04:00
|
|
|
static char *PrependDir(IOR_param_t *, char *);
|
|
|
|
static char **ParseFileName(char *, int *);
|
2021-01-20 18:00:33 +03:00
|
|
|
static void InitTests(IOR_test_t *);
|
2011-12-13 09:00:18 +04:00
|
|
|
static void TestIoSys(IOR_test_t *);
|
2021-01-20 17:57:21 +03:00
|
|
|
static void ValidateTests(IOR_param_t * params, MPI_Comm com);
|
2018-09-19 20:06:05 +03:00
|
|
|
static IOR_offset_t WriteOrRead(IOR_param_t *test, IOR_results_t *results,
|
2020-05-31 14:11:00 +03:00
|
|
|
aiori_fd_t *fd, const int access,
|
2018-09-19 20:06:05 +03:00
|
|
|
IOR_io_buffers *ioBuffers);
|
2011-11-12 04:40:45 +04:00
|
|
|
|
2020-05-31 14:50:03 +03:00
|
|
|
static void ior_set_xfer_hints(IOR_param_t * p){
|
|
|
|
aiori_xfer_hint_t * hints = & p->hints;
|
|
|
|
hints->dryRun = p->dryRun;
|
|
|
|
hints->filePerProc = p->filePerProc;
|
|
|
|
hints->collective = p->collective;
|
|
|
|
hints->numTasks = p->numTasks;
|
|
|
|
hints->numNodes = p->numNodes;
|
|
|
|
hints->randomOffset = p->randomOffset;
|
|
|
|
hints->fsyncPerWrite = p->fsyncPerWrite;
|
|
|
|
hints->segmentCount = p->segmentCount;
|
|
|
|
hints->blockSize = p->blockSize;
|
|
|
|
hints->transferSize = p->transferSize;
|
|
|
|
hints->expectedAggFileSize = p->expectedAggFileSize;
|
|
|
|
hints->singleXferAttempt = p->singleXferAttempt;
|
|
|
|
|
|
|
|
if(backend->xfer_hints){
|
|
|
|
backend->xfer_hints(hints);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-24 13:10:42 +03:00
|
|
|
int aiori_warning_as_errors = 0;
|
|
|
|
|
2021-01-29 17:00:17 +03:00
|
|
|
/*
|
|
|
|
Returns 1 if the process participates in the test
|
|
|
|
*/
|
|
|
|
static int test_initialize(IOR_test_t * test){
|
2021-01-29 12:08:54 +03:00
|
|
|
int range[3];
|
|
|
|
IOR_param_t *params = &test->params;
|
|
|
|
MPI_Group orig_group, new_group;
|
|
|
|
|
|
|
|
/* set up communicator for test */
|
|
|
|
MPI_CHECK(MPI_Comm_group(params->mpi_comm_world, &orig_group),
|
|
|
|
"MPI_Comm_group() error");
|
|
|
|
range[0] = 0; /* first rank */
|
|
|
|
range[1] = params->numTasks - 1; /* last rank */
|
|
|
|
range[2] = 1; /* stride */
|
|
|
|
MPI_CHECK(MPI_Group_range_incl(orig_group, 1, &range, &new_group),
|
|
|
|
"MPI_Group_range_incl() error");
|
|
|
|
MPI_CHECK(MPI_Comm_create(params->mpi_comm_world, new_group, & params->testComm),
|
|
|
|
"MPI_Comm_create() error");
|
|
|
|
MPI_CHECK(MPI_Group_free(&orig_group), "MPI_Group_Free() error");
|
|
|
|
MPI_CHECK(MPI_Group_free(&new_group), "MPI_Group_Free() error");
|
|
|
|
|
2021-01-29 17:00:17 +03:00
|
|
|
|
|
|
|
if (params->testComm == MPI_COMM_NULL) {
|
2021-02-08 16:28:13 +03:00
|
|
|
/* tasks not in the group do not participate in this test, this matches the proceses in test_finalize() that participate */
|
2021-01-29 17:00:17 +03:00
|
|
|
MPI_CHECK(MPI_Barrier(params->mpi_comm_world), "barrier error");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-01-29 12:08:54 +03:00
|
|
|
/* Setup global variables */
|
|
|
|
testComm = params->testComm;
|
2020-05-30 20:19:48 +03:00
|
|
|
verbose = test->params.verbose;
|
|
|
|
backend = test->params.backend;
|
2021-01-29 12:08:54 +03:00
|
|
|
|
2021-02-18 13:40:42 +03:00
|
|
|
#ifdef HAVE_CUDA
|
|
|
|
cudaError_t cret = cudaSetDevice(test->params.gpuID);
|
|
|
|
if(cret != cudaSuccess){
|
|
|
|
EWARNF("cudaSetDevice(%d) error: %s", test->params.gpuID, cudaGetErrorString(cret));
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2020-05-30 20:19:48 +03:00
|
|
|
if(backend->initialize){
|
|
|
|
backend->initialize(test->params.backend_options);
|
|
|
|
}
|
2020-05-31 14:50:03 +03:00
|
|
|
ior_set_xfer_hints(& test->params);
|
2020-06-24 13:10:42 +03:00
|
|
|
aiori_warning_as_errors = test->params.warningAsErrors;
|
2020-05-31 15:30:31 +03:00
|
|
|
|
|
|
|
if (rank == 0 && verbose >= VERBOSE_0) {
|
|
|
|
ShowTestStart(& test->params);
|
|
|
|
}
|
2021-01-29 17:00:17 +03:00
|
|
|
return 1;
|
2020-05-30 20:19:48 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static void test_finalize(IOR_test_t * test){
|
|
|
|
backend = test->params.backend;
|
|
|
|
if(backend->finalize){
|
|
|
|
backend->finalize(test->params.backend_options);
|
|
|
|
}
|
2021-02-08 16:28:13 +03:00
|
|
|
MPI_CHECK(MPI_Barrier(test->params.mpi_comm_world), "barrier error");
|
2021-01-29 17:00:17 +03:00
|
|
|
MPI_CHECK(MPI_Comm_free(& testComm), "MPI_Comm_free() error");
|
2020-05-30 20:19:48 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2018-07-07 13:42:21 +03:00
|
|
|
IOR_test_t * ior_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out){
|
2011-12-13 09:00:18 +04:00
|
|
|
IOR_test_t *tests_head;
|
|
|
|
IOR_test_t *tptr;
|
2018-07-07 13:42:21 +03:00
|
|
|
out_logfile = world_out;
|
2018-07-08 15:47:55 +03:00
|
|
|
out_resultfile = world_out;
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2021-01-20 17:57:21 +03:00
|
|
|
MPI_CHECK(MPI_Comm_rank(world_com, &rank), "cannot get rank");
|
2012-09-10 21:50:18 +04:00
|
|
|
|
2015-05-19 18:36:28 +03:00
|
|
|
/* setup tests, and validate parameters */
|
2021-01-20 17:57:21 +03:00
|
|
|
tests_head = ParseCommandLine(argc, argv, world_com);
|
2021-01-20 18:00:33 +03:00
|
|
|
InitTests(tests_head);
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
PrintHeader(argc, argv);
|
2011-12-11 13:50:19 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
/* perform each test */
|
2014-08-14 02:53:24 +04:00
|
|
|
for (tptr = tests_head; tptr != NULL; tptr = tptr->next) {
|
2021-01-29 17:00:17 +03:00
|
|
|
int participate = test_initialize(tptr);
|
|
|
|
if( ! participate ) continue;
|
2018-07-07 13:42:21 +03:00
|
|
|
totalErrorCount = 0;
|
2011-12-13 09:00:18 +04:00
|
|
|
TestIoSys(tptr);
|
2018-07-07 13:42:21 +03:00
|
|
|
tptr->results->errors = totalErrorCount;
|
2018-07-08 16:59:54 +03:00
|
|
|
ShowTestEnd(tptr);
|
2020-05-30 20:19:48 +03:00
|
|
|
test_finalize(tptr);
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
PrintLongSummaryAllTests(tests_head);
|
2011-12-13 09:00:18 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
/* display finish time */
|
2018-07-08 16:59:54 +03:00
|
|
|
PrintTestEnds();
|
2018-07-07 13:42:21 +03:00
|
|
|
return tests_head;
|
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2011-12-13 09:00:18 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2018-07-07 13:42:21 +03:00
|
|
|
int ior_main(int argc, char **argv)
|
|
|
|
{
|
|
|
|
IOR_test_t *tests_head;
|
|
|
|
IOR_test_t *tptr;
|
2018-07-08 15:07:32 +03:00
|
|
|
|
2018-07-07 13:42:21 +03:00
|
|
|
out_logfile = stdout;
|
2018-07-08 15:47:55 +03:00
|
|
|
out_resultfile = stdout;
|
2018-07-07 13:42:21 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* check -h option from commandline without starting MPI;
|
|
|
|
*/
|
2021-01-20 17:57:21 +03:00
|
|
|
tests_head = ParseCommandLine(argc, argv, MPI_COMM_WORLD);
|
2018-07-07 13:42:21 +03:00
|
|
|
|
|
|
|
/* start the MPI code */
|
|
|
|
MPI_CHECK(MPI_Init(&argc, &argv), "cannot initialize MPI");
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
|
2021-01-20 17:57:21 +03:00
|
|
|
MPI_CHECK(MPI_Comm_rank(MPI_COMM_WORLD, &rank), "cannot get rank");
|
2018-07-08 15:07:32 +03:00
|
|
|
|
2018-07-07 13:42:21 +03:00
|
|
|
/* set error-handling */
|
|
|
|
/*MPI_CHECK(MPI_Errhandler_set(mpi_comm_world, MPI_ERRORS_RETURN),
|
|
|
|
"cannot set errhandler"); */
|
|
|
|
|
|
|
|
/* setup tests, and validate parameters */
|
2021-01-20 18:00:33 +03:00
|
|
|
InitTests(tests_head);
|
2018-07-07 13:42:21 +03:00
|
|
|
|
|
|
|
PrintHeader(argc, argv);
|
|
|
|
|
|
|
|
/* perform each test */
|
|
|
|
for (tptr = tests_head; tptr != NULL; tptr = tptr->next) {
|
2021-01-29 17:00:17 +03:00
|
|
|
int participate = test_initialize(tptr);
|
|
|
|
if( ! participate ) continue;
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
|
2018-07-07 13:42:21 +03:00
|
|
|
// This is useful for trapping a running MPI process. While
|
|
|
|
// this is sleeping, run the script 'testing/hdfs/gdb.attach'
|
|
|
|
if (verbose >= VERBOSE_4) {
|
|
|
|
fprintf(out_logfile, "\trank %d: sleeping\n", rank);
|
|
|
|
sleep(5);
|
|
|
|
fprintf(out_logfile, "\trank %d: awake.\n", rank);
|
|
|
|
}
|
2018-08-30 00:14:19 +03:00
|
|
|
|
2018-07-07 13:42:21 +03:00
|
|
|
TestIoSys(tptr);
|
2018-07-08 16:59:54 +03:00
|
|
|
ShowTestEnd(tptr);
|
2020-05-30 20:19:48 +03:00
|
|
|
test_finalize(tptr);
|
2018-07-07 13:42:21 +03:00
|
|
|
}
|
|
|
|
|
2019-10-24 01:17:37 +03:00
|
|
|
if (verbose <= VERBOSE_0)
|
2018-07-07 13:42:21 +03:00
|
|
|
/* always print final summary */
|
2019-10-24 01:17:37 +03:00
|
|
|
verbose = VERBOSE_1;
|
2018-07-07 13:42:21 +03:00
|
|
|
PrintLongSummaryAllTests(tests_head);
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
|
2018-07-07 13:42:21 +03:00
|
|
|
/* display finish time */
|
2018-07-08 16:59:54 +03:00
|
|
|
PrintTestEnds();
|
2018-07-07 13:42:21 +03:00
|
|
|
|
|
|
|
MPI_CHECK(MPI_Finalize(), "cannot finalize MPI");
|
|
|
|
|
2018-09-17 12:38:57 +03:00
|
|
|
DestroyTests(tests_head);
|
2018-07-07 13:42:21 +03:00
|
|
|
|
|
|
|
return totalErrorCount;
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
|
|
|
/***************************** F U N C T I O N S ******************************/
|
|
|
|
|
2011-11-10 04:13:44 +04:00
|
|
|
/*
|
|
|
|
* Initialize an IOR_param_t structure to the defaults
|
|
|
|
*/
|
2021-01-20 17:57:21 +03:00
|
|
|
void init_IOR_Param_t(IOR_param_t * p, MPI_Comm com)
|
2011-11-10 04:13:44 +04:00
|
|
|
{
|
2017-10-20 00:26:52 +03:00
|
|
|
const char *default_aiori = aiori_default ();
|
|
|
|
assert (NULL != default_aiori);
|
|
|
|
|
2011-11-10 04:13:44 +04:00
|
|
|
memset(p, 0, sizeof(IOR_param_t));
|
2018-07-14 10:41:35 +03:00
|
|
|
p->api = strdup(default_aiori);
|
|
|
|
p->platform = strdup("HOST(OSTYPE)");
|
|
|
|
p->testFileName = strdup("testFile");
|
|
|
|
|
|
|
|
p->writeFile = p->readFile = FALSE;
|
|
|
|
p->checkWrite = p->checkRead = FALSE;
|
2014-08-14 02:53:24 +04:00
|
|
|
|
2019-08-31 01:45:03 +03:00
|
|
|
/*
|
|
|
|
* These can be overridden from the command-line but otherwise will be
|
|
|
|
* set from MPI.
|
|
|
|
*/
|
|
|
|
p->numTasks = -1;
|
|
|
|
p->numNodes = -1;
|
|
|
|
p->numTasksOnNode0 = -1;
|
|
|
|
|
2011-11-10 04:13:44 +04:00
|
|
|
p->repetitions = 1;
|
|
|
|
p->repCounter = -1;
|
|
|
|
p->open = WRITE;
|
|
|
|
p->taskPerNodeOffset = 1;
|
|
|
|
p->segmentCount = 1;
|
|
|
|
p->blockSize = 1048576;
|
|
|
|
p->transferSize = 262144;
|
|
|
|
p->randomSeed = -1;
|
2015-05-21 21:05:56 +03:00
|
|
|
p->incompressibleSeed = 573;
|
2021-01-20 17:57:21 +03:00
|
|
|
p->testComm = com; // this com might change for smaller tests
|
|
|
|
p->mpi_comm_world = com;
|
2014-08-29 01:39:44 +04:00
|
|
|
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
p->URI = NULL;
|
2011-11-10 04:13:44 +04:00
|
|
|
}
|
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
static void
|
2011-11-12 02:22:17 +04:00
|
|
|
DisplayOutliers(int numTasks,
|
2011-06-17 23:20:43 +04:00
|
|
|
double timerVal,
|
2011-11-12 02:22:17 +04:00
|
|
|
char *timeString, int access, int outlierThreshold)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2011-11-12 02:22:17 +04:00
|
|
|
char accessString[MAX_STR];
|
|
|
|
double sum, mean, sqrDiff, var, sd;
|
|
|
|
|
|
|
|
/* for local timerVal, don't compensate for wall clock delta */
|
2021-01-20 22:38:54 +03:00
|
|
|
//timerVal += wall_clock_delta;
|
2011-11-12 02:22:17 +04:00
|
|
|
|
|
|
|
MPI_CHECK(MPI_Allreduce
|
|
|
|
(&timerVal, &sum, 1, MPI_DOUBLE, MPI_SUM, testComm),
|
|
|
|
"MPI_Allreduce()");
|
|
|
|
mean = sum / numTasks;
|
|
|
|
sqrDiff = pow((mean - timerVal), 2);
|
|
|
|
MPI_CHECK(MPI_Allreduce
|
|
|
|
(&sqrDiff, &var, 1, MPI_DOUBLE, MPI_SUM, testComm),
|
|
|
|
"MPI_Allreduce()");
|
|
|
|
var = var / numTasks;
|
|
|
|
sd = sqrt(var);
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
if (access == WRITE) {
|
|
|
|
strcpy(accessString, "write");
|
|
|
|
} else { /* READ */
|
|
|
|
strcpy(accessString, "read");
|
|
|
|
}
|
|
|
|
if (fabs(timerVal - mean) > (double)outlierThreshold) {
|
2020-03-17 18:31:56 +03:00
|
|
|
char hostname[MAX_STR];
|
|
|
|
int ret = gethostname(hostname, MAX_STR);
|
|
|
|
if (ret != 0)
|
|
|
|
strcpy(hostname, "unknown");
|
|
|
|
|
2020-06-24 13:10:42 +03:00
|
|
|
EWARNF("for %s, task %d, %s %s is %f (mean=%f, stddev=%f)\n",
|
|
|
|
hostname, rank, accessString, timeString, timerVal, mean, sd);
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2011-11-12 04:40:45 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Check for outliers in start/end times and elapsed create/xfer/close times.
|
|
|
|
*/
|
2018-09-19 20:06:05 +03:00
|
|
|
static void
|
|
|
|
CheckForOutliers(IOR_param_t *test, const double *timer, const int access)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2018-09-19 20:06:05 +03:00
|
|
|
DisplayOutliers(test->numTasks, timer[0],
|
2011-11-12 02:22:17 +04:00
|
|
|
"start time", access, test->outlierThreshold);
|
|
|
|
DisplayOutliers(test->numTasks,
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[1] - timer[0],
|
2011-11-12 02:22:17 +04:00
|
|
|
"elapsed create time", access, test->outlierThreshold);
|
|
|
|
DisplayOutliers(test->numTasks,
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[3] - timer[2],
|
2011-11-12 02:22:17 +04:00
|
|
|
"elapsed transfer time", access,
|
|
|
|
test->outlierThreshold);
|
|
|
|
DisplayOutliers(test->numTasks,
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[5] - timer[4],
|
2011-11-12 02:22:17 +04:00
|
|
|
"elapsed close time", access, test->outlierThreshold);
|
2018-09-19 20:06:05 +03:00
|
|
|
DisplayOutliers(test->numTasks, timer[5], "end time",
|
2011-11-12 02:22:17 +04:00
|
|
|
access, test->outlierThreshold);
|
2011-11-12 04:40:45 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Check if actual file size equals expected size; if not use actual for
|
|
|
|
* calculating performance rate.
|
|
|
|
*/
|
2020-06-30 14:33:56 +03:00
|
|
|
static void CheckFileSize(IOR_test_t *test, char * testFilename, IOR_offset_t dataMoved, int rep, const int access)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2014-08-14 02:53:24 +04:00
|
|
|
IOR_param_t *params = &test->params;
|
|
|
|
IOR_results_t *results = test->results;
|
2018-09-20 00:39:25 +03:00
|
|
|
IOR_point_t *point = (access == WRITE) ? &results[rep].write :
|
|
|
|
&results[rep].read;
|
2011-12-13 09:00:18 +04:00
|
|
|
|
2020-06-30 14:03:05 +03:00
|
|
|
/* get the size of the file */
|
|
|
|
IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum;
|
2020-06-30 14:33:56 +03:00
|
|
|
aggFileSizeFromStat = backend->get_file_size(params->backend_options, testFilename);
|
2020-06-30 14:03:05 +03:00
|
|
|
|
|
|
|
if (params->hints.filePerProc == TRUE) {
|
|
|
|
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1,
|
|
|
|
MPI_LONG_LONG_INT, MPI_SUM, testComm),
|
|
|
|
"cannot reduce total data moved");
|
|
|
|
aggFileSizeFromStat = tmpSum;
|
|
|
|
} else {
|
|
|
|
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1,
|
|
|
|
MPI_LONG_LONG_INT, MPI_MIN, testComm),
|
|
|
|
"cannot reduce total data moved");
|
|
|
|
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1,
|
|
|
|
MPI_LONG_LONG_INT, MPI_MAX, testComm),
|
|
|
|
"cannot reduce total data moved");
|
|
|
|
if (tmpMin != tmpMax) {
|
|
|
|
if (rank == 0) {
|
|
|
|
WARN("inconsistent file size by different tasks");
|
|
|
|
}
|
|
|
|
/* incorrect, but now consistent across tasks */
|
|
|
|
aggFileSizeFromStat = tmpMin;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
point->aggFileSizeFromStat = aggFileSizeFromStat;
|
|
|
|
|
2018-09-20 00:39:25 +03:00
|
|
|
MPI_CHECK(MPI_Allreduce(&dataMoved, &point->aggFileSizeFromXfer,
|
2011-11-12 02:22:17 +04:00
|
|
|
1, MPI_LONG_LONG_INT, MPI_SUM, testComm),
|
|
|
|
"cannot total data moved");
|
|
|
|
|
2020-11-03 13:01:09 +03:00
|
|
|
if (strcasecmp(params->api, "HDF5") != 0 && strcasecmp(params->api, "NCMPI") != 0) {
|
2011-11-12 02:22:17 +04:00
|
|
|
if (verbose >= VERBOSE_0 && rank == 0) {
|
2011-12-13 09:00:18 +04:00
|
|
|
if ((params->expectedAggFileSize
|
2018-09-20 00:39:25 +03:00
|
|
|
!= point->aggFileSizeFromXfer)
|
|
|
|
|| (point->aggFileSizeFromStat
|
|
|
|
!= point->aggFileSizeFromXfer)) {
|
2020-06-24 13:10:42 +03:00
|
|
|
EWARNF("Expected aggregate file size = %lld", (long long) params->expectedAggFileSize);
|
|
|
|
EWARNF("Stat() of aggregate file size = %lld", (long long) point->aggFileSizeFromStat);
|
|
|
|
EWARNF("Using actual aggregate bytes moved = %lld", (long long) point->aggFileSizeFromXfer);
|
2017-10-20 19:02:24 +03:00
|
|
|
if(params->deadlineForStonewalling){
|
2020-06-24 13:10:42 +03:00
|
|
|
EWARN("Maybe caused by deadlineForStonewalling");
|
2017-10-20 19:02:24 +03:00
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
}
|
2018-09-20 00:39:25 +03:00
|
|
|
|
|
|
|
point->aggFileSizeForBW = point->aggFileSizeFromXfer;
|
2011-11-12 04:40:45 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Compare buffers after reading/writing each transfer. Displays only first
|
|
|
|
* difference in buffers and returns total errors counted.
|
|
|
|
*/
|
2011-11-12 04:40:45 +04:00
|
|
|
static size_t
|
2020-09-28 22:09:48 +03:00
|
|
|
CompareData(void *expectedBuffer, size_t size, IOR_offset_t transferCount, IOR_param_t *test, IOR_offset_t offset, int fillrank, int access)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2021-01-20 01:28:34 +03:00
|
|
|
assert(access == WRITECHECK || access == READCHECK);
|
2021-03-18 23:42:50 +03:00
|
|
|
return verify_memory_pattern(offset, expectedBuffer, transferCount, test->setTimeStampSignature, fillrank, test->dataPacketType);
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2011-11-12 03:11:28 +04:00
|
|
|
/*
|
|
|
|
* Count all errors across all tasks; report errors found.
|
|
|
|
*/
|
2011-11-12 04:40:45 +04:00
|
|
|
static int CountErrors(IOR_param_t * test, int access, int errors)
|
2011-11-12 02:22:17 +04:00
|
|
|
{
|
|
|
|
int allErrors = 0;
|
|
|
|
|
|
|
|
if (test->checkWrite || test->checkRead) {
|
|
|
|
MPI_CHECK(MPI_Reduce(&errors, &allErrors, 1, MPI_INT, MPI_SUM,
|
|
|
|
0, testComm), "cannot reduce errors");
|
|
|
|
MPI_CHECK(MPI_Bcast(&allErrors, 1, MPI_INT, 0, testComm),
|
|
|
|
"cannot broadcast allErrors value");
|
|
|
|
if (allErrors != 0) {
|
|
|
|
totalErrorCount += allErrors;
|
|
|
|
test->errorFound = TRUE;
|
|
|
|
}
|
|
|
|
if (rank == 0 && allErrors != 0) {
|
|
|
|
if (allErrors < 0) {
|
|
|
|
WARN("overflow in errors counted");
|
|
|
|
allErrors = -1;
|
|
|
|
}
|
2020-06-24 13:10:42 +03:00
|
|
|
EWARNF("Incorrect data on %s (%d errors found).\n",
|
2014-08-14 02:53:24 +04:00
|
|
|
access == WRITECHECK ? "write" : "read", allErrors);
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile,
|
2011-11-12 02:22:17 +04:00
|
|
|
"Used Time Stamp %u (0x%x) for Data Signature\n",
|
|
|
|
test->timeStampSignatureValue,
|
|
|
|
test->timeStampSignatureValue);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return (allErrors);
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2018-08-29 19:49:41 +03:00
|
|
|
void AllocResults(IOR_test_t *test)
|
2011-12-13 09:00:18 +04:00
|
|
|
{
|
2018-07-15 21:38:17 +03:00
|
|
|
int reps;
|
|
|
|
if (test->results != NULL)
|
2019-03-27 23:04:48 +03:00
|
|
|
return;
|
|
|
|
|
2018-07-15 21:38:17 +03:00
|
|
|
reps = test->params.repetitions;
|
|
|
|
test->results = (IOR_results_t *) safeMalloc(sizeof(IOR_results_t) * reps);
|
2011-12-13 09:00:18 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void FreeResults(IOR_test_t *test)
|
|
|
|
{
|
2018-07-15 21:38:17 +03:00
|
|
|
if (test->results != NULL) {
|
|
|
|
free(test->results);
|
|
|
|
}
|
2011-12-13 09:00:18 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-07-25 13:51:54 +03:00
|
|
|
/**
|
2011-06-17 23:20:43 +04:00
|
|
|
* Create new test for list of tests.
|
|
|
|
*/
|
2011-12-13 09:00:18 +04:00
|
|
|
IOR_test_t *CreateTest(IOR_param_t *init_params, int test_num)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2011-12-13 09:00:18 +04:00
|
|
|
IOR_test_t *newTest = NULL;
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2011-12-13 09:00:18 +04:00
|
|
|
newTest = (IOR_test_t *) malloc(sizeof(IOR_test_t));
|
2011-11-12 02:22:17 +04:00
|
|
|
if (newTest == NULL)
|
2011-12-13 09:00:18 +04:00
|
|
|
ERR("malloc() of IOR_test_t failed");
|
|
|
|
newTest->params = *init_params;
|
2018-07-14 10:41:35 +03:00
|
|
|
newTest->params.platform = GetPlatformName();
|
2011-12-13 09:00:18 +04:00
|
|
|
newTest->params.id = test_num;
|
|
|
|
newTest->next = NULL;
|
2014-08-14 02:53:24 +04:00
|
|
|
newTest->results = NULL;
|
2018-07-14 10:41:35 +03:00
|
|
|
|
2011-12-13 09:00:18 +04:00
|
|
|
return newTest;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void DestroyTest(IOR_test_t *test)
|
|
|
|
{
|
2014-08-14 02:53:24 +04:00
|
|
|
FreeResults(test);
|
|
|
|
free(test);
|
2011-12-13 09:00:18 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
static void DestroyTests(IOR_test_t *tests_head)
|
|
|
|
{
|
2014-08-14 02:53:24 +04:00
|
|
|
IOR_test_t *tptr, *next;
|
2011-12-13 09:00:18 +04:00
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
for (tptr = tests_head; tptr != NULL; tptr = next) {
|
|
|
|
next = tptr->next;
|
|
|
|
DestroyTest(tptr);
|
|
|
|
}
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Distribute IOR_HINTs to all tasks' environments.
|
|
|
|
*/
|
2021-01-20 17:57:21 +03:00
|
|
|
static void DistributeHints(MPI_Comm com)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2011-11-12 02:22:17 +04:00
|
|
|
char hint[MAX_HINTS][MAX_STR], fullHint[MAX_STR], hintVariable[MAX_STR];
|
|
|
|
int hintCount = 0, i;
|
|
|
|
|
|
|
|
if (rank == 0) {
|
|
|
|
for (i = 0; environ[i] != NULL; i++) {
|
|
|
|
if (strncmp(environ[i], "IOR_HINT", strlen("IOR_HINT"))
|
|
|
|
== 0) {
|
|
|
|
hintCount++;
|
|
|
|
if (hintCount == MAX_HINTS) {
|
|
|
|
WARN("exceeded max hints; reset MAX_HINTS and recompile");
|
|
|
|
hintCount = MAX_HINTS;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
/* assume no IOR_HINT is greater than MAX_STR in length */
|
|
|
|
strncpy(hint[hintCount - 1], environ[i],
|
|
|
|
MAX_STR - 1);
|
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-01-20 17:57:21 +03:00
|
|
|
MPI_CHECK(MPI_Bcast(&hintCount, sizeof(hintCount), MPI_BYTE, 0, com), "cannot broadcast hints");
|
2011-11-12 02:22:17 +04:00
|
|
|
for (i = 0; i < hintCount; i++) {
|
2021-01-20 17:57:21 +03:00
|
|
|
MPI_CHECK(MPI_Bcast(&hint[i], MAX_STR, MPI_BYTE, 0, com),
|
2011-11-12 02:22:17 +04:00
|
|
|
"cannot broadcast hints");
|
|
|
|
strcpy(fullHint, hint[i]);
|
|
|
|
strcpy(hintVariable, strtok(fullHint, "="));
|
|
|
|
if (getenv(hintVariable) == NULL) {
|
|
|
|
/* doesn't exist in this task's environment; better set it */
|
|
|
|
if (putenv(hint[i]) != 0)
|
|
|
|
WARN("cannot set environment variable");
|
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
}
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Return string describing machine name and type.
|
|
|
|
*/
|
2018-07-14 10:41:35 +03:00
|
|
|
char * GetPlatformName()
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2011-11-12 02:22:17 +04:00
|
|
|
char nodeName[MAX_STR], *p, *start, sysName[MAX_STR];
|
2018-07-14 10:41:35 +03:00
|
|
|
char platformName[MAX_STR];
|
2011-11-12 02:22:17 +04:00
|
|
|
struct utsname name;
|
|
|
|
|
|
|
|
if (uname(&name) != 0) {
|
2011-12-15 01:40:25 +04:00
|
|
|
EWARN("cannot get platform name");
|
2011-11-12 02:22:17 +04:00
|
|
|
sprintf(sysName, "%s", "Unknown");
|
|
|
|
sprintf(nodeName, "%s", "Unknown");
|
2011-06-17 23:20:43 +04:00
|
|
|
} else {
|
2011-11-12 02:22:17 +04:00
|
|
|
sprintf(sysName, "%s", name.sysname);
|
|
|
|
sprintf(nodeName, "%s", name.nodename);
|
2011-06-17 23:20:43 +04:00
|
|
|
}
|
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
start = nodeName;
|
|
|
|
if (strlen(nodeName) == 0) {
|
|
|
|
p = start;
|
|
|
|
} else {
|
|
|
|
/* point to one character back from '\0' */
|
|
|
|
p = start + strlen(nodeName) - 1;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* to cut off trailing node number, search backwards
|
|
|
|
* for the first non-numeric character
|
|
|
|
*/
|
|
|
|
while (p != start) {
|
|
|
|
if (*p < '0' || *p > '9') {
|
|
|
|
*(p + 1) = '\0';
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
p--;
|
|
|
|
}
|
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
sprintf(platformName, "%s(%s)", nodeName, sysName);
|
2018-07-14 10:41:35 +03:00
|
|
|
return strdup(platformName);
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2011-06-17 23:20:43 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Parse file name.
|
|
|
|
*/
|
2011-11-12 04:40:45 +04:00
|
|
|
static char **ParseFileName(char *name, int *count)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2011-11-12 02:22:17 +04:00
|
|
|
char **fileNames, *tmp, *token;
|
|
|
|
char delimiterString[3] = { FILENAME_DELIMITER, '\n', '\0' };
|
|
|
|
int i = 0;
|
|
|
|
|
|
|
|
*count = 0;
|
|
|
|
tmp = name;
|
|
|
|
|
|
|
|
/* pass one */
|
|
|
|
/* if something there, count the first item */
|
|
|
|
if (*tmp != '\0') {
|
|
|
|
(*count)++;
|
|
|
|
}
|
|
|
|
/* count the rest of the filenames */
|
|
|
|
while (*tmp != '\0') {
|
|
|
|
if (*tmp == FILENAME_DELIMITER) {
|
|
|
|
(*count)++;
|
|
|
|
}
|
|
|
|
tmp++;
|
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
fileNames = (char **)malloc((*count) * sizeof(char **));
|
|
|
|
if (fileNames == NULL)
|
|
|
|
ERR("out of memory");
|
|
|
|
|
|
|
|
/* pass two */
|
|
|
|
token = strtok(name, delimiterString);
|
|
|
|
while (token != NULL) {
|
|
|
|
fileNames[i] = token;
|
|
|
|
token = strtok(NULL, delimiterString);
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
return (fileNames);
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2018-07-08 15:38:05 +03:00
|
|
|
|
2011-06-17 23:20:43 +04:00
|
|
|
/*
|
2018-07-08 15:38:05 +03:00
|
|
|
* Return test file name to access.
|
|
|
|
* for single shared file, fileNames[0] is returned in testFileName
|
2011-06-17 23:20:43 +04:00
|
|
|
*/
|
2018-07-08 15:38:05 +03:00
|
|
|
void GetTestFileName(char *testFileName, IOR_param_t * test)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2018-07-08 15:38:05 +03:00
|
|
|
char **fileNames;
|
|
|
|
char initialTestFileName[MAX_PATHLEN];
|
|
|
|
char testFileNameRoot[MAX_STR];
|
|
|
|
char tmpString[MAX_STR];
|
2019-11-05 17:37:54 +03:00
|
|
|
int count;
|
|
|
|
int socket, core;
|
2012-01-09 03:46:43 +04:00
|
|
|
|
2018-07-08 15:38:05 +03:00
|
|
|
/* parse filename for multiple file systems */
|
|
|
|
strcpy(initialTestFileName, test->testFileName);
|
2019-11-05 17:37:54 +03:00
|
|
|
if(test->dualMount){
|
|
|
|
GetProcessorAndCore(&socket, &core);
|
2020-08-03 14:30:21 +03:00
|
|
|
sprintf(tmpString, "%s%d/%s",initialTestFileName, socket, "data");
|
2019-11-05 17:37:54 +03:00
|
|
|
strcpy(initialTestFileName, tmpString);
|
|
|
|
}
|
2018-07-08 15:38:05 +03:00
|
|
|
fileNames = ParseFileName(initialTestFileName, &count);
|
|
|
|
if (count > 1 && test->uniqueDir == TRUE)
|
|
|
|
ERR("cannot use multiple file names with unique directories");
|
|
|
|
if (test->filePerProc) {
|
|
|
|
strcpy(testFileNameRoot,
|
|
|
|
fileNames[((rank +
|
|
|
|
rankOffset) % test->numTasks) % count]);
|
|
|
|
} else {
|
|
|
|
strcpy(testFileNameRoot, fileNames[0]);
|
2011-06-17 23:20:43 +04:00
|
|
|
}
|
2012-01-09 03:46:43 +04:00
|
|
|
|
2018-07-08 15:38:05 +03:00
|
|
|
/* give unique name if using multiple files */
|
|
|
|
if (test->filePerProc) {
|
|
|
|
/*
|
|
|
|
* prepend rank subdirectory before filename
|
|
|
|
* e.g., /dir/file => /dir/<rank>/file
|
|
|
|
*/
|
|
|
|
if (test->uniqueDir == TRUE) {
|
|
|
|
strcpy(testFileNameRoot,
|
|
|
|
PrependDir(test, testFileNameRoot));
|
|
|
|
}
|
|
|
|
sprintf(testFileName, "%s.%08d", testFileNameRoot,
|
|
|
|
(rank + rankOffset) % test->numTasks);
|
|
|
|
} else {
|
|
|
|
strcpy(testFileName, testFileNameRoot);
|
|
|
|
}
|
2012-01-09 03:46:43 +04:00
|
|
|
|
2018-07-08 15:38:05 +03:00
|
|
|
/* add suffix for multiple files */
|
|
|
|
if (test->repCounter > -1) {
|
|
|
|
sprintf(tmpString, ".%d", test->repCounter);
|
|
|
|
strcat(testFileName, tmpString);
|
|
|
|
}
|
|
|
|
free (fileNames);
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* From absolute directory, insert rank as subdirectory. Allows each task
|
|
|
|
* to write to its own directory. E.g., /dir/file => /dir/<rank>/file.
|
|
|
|
*/
|
2011-11-12 04:40:45 +04:00
|
|
|
static char *PrependDir(IOR_param_t * test, char *rootDir)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2011-11-12 02:22:17 +04:00
|
|
|
char *dir;
|
2019-08-31 00:31:23 +03:00
|
|
|
char *fname;
|
2011-11-12 02:22:17 +04:00
|
|
|
int i;
|
|
|
|
|
|
|
|
dir = (char *)malloc(MAX_STR + 1);
|
|
|
|
if (dir == NULL)
|
|
|
|
ERR("out of memory");
|
|
|
|
|
|
|
|
/* get dir name */
|
|
|
|
strcpy(dir, rootDir);
|
|
|
|
i = strlen(dir) - 1;
|
|
|
|
while (i > 0) {
|
|
|
|
if (dir[i] == '\0' || dir[i] == '/') {
|
|
|
|
dir[i] = '/';
|
|
|
|
dir[i + 1] = '\0';
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
i--;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* get file name */
|
2019-08-31 00:31:23 +03:00
|
|
|
fname = rootDir + i + 1;
|
2011-11-12 02:22:17 +04:00
|
|
|
|
|
|
|
/* create directory with rank as subdirectory */
|
2019-08-31 00:31:23 +03:00
|
|
|
sprintf(dir + i + 1, "%d", (rank + rankOffset) % test->numTasks);
|
2011-11-12 02:22:17 +04:00
|
|
|
|
|
|
|
/* dir doesn't exist, so create */
|
2020-05-31 13:50:15 +03:00
|
|
|
if (backend->access(dir, F_OK, test->backend_options) != 0) {
|
|
|
|
if (backend->mkdir(dir, S_IRWXU, test->backend_options) < 0) {
|
2019-08-31 00:11:25 +03:00
|
|
|
ERRF("cannot create directory: %s", dir);
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* check if correct permissions */
|
2020-05-31 13:50:15 +03:00
|
|
|
} else if (backend->access(dir, R_OK, test->backend_options) != 0 ||
|
|
|
|
backend->access(dir, W_OK, test->backend_options) != 0 ||
|
|
|
|
backend->access(dir, X_OK, test->backend_options) != 0) {
|
2019-08-31 00:11:25 +03:00
|
|
|
ERRF("invalid directory permissions: %s", dir);
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/* concatenate dir and file names */
|
|
|
|
strcat(dir, "/");
|
2019-08-31 00:31:23 +03:00
|
|
|
strcat(dir, fname);
|
2011-11-12 02:22:17 +04:00
|
|
|
|
|
|
|
return dir;
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
|
|
|
/******************************************************************************/
|
|
|
|
/*
|
|
|
|
* Reduce test results, and show if verbose set.
|
|
|
|
*/
|
2018-09-19 20:06:05 +03:00
|
|
|
static void
|
|
|
|
ReduceIterResults(IOR_test_t *test, double *timer, const int rep, const int access)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2018-09-19 20:06:05 +03:00
|
|
|
double reduced[IOR_NB_TIMERS] = { 0 };
|
|
|
|
double diff[IOR_NB_TIMERS / 2 + 1];
|
2019-09-10 21:39:31 +03:00
|
|
|
double totalTime, accessTime;
|
|
|
|
IOR_param_t *params = &test->params;
|
|
|
|
double bw, iops, latency, minlatency;
|
2018-09-19 20:06:05 +03:00
|
|
|
int i;
|
|
|
|
MPI_Op op;
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2018-09-19 20:06:05 +03:00
|
|
|
assert(access == WRITE || access == READ);
|
2011-12-11 08:45:19 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
/* Find the minimum start time of the even numbered timers, and the
|
|
|
|
maximum finish time for the odd numbered timers */
|
2018-09-19 20:06:05 +03:00
|
|
|
for (i = 0; i < IOR_NB_TIMERS; i++) {
|
2011-11-12 02:22:17 +04:00
|
|
|
op = i % 2 ? MPI_MAX : MPI_MIN;
|
2018-09-19 20:06:05 +03:00
|
|
|
MPI_CHECK(MPI_Reduce(&timer[i], &reduced[i], 1, MPI_DOUBLE,
|
2011-11-12 02:22:17 +04:00
|
|
|
op, 0, testComm), "MPI_Reduce()");
|
2011-06-17 23:20:43 +04:00
|
|
|
}
|
|
|
|
|
2018-09-19 20:06:05 +03:00
|
|
|
/* Calculate elapsed times and throughput numbers */
|
|
|
|
for (i = 0; i < IOR_NB_TIMERS / 2; i++)
|
|
|
|
diff[i] = reduced[2 * i + 1] - reduced[2 * i];
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2018-09-19 20:06:05 +03:00
|
|
|
totalTime = reduced[5] - reduced[0];
|
2019-09-10 21:39:31 +03:00
|
|
|
accessTime = reduced[3] - reduced[2];
|
2014-08-14 02:53:24 +04:00
|
|
|
|
2018-09-20 00:39:25 +03:00
|
|
|
IOR_point_t *point = (access == WRITE) ? &test->results[rep].write :
|
|
|
|
&test->results[rep].read;
|
2014-08-14 02:53:24 +04:00
|
|
|
|
2018-09-20 00:39:25 +03:00
|
|
|
point->time = totalTime;
|
2012-01-09 06:55:46 +04:00
|
|
|
|
2018-09-19 20:06:05 +03:00
|
|
|
if (verbose < VERBOSE_0)
|
|
|
|
return;
|
|
|
|
|
2018-09-20 00:39:25 +03:00
|
|
|
bw = (double)point->aggFileSizeForBW / totalTime;
|
2019-09-10 21:39:31 +03:00
|
|
|
|
|
|
|
/* For IOPS in this iteration, we divide the total amount of IOs from
|
|
|
|
* all ranks over the entire access time (first start -> last end). */
|
|
|
|
iops = (point->aggFileSizeForBW / params->transferSize) / accessTime;
|
|
|
|
|
|
|
|
/* For Latency, we divide the total access time for each task over the
|
|
|
|
* number of I/Os issued from that task; then reduce and display the
|
|
|
|
* minimum (best) latency achieved. So what is reported is the average
|
|
|
|
* latency of all ops from a single task, then taking the minimum of
|
2019-10-30 17:43:43 +03:00
|
|
|
* that between all tasks. */
|
2019-09-10 21:39:31 +03:00
|
|
|
latency = (timer[3] - timer[2]) / (params->blockSize / params->transferSize);
|
|
|
|
MPI_CHECK(MPI_Reduce(&latency, &minlatency, 1, MPI_DOUBLE,
|
|
|
|
MPI_MIN, 0, testComm), "MPI_Reduce()");
|
|
|
|
|
|
|
|
/* Only rank 0 tallies and prints the results. */
|
|
|
|
if (rank != 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
PrintReducedResult(test, access, bw, iops, latency, diff, totalTime, rep);
|
2012-01-09 06:55:46 +04:00
|
|
|
}
|
|
|
|
|
2011-06-17 23:20:43 +04:00
|
|
|
/*
|
|
|
|
* Check for file(s), then remove all files if file-per-proc, else single file.
|
Algorithms 'S3', 'S3_plus', and 'S3_EMC' all available.
These are variants on S3. S3 uses the "pure" S3 interface, e.g. using
Multi-Part-Upload. The "plus" variant enables EMC-extensions in the aws4c
library. This allows the N:N case to use "append", in the case where
"transfer_size" != "block_size" for IOR. In pure S3, the N:N case will
fail, because the EMC-extensions won't be enabled, and appending (which
attempts to use the EMC byte-range tricks to do this) will throw an error.
In the S3_EMC alg, N:1 uses EMCs other byte-range tricks to write different
parts of an N:1 file, and also uses append to write the parts of an N:N
file. Preliminary tests show these EMC extensions look to improve BW by
~20%.
I put all three algs in aiori-S3.c, because it seemed some code was getting
reused. Not sure if that's still going to make sense after the TBD, below.
TBD: Recently realized that the "pure' S3 shouldn't be trying to use
appends for anything. In the N:N case, it should just use MPU, within each
file. Then, there's no need for S3_plus. We just have S3, which does MPU
for all writes where transfer_size != block_size, and uses (standard)
byte-range reads for reading. Then S3_EMC uses "appends for N:N writes,
and byte-range writes for N:1 writes. This separates the code for the two
algs a little more, but we might still want them in the same file.
2014-10-30 01:04:30 +03:00
|
|
|
*
|
2011-06-17 23:20:43 +04:00
|
|
|
*/
|
2011-11-12 04:40:45 +04:00
|
|
|
static void RemoveFile(char *testFileName, int filePerProc, IOR_param_t * test)
|
2011-11-12 02:22:17 +04:00
|
|
|
{
|
2018-07-07 12:29:27 +03:00
|
|
|
int tmpRankOffset = 0;
|
2011-11-12 02:22:17 +04:00
|
|
|
if (filePerProc) {
|
|
|
|
/* in random tasks, delete own file */
|
|
|
|
if (test->reorderTasksRandom == TRUE) {
|
|
|
|
tmpRankOffset = rankOffset;
|
|
|
|
rankOffset = 0;
|
|
|
|
GetTestFileName(testFileName, test);
|
|
|
|
}
|
2020-05-31 13:50:15 +03:00
|
|
|
if (backend->access(testFileName, F_OK, test->backend_options) == 0) {
|
2019-08-31 00:11:25 +03:00
|
|
|
if (verbose >= VERBOSE_3) {
|
|
|
|
fprintf(out_logfile, "task %d removing %s\n", rank,
|
|
|
|
testFileName);
|
|
|
|
}
|
2020-05-31 13:50:15 +03:00
|
|
|
backend->delete(testFileName, test->backend_options);
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
if (test->reorderTasksRandom == TRUE) {
|
|
|
|
rankOffset = tmpRankOffset;
|
|
|
|
GetTestFileName(testFileName, test);
|
|
|
|
}
|
|
|
|
} else {
|
2020-05-31 13:50:15 +03:00
|
|
|
if ((rank == 0) && (backend->access(testFileName, F_OK, test->backend_options) == 0)) {
|
2019-08-31 00:11:25 +03:00
|
|
|
if (verbose >= VERBOSE_3) {
|
|
|
|
fprintf(out_logfile, "task %d removing %s\n", rank,
|
|
|
|
testFileName);
|
|
|
|
}
|
2020-05-31 13:50:15 +03:00
|
|
|
backend->delete(testFileName, test->backend_options);
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
}
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Setup tests by parsing commandline and creating test script.
|
2015-05-19 18:36:28 +03:00
|
|
|
* Perform a sanity-check on the configured parameters.
|
2011-06-17 23:20:43 +04:00
|
|
|
*/
|
2021-01-20 18:00:33 +03:00
|
|
|
static void InitTests(IOR_test_t *tests)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2021-01-20 18:00:33 +03:00
|
|
|
if(tests == NULL){
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
MPI_Comm com = tests->params.mpi_comm_world;
|
2019-08-31 01:45:03 +03:00
|
|
|
int mpiNumNodes = 0;
|
|
|
|
int mpiNumTasks = 0;
|
|
|
|
int mpiNumTasksOnNode0 = 0;
|
2018-07-14 10:41:35 +03:00
|
|
|
|
2020-06-24 13:10:42 +03:00
|
|
|
verbose = tests->params.verbose;
|
|
|
|
aiori_warning_as_errors = tests->params.warningAsErrors;
|
|
|
|
|
2019-08-31 01:45:03 +03:00
|
|
|
/*
|
|
|
|
* These default values are the same for every test and expensive to
|
|
|
|
* retrieve so just do it once.
|
|
|
|
*/
|
|
|
|
mpiNumNodes = GetNumNodes(com);
|
|
|
|
mpiNumTasks = GetNumTasks(com);
|
|
|
|
mpiNumTasksOnNode0 = GetNumTasksOnNode0(com);
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
/*
|
|
|
|
* Since there is no guarantee that anyone other than
|
|
|
|
* task 0 has the environment settings for the hints, pass
|
2018-07-07 13:42:21 +03:00
|
|
|
* the hint=value pair to everyone else in mpi_comm_world
|
2011-11-12 02:22:17 +04:00
|
|
|
*/
|
2021-01-20 17:57:21 +03:00
|
|
|
DistributeHints(com);
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
/* check validity of tests and create test queue */
|
|
|
|
while (tests != NULL) {
|
2018-07-14 10:41:35 +03:00
|
|
|
IOR_param_t *params = & tests->params;
|
|
|
|
params->testComm = com;
|
2019-08-31 01:45:03 +03:00
|
|
|
|
|
|
|
/* use MPI values if not overridden on command-line */
|
|
|
|
if (params->numNodes == -1) {
|
|
|
|
params->numNodes = mpiNumNodes;
|
|
|
|
}
|
|
|
|
if (params->numTasks == -1) {
|
|
|
|
params->numTasks = mpiNumTasks;
|
|
|
|
} else if (params->numTasks > mpiNumTasks) {
|
|
|
|
if (rank == 0) {
|
2020-06-24 13:10:42 +03:00
|
|
|
EWARNF("More tasks requested (%d) than available (%d),",
|
2019-08-31 01:45:03 +03:00
|
|
|
params->numTasks, mpiNumTasks);
|
2020-06-24 13:10:42 +03:00
|
|
|
EWARNF(" running with %d tasks.\n", mpiNumTasks);
|
2019-08-31 01:45:03 +03:00
|
|
|
}
|
|
|
|
params->numTasks = mpiNumTasks;
|
|
|
|
}
|
|
|
|
if (params->numTasksOnNode0 == -1) {
|
|
|
|
params->numTasksOnNode0 = mpiNumTasksOnNode0;
|
2018-07-14 10:41:35 +03:00
|
|
|
}
|
2019-08-31 01:45:03 +03:00
|
|
|
|
|
|
|
params->tasksBlockMapping = QueryNodeMapping(com,false);
|
2018-07-14 10:41:35 +03:00
|
|
|
params->expectedAggFileSize =
|
|
|
|
params->blockSize * params->segmentCount * params->numTasks;
|
|
|
|
|
2021-01-20 17:57:21 +03:00
|
|
|
ValidateTests(&tests->params, com);
|
2011-12-13 09:00:18 +04:00
|
|
|
tests = tests->next;
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2021-01-20 17:06:05 +03:00
|
|
|
init_clock(com);
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 03:11:28 +04:00
|
|
|
/*
|
|
|
|
* Setup transfer buffers, creating and filling as needed.
|
|
|
|
*/
|
2017-09-27 19:45:47 +03:00
|
|
|
static void XferBuffersSetup(IOR_io_buffers* ioBuffers, IOR_param_t* test,
|
2015-05-27 19:24:52 +03:00
|
|
|
int pretendRank)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2021-03-17 02:24:23 +03:00
|
|
|
ioBuffers->buffer = aligned_buffer_alloc(test->transferSize, test->gpuMemoryFlags);
|
2012-01-13 08:34:40 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Free transfer buffers.
|
|
|
|
*/
|
2015-05-27 19:24:52 +03:00
|
|
|
static void XferBuffersFree(IOR_io_buffers* ioBuffers, IOR_param_t* test)
|
|
|
|
|
2012-01-13 08:34:40 +04:00
|
|
|
{
|
2021-02-18 13:40:42 +03:00
|
|
|
aligned_buffer_free(ioBuffers->buffer, test->gpuMemoryFlags);
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2012-09-10 21:50:18 +04:00
|
|
|
|
2011-12-13 09:00:18 +04:00
|
|
|
|
2012-01-07 05:29:45 +04:00
|
|
|
/*
|
|
|
|
* malloc a buffer, touching every page in an attempt to defeat lazy allocation.
|
|
|
|
*/
|
|
|
|
static void *malloc_and_touch(size_t size)
|
|
|
|
{
|
|
|
|
size_t page_size;
|
|
|
|
char *buf;
|
|
|
|
char *ptr;
|
|
|
|
|
|
|
|
if (size == 0)
|
|
|
|
return NULL;
|
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
page_size = sysconf(_SC_PAGESIZE);
|
2012-01-07 05:29:45 +04:00
|
|
|
|
|
|
|
buf = (char *)malloc(size);
|
|
|
|
if (buf == NULL)
|
2012-01-09 06:41:30 +04:00
|
|
|
return NULL;
|
2012-01-07 05:29:45 +04:00
|
|
|
|
|
|
|
for (ptr = buf; ptr < buf+size; ptr += page_size) {
|
|
|
|
*ptr = (char)1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return (void *)buf;
|
|
|
|
}
|
|
|
|
|
2011-12-14 10:48:14 +04:00
|
|
|
static void file_hits_histogram(IOR_param_t *params)
|
|
|
|
{
|
2018-07-07 12:29:27 +03:00
|
|
|
int *rankoffs = NULL;
|
|
|
|
int *filecont = NULL;
|
|
|
|
int *filehits = NULL;
|
2014-08-14 02:53:24 +04:00
|
|
|
int ifile;
|
|
|
|
int jfile;
|
|
|
|
|
|
|
|
if (rank == 0) {
|
|
|
|
rankoffs = (int *)malloc(params->numTasks * sizeof(int));
|
|
|
|
filecont = (int *)malloc(params->numTasks * sizeof(int));
|
|
|
|
filehits = (int *)malloc(params->numTasks * sizeof(int));
|
|
|
|
}
|
|
|
|
|
|
|
|
MPI_CHECK(MPI_Gather(&rankOffset, 1, MPI_INT, rankoffs,
|
2021-01-20 17:57:21 +03:00
|
|
|
1, MPI_INT, 0, params->testComm),
|
2014-08-14 02:53:24 +04:00
|
|
|
"MPI_Gather error");
|
|
|
|
|
|
|
|
if (rank != 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
memset((void *)filecont, 0, params->numTasks * sizeof(int));
|
|
|
|
for (ifile = 0; ifile < params->numTasks; ifile++) {
|
|
|
|
filecont[(ifile + rankoffs[ifile]) % params->numTasks]++;
|
|
|
|
}
|
|
|
|
memset((void *)filehits, 0, params->numTasks * sizeof(int));
|
|
|
|
for (ifile = 0; ifile < params->numTasks; ifile++)
|
|
|
|
for (jfile = 0; jfile < params->numTasks; jfile++) {
|
|
|
|
if (ifile == filecont[jfile])
|
|
|
|
filehits[ifile]++;
|
|
|
|
}
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile, "#File Hits Dist:");
|
2014-08-14 02:53:24 +04:00
|
|
|
jfile = 0;
|
|
|
|
ifile = 0;
|
|
|
|
while (jfile < params->numTasks && ifile < params->numTasks) {
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile, " %d", filehits[ifile]);
|
2014-08-14 02:53:24 +04:00
|
|
|
jfile += filehits[ifile], ifile++;
|
|
|
|
}
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile, "\n");
|
2014-08-14 02:53:24 +04:00
|
|
|
free(rankoffs);
|
|
|
|
free(filecont);
|
|
|
|
free(filehits);
|
2011-12-14 10:48:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2012-01-09 00:30:05 +04:00
|
|
|
int test_time_elapsed(IOR_param_t *params, double startTime)
|
|
|
|
{
|
2014-08-14 02:53:24 +04:00
|
|
|
double endTime;
|
2012-01-09 00:30:05 +04:00
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
if (params->maxTimeDuration == 0)
|
|
|
|
return 0;
|
2012-01-09 00:30:05 +04:00
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
endTime = startTime + (params->maxTimeDuration * 60);
|
2012-01-09 00:30:05 +04:00
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
return GetTimeStamp() >= endTime;
|
2012-01-09 00:30:05 +04:00
|
|
|
}
|
|
|
|
|
2012-01-09 06:41:30 +04:00
|
|
|
/*
|
|
|
|
* hog some memory as a rough simulation of a real application's memory use
|
|
|
|
*/
|
|
|
|
static void *HogMemory(IOR_param_t *params)
|
|
|
|
{
|
|
|
|
size_t size;
|
|
|
|
void *buf;
|
|
|
|
|
|
|
|
if (params->memoryPerTask != 0) {
|
|
|
|
size = params->memoryPerTask;
|
|
|
|
} else if (params->memoryPerNode != 0) {
|
|
|
|
if (verbose >= VERBOSE_3)
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile, "This node hogging %ld bytes of memory\n",
|
2012-01-09 06:41:30 +04:00
|
|
|
params->memoryPerNode);
|
2019-08-31 01:45:03 +03:00
|
|
|
size = params->memoryPerNode / params->numTasksOnNode0;
|
2012-01-09 06:41:30 +04:00
|
|
|
} else {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (verbose >= VERBOSE_3)
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile, "This task hogging %ld bytes of memory\n", size);
|
2012-01-09 06:41:30 +04:00
|
|
|
|
|
|
|
buf = malloc_and_touch(size);
|
|
|
|
if (buf == NULL)
|
|
|
|
ERR("malloc of simulated applciation buffer failed");
|
|
|
|
|
|
|
|
return buf;
|
|
|
|
}
|
2018-09-19 20:06:05 +03:00
|
|
|
/*
|
|
|
|
* Write times taken during each iteration of the test.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
WriteTimes(IOR_param_t *test, const double *timer, const int iteration,
|
|
|
|
const int access)
|
|
|
|
{
|
|
|
|
char timerName[MAX_STR];
|
2012-01-09 06:41:30 +04:00
|
|
|
|
2018-09-19 20:06:05 +03:00
|
|
|
for (int i = 0; i < IOR_NB_TIMERS; i++) {
|
|
|
|
|
|
|
|
if (access == WRITE) {
|
|
|
|
switch (i) {
|
|
|
|
case 0:
|
|
|
|
strcpy(timerName, "write open start");
|
|
|
|
break;
|
|
|
|
case 1:
|
|
|
|
strcpy(timerName, "write open stop");
|
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
strcpy(timerName, "write start");
|
|
|
|
break;
|
|
|
|
case 3:
|
|
|
|
strcpy(timerName, "write stop");
|
|
|
|
break;
|
|
|
|
case 4:
|
|
|
|
strcpy(timerName, "write close start");
|
|
|
|
break;
|
|
|
|
case 5:
|
|
|
|
strcpy(timerName, "write close stop");
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
strcpy(timerName, "invalid timer");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
switch (i) {
|
|
|
|
case 0:
|
|
|
|
strcpy(timerName, "read open start");
|
|
|
|
break;
|
|
|
|
case 1:
|
|
|
|
strcpy(timerName, "read open stop");
|
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
strcpy(timerName, "read start");
|
|
|
|
break;
|
|
|
|
case 3:
|
|
|
|
strcpy(timerName, "read stop");
|
|
|
|
break;
|
|
|
|
case 4:
|
|
|
|
strcpy(timerName, "read close start");
|
|
|
|
break;
|
|
|
|
case 5:
|
|
|
|
strcpy(timerName, "read close stop");
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
strcpy(timerName, "invalid timer");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fprintf(out_logfile, "Test %d: Iter=%d, Task=%d, Time=%f, %s\n",
|
|
|
|
test->id, iteration, (int)rank, timer[i],
|
|
|
|
timerName);
|
|
|
|
}
|
|
|
|
}
|
2021-02-09 20:54:14 +03:00
|
|
|
|
|
|
|
static void StoreRankInformation(IOR_test_t *test, double *timer, const int rep, const int access){
|
|
|
|
IOR_param_t *params = &test->params;
|
|
|
|
double totalTime = timer[5] - timer[0];
|
|
|
|
double accessTime = timer[3] - timer[2];
|
|
|
|
double times[] = {totalTime, accessTime};
|
|
|
|
|
|
|
|
if(rank == 0){
|
|
|
|
FILE* fd = fopen(params->saveRankDetailsCSV, "a");
|
|
|
|
if (fd == NULL){
|
|
|
|
FAIL("Cannot open saveRankPerformanceDetailsCSV file for writes!");
|
|
|
|
}
|
|
|
|
int size;
|
|
|
|
MPI_Comm_size(params->testComm, & size);
|
|
|
|
double *all_times = malloc(2* size * sizeof(double));
|
|
|
|
MPI_Gather(times, 2, MPI_DOUBLE, all_times, 2, MPI_DOUBLE, 0, params->testComm);
|
|
|
|
IOR_point_t *point = (access == WRITE) ? &test->results[rep].write : &test->results[rep].read;
|
|
|
|
double file_size = ((double) point->aggFileSizeForBW) / size;
|
|
|
|
|
|
|
|
for(int i=0; i < size; i++){
|
|
|
|
char buff[1024];
|
|
|
|
sprintf(buff, "%s,%d,%.10e,%.10e,%.10e,%.10e\n", access==WRITE ? "write" : "read", i, all_times[i*2], all_times[i*2+1], file_size/all_times[i*2], file_size/all_times[i*2+1] );
|
|
|
|
int ret = fwrite(buff, strlen(buff), 1, fd);
|
|
|
|
if(ret != 1){
|
|
|
|
WARN("Couln't append to saveRankPerformanceDetailsCSV file\n");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fclose(fd);
|
|
|
|
}else{
|
|
|
|
MPI_Gather(& times, 2, MPI_DOUBLE, NULL, 2, MPI_DOUBLE, 0, testComm);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void ProcessIterResults(IOR_test_t *test, double *timer, const int rep, const int access){
|
|
|
|
IOR_param_t *params = &test->params;
|
|
|
|
|
|
|
|
if (verbose >= VERBOSE_3)
|
|
|
|
WriteTimes(params, timer, rep, access);
|
|
|
|
ReduceIterResults(test, timer, rep, access);
|
|
|
|
if (params->outlierThreshold) {
|
|
|
|
CheckForOutliers(params, timer, access);
|
|
|
|
}
|
|
|
|
|
|
|
|
if(params->saveRankDetailsCSV){
|
|
|
|
StoreRankInformation(test, timer, rep, access);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-06-17 23:20:43 +04:00
|
|
|
/*
|
|
|
|
* Using the test parameters, run iteration(s) of single test.
|
|
|
|
*/
|
2011-12-13 09:00:18 +04:00
|
|
|
static void TestIoSys(IOR_test_t *test)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2014-08-14 02:53:24 +04:00
|
|
|
IOR_param_t *params = &test->params;
|
|
|
|
IOR_results_t *results = test->results;
|
2011-11-12 02:22:17 +04:00
|
|
|
char testFileName[MAX_STR];
|
2018-09-19 20:06:05 +03:00
|
|
|
double timer[IOR_NB_TIMERS];
|
2011-11-12 02:22:17 +04:00
|
|
|
double startTime;
|
2015-05-27 19:24:52 +03:00
|
|
|
int pretendRank;
|
2018-09-19 20:06:05 +03:00
|
|
|
int rep;
|
2020-05-31 14:11:00 +03:00
|
|
|
aiori_fd_t *fd;
|
2011-11-12 02:22:17 +04:00
|
|
|
IOR_offset_t dataMoved; /* for data rate calculation */
|
2012-01-07 05:29:45 +04:00
|
|
|
void *hog_buf;
|
2015-05-27 19:24:52 +03:00
|
|
|
IOR_io_buffers ioBuffers;
|
2011-11-12 02:22:17 +04:00
|
|
|
|
|
|
|
if (rank == 0 && verbose >= VERBOSE_1) {
|
2019-10-24 01:17:37 +03:00
|
|
|
fprintf(out_logfile, "Participating tasks : %d\n", params->numTasks);
|
2018-07-07 13:42:21 +03:00
|
|
|
fflush(out_logfile);
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2011-12-13 09:00:18 +04:00
|
|
|
if (rank == 0 && params->reorderTasks == TRUE && verbose >= VERBOSE_1) {
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile,
|
2019-08-01 03:42:03 +03:00
|
|
|
"Using reorderTasks '-C' (useful to avoid read cache in client)\n");
|
2018-07-07 13:42:21 +03:00
|
|
|
fflush(out_logfile);
|
2011-06-17 23:20:43 +04:00
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
/* show test setup */
|
|
|
|
if (rank == 0 && verbose >= VERBOSE_0)
|
2011-12-13 09:00:18 +04:00
|
|
|
ShowSetup(params);
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2012-01-09 06:41:30 +04:00
|
|
|
hog_buf = HogMemory(params);
|
2012-01-07 05:29:45 +04:00
|
|
|
|
2015-05-27 19:24:52 +03:00
|
|
|
pretendRank = (rank + rankOffset) % params->numTasks;
|
|
|
|
|
|
|
|
/* IO Buffer Setup */
|
|
|
|
|
2017-09-27 19:45:47 +03:00
|
|
|
if (params->setTimeStampSignature) { // initialize the buffer properly
|
2019-12-22 14:21:40 +03:00
|
|
|
params->timeStampSignatureValue = (unsigned int) params->setTimeStampSignature;
|
2017-09-27 19:45:47 +03:00
|
|
|
}
|
2015-05-27 19:24:52 +03:00
|
|
|
XferBuffersSetup(&ioBuffers, params, pretendRank);
|
2021-03-18 23:42:50 +03:00
|
|
|
|
2015-05-27 19:24:52 +03:00
|
|
|
/* Initial time stamp */
|
2011-11-12 02:22:17 +04:00
|
|
|
startTime = GetTimeStamp();
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
/* loop over test iterations */
|
2018-07-15 21:38:17 +03:00
|
|
|
uint64_t params_saved_wearout = params->stoneWallingWearOutIterations;
|
2020-12-01 21:24:09 +03:00
|
|
|
|
|
|
|
/* Check if the file exists and warn users */
|
2020-12-03 23:54:51 +03:00
|
|
|
if((params->writeFile || params->checkWrite) && (params->hints.filePerProc || rank == 0)){
|
2020-12-02 13:14:28 +03:00
|
|
|
struct stat sb;
|
|
|
|
GetTestFileName(testFileName, params);
|
|
|
|
int ret = backend->stat(testFileName, & sb, params->backend_options);
|
|
|
|
if(ret == 0) {
|
|
|
|
EWARNF("The file \"%s\" exists already and will be overwritten", testFileName);
|
|
|
|
}
|
2020-12-01 21:24:09 +03:00
|
|
|
}
|
|
|
|
|
2011-12-13 09:00:18 +04:00
|
|
|
for (rep = 0; rep < params->repetitions; rep++) {
|
2011-11-12 02:22:17 +04:00
|
|
|
/* Get iteration start time in seconds in task 0 and broadcast to
|
|
|
|
all tasks */
|
|
|
|
if (rank == 0) {
|
2017-09-27 19:45:47 +03:00
|
|
|
if (! params->setTimeStampSignature) {
|
2011-11-12 02:22:17 +04:00
|
|
|
time_t currentTime;
|
|
|
|
if ((currentTime = time(NULL)) == -1) {
|
|
|
|
ERR("cannot get current time");
|
|
|
|
}
|
2011-12-13 09:00:18 +04:00
|
|
|
params->timeStampSignatureValue =
|
2014-08-14 02:53:24 +04:00
|
|
|
(unsigned int)currentTime;
|
2019-10-24 01:17:37 +03:00
|
|
|
}
|
|
|
|
if (verbose >= VERBOSE_2) {
|
|
|
|
fprintf(out_logfile,
|
|
|
|
"Using Time Stamp %u (0x%x) for Data Signature\n",
|
|
|
|
params->timeStampSignatureValue,
|
|
|
|
params->timeStampSignatureValue);
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2014-08-14 02:53:24 +04:00
|
|
|
if (rep == 0 && verbose >= VERBOSE_0) {
|
2018-07-08 15:47:55 +03:00
|
|
|
PrintTableHeader();
|
2014-08-14 02:53:24 +04:00
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
MPI_CHECK(MPI_Bcast
|
2011-12-13 09:00:18 +04:00
|
|
|
(¶ms->timeStampSignatureValue, 1, MPI_UNSIGNED, 0,
|
2011-11-12 02:22:17 +04:00
|
|
|
testComm), "cannot broadcast start time value");
|
2017-12-09 13:52:13 +03:00
|
|
|
|
2021-03-18 23:42:50 +03:00
|
|
|
generate_memory_pattern((char*) ioBuffers.buffer, params->transferSize, params->setTimeStampSignature, pretendRank, params->dataPacketType);
|
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
/* use repetition count for number of multiple files */
|
2011-12-13 09:00:18 +04:00
|
|
|
if (params->multiFile)
|
|
|
|
params->repCounter = rep;
|
2011-11-12 02:22:17 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* write the file(s), getting timing between I/O calls
|
|
|
|
*/
|
|
|
|
|
2012-01-09 00:30:05 +04:00
|
|
|
if (params->writeFile && !test_time_elapsed(params, startTime)) {
|
2011-12-13 09:00:18 +04:00
|
|
|
GetTestFileName(testFileName, params);
|
2011-11-12 02:22:17 +04:00
|
|
|
if (verbose >= VERBOSE_3) {
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile, "task %d writing %s\n", rank,
|
2011-11-12 02:22:17 +04:00
|
|
|
testFileName);
|
|
|
|
}
|
2011-12-13 09:00:18 +04:00
|
|
|
DelaySecs(params->interTestDelay);
|
|
|
|
if (params->useExistingTestFile == FALSE) {
|
|
|
|
RemoveFile(testFileName, params->filePerProc,
|
|
|
|
params);
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2018-07-15 21:38:17 +03:00
|
|
|
|
|
|
|
params->stoneWallingWearOutIterations = params_saved_wearout;
|
2011-11-12 02:22:17 +04:00
|
|
|
MPI_CHECK(MPI_Barrier(testComm), "barrier error");
|
2011-12-13 09:00:18 +04:00
|
|
|
params->open = WRITE;
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[0] = GetTimeStamp();
|
2020-05-30 21:01:20 +03:00
|
|
|
fd = backend->create(testFileName, IOR_WRONLY | IOR_CREAT | IOR_TRUNC, params->backend_options);
|
2020-06-30 21:41:49 +03:00
|
|
|
if(fd == NULL) FAIL("Cannot create file");
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[1] = GetTimeStamp();
|
2011-12-13 09:00:18 +04:00
|
|
|
if (params->intraTestBarriers)
|
2011-11-12 02:22:17 +04:00
|
|
|
MPI_CHECK(MPI_Barrier(testComm),
|
|
|
|
"barrier error");
|
|
|
|
if (rank == 0 && verbose >= VERBOSE_1) {
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile,
|
2011-12-13 13:07:55 +04:00
|
|
|
"Commencing write performance test: %s",
|
2014-08-14 02:53:24 +04:00
|
|
|
CurrentTimeString());
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[2] = GetTimeStamp();
|
2018-09-20 00:39:25 +03:00
|
|
|
dataMoved = WriteOrRead(params, &results[rep], fd, WRITE, &ioBuffers);
|
2014-08-29 01:39:44 +04:00
|
|
|
if (params->verbose >= VERBOSE_4) {
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile, "* data moved = %llu\n", dataMoved);
|
|
|
|
fflush(out_logfile);
|
2014-08-29 01:39:44 +04:00
|
|
|
}
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[3] = GetTimeStamp();
|
2011-12-13 09:00:18 +04:00
|
|
|
if (params->intraTestBarriers)
|
2011-11-12 02:22:17 +04:00
|
|
|
MPI_CHECK(MPI_Barrier(testComm),
|
|
|
|
"barrier error");
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[4] = GetTimeStamp();
|
2020-05-31 13:50:15 +03:00
|
|
|
backend->close(fd, params->backend_options);
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[5] = GetTimeStamp();
|
2011-11-12 02:22:17 +04:00
|
|
|
MPI_CHECK(MPI_Barrier(testComm), "barrier error");
|
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
/* check if stat() of file doesn't equal expected file size,
|
|
|
|
use actual amount of byte moved */
|
2020-06-30 14:03:05 +03:00
|
|
|
CheckFileSize(test, testFileName, dataMoved, rep, WRITE);
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2021-02-09 20:54:14 +03:00
|
|
|
ProcessIterResults(test, timer, rep, WRITE);
|
2018-07-15 21:38:17 +03:00
|
|
|
|
|
|
|
/* check if in this round we run write with stonewalling */
|
|
|
|
if(params->deadlineForStonewalling > 0){
|
2018-09-20 00:39:25 +03:00
|
|
|
params->stoneWallingWearOutIterations = results[rep].write.pairs_accessed;
|
2018-07-15 21:38:17 +03:00
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
/*
|
|
|
|
* perform a check of data, reading back data and comparing
|
|
|
|
* against what was expected to be written
|
|
|
|
*/
|
2012-01-09 00:30:05 +04:00
|
|
|
if (params->checkWrite && !test_time_elapsed(params, startTime)) {
|
2011-11-12 02:22:17 +04:00
|
|
|
MPI_CHECK(MPI_Barrier(testComm), "barrier error");
|
|
|
|
if (rank == 0 && verbose >= VERBOSE_1) {
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile,
|
2011-11-12 02:22:17 +04:00
|
|
|
"Verifying contents of the file(s) just written.\n");
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile, "%s\n", CurrentTimeString());
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2011-12-13 09:00:18 +04:00
|
|
|
if (params->reorderTasks) {
|
2011-11-12 02:22:17 +04:00
|
|
|
/* move two nodes away from writing node */
|
2019-07-28 19:25:42 +03:00
|
|
|
int shift = 1; /* assume a by-node (round-robin) mapping of tasks to nodes */
|
|
|
|
if (params->tasksBlockMapping) {
|
2019-08-31 01:45:03 +03:00
|
|
|
shift = params->numTasksOnNode0; /* switch to by-slot (contiguous block) mapping */
|
2019-07-28 00:27:20 +03:00
|
|
|
}
|
|
|
|
rankOffset = (2 * shift) % params->numTasks;
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2021-03-18 23:42:50 +03:00
|
|
|
|
2011-12-13 09:00:18 +04:00
|
|
|
GetTestFileName(testFileName, params);
|
|
|
|
params->open = WRITECHECK;
|
2020-05-31 13:50:15 +03:00
|
|
|
fd = backend->open(testFileName, IOR_RDONLY, params->backend_options);
|
2020-06-30 21:41:49 +03:00
|
|
|
if(fd == NULL) FAIL("Cannot open file");
|
2018-09-20 00:39:25 +03:00
|
|
|
dataMoved = WriteOrRead(params, &results[rep], fd, WRITECHECK, &ioBuffers);
|
2020-05-31 13:50:15 +03:00
|
|
|
backend->close(fd, params->backend_options);
|
2011-11-12 02:22:17 +04:00
|
|
|
rankOffset = 0;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* read the file(s), getting timing between I/O calls
|
|
|
|
*/
|
2017-09-27 19:45:47 +03:00
|
|
|
if ((params->readFile || params->checkRead ) && !test_time_elapsed(params, startTime)) {
|
2018-07-15 21:38:17 +03:00
|
|
|
/* check for stonewall */
|
|
|
|
if(params->stoneWallingStatusFile){
|
2021-01-20 17:06:05 +03:00
|
|
|
params->stoneWallingWearOutIterations = ReadStoneWallingIterations(params->stoneWallingStatusFile, params->testComm);
|
2018-07-15 21:38:17 +03:00
|
|
|
if(params->stoneWallingWearOutIterations == -1 && rank == 0){
|
2020-06-24 13:10:42 +03:00
|
|
|
WARN("Could not read back the stonewalling status from the file!");
|
2018-07-15 21:38:17 +03:00
|
|
|
params->stoneWallingWearOutIterations = 0;
|
|
|
|
}
|
|
|
|
}
|
2017-09-27 19:45:47 +03:00
|
|
|
int operation_flag = READ;
|
|
|
|
if ( params->checkRead ){
|
|
|
|
// actually read and then compare the buffer
|
|
|
|
operation_flag = READCHECK;
|
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
/* Get rankOffset [file offset] for this process to read, based on -C,-Z,-Q,-X options */
|
|
|
|
/* Constant process offset reading */
|
2011-12-13 09:00:18 +04:00
|
|
|
if (params->reorderTasks) {
|
2019-08-01 19:20:01 +03:00
|
|
|
/* move one node away from writing node */
|
2019-07-28 19:25:42 +03:00
|
|
|
int shift = 1; /* assume a by-node (round-robin) mapping of tasks to nodes */
|
|
|
|
if (params->tasksBlockMapping) {
|
2019-08-31 01:45:03 +03:00
|
|
|
shift=params->numTasksOnNode0; /* switch to a by-slot (contiguous block) mapping */
|
2019-07-28 00:27:20 +03:00
|
|
|
}
|
|
|
|
rankOffset = (params->taskPerNodeOffset * shift) % params->numTasks;
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
/* random process offset reading */
|
2011-12-13 09:00:18 +04:00
|
|
|
if (params->reorderTasksRandom) {
|
2011-11-12 02:22:17 +04:00
|
|
|
/* this should not intefere with randomOffset within a file because GetOffsetArrayRandom */
|
2018-10-08 23:47:28 +03:00
|
|
|
/* seeds every rand() call */
|
2014-08-14 02:53:24 +04:00
|
|
|
int nodeoffset;
|
2011-11-12 02:22:17 +04:00
|
|
|
unsigned int iseed0;
|
2011-12-13 09:00:18 +04:00
|
|
|
nodeoffset = params->taskPerNodeOffset;
|
2019-08-31 01:45:03 +03:00
|
|
|
nodeoffset = (nodeoffset < params->numNodes) ? nodeoffset : params->numNodes - 1;
|
2011-12-14 10:48:14 +04:00
|
|
|
if (params->reorderTasksRandomSeed < 0)
|
2014-08-14 02:53:24 +04:00
|
|
|
iseed0 = -1 * params->reorderTasksRandomSeed + rep;
|
|
|
|
else
|
|
|
|
iseed0 = params->reorderTasksRandomSeed;
|
2011-11-12 02:22:17 +04:00
|
|
|
srand(rank + iseed0);
|
|
|
|
{
|
2011-12-13 09:00:18 +04:00
|
|
|
rankOffset = rand() % params->numTasks;
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
while (rankOffset <
|
2019-08-31 01:45:03 +03:00
|
|
|
(nodeoffset * params->numTasksOnNode0)) {
|
2011-12-13 09:00:18 +04:00
|
|
|
rankOffset = rand() % params->numTasks;
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
/* Get more detailed stats if requested by verbose level */
|
|
|
|
if (verbose >= VERBOSE_2) {
|
2014-08-14 02:53:24 +04:00
|
|
|
file_hits_histogram(params);
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
/* Using globally passed rankOffset, following function generates testFileName to read */
|
2011-12-13 09:00:18 +04:00
|
|
|
GetTestFileName(testFileName, params);
|
2011-11-12 02:22:17 +04:00
|
|
|
|
|
|
|
if (verbose >= VERBOSE_3) {
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile, "task %d reading %s\n", rank,
|
2011-11-12 02:22:17 +04:00
|
|
|
testFileName);
|
|
|
|
}
|
2011-12-13 09:00:18 +04:00
|
|
|
DelaySecs(params->interTestDelay);
|
2011-11-12 02:22:17 +04:00
|
|
|
MPI_CHECK(MPI_Barrier(testComm), "barrier error");
|
2011-12-13 09:00:18 +04:00
|
|
|
params->open = READ;
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[0] = GetTimeStamp();
|
2020-05-31 13:50:15 +03:00
|
|
|
fd = backend->open(testFileName, IOR_RDONLY, params->backend_options);
|
2020-06-30 21:41:49 +03:00
|
|
|
if(fd == NULL) FAIL("Cannot open file");
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[1] = GetTimeStamp();
|
2011-12-13 09:00:18 +04:00
|
|
|
if (params->intraTestBarriers)
|
2011-11-12 02:22:17 +04:00
|
|
|
MPI_CHECK(MPI_Barrier(testComm),
|
|
|
|
"barrier error");
|
|
|
|
if (rank == 0 && verbose >= VERBOSE_1) {
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile,
|
2019-08-31 01:45:03 +03:00
|
|
|
"Commencing read performance test: %s\n",
|
2014-08-14 02:53:24 +04:00
|
|
|
CurrentTimeString());
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[2] = GetTimeStamp();
|
2018-09-20 00:39:25 +03:00
|
|
|
dataMoved = WriteOrRead(params, &results[rep], fd, operation_flag, &ioBuffers);
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[3] = GetTimeStamp();
|
2011-12-13 09:00:18 +04:00
|
|
|
if (params->intraTestBarriers)
|
2011-11-12 02:22:17 +04:00
|
|
|
MPI_CHECK(MPI_Barrier(testComm),
|
|
|
|
"barrier error");
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[4] = GetTimeStamp();
|
2020-05-31 13:50:15 +03:00
|
|
|
backend->close(fd, params->backend_options);
|
2018-09-19 20:06:05 +03:00
|
|
|
timer[5] = GetTimeStamp();
|
2011-11-12 02:22:17 +04:00
|
|
|
|
|
|
|
/* check if stat() of file doesn't equal expected file size,
|
|
|
|
use actual amount of byte moved */
|
2020-06-30 14:03:05 +03:00
|
|
|
CheckFileSize(test, testFileName, dataMoved, rep, READ);
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2021-02-09 20:54:14 +03:00
|
|
|
ProcessIterResults(test, timer, rep, READ);
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
|
2011-12-13 09:00:18 +04:00
|
|
|
if (!params->keepFile
|
2012-01-09 06:55:46 +04:00
|
|
|
&& !(params->errorFound && params->keepFileWithError)) {
|
|
|
|
double start, finish;
|
|
|
|
start = GetTimeStamp();
|
|
|
|
MPI_CHECK(MPI_Barrier(testComm), "barrier error");
|
2011-12-13 09:00:18 +04:00
|
|
|
RemoveFile(testFileName, params->filePerProc, params);
|
2012-01-09 06:55:46 +04:00
|
|
|
MPI_CHECK(MPI_Barrier(testComm), "barrier error");
|
|
|
|
finish = GetTimeStamp();
|
|
|
|
PrintRemoveTiming(start, finish, rep);
|
|
|
|
} else {
|
|
|
|
MPI_CHECK(MPI_Barrier(testComm), "barrier error");
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2011-12-13 09:00:18 +04:00
|
|
|
params->errorFound = FALSE;
|
2011-11-12 02:22:17 +04:00
|
|
|
rankOffset = 0;
|
2018-07-08 18:47:23 +03:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2020-10-28 12:38:17 +03:00
|
|
|
PrintRepeatEnd();
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2012-01-14 01:27:55 +04:00
|
|
|
if (params->summary_every_test) {
|
|
|
|
PrintLongSummaryHeader();
|
|
|
|
PrintLongSummaryOneTest(test);
|
|
|
|
} else {
|
|
|
|
PrintShortSummary(test);
|
|
|
|
}
|
2011-12-13 09:00:18 +04:00
|
|
|
|
2015-05-27 19:24:52 +03:00
|
|
|
XferBuffersFree(&ioBuffers, params);
|
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
if (hog_buf != NULL)
|
|
|
|
free(hog_buf);
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Determine if valid tests from parameters.
|
|
|
|
*/
|
2021-01-20 17:57:21 +03:00
|
|
|
static void ValidateTests(IOR_param_t * test, MPI_Comm com)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2011-11-12 02:22:17 +04:00
|
|
|
IOR_param_t defaults;
|
2021-01-20 17:57:21 +03:00
|
|
|
init_IOR_Param_t(&defaults, com);
|
2015-05-19 18:36:28 +03:00
|
|
|
|
2021-04-13 12:01:30 +03:00
|
|
|
if (test->stoneWallingStatusFile && test->keepFile == 0)
|
|
|
|
ERR("a StoneWallingStatusFile is only sensible when splitting write/read into multiple executions of ior, please use -k");
|
|
|
|
if (test->stoneWallingStatusFile && test->stoneWallingWearOut == 0 && test->writeFile)
|
|
|
|
ERR("the StoneWallingStatusFile is only sensible for a write test when using stoneWallingWearOut");
|
|
|
|
if (test->deadlineForStonewalling == 0 && test->stoneWallingWearOut > 0)
|
|
|
|
ERR("the stoneWallingWearOut is only sensible when setting a stonewall deadline with -D");
|
|
|
|
if (test->stoneWallingStatusFile && test->testscripts)
|
|
|
|
WARN("the StoneWallingStatusFile only preserves the last experiment, make sure that each run uses a separate status file!");
|
2011-11-12 02:22:17 +04:00
|
|
|
if (test->repetitions <= 0)
|
|
|
|
WARN_RESET("too few test repetitions",
|
|
|
|
test, &defaults, repetitions);
|
|
|
|
if (test->numTasks <= 0)
|
|
|
|
ERR("too few tasks for testing");
|
|
|
|
if (test->interTestDelay < 0)
|
|
|
|
WARN_RESET("inter-test delay must be nonnegative value",
|
|
|
|
test, &defaults, interTestDelay);
|
|
|
|
if (test->readFile != TRUE && test->writeFile != TRUE
|
|
|
|
&& test->checkRead != TRUE && test->checkWrite != TRUE)
|
2017-12-09 19:22:58 +03:00
|
|
|
ERR("test must write, read, or check read/write file");
|
|
|
|
if(! test->setTimeStampSignature && test->writeFile != TRUE && test->checkRead == TRUE)
|
2017-12-09 19:24:31 +03:00
|
|
|
ERR("using readCheck only requires to write a timeStampSignature -- use -G");
|
2011-11-12 02:22:17 +04:00
|
|
|
if (test->segmentCount < 0)
|
|
|
|
ERR("segment count must be positive value");
|
|
|
|
if ((test->blockSize % sizeof(IOR_size_t)) != 0)
|
|
|
|
ERR("block size must be a multiple of access size");
|
|
|
|
if (test->blockSize < 0)
|
|
|
|
ERR("block size must be non-negative integer");
|
|
|
|
if ((test->transferSize % sizeof(IOR_size_t)) != 0)
|
|
|
|
ERR("transfer size must be a multiple of access size");
|
|
|
|
if (test->transferSize < 0)
|
|
|
|
ERR("transfer size must be non-negative integer");
|
|
|
|
if (test->transferSize == 0) {
|
|
|
|
ERR("test will not complete with zero transfer size");
|
2011-06-17 23:20:43 +04:00
|
|
|
} else {
|
2011-11-12 02:22:17 +04:00
|
|
|
if ((test->blockSize % test->transferSize) != 0)
|
|
|
|
ERR("block size must be a multiple of transfer size");
|
|
|
|
}
|
|
|
|
if (test->blockSize < test->transferSize)
|
|
|
|
ERR("block size must not be smaller than transfer size");
|
2020-11-06 13:38:50 +03:00
|
|
|
if (test->randomOffset && test->blockSize == test->transferSize)
|
|
|
|
ERR("IOR will randomize access within a block and repeats the same pattern for all segments, therefore choose blocksize > transferSize");
|
2020-12-01 16:52:29 +03:00
|
|
|
if (! test->randomOffset && test->randomPrefillBlocksize)
|
|
|
|
ERR("Setting the randomPrefill option without using random is not useful");
|
|
|
|
if (test->randomPrefillBlocksize && (test->blockSize % test->randomPrefillBlocksize != 0))
|
|
|
|
ERR("The randomPrefill option must divide the blockSize");
|
2015-05-19 18:36:28 +03:00
|
|
|
/* specific APIs */
|
2017-12-05 02:20:31 +03:00
|
|
|
if ((strcasecmp(test->api, "MPIIO") == 0)
|
2011-11-12 02:22:17 +04:00
|
|
|
&& (test->blockSize < sizeof(IOR_size_t)
|
|
|
|
|| test->transferSize < sizeof(IOR_size_t)))
|
|
|
|
ERR("block/transfer size may not be smaller than IOR_size_t for MPIIO");
|
2017-12-05 02:20:31 +03:00
|
|
|
if ((strcasecmp(test->api, "HDF5") == 0)
|
2011-11-12 02:22:17 +04:00
|
|
|
&& (test->blockSize < sizeof(IOR_size_t)
|
|
|
|
|| test->transferSize < sizeof(IOR_size_t)))
|
|
|
|
ERR("block/transfer size may not be smaller than IOR_size_t for HDF5");
|
2017-12-05 02:20:31 +03:00
|
|
|
if ((strcasecmp(test->api, "NCMPI") == 0)
|
2011-11-12 02:22:17 +04:00
|
|
|
&& (test->blockSize < sizeof(IOR_size_t)
|
|
|
|
|| test->transferSize < sizeof(IOR_size_t)))
|
|
|
|
ERR("block/transfer size may not be smaller than IOR_size_t for NCMPI");
|
2018-04-27 20:44:20 +03:00
|
|
|
if (((strcasecmp(test->api, "POSIX") != 0)
|
|
|
|
&& (strcasecmp(test->api, "MPIIO") != 0)
|
2018-07-12 18:13:40 +03:00
|
|
|
&& (strcasecmp(test->api, "MMAP") != 0)
|
2018-08-15 01:08:04 +03:00
|
|
|
&& (strcasecmp(test->api, "HDFS") != 0)
|
2020-05-31 19:20:58 +03:00
|
|
|
&& (strcasecmp(test->api, "DFS") != 0)
|
2019-08-02 07:03:59 +03:00
|
|
|
&& (strcasecmp(test->api, "Gfarm") != 0)
|
2020-03-05 04:40:05 +03:00
|
|
|
&& (strcasecmp(test->api, "RADOS") != 0)
|
|
|
|
&& (strcasecmp(test->api, "CEPHFS") != 0)) && test->fsync)
|
2018-04-27 20:44:20 +03:00
|
|
|
WARN_RESET("fsync() not supported in selected backend",
|
2011-11-12 02:22:17 +04:00
|
|
|
test, &defaults, fsync);
|
2020-07-03 10:09:40 +03:00
|
|
|
/* parameter consistency */
|
2011-11-12 02:22:17 +04:00
|
|
|
if (test->reorderTasks == TRUE && test->reorderTasksRandom == TRUE)
|
|
|
|
ERR("Both Constant and Random task re-ordering specified. Choose one and resubmit");
|
|
|
|
if (test->randomOffset && test->reorderTasksRandom
|
|
|
|
&& test->filePerProc == FALSE)
|
|
|
|
ERR("random offset and random reorder tasks specified with single-shared-file. Choose one and resubmit");
|
|
|
|
if (test->randomOffset && test->reorderTasks
|
|
|
|
&& test->filePerProc == FALSE)
|
|
|
|
ERR("random offset and constant reorder tasks specified with single-shared-file. Choose one and resubmit");
|
2020-11-04 16:47:35 +03:00
|
|
|
if (test->randomOffset && test->checkRead && test->randomSeed == -1)
|
|
|
|
ERR("random offset with read check option requires to set the random seed");
|
2021-03-18 23:42:50 +03:00
|
|
|
if (test->randomOffset && test->dataPacketType == DATA_OFFSET)
|
2011-11-12 02:22:17 +04:00
|
|
|
ERR("random offset not available with store file offset option)");
|
2017-12-05 02:20:31 +03:00
|
|
|
if ((strcasecmp(test->api, "HDF5") == 0) && test->randomOffset)
|
2011-11-12 02:22:17 +04:00
|
|
|
ERR("random offset not available with HDF5");
|
2017-12-05 02:20:31 +03:00
|
|
|
if ((strcasecmp(test->api, "NCMPI") == 0) && test->randomOffset)
|
2011-11-12 02:22:17 +04:00
|
|
|
ERR("random offset not available with NCMPI");
|
2017-12-05 02:20:31 +03:00
|
|
|
if ((strcasecmp(test->api, "NCMPI") == 0) && test->filePerProc)
|
2011-11-12 02:22:17 +04:00
|
|
|
ERR("file-per-proc not available in current NCMPI");
|
2015-05-19 18:36:28 +03:00
|
|
|
|
2020-05-31 14:50:03 +03:00
|
|
|
backend = test->backend;
|
|
|
|
ior_set_xfer_hints(test);
|
2019-09-01 17:59:52 +03:00
|
|
|
/* allow the backend to validate the options */
|
|
|
|
if(test->backend->check_params){
|
2020-05-30 22:09:37 +03:00
|
|
|
int check = test->backend->check_params(test->backend_options);
|
2020-05-30 20:19:48 +03:00
|
|
|
if (check){
|
2019-09-01 17:59:52 +03:00
|
|
|
ERR("The backend returned that the test parameters are invalid.");
|
|
|
|
}
|
2015-05-19 18:36:28 +03:00
|
|
|
}
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2018-02-28 20:50:30 +03:00
|
|
|
/**
|
|
|
|
* Returns a precomputed array of IOR_offset_t for the inner benchmark loop.
|
2020-11-06 12:52:35 +03:00
|
|
|
* They get created sequentially and mixed up in the end.
|
|
|
|
* It should be noted that as the seeds get synchronised across all processes if not FilePerProcess is set
|
|
|
|
* every process computes the same random order.
|
2018-02-28 20:50:30 +03:00
|
|
|
* For a shared file all transfers get randomly assigned to ranks. The processes
|
|
|
|
* can also have differen't numbers of transfers. This might lead to a bigger
|
|
|
|
* diversion in accesse as it dose with filePerProc. This is expected but
|
|
|
|
* should be mined.
|
|
|
|
* @param test IOR_param_t for getting transferSize, blocksize and SegmentCount
|
2020-07-03 10:09:40 +03:00
|
|
|
* @param pretendRank int pretended Rank for shifting the offsets correctly
|
2018-02-28 20:50:30 +03:00
|
|
|
* @return IOR_offset_t
|
|
|
|
*/
|
2020-11-06 13:27:11 +03:00
|
|
|
IOR_offset_t *GetOffsetArrayRandom(IOR_param_t * test, int pretendRank, IOR_offset_t * out_count)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2011-11-12 02:22:17 +04:00
|
|
|
int seed;
|
2020-11-06 12:52:35 +03:00
|
|
|
IOR_offset_t i;
|
|
|
|
IOR_offset_t offsets;
|
2011-11-12 03:11:28 +04:00
|
|
|
IOR_offset_t offsetCnt = 0;
|
2011-11-12 02:22:17 +04:00
|
|
|
IOR_offset_t *offsetArray;
|
|
|
|
|
2020-11-06 12:52:35 +03:00
|
|
|
if (test->filePerProc) {
|
|
|
|
/* set up seed, each process can determine which regions to access individually */
|
|
|
|
if (test->randomSeed == -1) {
|
|
|
|
seed = time(NULL);
|
|
|
|
test->randomSeed = seed;
|
|
|
|
} else {
|
|
|
|
seed = test->randomSeed + pretendRank;
|
|
|
|
}
|
|
|
|
}else{
|
|
|
|
/* Shared file requires that the seed is synchronized */
|
|
|
|
if (test->randomSeed == -1) {
|
|
|
|
// all processes need to have the same seed.
|
|
|
|
if(rank == 0){
|
|
|
|
seed = time(NULL);
|
|
|
|
}
|
|
|
|
MPI_CHECK(MPI_Bcast(& seed, 1, MPI_INT, 0, test->testComm), "cannot broadcast random seed value");
|
|
|
|
test->randomSeed = seed;
|
|
|
|
}else{
|
|
|
|
seed = test->randomSeed;
|
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
}
|
2020-11-06 12:52:35 +03:00
|
|
|
srandom(seed);
|
2011-11-12 02:22:17 +04:00
|
|
|
|
|
|
|
/* count needed offsets (pass 1) */
|
2020-11-06 13:27:11 +03:00
|
|
|
if (test->filePerProc) {
|
|
|
|
offsets = test->blockSize / test->transferSize;
|
|
|
|
}else{
|
2020-11-06 12:52:35 +03:00
|
|
|
offsets = 0;
|
2020-11-06 13:27:11 +03:00
|
|
|
for (i = 0; i < test->blockSize * test->numTasks; i += test->transferSize) {
|
2020-11-06 12:52:35 +03:00
|
|
|
// this counts which process get how many transferes in the shared file
|
|
|
|
if ((rand() % test->numTasks) == pretendRank) {
|
|
|
|
offsets++;
|
|
|
|
}
|
2020-11-04 16:47:35 +03:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
}
|
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
/* setup empty array */
|
2020-11-06 13:27:11 +03:00
|
|
|
offsetArray = (IOR_offset_t *) safeMalloc(offsets * sizeof(IOR_offset_t));
|
2020-11-06 12:52:35 +03:00
|
|
|
|
2020-11-06 13:27:11 +03:00
|
|
|
*out_count = offsets;
|
2011-11-12 02:22:17 +04:00
|
|
|
|
|
|
|
if (test->filePerProc) {
|
2020-11-06 12:52:35 +03:00
|
|
|
/* fill array */
|
|
|
|
for (i = 0; i < offsets; i++) {
|
2020-11-06 13:27:11 +03:00
|
|
|
offsetArray[i] = i * test->transferSize;
|
2020-11-06 12:52:35 +03:00
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
} else {
|
2020-11-06 12:52:35 +03:00
|
|
|
/* fill with offsets (pass 2) */
|
|
|
|
srandom(seed); /* need same seed to get same transfers as counted in the beginning*/
|
2020-11-06 13:27:11 +03:00
|
|
|
for (i = 0; i < test->blockSize * test->numTasks; i += test->transferSize) {
|
2020-11-06 12:52:35 +03:00
|
|
|
if ((rand() % test->numTasks) == pretendRank) {
|
|
|
|
offsetArray[offsetCnt] = i;
|
|
|
|
offsetCnt++;
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2020-11-06 12:52:35 +03:00
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
|
|
|
/* reorder array */
|
|
|
|
for (i = 0; i < offsets; i++) {
|
2020-11-06 12:52:35 +03:00
|
|
|
IOR_offset_t value, tmp;
|
2018-10-08 23:47:28 +03:00
|
|
|
value = rand() % offsets;
|
2011-11-12 02:22:17 +04:00
|
|
|
tmp = offsetArray[value];
|
|
|
|
offsetArray[value] = offsetArray[i];
|
|
|
|
offsetArray[i] = tmp;
|
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
return (offsetArray);
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2020-12-01 16:52:29 +03:00
|
|
|
static IOR_offset_t WriteOrReadSingle(IOR_offset_t offset, int pretendRank, IOR_offset_t transfer, IOR_offset_t * transferCount, int * errors, IOR_param_t * test, aiori_fd_t * fd, IOR_io_buffers* ioBuffers, int access){
|
2018-07-07 12:29:27 +03:00
|
|
|
IOR_offset_t amtXferred = 0;
|
2017-10-20 19:02:24 +03:00
|
|
|
|
|
|
|
void *buffer = ioBuffers->buffer;
|
|
|
|
if (access == WRITE) {
|
2017-11-30 13:56:26 +03:00
|
|
|
/* fills each transfer with a unique pattern
|
|
|
|
* containing the offset into the file */
|
2021-03-18 23:42:50 +03:00
|
|
|
update_write_memory_pattern(offset, ioBuffers->buffer, transfer, test->setTimeStampSignature, pretendRank, test->dataPacketType);
|
2020-06-10 19:47:07 +03:00
|
|
|
amtXferred = backend->xfer(access, fd, buffer, transfer, offset, test->backend_options);
|
2017-10-20 19:02:24 +03:00
|
|
|
if (amtXferred != transfer)
|
|
|
|
ERR("cannot write to file");
|
2020-05-30 20:19:48 +03:00
|
|
|
if (test->fsyncPerWrite)
|
|
|
|
backend->fsync(fd, test->backend_options);
|
2018-09-30 12:01:21 +03:00
|
|
|
if (test->interIODelay > 0){
|
2018-10-06 19:30:00 +03:00
|
|
|
struct timespec wait = {test->interIODelay / 1000 / 1000, 1000l * (test->interIODelay % 1000000)};
|
|
|
|
nanosleep( & wait, NULL);
|
2018-09-30 12:01:21 +03:00
|
|
|
}
|
2017-10-20 19:02:24 +03:00
|
|
|
} else if (access == READ) {
|
2020-06-10 19:47:07 +03:00
|
|
|
amtXferred = backend->xfer(access, fd, buffer, transfer, offset, test->backend_options);
|
2017-10-20 19:02:24 +03:00
|
|
|
if (amtXferred != transfer)
|
|
|
|
ERR("cannot read from file");
|
2018-09-30 12:01:21 +03:00
|
|
|
if (test->interIODelay > 0){
|
2018-10-06 19:30:00 +03:00
|
|
|
struct timespec wait = {test->interIODelay / 1000 / 1000, 1000l * (test->interIODelay % 1000000)};
|
|
|
|
nanosleep( & wait, NULL);
|
2018-09-30 12:01:21 +03:00
|
|
|
}
|
2017-10-20 19:02:24 +03:00
|
|
|
} else if (access == WRITECHECK) {
|
2020-09-28 22:09:48 +03:00
|
|
|
((long long int*) buffer)[0] = ~((long long int*) buffer)[0]; // changes the buffer, no memset to reduce the memory pressure
|
|
|
|
amtXferred = backend->xfer(access, fd, buffer, transfer, offset, test->backend_options);
|
2017-10-20 19:02:24 +03:00
|
|
|
if (amtXferred != transfer)
|
|
|
|
ERR("cannot read from file write check");
|
2020-09-28 22:09:48 +03:00
|
|
|
*errors += CompareData(buffer, transfer, *transferCount, test, offset, pretendRank, WRITECHECK);
|
2017-10-20 19:02:24 +03:00
|
|
|
} else if (access == READCHECK) {
|
2020-09-28 22:09:48 +03:00
|
|
|
((long long int*) buffer)[0] = ~((long long int*) buffer)[0]; // changes the buffer, no memset to reduce the memory pressure
|
|
|
|
amtXferred = backend->xfer(access, fd, buffer, transfer, offset, test->backend_options);
|
2017-10-20 19:02:24 +03:00
|
|
|
if (amtXferred != transfer){
|
|
|
|
ERR("cannot read from file");
|
|
|
|
}
|
2020-09-28 22:09:48 +03:00
|
|
|
*errors += CompareData(buffer, transfer, *transferCount, test, offset, pretendRank, READCHECK);
|
2017-10-20 19:02:24 +03:00
|
|
|
}
|
|
|
|
return amtXferred;
|
|
|
|
}
|
|
|
|
|
2020-12-01 17:45:07 +03:00
|
|
|
static void prefillSegment(IOR_param_t *test, void * randomPrefillBuffer, int pretendRank, aiori_fd_t *fd, IOR_io_buffers *ioBuffers, int startSegment, int endSegment){
|
|
|
|
// prefill the whole file already with an invalid pattern
|
|
|
|
int offsets = test->blockSize / test->randomPrefillBlocksize;
|
|
|
|
void * oldBuffer = ioBuffers->buffer;
|
|
|
|
IOR_offset_t transferCount;
|
|
|
|
int errors;
|
|
|
|
ioBuffers->buffer = randomPrefillBuffer;
|
|
|
|
for (int i = startSegment; i < endSegment; i++){
|
|
|
|
for (int j = 0; j < offsets; j++) {
|
|
|
|
IOR_offset_t offset = j * test->randomPrefillBlocksize;
|
|
|
|
if (test->filePerProc) {
|
|
|
|
offset += i * test->blockSize;
|
|
|
|
} else {
|
|
|
|
offset += (i * test->numTasks * test->blockSize) + (pretendRank * test->blockSize);
|
|
|
|
}
|
|
|
|
WriteOrReadSingle(offset, pretendRank, test->randomPrefillBlocksize, & transferCount, & errors, test, fd, ioBuffers, WRITE);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ioBuffers->buffer = oldBuffer;
|
|
|
|
}
|
|
|
|
|
2011-06-17 23:20:43 +04:00
|
|
|
/*
|
|
|
|
* Write or Read data to file(s). This loops through the strides, writing
|
|
|
|
* out the data to each block in transfer sizes, until the remainder left is 0.
|
|
|
|
*/
|
2018-09-19 20:06:05 +03:00
|
|
|
static IOR_offset_t WriteOrRead(IOR_param_t *test, IOR_results_t *results,
|
2020-05-31 14:11:00 +03:00
|
|
|
aiori_fd_t *fd, const int access, IOR_io_buffers *ioBuffers)
|
2011-06-17 23:20:43 +04:00
|
|
|
{
|
2011-11-12 02:22:17 +04:00
|
|
|
int errors = 0;
|
2012-01-13 08:34:40 +04:00
|
|
|
IOR_offset_t transferCount = 0;
|
2018-07-07 13:42:21 +03:00
|
|
|
uint64_t pairCnt = 0;
|
2011-11-12 02:22:17 +04:00
|
|
|
int pretendRank;
|
|
|
|
IOR_offset_t dataMoved = 0; /* for data rate calculation */
|
|
|
|
double startForStonewall;
|
|
|
|
int hitStonewall;
|
2020-11-05 17:00:09 +03:00
|
|
|
int i, j;
|
2018-09-20 00:39:25 +03:00
|
|
|
IOR_point_t *point = ((access == WRITE) || (access == WRITECHECK)) ?
|
|
|
|
&results->write : &results->read;
|
2011-11-12 02:22:17 +04:00
|
|
|
|
|
|
|
/* initialize values */
|
|
|
|
pretendRank = (rank + rankOffset) % test->numTasks;
|
|
|
|
|
2020-11-05 17:00:09 +03:00
|
|
|
// offsetArray = GetOffsetArraySequential(test, pretendRank);
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2020-11-06 13:27:11 +03:00
|
|
|
IOR_offset_t offsets;
|
|
|
|
IOR_offset_t * offsets_rnd;
|
2011-11-12 02:22:17 +04:00
|
|
|
if (test->randomOffset) {
|
2020-11-06 13:27:11 +03:00
|
|
|
offsets_rnd = GetOffsetArrayRandom(test, pretendRank, & offsets);
|
|
|
|
}else{
|
|
|
|
offsets = (test->blockSize / test->transferSize);
|
|
|
|
}
|
|
|
|
|
2020-12-01 17:45:07 +03:00
|
|
|
void * randomPrefillBuffer = NULL;
|
|
|
|
if(test->randomPrefillBlocksize && (access == WRITE || access == WRITECHECK)){
|
2021-02-18 13:40:42 +03:00
|
|
|
randomPrefillBuffer = aligned_buffer_alloc(test->randomPrefillBlocksize, test->gpuMemoryFlags);
|
2020-12-01 17:45:07 +03:00
|
|
|
// store invalid data into the buffer
|
|
|
|
memset(randomPrefillBuffer, -1, test->randomPrefillBlocksize);
|
2011-06-17 23:20:43 +04:00
|
|
|
}
|
|
|
|
|
2020-12-01 16:52:29 +03:00
|
|
|
// start timer after random offset was generated
|
2011-11-12 02:22:17 +04:00
|
|
|
startForStonewall = GetTimeStamp();
|
2018-07-15 21:38:17 +03:00
|
|
|
hitStonewall = 0;
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2020-12-01 17:45:07 +03:00
|
|
|
if(randomPrefillBuffer && test->deadlineForStonewalling == 0){
|
|
|
|
double t_start = GetTimeStamp();
|
|
|
|
prefillSegment(test, randomPrefillBuffer, pretendRank, fd, ioBuffers, 0, test->segmentCount);
|
|
|
|
if(rank == 0 && verbose > VERBOSE_1){
|
|
|
|
fprintf(out_logfile, "Random prefill took: %fs\n", GetTimeStamp() - t_start);
|
2020-12-01 16:52:29 +03:00
|
|
|
}
|
2020-12-01 17:47:57 +03:00
|
|
|
// must synchronize processes to ensure they are not running ahead
|
|
|
|
MPI_Barrier(test->testComm);
|
2020-12-01 16:52:29 +03:00
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
|
2020-11-05 17:00:09 +03:00
|
|
|
for (i = 0; i < test->segmentCount && !hitStonewall; i++) {
|
2020-12-01 17:45:07 +03:00
|
|
|
if(randomPrefillBuffer && test->deadlineForStonewalling != 0){
|
|
|
|
// prefill the whole segment with data, this needs to be done collectively
|
|
|
|
double t_start = GetTimeStamp();
|
|
|
|
prefillSegment(test, randomPrefillBuffer, pretendRank, fd, ioBuffers, i, i+1);
|
2020-12-01 17:47:57 +03:00
|
|
|
MPI_Barrier(test->testComm);
|
2020-12-01 17:45:07 +03:00
|
|
|
if(rank == 0 && verbose > VERBOSE_1){
|
|
|
|
fprintf(out_logfile, "Random: synchronizing segment count with barrier and prefill took: %fs\n", GetTimeStamp() - t_start);
|
|
|
|
}
|
2020-12-01 16:52:29 +03:00
|
|
|
}
|
2020-11-06 13:27:11 +03:00
|
|
|
for (j = 0; j < offsets && !hitStonewall ; j++) {
|
2020-11-05 17:00:09 +03:00
|
|
|
IOR_offset_t offset;
|
|
|
|
if (test->randomOffset) {
|
2020-11-06 13:27:11 +03:00
|
|
|
if(test->filePerProc){
|
|
|
|
offset = offsets_rnd[j] + (i * test->blockSize);
|
|
|
|
}else{
|
|
|
|
offset = offsets_rnd[j] + (i * test->numTasks * test->blockSize);
|
|
|
|
}
|
2020-11-05 17:00:09 +03:00
|
|
|
}else{
|
|
|
|
offset = j * test->transferSize;
|
|
|
|
if (test->filePerProc) {
|
|
|
|
offset += i * test->blockSize;
|
|
|
|
} else {
|
|
|
|
offset += (i * test->numTasks * test->blockSize) + (pretendRank * test->blockSize);
|
|
|
|
}
|
|
|
|
}
|
2020-12-01 16:52:29 +03:00
|
|
|
dataMoved += WriteOrReadSingle(offset, pretendRank, test->transferSize, & transferCount, & errors, test, fd, ioBuffers, access);
|
2020-11-05 17:00:09 +03:00
|
|
|
pairCnt++;
|
2019-12-19 16:42:34 +03:00
|
|
|
|
2020-11-05 17:00:09 +03:00
|
|
|
hitStonewall = ((test->deadlineForStonewalling != 0
|
|
|
|
&& (GetTimeStamp() - startForStonewall) > test->deadlineForStonewalling))
|
|
|
|
|| (test->stoneWallingWearOutIterations != 0 && pairCnt == test->stoneWallingWearOutIterations) ;
|
2019-12-19 16:42:34 +03:00
|
|
|
|
2020-11-05 17:00:09 +03:00
|
|
|
if ( test->collective && test->deadlineForStonewalling ) {
|
|
|
|
// if collective-mode, you'll get a HANG, if some rank 'accidentally' leave this loop
|
|
|
|
// it absolutely must be an 'all or none':
|
2021-01-20 17:35:06 +03:00
|
|
|
MPI_CHECK(MPI_Bcast(&hitStonewall, 1, MPI_INT, 0, testComm), "hitStonewall broadcast failed");
|
2020-11-05 17:00:09 +03:00
|
|
|
}
|
|
|
|
}
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2017-10-20 19:02:24 +03:00
|
|
|
if (test->stoneWallingWearOut){
|
2018-07-07 13:42:21 +03:00
|
|
|
if (verbose >= VERBOSE_1){
|
|
|
|
fprintf(out_logfile, "%d: stonewalling pairs accessed: %lld\n", rank, (long long) pairCnt);
|
|
|
|
}
|
|
|
|
long long data_moved_ll = (long long) dataMoved;
|
|
|
|
long long pairs_accessed_min = 0;
|
2018-09-20 00:39:25 +03:00
|
|
|
MPI_CHECK(MPI_Allreduce(& pairCnt, &point->pairs_accessed,
|
2017-10-20 19:02:24 +03:00
|
|
|
1, MPI_LONG_LONG_INT, MPI_MAX, testComm), "cannot reduce pairs moved");
|
2018-07-07 13:42:21 +03:00
|
|
|
double stonewall_runtime = GetTimeStamp() - startForStonewall;
|
2018-09-20 00:39:25 +03:00
|
|
|
point->stonewall_time = stonewall_runtime;
|
2018-07-07 13:42:21 +03:00
|
|
|
MPI_CHECK(MPI_Reduce(& pairCnt, & pairs_accessed_min,
|
|
|
|
1, MPI_LONG_LONG_INT, MPI_MIN, 0, testComm), "cannot reduce pairs moved");
|
2018-09-20 00:39:25 +03:00
|
|
|
MPI_CHECK(MPI_Reduce(& data_moved_ll, &point->stonewall_min_data_accessed,
|
2018-07-07 13:42:21 +03:00
|
|
|
1, MPI_LONG_LONG_INT, MPI_MIN, 0, testComm), "cannot reduce pairs moved");
|
2020-12-04 00:07:45 +03:00
|
|
|
MPI_CHECK(MPI_Reduce(& data_moved_ll, &point->stonewall_total_data_accessed,
|
2018-07-07 13:42:21 +03:00
|
|
|
1, MPI_LONG_LONG_INT, MPI_SUM, 0, testComm), "cannot reduce pairs moved");
|
|
|
|
|
|
|
|
if(rank == 0){
|
2020-12-04 00:07:45 +03:00
|
|
|
point->stonewall_avg_data_accessed = point->stonewall_total_data_accessed / test->numTasks;
|
2018-07-07 13:42:21 +03:00
|
|
|
fprintf(out_logfile, "stonewalling pairs accessed min: %lld max: %zu -- min data: %.1f GiB mean data: %.1f GiB time: %.1fs\n",
|
2018-09-20 00:39:25 +03:00
|
|
|
pairs_accessed_min, point->pairs_accessed,
|
2020-12-04 00:07:45 +03:00
|
|
|
point->stonewall_min_data_accessed /1024.0 / 1024 / 1024, point->stonewall_avg_data_accessed / 1024.0 / 1024 / 1024 , point->stonewall_time);
|
2018-07-07 13:42:21 +03:00
|
|
|
}
|
2018-09-20 00:39:25 +03:00
|
|
|
if(pairCnt != point->pairs_accessed){
|
2021-01-20 01:28:34 +03:00
|
|
|
// some work needs still to be done, complete the current block !
|
|
|
|
i--;
|
|
|
|
if(j == offsets){
|
|
|
|
j = 0; // current block is completed
|
2021-01-20 02:19:01 +03:00
|
|
|
i++;
|
2021-01-20 01:28:34 +03:00
|
|
|
}
|
2020-11-05 17:00:09 +03:00
|
|
|
for ( ; pairCnt < point->pairs_accessed; i++) {
|
2021-01-20 01:28:34 +03:00
|
|
|
for ( ; j < offsets && pairCnt < point->pairs_accessed ; j++) {
|
2020-11-05 17:00:09 +03:00
|
|
|
IOR_offset_t offset;
|
|
|
|
if (test->randomOffset) {
|
2020-11-06 13:27:11 +03:00
|
|
|
if(test->filePerProc){
|
|
|
|
offset = offsets_rnd[j] + (i * test->blockSize);
|
|
|
|
}else{
|
|
|
|
offset = offsets_rnd[j] + (i * test->numTasks * test->blockSize);
|
|
|
|
}
|
2020-11-05 17:00:09 +03:00
|
|
|
}else{
|
|
|
|
offset = j * test->transferSize;
|
|
|
|
if (test->filePerProc) {
|
|
|
|
offset += i * test->blockSize;
|
|
|
|
} else {
|
|
|
|
offset += (i * test->numTasks * test->blockSize) + (pretendRank * test->blockSize);
|
|
|
|
}
|
|
|
|
}
|
2020-12-01 16:52:29 +03:00
|
|
|
dataMoved += WriteOrReadSingle(offset, pretendRank, test->transferSize, & transferCount, & errors, test, fd, ioBuffers, access);
|
2020-11-05 17:00:09 +03:00
|
|
|
pairCnt++;
|
|
|
|
}
|
2021-01-20 01:28:34 +03:00
|
|
|
j = 0;
|
2017-10-20 19:02:24 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}else{
|
2018-09-20 00:39:25 +03:00
|
|
|
point->pairs_accessed = pairCnt;
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
totalErrorCount += CountErrors(test, access, errors);
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
if (access == WRITE && test->fsync == TRUE) {
|
2020-05-31 13:50:15 +03:00
|
|
|
backend->fsync(fd, test->backend_options); /*fsync after all accesses */
|
2011-11-12 02:22:17 +04:00
|
|
|
}
|
2020-12-01 17:45:07 +03:00
|
|
|
if(randomPrefillBuffer){
|
2021-02-18 13:40:42 +03:00
|
|
|
aligned_buffer_free(randomPrefillBuffer, test->gpuMemoryFlags);
|
2020-12-01 17:45:07 +03:00
|
|
|
}
|
|
|
|
|
2011-11-12 02:22:17 +04:00
|
|
|
return (dataMoved);
|
2011-11-12 03:11:28 +04:00
|
|
|
}
|