diff --git a/src/ior.c b/src/ior.c index 3509cc5..3cf435b 100755 --- a/src/ior.c +++ b/src/ior.c @@ -53,7 +53,7 @@ static char *PrependDir(IOR_param_t *, char *); static char **ParseFileName(char *, int *); static void InitTests(IOR_test_t * , MPI_Comm); static void TestIoSys(IOR_test_t *); -static void ValidateTests(IOR_param_t *); +static void ValidateTests(IOR_param_t * params, MPI_Comm com); static IOR_offset_t WriteOrRead(IOR_param_t *test, IOR_results_t *results, aiori_fd_t *fd, const int access, IOR_io_buffers *ioBuffers); @@ -107,12 +107,11 @@ IOR_test_t * ior_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out IOR_test_t *tptr; out_logfile = world_out; out_resultfile = world_out; - mpi_comm_world = world_com; - MPI_CHECK(MPI_Comm_rank(mpi_comm_world, &rank), "cannot get rank"); + MPI_CHECK(MPI_Comm_rank(world_com, &rank), "cannot get rank"); /* setup tests, and validate parameters */ - tests_head = ParseCommandLine(argc, argv); + tests_head = ParseCommandLine(argc, argv, world_com); InitTests(tests_head, world_com); PrintHeader(argc, argv); @@ -147,20 +146,19 @@ int ior_main(int argc, char **argv) /* * check -h option from commandline without starting MPI; */ - tests_head = ParseCommandLine(argc, argv); + tests_head = ParseCommandLine(argc, argv, MPI_COMM_WORLD); /* start the MPI code */ MPI_CHECK(MPI_Init(&argc, &argv), "cannot initialize MPI"); - mpi_comm_world = MPI_COMM_WORLD; - MPI_CHECK(MPI_Comm_rank(mpi_comm_world, &rank), "cannot get rank"); + MPI_CHECK(MPI_Comm_rank(MPI_COMM_WORLD, &rank), "cannot get rank"); /* set error-handling */ /*MPI_CHECK(MPI_Errhandler_set(mpi_comm_world, MPI_ERRORS_RETURN), "cannot set errhandler"); */ /* setup tests, and validate parameters */ - InitTests(tests_head, mpi_comm_world); + InitTests(tests_head, MPI_COMM_WORLD); PrintHeader(argc, argv); @@ -201,7 +199,7 @@ int ior_main(int argc, char **argv) /* * Initialize an IOR_param_t structure to the defaults */ -void init_IOR_Param_t(IOR_param_t * p) +void init_IOR_Param_t(IOR_param_t * p, MPI_Comm com) { const char *default_aiori = aiori_default (); assert (NULL != default_aiori); @@ -231,7 +229,8 @@ void init_IOR_Param_t(IOR_param_t * p) p->transferSize = 262144; p->randomSeed = -1; p->incompressibleSeed = 573; - p->testComm = mpi_comm_world; + p->testComm = com; // this com might change for smaller tests + p->mpi_comm_world = com; p->URI = NULL; } @@ -567,7 +566,7 @@ static void DestroyTests(IOR_test_t *tests_head) /* * Distribute IOR_HINTs to all tasks' environments. */ -void DistributeHints(void) +static void DistributeHints(MPI_Comm com) { char hint[MAX_HINTS][MAX_STR], fullHint[MAX_STR], hintVariable[MAX_STR]; int hintCount = 0, i; @@ -589,9 +588,9 @@ void DistributeHints(void) } } - MPI_CHECK(MPI_Bcast(&hintCount, sizeof(hintCount), MPI_BYTE, 0, testComm), "cannot broadcast hints"); + MPI_CHECK(MPI_Bcast(&hintCount, sizeof(hintCount), MPI_BYTE, 0, com), "cannot broadcast hints"); for (i = 0; i < hintCount; i++) { - MPI_CHECK(MPI_Bcast(&hint[i], MAX_STR, MPI_BYTE, 0, testComm), + MPI_CHECK(MPI_Bcast(&hint[i], MAX_STR, MPI_BYTE, 0, com), "cannot broadcast hints"); strcpy(fullHint, hint[i]); strcpy(hintVariable, strtok(fullHint, "=")); @@ -973,7 +972,7 @@ static void InitTests(IOR_test_t *tests, MPI_Comm com) * task 0 has the environment settings for the hints, pass * the hint=value pair to everyone else in mpi_comm_world */ - DistributeHints(); + DistributeHints(com); /* check validity of tests and create test queue */ while (tests != NULL) { @@ -1002,7 +1001,7 @@ static void InitTests(IOR_test_t *tests, MPI_Comm com) params->expectedAggFileSize = params->blockSize * params->segmentCount * params->numTasks; - ValidateTests(&tests->params); + ValidateTests(&tests->params, com); tests = tests->next; } @@ -1069,7 +1068,7 @@ static void file_hits_histogram(IOR_param_t *params) } MPI_CHECK(MPI_Gather(&rankOffset, 1, MPI_INT, rankoffs, - 1, MPI_INT, 0, mpi_comm_world), + 1, MPI_INT, 0, params->testComm), "MPI_Gather error"); if (rank != 0) @@ -1225,21 +1224,21 @@ static void TestIoSys(IOR_test_t *test) IOR_io_buffers ioBuffers; /* set up communicator for test */ - MPI_CHECK(MPI_Comm_group(mpi_comm_world, &orig_group), + MPI_CHECK(MPI_Comm_group(params->mpi_comm_world, &orig_group), "MPI_Comm_group() error"); range[0] = 0; /* first rank */ range[1] = params->numTasks - 1; /* last rank */ range[2] = 1; /* stride */ MPI_CHECK(MPI_Group_range_incl(orig_group, 1, &range, &new_group), "MPI_Group_range_incl() error"); - MPI_CHECK(MPI_Comm_create(mpi_comm_world, new_group, &testComm), + MPI_CHECK(MPI_Comm_create(params->mpi_comm_world, new_group, &testComm), "MPI_Comm_create() error"); MPI_CHECK(MPI_Group_free(&orig_group), "MPI_Group_Free() error"); MPI_CHECK(MPI_Group_free(&new_group), "MPI_Group_Free() error"); params->testComm = testComm; if (testComm == MPI_COMM_NULL) { /* tasks not in the group do not participate in this test */ - MPI_CHECK(MPI_Barrier(mpi_comm_world), "barrier error"); + MPI_CHECK(MPI_Barrier(params->mpi_comm_world), "barrier error"); return; } if (rank == 0 && verbose >= VERBOSE_1) { @@ -1536,17 +1535,16 @@ static void TestIoSys(IOR_test_t *test) free(hog_buf); /* Sync with the tasks that did not participate in this test */ - MPI_CHECK(MPI_Barrier(mpi_comm_world), "barrier error"); - + MPI_CHECK(MPI_Barrier(params->mpi_comm_world), "barrier error"); } /* * Determine if valid tests from parameters. */ -static void ValidateTests(IOR_param_t * test) +static void ValidateTests(IOR_param_t * test, MPI_Comm com) { IOR_param_t defaults; - init_IOR_Param_t(&defaults); + init_IOR_Param_t(&defaults, com); if (test->repetitions <= 0) WARN_RESET("too few test repetitions", diff --git a/src/ior.h b/src/ior.h index 87940a0..6252f78 100755 --- a/src/ior.h +++ b/src/ior.h @@ -98,7 +98,8 @@ typedef struct char * options; /* options string */ // intermediate options int collective; /* collective I/O */ - MPI_Comm testComm; /* MPI communicator */ + MPI_Comm testComm; /* Current MPI communicator */ + MPI_Comm mpi_comm_world; /* The global MPI communicator */ int dryRun; /* do not perform any I/Os just run evtl. inputs print dummy output */ int dualMount; /* dual mount points */ int numTasks; /* number of tasks for test */ @@ -205,7 +206,7 @@ IOR_test_t *CreateTest(IOR_param_t *init_params, int test_num); void AllocResults(IOR_test_t *test); char * GetPlatformName(void); -void init_IOR_Param_t(IOR_param_t *p); +void init_IOR_Param_t(IOR_param_t *p, MPI_Comm global_com); /* * This function runs IOR given by command line, useful for testing diff --git a/src/mdtest.c b/src/mdtest.c index 5386739..74cb297 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1944,7 +1944,6 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * testComm = world_com; out_logfile = world_out; out_resultfile = world_out; - mpi_comm_world = world_com; init_clock(world_com); diff --git a/src/parse_options.c b/src/parse_options.c index 1a2ad7e..05fa78f 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -32,7 +32,7 @@ #include "option.h" #include "aiori.h" -IOR_param_t initialTestParams; +static IOR_param_t initialTestParams; option_help * createGlobalOptions(IOR_param_t * params); @@ -451,9 +451,9 @@ option_help * createGlobalOptions(IOR_param_t * params){ /* * Parse Commandline. */ -IOR_test_t *ParseCommandLine(int argc, char **argv) +IOR_test_t *ParseCommandLine(int argc, char **argv, MPI_Comm com) { - init_IOR_Param_t(& initialTestParams); + init_IOR_Param_t(& initialTestParams, com); IOR_test_t *tests = NULL; diff --git a/src/parse_options.h b/src/parse_options.h index 45b93ca..b12dd78 100755 --- a/src/parse_options.h +++ b/src/parse_options.h @@ -13,8 +13,6 @@ #include "ior.h" -extern IOR_param_t initialTestParams; - -IOR_test_t *ParseCommandLine(int argc, char **argv); +IOR_test_t *ParseCommandLine(int argc, char **argv, MPI_Comm com); #endif /* !_PARSE_OPTIONS_H */ diff --git a/src/utilities.c b/src/utilities.c index cb2deae..cf79e00 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -65,7 +65,6 @@ int rank = 0; int rankOffset = 0; int verbose = VERBOSE_0; /* verbose output */ MPI_Comm testComm; -MPI_Comm mpi_comm_world; FILE * out_logfile = NULL; FILE * out_resultfile = NULL; enum OutputFormat_t outputFormat; diff --git a/src/utilities.h b/src/utilities.h index dd6d16f..1eff2d2 100755 --- a/src/utilities.h +++ b/src/utilities.h @@ -22,7 +22,6 @@ extern int rank; extern int rankOffset; extern int verbose; extern MPI_Comm testComm; -extern MPI_Comm mpi_comm_world; extern FILE * out_resultfile; extern enum OutputFormat_t outputFormat; /* format of the output */ diff --git a/testing/complex-tests.sh b/testing/complex-tests.sh index c314cf9..a04c14d 100755 --- a/testing/complex-tests.sh +++ b/testing/complex-tests.sh @@ -19,13 +19,13 @@ MDTEST 2 -I 20 -a DUMMY -x stonewall-md.log -T -v MDTEST 2 -I 20 -a DUMMY -x stonewall-md.log -D -v #shared tests -IOR 2 -a POSIX -w -z -Y -e -i1 -m -t 100k -b 100k -IOR 2 -a POSIX -w -k -e -i1 -m -t 100k -b 100k -IOR 2 -a POSIX -r -z-k -e -i1 -m -t 100k -b 100k +IOR 2 -a POSIX -w -z -Y -e -i1 -m -t 100k -b 200k +IOR 2 -a POSIX -w -k -e -i1 -m -t 100k -b 200k +IOR 2 -a POSIX -r -z-k -e -i1 -m -t 100k -b 200k #test mutually exclusive options -IOR 2 -a POSIX -w -z -k -e -i1 -m -t 100k -b 100k -IOR 2 -a POSIX -w -z -k -e -i1 -m -t 100k -b 100k +IOR 2 -a POSIX -w -z -k -e -i1 -m -t 100k -b 200k +IOR 2 -a POSIX -w -z -k -e -i1 -m -t 100k -b 200k IOR 2 -a POSIX -w -Z -i1 -m -t 100k -b 100k -d 0.1 # Now set the num tasks per node to 1: