vitalif
/
mdtest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support IO redirect from main()

master
Julian M. Kunkel 2018-07-08 13:07:32 +01:00
parent 0052bff3e9
commit 0f7a1f14b9
7 changed files with 45 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
doc/USER_GUIDE
View File

@ -23,10 +23,10 @@ Index:
*******************
* 1. DESCRIPTION *
*******************
IOR can be used for testing performance of parallel file systems using various
interfaces and access patterns. IOR uses MPI for process synchronization.
IOR version 2 is a complete rewrite of the original IOR (Interleaved-Or-Random)
version 1 code.
IOR can be used for testing performance of parallel file systems using various
interfaces and access patterns. IOR uses MPI for process synchronization.
IOR version 2 is a complete rewrite of the original IOR (Interleaved-Or-Random)
version 1 code.
******************
@ -39,7 +39,7 @@ Two ways to run IOR:
E.g., to execute: IOR -w -r -o filename
This performs a write and a read to the file 'filename'.
* Command line with scripts -- any arguments on the command line will
* Command line with scripts -- any arguments on the command line will
establish the default for the test run, but a script may be used in
conjunction with this for varying specific tests during an execution of the
code.
@ -125,7 +125,7 @@ GENERAL:
S3_EMC, or NCMPI, depending on test [POSIX]
* testFile - name of the output file [testFile]
NOTE: with filePerProc set, the tasks can round
NOTE: with filePerProc set, the tasks can round
robin across multiple file names '-o S@S@S'
* hintsFileName - name of the hints file []
@ -267,7 +267,7 @@ GENERAL:
data, this option measures the amount of
data moved in a fixed amount of time. The
objective is to prevent tasks slow to
complete from skewing the performance.
complete from skewing the performance.
* setting this to zero (0) unsets this option
* this option is incompatible w/data checking
@ -280,7 +280,8 @@ GENERAL:
* summaryAlways - Always print the long summary for each test.
Useful for long runs that may be interrupted, preventing
the final long summary for ALL tests to be printed.
* summaryFile=File - Output the summary to the file instead on stdout/stderr.
* summaryFormat=FMT - Choose the output format -- default, JSON, CSV
POSIX-ONLY:
===========
@ -319,7 +320,7 @@ HDF5-ONLY:
NOTE: default IOR creates a dataset the size of
numTasks * blockSize to be accessed by all
tasks
* noFill - no pre-filling of data in HDF5 file creation [0=FALSE]
* setAlignment - HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g) [1]
@ -483,8 +484,8 @@ zip, gzip, and bzip.
2) gzip: For gzipped files, a transfer size of 1k is sufficient.
3) bzip2: For bziped files a transfer size of 1k is insufficient (~50% compressed).
To avoid compression a transfer size of greater than the bzip block size is required
3) bzip2: For bziped files a transfer size of 1k is insufficient (~50% compressed).
To avoid compression a transfer size of greater than the bzip block size is required
(default = 900KB). I suggest a transfer size of greather than 1MB to avoid bzip2 compression.
Be aware of the block size your compression algorithm will look at, and adjust the transfer size
@ -508,9 +509,9 @@ HOW DO I PERFORM MULTIPLE DATA CHECKS ON AN EXISTING FILE?
and -r implied using both. This semantic has been subsequently altered to be
omitting -w, -r, -W, and -R implied using both -w and -r.)
If you're running new tests to create a file and want repeat data checking on
this file multiple times, there is an undocumented option for this. It's -O
multiReRead=1, and you'd need to have an IOR version compiled with the
If you're running new tests to create a file and want repeat data checking on
this file multiple times, there is an undocumented option for this. It's -O
multiReRead=1, and you'd need to have an IOR version compiled with the
USE_UNDOC_OPT=1 (in iordef.h). The command line would look like this:
IOR -k -E -w -W -i 5 -o file -O multiReRead=1
@ -586,7 +587,7 @@ HOW DO I USE STONEWALLING?
actually reading the same amount from disk in the allotted time, but they
are also reading the cached data from the previous test each time to get the
increased performance. Setting -D high enough so that the cache is
overfilled will prevent this.
overfilled will prevent this.
HOW DO I BYPASS CACHING WHEN READING BACK A FILE I'VE JUST WRITTEN?

11
src/ior.c
View File

@ -90,6 +90,7 @@ IOR_test_t * ior_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out
for (tptr = tests_head; tptr != NULL; tptr = tptr->next) {
totalErrorCount = 0;
verbose = tptr->params.verbose;
tptr->params.testComm = world_com;
if (rank == 0 && verbose >= VERBOSE_0) {
ShowTestInfo(&tptr->params);
}
@ -125,6 +126,7 @@ int ior_main(int argc, char **argv)
int i;
IOR_test_t *tests_head;
IOR_test_t *tptr;
out_logfile = stdout;
/*
@ -153,6 +155,7 @@ int ior_main(int argc, char **argv)
MPI_CHECK(MPI_Comm_size(mpi_comm_world, &numTasksWorld),
"cannot get number of tasks");
MPI_CHECK(MPI_Comm_rank(mpi_comm_world, &rank), "cannot get rank");
PrintEarlyHeader();
/* set error-handling */
@ -230,6 +233,8 @@ int ior_main(int argc, char **argv)
aws_cleanup();
#endif
fflush(out_logfile);
return totalErrorCount;
}
@ -1668,7 +1673,6 @@ static void PrintLongSummaryOneOperation(IOR_test_t *test, double *times, char *
struct results *bw;
struct results *ops;
int reps;
if (rank != 0 || verbose < VERBOSE_0)
return;
@ -1682,14 +1686,12 @@ static void PrintLongSummaryOneOperation(IOR_test_t *test, double *times, char *
fprintf(out_logfile, "%10.2f ", bw->max / MEBIBYTE);
fprintf(out_logfile, "%10.2f ", bw->min / MEBIBYTE);
fprintf(out_logfile, "%10.2f ", bw->mean / MEBIBYTE);
fprintf(out_logfile, "%10.2f ", bw->sd / MEBIBYTE);
fprintf(out_logfile, "%10.2f ", ops->max);
fprintf(out_logfile, "%10.2f ", ops->min);
fprintf(out_logfile, "%10.2f ", ops->mean);
fprintf(out_logfile, "%10.2f ", ops->sd);
fprintf(out_logfile, "%10.5f ",
mean_of_array_of_doubles(times, reps));
fprintf(out_logfile, "%10.5f ", mean_of_array_of_doubles(times, reps));
fprintf(out_logfile, "%5d ", params->id);
fprintf(out_logfile, "%6d ", params->numTasks);
fprintf(out_logfile, "%3d ", params->tasksPerNode);
@ -1742,7 +1744,6 @@ static void PrintLongSummaryHeader()
static void PrintLongSummaryAllTests(IOR_test_t *tests_head)
{
IOR_test_t *tptr;
if (rank != 0 || verbose < VERBOSE_0)
return;

1
src/ior.h
View File

@ -82,7 +82,6 @@ typedef struct IO_BUFFERS
typedef struct
{
FILE * out_logfile;
char debug[MAX_STR]; /* debug info string */
unsigned int mode; /* file permissions */
unsigned int openFlags; /* open flags (see also <open>) */

6
src/iordef.h
View File

@ -61,6 +61,12 @@ extern int verbose; /* verbose output */
/*************************** D E F I N I T I O N S ****************************/
enum OutputFormat_t{
OUTPUT_DEFAULT,
OUTPUT_CSV,
OUTPUT_JSON
};
#ifndef FALSE
# define FALSE 0
#endif /* not FALSE */

15
src/parse_options.c
View File

@ -170,6 +170,21 @@ void DecodeDirective(char *line, IOR_param_t *params)
}
if (strcasecmp(option, "api") == 0) {
strcpy(params->api, value);
} else if (strcasecmp(option, "summaryFile") == 0) {
out_logfile = fopen(value, "w");
if (out_logfile == NULL){
FAIL("Cannot open output file for writes!");
}
} else if (strcasecmp(option, "summaryFormat") == 0) {
if(strcasecmp(value, "default")){
outputFormat = OUTPUT_DEFAULT;
}else if(strcasecmp(value, "JSON")){
outputFormat = OUTPUT_JSON;
}else if(strcasecmp(value, "CSV")){
outputFormat = OUTPUT_CSV;
}else{
FAIL("Unknown summaryFormat");
}
} else if (strcasecmp(option, "refnum") == 0) {
params->referenceNumber = atoi(value);
} else if (strcasecmp(option, "debug") == 0) {

1
src/utilities.c
View File

@ -57,6 +57,7 @@ int verbose = VERBOSE_0; /* verbose output */
MPI_Comm testComm;
MPI_Comm mpi_comm_world;
FILE * out_logfile;
enum OutputFormat_t outputFormat;
/***************************** F U N C T I O N S ******************************/

1
src/utilities.h
View File

@ -26,6 +26,7 @@ extern int verbose;
extern MPI_Comm testComm;
extern MPI_Comm mpi_comm_world;
extern FILE * out_logfile;
extern enum OutputFormat_t outputFormat; /* format of the output */
/*
* Try using the system's PATH_MAX, which is what realpath and such use.