Updated HDF5 to new module specification.
Julian M. Kunkel
parent
aa9e2ad815
commit
930ccdc68d
202
src/aiori-HDF5.c
202
src/aiori-HDF5.c
|
|
@ -81,22 +81,29 @@
|
|||
#endif /* H5_VERS_MAJOR > 1 && H5_VERS_MINOR > 6 */
|
||||
/**************************** P R O T O T Y P E S *****************************/
|
||||
|
||||
static IOR_offset_t SeekOffset(void *, IOR_offset_t, IOR_param_t *);
|
||||
static void SetupDataSet(void *, IOR_param_t *);
|
||||
static void *HDF5_Create(char *, IOR_param_t *);
|
||||
static void *HDF5_Open(char *, IOR_param_t *);
|
||||
static IOR_offset_t SeekOffset(void *, IOR_offset_t, void *);
|
||||
static void SetupDataSet(void *, void *);
|
||||
static void *HDF5_Create(char *, int flags, void *);
|
||||
static void *HDF5_Open(char *, int flags, void *);
|
||||
static IOR_offset_t HDF5_Xfer(int, void *, IOR_size_t *,
|
||||
IOR_offset_t, IOR_param_t *);
|
||||
static void HDF5_Close(void *, IOR_param_t *);
|
||||
static void HDF5_Delete(char *, IOR_param_t *);
|
||||
IOR_offset_t, void *);
|
||||
static void HDF5_Close(void *, void *);
|
||||
static void HDF5_Delete(char *, void *);
|
||||
static char* HDF5_GetVersion();
|
||||
static void HDF5_Fsync(void *, IOR_param_t *);
|
||||
static IOR_offset_t HDF5_GetFileSize(IOR_param_t *, MPI_Comm, char *);
|
||||
static int HDF5_Access(const char *, int, IOR_param_t *);
|
||||
static void HDF5_Fsync(void *, void *);
|
||||
static IOR_offset_t HDF5_GetFileSize(void *, MPI_Comm, char *);
|
||||
static int HDF5_Access(const char *, int, void *);
|
||||
static void HDF5_init_xfer_options(IOR_param_t * params);
|
||||
static int HDF5_check_params(void * options);
|
||||
|
||||
/************************** O P T I O N S *****************************/
|
||||
typedef struct{
|
||||
int collective_md;
|
||||
char * hintsFileName; /* full name for hints file */
|
||||
int showHints; /* show hints */
|
||||
int individualDataSets; /* datasets not shared by all procs */
|
||||
int noFill; /* no fill in file creation */
|
||||
IOR_offset_t setAlignment; /* alignment in bytes */
|
||||
} HDF5_options_t;
|
||||
/***************************** F U N C T I O N S ******************************/
|
||||
|
||||
|
|
@ -106,14 +113,21 @@ static option_help * HDF5_options(void ** init_backend_options, void * init_valu
|
|||
if (init_values != NULL){
|
||||
memcpy(o, init_values, sizeof(HDF5_options_t));
|
||||
}else{
|
||||
memset(o, 0, sizeof(HDF5_options_t));
|
||||
/* initialize the options properly */
|
||||
o->collective_md = 0;
|
||||
o->setAlignment = 1;
|
||||
}
|
||||
|
||||
*init_backend_options = o;
|
||||
|
||||
option_help h [] = {
|
||||
{0, "hdf5.collectiveMetadata", "Use collectiveMetadata (available since HDF5-1.10.0)", OPTION_FLAG, 'd', & o->collective_md},
|
||||
{0, "hdf5.hintsFileName","Full name for hints file", OPTION_OPTIONAL_ARGUMENT, 's', & o->hintsFileName},
|
||||
{0, "hdf5.showHints", "Show MPI hints", OPTION_FLAG, 'd', & o->showHints},
|
||||
{0, "hdf5.individualDataSets", "Datasets not shared by all procs [not working]", OPTION_FLAG, 'd', & o->individualDataSets},
|
||||
{0, "hdf5.setAlignment", "HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'd', & o->setAlignment},
|
||||
{0, "hdf5.noFill", "No fill in HDF5 file creation", OPTION_FLAG, 'd', & o->noFill},
|
||||
LAST_OPTION
|
||||
};
|
||||
option_help * help = malloc(sizeof(h));
|
||||
|
|
@ -133,6 +147,7 @@ ior_aiori_t hdf5_aiori = {
|
|||
.close = HDF5_Close,
|
||||
.delete = HDF5_Delete,
|
||||
.get_version = HDF5_GetVersion,
|
||||
.init_xfer_options = HDF5_init_xfer_options,
|
||||
.fsync = HDF5_Fsync,
|
||||
.get_file_size = HDF5_GetFileSize,
|
||||
.statfs = aiori_posix_statfs,
|
||||
|
|
@ -140,7 +155,8 @@ ior_aiori_t hdf5_aiori = {
|
|||
.rmdir = aiori_posix_rmdir,
|
||||
.access = HDF5_Access,
|
||||
.stat = aiori_posix_stat,
|
||||
.get_options = HDF5_options
|
||||
.get_options = HDF5_options,
|
||||
.check_params = HDF5_check_params
|
||||
};
|
||||
|
||||
static hid_t xferPropList; /* xfer property list */
|
||||
|
|
@ -151,20 +167,47 @@ hid_t memDataSpace; /* memory data space id */
|
|||
int newlyOpenedFile; /* newly opened file */
|
||||
|
||||
/***************************** F U N C T I O N S ******************************/
|
||||
static IOR_param_t * ior_param = NULL;
|
||||
|
||||
static void HDF5_init_xfer_options(IOR_param_t * params){
|
||||
ior_param = params;
|
||||
}
|
||||
|
||||
static int HDF5_check_params(void * options){
|
||||
HDF5_options_t *o = (HDF5_options_t*) options;
|
||||
if (o->setAlignment < 0)
|
||||
ERR("alignment must be non-negative integer");
|
||||
if (o->individualDataSets)
|
||||
ERR("individual data sets not implemented");
|
||||
if (o->noFill) {
|
||||
/* check if hdf5 available */
|
||||
#if defined (H5_VERS_MAJOR) && defined (H5_VERS_MINOR)
|
||||
/* no-fill option not available until hdf5-1.6.x */
|
||||
#if (H5_VERS_MAJOR > 0 && H5_VERS_MINOR > 5)
|
||||
#else
|
||||
ERRF("'no fill' option not available in HDF5");
|
||||
#endif
|
||||
#else
|
||||
WARN("unable to determine HDF5 version for 'no fill' usage");
|
||||
#endif
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Create and open a file through the HDF5 interface.
|
||||
*/
|
||||
static void *HDF5_Create(char *testFileName, IOR_param_t * param)
|
||||
static void *HDF5_Create(char *testFileName, int flags, void * param)
|
||||
{
|
||||
return HDF5_Open(testFileName, param);
|
||||
return HDF5_Open(testFileName, flags, param);
|
||||
}
|
||||
|
||||
/*
|
||||
* Open a file through the HDF5 interface.
|
||||
*/
|
||||
static void *HDF5_Open(char *testFileName, IOR_param_t * param)
|
||||
static void *HDF5_Open(char *testFileName, int flags, void * param)
|
||||
{
|
||||
HDF5_options_t *o = (HDF5_options_t*) param;
|
||||
hid_t accessPropList, createPropList;
|
||||
hsize_t memStart[NUM_DIMS],
|
||||
dataSetDims[NUM_DIMS],
|
||||
|
|
@ -182,36 +225,27 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
|
|||
/*
|
||||
* HDF5 uses different flags than those for POSIX/MPIIO
|
||||
*/
|
||||
if (param->open == WRITE) { /* WRITE flags */
|
||||
param->openFlags = IOR_TRUNC;
|
||||
} else { /* READ or check WRITE/READ flags */
|
||||
param->openFlags = IOR_RDONLY;
|
||||
}
|
||||
|
||||
/* set IOR file flags to HDF5 flags */
|
||||
/* -- file open flags -- */
|
||||
if (param->openFlags & IOR_RDONLY) {
|
||||
if (flags & IOR_RDONLY) {
|
||||
fd_mode |= H5F_ACC_RDONLY;
|
||||
}
|
||||
if (param->openFlags & IOR_WRONLY) {
|
||||
fprintf(stdout, "File write only not implemented in HDF5\n");
|
||||
}
|
||||
if (param->openFlags & IOR_RDWR) {
|
||||
if (flags & IOR_WRONLY || flags & IOR_RDWR) {
|
||||
fd_mode |= H5F_ACC_RDWR;
|
||||
}
|
||||
if (param->openFlags & IOR_APPEND) {
|
||||
if (flags & IOR_APPEND) {
|
||||
fprintf(stdout, "File append not implemented in HDF5\n");
|
||||
}
|
||||
if (param->openFlags & IOR_CREAT) {
|
||||
if (flags & IOR_CREAT) {
|
||||
fd_mode |= H5F_ACC_CREAT;
|
||||
}
|
||||
if (param->openFlags & IOR_EXCL) {
|
||||
if (flags & IOR_EXCL) {
|
||||
fd_mode |= H5F_ACC_EXCL;
|
||||
}
|
||||
if (param->openFlags & IOR_TRUNC) {
|
||||
if (flags & IOR_TRUNC) {
|
||||
fd_mode |= H5F_ACC_TRUNC;
|
||||
}
|
||||
if (param->openFlags & IOR_DIRECT) {
|
||||
if (flags & IOR_DIRECT) {
|
||||
fprintf(stdout, "O_DIRECT not implemented in HDF5\n");
|
||||
}
|
||||
|
||||
|
|
@ -231,13 +265,13 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
|
|||
* someday HDF5 implementation will allow subsets of MPI_COMM_WORLD
|
||||
*/
|
||||
/* store MPI communicator info for the file access property list */
|
||||
if (param->filePerProc) {
|
||||
if (ior_param->filePerProc) {
|
||||
comm = MPI_COMM_SELF;
|
||||
} else {
|
||||
comm = testComm;
|
||||
}
|
||||
|
||||
SetHints(&mpiHints, param->hintsFileName);
|
||||
SetHints(&mpiHints, o->hintsFileName);
|
||||
/*
|
||||
* note that with MP_HINTS_FILTERED=no, all key/value pairs will
|
||||
* be in the info object. The info object that is attached to
|
||||
|
|
@ -245,7 +279,7 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
|
|||
* deemed valid by the implementation.
|
||||
*/
|
||||
/* show hints passed to file */
|
||||
if (rank == 0 && param->showHints) {
|
||||
if (rank == 0 && o->showHints) {
|
||||
fprintf(stdout, "\nhints passed to access property list {\n");
|
||||
ShowHints(&mpiHints);
|
||||
fprintf(stdout, "}\n");
|
||||
|
|
@ -254,12 +288,11 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
|
|||
"cannot set file access property list");
|
||||
|
||||
/* set alignment */
|
||||
HDF5_CHECK(H5Pset_alignment(accessPropList, param->setAlignment,
|
||||
param->setAlignment),
|
||||
HDF5_CHECK(H5Pset_alignment(accessPropList, o->setAlignment, o->setAlignment),
|
||||
"cannot set alignment");
|
||||
|
||||
#ifdef HAVE_H5PSET_ALL_COLL_METADATA_OPS
|
||||
HDF5_options_t *o = (HDF5_options_t*) param->backend_options;
|
||||
|
||||
if (o->collective_md) {
|
||||
/* more scalable metadata */
|
||||
|
||||
|
|
@ -271,10 +304,9 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
|
|||
#endif
|
||||
|
||||
/* open file */
|
||||
if(! param->dryRun){
|
||||
if (param->open == WRITE) { /* WRITE */
|
||||
*fd = H5Fcreate(testFileName, fd_mode,
|
||||
createPropList, accessPropList);
|
||||
if(! ior_param->dryRun){
|
||||
if (ior_param->open == WRITE) { /* WRITE */
|
||||
*fd = H5Fcreate(testFileName, H5F_ACC_TRUNC, createPropList, accessPropList);
|
||||
HDF5_CHECK(*fd, "cannot create file");
|
||||
} else { /* READ or CHECK */
|
||||
*fd = H5Fopen(testFileName, fd_mode, accessPropList);
|
||||
|
|
@ -283,9 +315,8 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
|
|||
}
|
||||
|
||||
/* show hints actually attached to file handle */
|
||||
if (param->showHints || (1) /* WEL - this needs fixing */ ) {
|
||||
if (rank == 0
|
||||
&& (param->showHints) /* WEL - this needs fixing */ ) {
|
||||
if (o->showHints || (1) /* WEL - this needs fixing */ ) {
|
||||
if (rank == 0 && (o->showHints) /* WEL - this needs fixing */ ) {
|
||||
WARN("showHints not working for HDF5");
|
||||
}
|
||||
} else {
|
||||
|
|
@ -334,7 +365,7 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
|
|||
HDF5_CHECK(xferPropList, "cannot create transfer property list");
|
||||
|
||||
/* set data transfer mode */
|
||||
if (param->collective) {
|
||||
if (ior_param->collective) {
|
||||
HDF5_CHECK(H5Pset_dxpl_mpio(xferPropList, H5FD_MPIO_COLLECTIVE),
|
||||
"cannot set collective data transfer mode");
|
||||
} else {
|
||||
|
|
@ -346,9 +377,9 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
|
|||
/* set up memory data space for transfer */
|
||||
memStart[0] = (hsize_t) 0;
|
||||
memCount[0] = (hsize_t) 1;
|
||||
memStride[0] = (hsize_t) (param->transferSize / sizeof(IOR_size_t));
|
||||
memBlock[0] = (hsize_t) (param->transferSize / sizeof(IOR_size_t));
|
||||
memDataSpaceDims[0] = (hsize_t) param->transferSize;
|
||||
memStride[0] = (hsize_t) (ior_param->transferSize / sizeof(IOR_size_t));
|
||||
memBlock[0] = (hsize_t) (ior_param->transferSize / sizeof(IOR_size_t));
|
||||
memDataSpaceDims[0] = (hsize_t) ior_param->transferSize;
|
||||
memDataSpace = H5Screate_simple(NUM_DIMS, memDataSpaceDims, NULL);
|
||||
HDF5_CHECK(memDataSpace, "cannot create simple memory data space");
|
||||
|
||||
|
|
@ -358,18 +389,18 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
|
|||
memBlock), "cannot create hyperslab");
|
||||
|
||||
/* set up parameters for fpp or different dataset count */
|
||||
if (param->filePerProc) {
|
||||
if (ior_param->filePerProc) {
|
||||
tasksPerDataSet = 1;
|
||||
} else {
|
||||
if (param->individualDataSets) {
|
||||
if (o->individualDataSets) {
|
||||
/* each task in segment has single data set */
|
||||
tasksPerDataSet = 1;
|
||||
} else {
|
||||
/* share single data set across all tasks in segment */
|
||||
tasksPerDataSet = param->numTasks;
|
||||
tasksPerDataSet = ior_param->numTasks;
|
||||
}
|
||||
}
|
||||
dataSetDims[0] = (hsize_t) ((param->blockSize / sizeof(IOR_size_t))
|
||||
dataSetDims[0] = (hsize_t) ((ior_param->blockSize / sizeof(IOR_size_t))
|
||||
* tasksPerDataSet);
|
||||
|
||||
/* create a simple data space containing information on size
|
||||
|
|
@ -386,7 +417,7 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
|
|||
* Write or read access to file using the HDF5 interface.
|
||||
*/
|
||||
static IOR_offset_t HDF5_Xfer(int access, void *fd, IOR_size_t * buffer,
|
||||
IOR_offset_t length, IOR_param_t * param)
|
||||
IOR_offset_t length, void * param)
|
||||
{
|
||||
static int firstReadCheck = FALSE, startNewDataSet;
|
||||
IOR_offset_t segmentPosition, segmentSize;
|
||||
|
|
@ -405,17 +436,16 @@ static IOR_offset_t HDF5_Xfer(int access, void *fd, IOR_size_t * buffer,
|
|||
}
|
||||
|
||||
/* determine by offset if need to start new data set */
|
||||
if (param->filePerProc == TRUE) {
|
||||
if (ior_param->filePerProc == TRUE) {
|
||||
segmentPosition = (IOR_offset_t) 0;
|
||||
segmentSize = param->blockSize;
|
||||
segmentSize = ior_param->blockSize;
|
||||
} else {
|
||||
segmentPosition =
|
||||
(IOR_offset_t) ((rank + rankOffset) % param->numTasks)
|
||||
* param->blockSize;
|
||||
segmentSize =
|
||||
(IOR_offset_t) (param->numTasks) * param->blockSize;
|
||||
(IOR_offset_t) ((rank + rankOffset) % ior_param->numTasks)
|
||||
* ior_param->blockSize;
|
||||
segmentSize = (IOR_offset_t) (ior_param->numTasks) * ior_param->blockSize;
|
||||
}
|
||||
if ((IOR_offset_t) ((param->offset - segmentPosition) % segmentSize) ==
|
||||
if ((IOR_offset_t) ((ior_param->offset - segmentPosition) % segmentSize) ==
|
||||
0) {
|
||||
/*
|
||||
* ordinarily start a new data set, unless this is the
|
||||
|
|
@ -427,7 +457,7 @@ static IOR_offset_t HDF5_Xfer(int access, void *fd, IOR_size_t * buffer,
|
|||
}
|
||||
}
|
||||
|
||||
if(param->dryRun)
|
||||
if(ior_param->dryRun)
|
||||
return length;
|
||||
|
||||
/* create new data set */
|
||||
|
|
@ -441,7 +471,7 @@ static IOR_offset_t HDF5_Xfer(int access, void *fd, IOR_size_t * buffer,
|
|||
SetupDataSet(fd, param);
|
||||
}
|
||||
|
||||
SeekOffset(fd, param->offset, param);
|
||||
SeekOffset(fd, ior_param->offset, param);
|
||||
|
||||
/* this is necessary to reset variables for reaccessing file */
|
||||
startNewDataSet = FALSE;
|
||||
|
|
@ -465,7 +495,7 @@ static IOR_offset_t HDF5_Xfer(int access, void *fd, IOR_size_t * buffer,
|
|||
/*
|
||||
* Perform fsync().
|
||||
*/
|
||||
static void HDF5_Fsync(void *fd, IOR_param_t * param)
|
||||
static void HDF5_Fsync(void *fd, void * param)
|
||||
{
|
||||
;
|
||||
}
|
||||
|
|
@ -473,11 +503,11 @@ static void HDF5_Fsync(void *fd, IOR_param_t * param)
|
|||
/*
|
||||
* Close a file through the HDF5 interface.
|
||||
*/
|
||||
static void HDF5_Close(void *fd, IOR_param_t * param)
|
||||
static void HDF5_Close(void *fd, void * param)
|
||||
{
|
||||
if(param->dryRun)
|
||||
if(ior_param->dryRun)
|
||||
return;
|
||||
if (param->fd_fppReadCheck == NULL) {
|
||||
//if (ior_param->fd_fppReadCheck == NULL) {
|
||||
HDF5_CHECK(H5Dclose(dataSet), "cannot close data set");
|
||||
HDF5_CHECK(H5Sclose(dataSpace), "cannot close data space");
|
||||
HDF5_CHECK(H5Sclose(fileDataSpace),
|
||||
|
|
@ -486,7 +516,7 @@ static void HDF5_Close(void *fd, IOR_param_t * param)
|
|||
"cannot close memory data space");
|
||||
HDF5_CHECK(H5Pclose(xferPropList),
|
||||
" cannot close transfer property list");
|
||||
}
|
||||
//}
|
||||
HDF5_CHECK(H5Fclose(*(hid_t *) fd), "cannot close file");
|
||||
free(fd);
|
||||
}
|
||||
|
|
@ -494,9 +524,9 @@ static void HDF5_Close(void *fd, IOR_param_t * param)
|
|||
/*
|
||||
* Delete a file through the HDF5 interface.
|
||||
*/
|
||||
static void HDF5_Delete(char *testFileName, IOR_param_t * param)
|
||||
static void HDF5_Delete(char *testFileName, void * param)
|
||||
{
|
||||
if(param->dryRun)
|
||||
if(ior_param->dryRun)
|
||||
return
|
||||
MPIIO_Delete(testFileName, param);
|
||||
return;
|
||||
|
|
@ -528,23 +558,24 @@ static char * HDF5_GetVersion()
|
|||
* Seek to offset in file using the HDF5 interface and set up hyperslab.
|
||||
*/
|
||||
static IOR_offset_t SeekOffset(void *fd, IOR_offset_t offset,
|
||||
IOR_param_t * param)
|
||||
void * param)
|
||||
{
|
||||
HDF5_options_t *o = (HDF5_options_t*) param;
|
||||
IOR_offset_t segmentSize;
|
||||
hsize_t hsStride[NUM_DIMS], hsCount[NUM_DIMS], hsBlock[NUM_DIMS];
|
||||
hsize_t hsStart[NUM_DIMS];
|
||||
|
||||
if (param->filePerProc == TRUE) {
|
||||
segmentSize = (IOR_offset_t) param->blockSize;
|
||||
if (ior_param->filePerProc == TRUE) {
|
||||
segmentSize = (IOR_offset_t) ior_param->blockSize;
|
||||
} else {
|
||||
segmentSize =
|
||||
(IOR_offset_t) (param->numTasks) * param->blockSize;
|
||||
(IOR_offset_t) (ior_param->numTasks) * ior_param->blockSize;
|
||||
}
|
||||
|
||||
/* create a hyperslab representing the file data space */
|
||||
if (param->individualDataSets) {
|
||||
if (o->individualDataSets) {
|
||||
/* start at zero offset if not */
|
||||
hsStart[0] = (hsize_t) ((offset % param->blockSize)
|
||||
hsStart[0] = (hsize_t) ((offset % ior_param->blockSize)
|
||||
/ sizeof(IOR_size_t));
|
||||
} else {
|
||||
/* start at a unique offset if shared */
|
||||
|
|
@ -552,8 +583,8 @@ static IOR_offset_t SeekOffset(void *fd, IOR_offset_t offset,
|
|||
(hsize_t) ((offset % segmentSize) / sizeof(IOR_size_t));
|
||||
}
|
||||
hsCount[0] = (hsize_t) 1;
|
||||
hsStride[0] = (hsize_t) (param->transferSize / sizeof(IOR_size_t));
|
||||
hsBlock[0] = (hsize_t) (param->transferSize / sizeof(IOR_size_t));
|
||||
hsStride[0] = (hsize_t) (ior_param->transferSize / sizeof(IOR_size_t));
|
||||
hsBlock[0] = (hsize_t) (ior_param->transferSize / sizeof(IOR_size_t));
|
||||
|
||||
/* retrieve data space from data set for hyperslab */
|
||||
fileDataSpace = H5Dget_space(dataSet);
|
||||
|
|
@ -567,8 +598,9 @@ static IOR_offset_t SeekOffset(void *fd, IOR_offset_t offset,
|
|||
/*
|
||||
* Create HDF5 data set.
|
||||
*/
|
||||
static void SetupDataSet(void *fd, IOR_param_t * param)
|
||||
static void SetupDataSet(void *fd, void * param)
|
||||
{
|
||||
HDF5_options_t *o = (HDF5_options_t*) param;
|
||||
char dataSetName[MAX_STR];
|
||||
hid_t dataSetPropList;
|
||||
int dataSetID;
|
||||
|
|
@ -582,8 +614,8 @@ static void SetupDataSet(void *fd, IOR_param_t * param)
|
|||
dataSetSuffix = 0;
|
||||
|
||||
/* may want to use individual access to each data set someday */
|
||||
if (param->individualDataSets) {
|
||||
dataSetID = (rank + rankOffset) % param->numTasks;
|
||||
if (o->individualDataSets) {
|
||||
dataSetID = (rank + rankOffset) % ior_param->numTasks;
|
||||
} else {
|
||||
dataSetID = 0;
|
||||
}
|
||||
|
|
@ -591,14 +623,14 @@ static void SetupDataSet(void *fd, IOR_param_t * param)
|
|||
sprintf(dataSetName, "%s-%04d.%04d", "Dataset", dataSetID,
|
||||
dataSetSuffix++);
|
||||
|
||||
if (param->open == WRITE) { /* WRITE */
|
||||
if (ior_param->open == WRITE) { /* WRITE */
|
||||
/* create data set */
|
||||
dataSetPropList = H5Pcreate(H5P_DATASET_CREATE);
|
||||
/* check if hdf5 available */
|
||||
#if defined (H5_VERS_MAJOR) && defined (H5_VERS_MINOR)
|
||||
/* no-fill option not available until hdf5-1.6.x */
|
||||
#if (H5_VERS_MAJOR > 0 && H5_VERS_MINOR > 5)
|
||||
if (param->noFill == TRUE) {
|
||||
if (o->noFill == TRUE) {
|
||||
if (rank == 0 && verbose >= VERBOSE_1) {
|
||||
fprintf(stdout, "\nusing 'no fill' option\n");
|
||||
}
|
||||
|
|
@ -629,9 +661,9 @@ static void SetupDataSet(void *fd, IOR_param_t * param)
|
|||
* Use MPIIO call to get file size.
|
||||
*/
|
||||
static IOR_offset_t
|
||||
HDF5_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName)
|
||||
HDF5_GetFileSize(void * test, MPI_Comm testComm, char *testFileName)
|
||||
{
|
||||
if(test->dryRun)
|
||||
if(ior_param->dryRun)
|
||||
return 0;
|
||||
return(MPIIO_GetFileSize(test, testComm, testFileName));
|
||||
}
|
||||
|
|
@ -639,9 +671,9 @@ HDF5_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName)
|
|||
/*
|
||||
* Use MPIIO call to check for access.
|
||||
*/
|
||||
static int HDF5_Access(const char *path, int mode, IOR_param_t *param)
|
||||
static int HDF5_Access(const char *path, int mode, void *param)
|
||||
{
|
||||
if(param->dryRun)
|
||||
if(ior_param->dryRun)
|
||||
return 0;
|
||||
return(MPIIO_Access(path, mode, param));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ typedef struct {
|
|||
int preallocate; /* preallocate file size */
|
||||
int useSharedFilePointer; /* use shared file pointer */
|
||||
int useStridedDatatype; /* put strided access into datatype */
|
||||
char * hintsFileName; /* full name for hints file */
|
||||
char * hintsFileName; /* full name for hints file */
|
||||
} mpiio_options_t;
|
||||
|
||||
static option_help * MPIIO_options(void ** init_backend_options, void * init_values){
|
||||
|
|
|
|||
|
|
@ -355,7 +355,6 @@ void ShowTestStart(IOR_param_t *test)
|
|||
PrintKeyValInt("fsyncperwrite", test->fsyncPerWrite);
|
||||
PrintKeyValInt("useExistingTestFile", test->useExistingTestFile);
|
||||
PrintKeyValInt("uniqueDir", test->uniqueDir);
|
||||
PrintKeyValInt("individualDataSets", test->individualDataSets);
|
||||
PrintKeyValInt("singleXferAttempt", test->singleXferAttempt);
|
||||
PrintKeyValInt("readFile", test->readFile);
|
||||
PrintKeyValInt("writeFile", test->writeFile);
|
||||
|
|
@ -366,7 +365,6 @@ void ShowTestStart(IOR_param_t *test)
|
|||
PrintKeyValInt("randomOffset", test->randomOffset);
|
||||
PrintKeyValInt("checkWrite", test->checkWrite);
|
||||
PrintKeyValInt("checkRead", test->checkRead);
|
||||
PrintKeyValInt("setAlignment", test->setAlignment);
|
||||
PrintKeyValInt("storeFileOffset", test->storeFileOffset);
|
||||
PrintKeyValInt("keepFile", test->keepFile);
|
||||
PrintKeyValInt("keepFileWithError", test->keepFileWithError);
|
||||
|
|
|
|||
31
src/ior.c
31
src/ior.c
|
|
@ -68,7 +68,7 @@ static void test_initialize(IOR_test_t * test){
|
|||
backend->initialize(test->params.backend_options);
|
||||
}
|
||||
if(backend->init_xfer_options){
|
||||
backend->init_xfer_options(test);
|
||||
backend->init_xfer_options(& test->params);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -214,7 +214,6 @@ void init_IOR_Param_t(IOR_param_t * p)
|
|||
p->randomSeed = -1;
|
||||
p->incompressibleSeed = 573;
|
||||
p->testComm = mpi_comm_world;
|
||||
p->setAlignment = 1;
|
||||
p->lustre_start_ost = -1;
|
||||
|
||||
hdfs_user = getenv("USER");
|
||||
|
|
@ -1343,7 +1342,7 @@ static void TestIoSys(IOR_test_t *test)
|
|||
MPI_CHECK(MPI_Barrier(testComm), "barrier error");
|
||||
params->open = WRITE;
|
||||
timer[0] = GetTimeStamp();
|
||||
fd = backend->create(testFileName, IOR_WRONLY | IOR_CREAT, params->backend_options);
|
||||
fd = backend->create(testFileName, IOR_WRONLY | IOR_CREAT | IOR_TRUNC, params->backend_options);
|
||||
timer[1] = GetTimeStamp();
|
||||
if (params->intraTestBarriers)
|
||||
MPI_CHECK(MPI_Barrier(testComm),
|
||||
|
|
@ -1592,8 +1591,6 @@ static void ValidateTests(IOR_param_t * test)
|
|||
ERR("block size must be non-negative integer");
|
||||
if ((test->transferSize % sizeof(IOR_size_t)) != 0)
|
||||
ERR("transfer size must be a multiple of access size");
|
||||
if (test->setAlignment < 0)
|
||||
ERR("alignment must be non-negative integer");
|
||||
if (test->transferSize < 0)
|
||||
ERR("transfer size must be non-negative integer");
|
||||
if (test->transferSize == 0) {
|
||||
|
|
@ -1647,32 +1644,8 @@ static void ValidateTests(IOR_param_t * test)
|
|||
ERR("random offset not available with HDF5");
|
||||
if ((strcasecmp(test->api, "NCMPI") == 0) && test->randomOffset)
|
||||
ERR("random offset not available with NCMPI");
|
||||
if ((strcasecmp(test->api, "HDF5") != 0) && test->individualDataSets)
|
||||
WARN_RESET("individual datasets only available in HDF5",
|
||||
test, &defaults, individualDataSets);
|
||||
if ((strcasecmp(test->api, "HDF5") == 0) && test->individualDataSets)
|
||||
WARN_RESET("individual data sets not implemented",
|
||||
test, &defaults, individualDataSets);
|
||||
if ((strcasecmp(test->api, "NCMPI") == 0) && test->filePerProc)
|
||||
ERR("file-per-proc not available in current NCMPI");
|
||||
if (test->noFill) {
|
||||
if (strcasecmp(test->api, "HDF5") != 0) {
|
||||
ERR("'no fill' option only available in HDF5");
|
||||
} else {
|
||||
/* check if hdf5 available */
|
||||
#if defined (H5_VERS_MAJOR) && defined (H5_VERS_MINOR)
|
||||
/* no-fill option not available until hdf5-1.6.x */
|
||||
#if (H5_VERS_MAJOR > 0 && H5_VERS_MINOR > 5)
|
||||
;
|
||||
#else
|
||||
ERRF("'no fill' option not available in %s",
|
||||
test->apiVersion);
|
||||
#endif
|
||||
#else
|
||||
WARN("unable to determine HDF5 version for 'no fill' usage");
|
||||
#endif
|
||||
}
|
||||
}
|
||||
if (test->useExistingTestFile && test->lustre_set_striping)
|
||||
ERR("Lustre stripe options are incompatible with useExistingTestFile");
|
||||
|
||||
|
|
|
|||
|
|
@ -159,11 +159,6 @@ typedef struct
|
|||
MPI_Datatype transferType; /* datatype for transfer */
|
||||
MPI_Datatype fileType; /* filetype for file view */
|
||||
|
||||
/* HDF5 variables */
|
||||
int individualDataSets; /* datasets not shared by all procs */
|
||||
int noFill; /* no fill in file creation */
|
||||
IOR_offset_t setAlignment; /* alignment in bytes */
|
||||
|
||||
/* HDFS variables */
|
||||
char * hdfs_user; /* copied from ENV, for now */
|
||||
const char* hdfs_name_node;
|
||||
|
|
|
|||
|
|
@ -179,16 +179,10 @@ void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_opt
|
|||
params->blockSize = string_to_bytes(value);
|
||||
} else if (strcasecmp(option, "transfersize") == 0) {
|
||||
params->transferSize = string_to_bytes(value);
|
||||
} else if (strcasecmp(option, "setalignment") == 0) {
|
||||
params->setAlignment = string_to_bytes(value);
|
||||
} else if (strcasecmp(option, "singlexferattempt") == 0) {
|
||||
params->singleXferAttempt = atoi(value);
|
||||
} else if (strcasecmp(option, "individualdatasets") == 0) {
|
||||
params->individualDataSets = atoi(value);
|
||||
} else if (strcasecmp(option, "intraTestBarriers") == 0) {
|
||||
params->intraTestBarriers = atoi(value);
|
||||
} else if (strcasecmp(option, "nofill") == 0) {
|
||||
params->noFill = atoi(value);
|
||||
} else if (strcasecmp(option, "verbose") == 0) {
|
||||
params->verbose = atoi(value);
|
||||
} else if (strcasecmp(option, "settimestampsignature") == 0) {
|
||||
|
|
@ -458,15 +452,12 @@ option_help * createGlobalOptions(IOR_param_t * params){
|
|||
*/
|
||||
{'G', NULL, "setTimeStampSignature -- set value for time stamp signature/random seed", OPTION_OPTIONAL_ARGUMENT, 'd', & params->setTimeStampSignature},
|
||||
{'i', NULL, "repetitions -- number of repetitions of test", OPTION_OPTIONAL_ARGUMENT, 'd', & params->repetitions},
|
||||
{'I', NULL, "individualDataSets -- datasets not shared by all procs [not working]", OPTION_FLAG, 'd', & params->individualDataSets},
|
||||
{'j', NULL, "outlierThreshold -- warn on outlier N seconds from mean", OPTION_OPTIONAL_ARGUMENT, 'd', & params->outlierThreshold},
|
||||
{'J', NULL, "setAlignment -- HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'd', & params->setAlignment},
|
||||
{'k', NULL, "keepFile -- don't remove the test file(s) on program exit", OPTION_FLAG, 'd', & params->keepFile},
|
||||
{'K', NULL, "keepFileWithError -- keep error-filled file(s) after data-checking", OPTION_FLAG, 'd', & params->keepFileWithError},
|
||||
{'l', NULL, "datapacket type-- type of packet that will be created [offset|incompressible|timestamp|o|i|t]", OPTION_OPTIONAL_ARGUMENT, 's', & params->buffer_type},
|
||||
{'m', NULL, "multiFile -- use number of reps (-i) for multiple file count", OPTION_FLAG, 'd', & params->multiFile},
|
||||
{'M', NULL, "memoryPerNode -- hog memory on the node (e.g.: 2g, 75%)", OPTION_OPTIONAL_ARGUMENT, 's', & params->memoryPerNodeStr},
|
||||
{'n', NULL, "noFill -- no fill in HDF5 file creation", OPTION_FLAG, 'd', & params->noFill},
|
||||
{'N', NULL, "numTasks -- number of tasks that are participating in the test (overrides MPI)", OPTION_OPTIONAL_ARGUMENT, 'd', & params->numTasks},
|
||||
{'o', NULL, "testFile -- full name for test", OPTION_OPTIONAL_ARGUMENT, 's', & params->testFileName},
|
||||
{'O', NULL, "string of IOR directives (e.g. -O checkRead=1,lustreStripeCount=32)", OPTION_OPTIONAL_ARGUMENT, 'p', & decodeDirectiveWrapper},
|
||||
|
|
|
|||
Loading…
Reference in New Issue