Extracted file size routine that is replicated in each module into IOR removing the MPI dependency from AIORI.
parent
3395fc621c
commit
82417128cd
|
@ -114,7 +114,7 @@ static void DFS_Delete(char *, aiori_mod_opt_t *);
|
|||
static char* DFS_GetVersion();
|
||||
static void DFS_Fsync(aiori_fd_t *, aiori_mod_opt_t *);
|
||||
static void DFS_Sync(aiori_mod_opt_t *);
|
||||
static IOR_offset_t DFS_GetFileSize(aiori_mod_opt_t *, MPI_Comm, char *);
|
||||
static IOR_offset_t DFS_GetFileSize(aiori_mod_opt_t *, char *);
|
||||
static int DFS_Statfs (const char *, ior_aiori_statfs_t *, aiori_mod_opt_t *);
|
||||
static int DFS_Stat (const char *, struct stat *, aiori_mod_opt_t *);
|
||||
static int DFS_Mkdir (const char *, mode_t, aiori_mod_opt_t *);
|
||||
|
@ -774,7 +774,7 @@ static char* DFS_GetVersion()
|
|||
* Use DFS stat() to return aggregate file size.
|
||||
*/
|
||||
static IOR_offset_t
|
||||
DFS_GetFileSize(aiori_mod_opt_t * test, MPI_Comm comm, char *testFileName)
|
||||
DFS_GetFileSize(aiori_mod_opt_t * test, char *testFileName)
|
||||
{
|
||||
dfs_obj_t *obj;
|
||||
daos_size_t fsize, tmpMin, tmpMax, tmpSum;
|
||||
|
@ -792,27 +792,6 @@ DFS_GetFileSize(aiori_mod_opt_t * test, MPI_Comm comm, char *testFileName)
|
|||
|
||||
dfs_release(obj);
|
||||
|
||||
if (hints->filePerProc == TRUE) {
|
||||
MPI_CHECK(MPI_Allreduce(&fsize, &tmpSum, 1,
|
||||
MPI_LONG_LONG_INT, MPI_SUM, comm),
|
||||
"cannot total data moved");
|
||||
fsize = tmpSum;
|
||||
} else {
|
||||
MPI_CHECK(MPI_Allreduce(&fsize, &tmpMin, 1,
|
||||
MPI_LONG_LONG_INT, MPI_MIN, comm),
|
||||
"cannot total data moved");
|
||||
MPI_CHECK(MPI_Allreduce(&fsize, &tmpMax, 1,
|
||||
MPI_LONG_LONG_INT, MPI_MAX, comm),
|
||||
"cannot total data moved");
|
||||
if (tmpMin != tmpMax) {
|
||||
if (rank == 0) {
|
||||
WARN("inconsistent file size by different tasks");
|
||||
}
|
||||
/* incorrect, but now consistent across tasks */
|
||||
fsize = tmpMin;
|
||||
}
|
||||
}
|
||||
|
||||
return (fsize);
|
||||
}
|
||||
|
||||
|
|
|
@ -108,7 +108,7 @@ static char * DUMMY_getVersion()
|
|||
return "0.5";
|
||||
}
|
||||
|
||||
static IOR_offset_t DUMMY_GetFileSize(aiori_mod_opt_t * options, MPI_Comm testComm, char *testFileName)
|
||||
static IOR_offset_t DUMMY_GetFileSize(aiori_mod_opt_t * options, char *testFileName)
|
||||
{
|
||||
if(verbose > 4){
|
||||
fprintf(out_logfile, "DUMMY getFileSize: %s\n", testFileName);
|
||||
|
|
|
@ -91,7 +91,7 @@ static void HDF5_Close(aiori_fd_t *, aiori_mod_opt_t *);
|
|||
static void HDF5_Delete(char *, aiori_mod_opt_t *);
|
||||
static char* HDF5_GetVersion();
|
||||
static void HDF5_Fsync(aiori_fd_t *, aiori_mod_opt_t *);
|
||||
static IOR_offset_t HDF5_GetFileSize(aiori_mod_opt_t *, MPI_Comm, char *);
|
||||
static IOR_offset_t HDF5_GetFileSize(aiori_mod_opt_t *, char *);
|
||||
static int HDF5_Access(const char *, int, aiori_mod_opt_t *);
|
||||
static void HDF5_init_xfer_options(aiori_xfer_hint_t * params);
|
||||
static int HDF5_check_params(aiori_mod_opt_t * options);
|
||||
|
@ -660,11 +660,11 @@ static void SetupDataSet(void *fd, int flags, aiori_mod_opt_t * param)
|
|||
* Use MPIIO call to get file size.
|
||||
*/
|
||||
static IOR_offset_t
|
||||
HDF5_GetFileSize(aiori_mod_opt_t * test, MPI_Comm testComm, char *testFileName)
|
||||
HDF5_GetFileSize(aiori_mod_opt_t * test, char *testFileName)
|
||||
{
|
||||
if(hints->dryRun)
|
||||
return 0;
|
||||
return(MPIIO_GetFileSize(test, testComm, testFileName));
|
||||
return(MPIIO_GetFileSize(test, testFileName));
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
@ -562,8 +562,7 @@ static IOR_offset_t SeekOffset(MPI_File fd, IOR_offset_t offset,
|
|||
* Use MPI_File_get_size() to return aggregate file size.
|
||||
* NOTE: This function is used by the HDF5 and NCMPI backends.
|
||||
*/
|
||||
IOR_offset_t MPIIO_GetFileSize(aiori_mod_opt_t * module_options, MPI_Comm testComm,
|
||||
char *testFileName)
|
||||
IOR_offset_t MPIIO_GetFileSize(aiori_mod_opt_t * module_options, char *testFileName)
|
||||
{
|
||||
mpiio_options_t * test = (mpiio_options_t*) module_options;
|
||||
if(hints->dryRun)
|
||||
|
@ -589,26 +588,5 @@ IOR_offset_t MPIIO_GetFileSize(aiori_mod_opt_t * module_options, MPI_Comm testCo
|
|||
if (mpiHints != MPI_INFO_NULL)
|
||||
MPI_CHECK(MPI_Info_free(&mpiHints), "MPI_Info_free failed");
|
||||
|
||||
if (hints->filePerProc == TRUE) {
|
||||
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1,
|
||||
MPI_LONG_LONG_INT, MPI_SUM, testComm),
|
||||
"cannot total data moved");
|
||||
aggFileSizeFromStat = tmpSum;
|
||||
} else {
|
||||
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1,
|
||||
MPI_LONG_LONG_INT, MPI_MIN, testComm),
|
||||
"cannot total data moved");
|
||||
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1,
|
||||
MPI_LONG_LONG_INT, MPI_MAX, testComm),
|
||||
"cannot total data moved");
|
||||
if (tmpMin != tmpMax) {
|
||||
if (rank == 0) {
|
||||
WARN("inconsistent file size by different tasks");
|
||||
}
|
||||
/* incorrect, but now consistent across tasks */
|
||||
aggFileSizeFromStat = tmpMin;
|
||||
}
|
||||
}
|
||||
|
||||
return (aggFileSizeFromStat);
|
||||
}
|
||||
|
|
|
@ -676,8 +676,7 @@ void POSIX_Delete(char *testFileName, aiori_mod_opt_t * param)
|
|||
/*
|
||||
* Use POSIX stat() to return aggregate file size.
|
||||
*/
|
||||
IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t * test, MPI_Comm testComm,
|
||||
char *testFileName)
|
||||
IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t * test, char *testFileName)
|
||||
{
|
||||
if(hints->dryRun)
|
||||
return 0;
|
||||
|
@ -689,26 +688,5 @@ IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t * test, MPI_Comm testComm,
|
|||
}
|
||||
aggFileSizeFromStat = stat_buf.st_size;
|
||||
|
||||
if (hints->filePerProc == TRUE) {
|
||||
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1,
|
||||
MPI_LONG_LONG_INT, MPI_SUM, testComm),
|
||||
"cannot total data moved");
|
||||
aggFileSizeFromStat = tmpSum;
|
||||
} else {
|
||||
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1,
|
||||
MPI_LONG_LONG_INT, MPI_MIN, testComm),
|
||||
"cannot total data moved");
|
||||
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1,
|
||||
MPI_LONG_LONG_INT, MPI_MAX, testComm),
|
||||
"cannot total data moved");
|
||||
if (tmpMin != tmpMax) {
|
||||
if (rank == 0) {
|
||||
WARN("inconsistent file size by different tasks");
|
||||
}
|
||||
/* incorrect, but now consistent across tasks */
|
||||
aggFileSizeFromStat = tmpMin;
|
||||
}
|
||||
}
|
||||
|
||||
return (aggFileSizeFromStat);
|
||||
}
|
||||
|
|
|
@ -4,6 +4,7 @@
|
|||
/* This file contains only debug relevant helpers */
|
||||
|
||||
#include <stdio.h>
|
||||
#include <mpi.h>
|
||||
|
||||
extern FILE * out_logfile;
|
||||
extern int verbose; /* verbose output */
|
||||
|
|
22
src/aiori.h
22
src/aiori.h
|
@ -15,12 +15,6 @@
|
|||
#ifndef _AIORI_H
|
||||
#define _AIORI_H
|
||||
|
||||
#include <mpi.h>
|
||||
|
||||
#ifndef MPI_FILE_NULL
|
||||
# include <mpio.h>
|
||||
#endif /* not MPI_FILE_NULL */
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
|
@ -101,12 +95,12 @@ typedef struct ior_aiori {
|
|||
*/
|
||||
void (*xfer_hints)(aiori_xfer_hint_t * params);
|
||||
IOR_offset_t (*xfer)(int access, aiori_fd_t *, IOR_size_t *,
|
||||
IOR_offset_t size, IOR_offset_t offset, aiori_mod_opt_t *);
|
||||
void (*close)(aiori_fd_t *, aiori_mod_opt_t *);
|
||||
void (*delete)(char *, aiori_mod_opt_t *);
|
||||
IOR_offset_t size, IOR_offset_t offset, aiori_mod_opt_t * module_options);
|
||||
void (*close)(aiori_fd_t *, aiori_mod_opt_t * module_options);
|
||||
void (*delete)(char *, aiori_mod_opt_t * module_options);
|
||||
char* (*get_version)(void);
|
||||
void (*fsync)(aiori_fd_t *, aiori_mod_opt_t *);
|
||||
IOR_offset_t (*get_file_size)(aiori_mod_opt_t * module_options, MPI_Comm, char *);
|
||||
void (*fsync)(aiori_fd_t *, aiori_mod_opt_t * module_options);
|
||||
IOR_offset_t (*get_file_size)(aiori_mod_opt_t * module_options, char * filename);
|
||||
int (*statfs) (const char *, ior_aiori_statfs_t *, aiori_mod_opt_t * module_options);
|
||||
int (*mkdir) (const char *path, mode_t mode, aiori_mod_opt_t * module_options);
|
||||
int (*rmdir) (const char *path, aiori_mod_opt_t * module_options);
|
||||
|
@ -164,7 +158,7 @@ void aiori_posix_xfer_hints(aiori_xfer_hint_t * params);
|
|||
aiori_fd_t *POSIX_Create(char *testFileName, int flags, aiori_mod_opt_t * module_options);
|
||||
int POSIX_Mknod(char *testFileName);
|
||||
aiori_fd_t *POSIX_Open(char *testFileName, int flags, aiori_mod_opt_t * module_options);
|
||||
IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t * test, MPI_Comm testComm, char *testFileName);
|
||||
IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t * test, char *testFileName);
|
||||
void POSIX_Delete(char *testFileName, aiori_mod_opt_t * module_options);
|
||||
void POSIX_Close(aiori_fd_t *fd, aiori_mod_opt_t * module_options);
|
||||
option_help * POSIX_options(aiori_mod_opt_t ** init_backend_options, aiori_mod_opt_t * init_values);
|
||||
|
@ -172,7 +166,7 @@ option_help * POSIX_options(aiori_mod_opt_t ** init_backend_options, aiori_mod_o
|
|||
|
||||
/* NOTE: these 3 MPI-IO functions are exported for reuse by HDF5/PNetCDF */
|
||||
void MPIIO_Delete(char *testFileName, aiori_mod_opt_t * module_options);
|
||||
IOR_offset_t MPIIO_GetFileSize(aiori_mod_opt_t * options, MPI_Comm testComm, char *testFileName);
|
||||
int MPIIO_Access(const char *, int, aiori_mod_opt_t *);
|
||||
IOR_offset_t MPIIO_GetFileSize(aiori_mod_opt_t * options, char *testFileName);
|
||||
int MPIIO_Access(const char *, int, aiori_mod_opt_t * module_options);
|
||||
|
||||
#endif /* not _AIORI_H */
|
||||
|
|
|
@ -313,8 +313,7 @@ CheckForOutliers(IOR_param_t *test, const double *timer, const int access)
|
|||
* Check if actual file size equals expected size; if not use actual for
|
||||
* calculating performance rate.
|
||||
*/
|
||||
static void CheckFileSize(IOR_test_t *test, char * testFilename, IOR_offset_t dataMoved, int rep,
|
||||
const int access)
|
||||
static void CheckFileSize(IOR_test_t *test, char * testFilename, IOR_offset_t dataMoved, int rep, const int access)
|
||||
{
|
||||
IOR_param_t *params = &test->params;
|
||||
IOR_results_t *results = test->results;
|
||||
|
@ -323,7 +322,7 @@ static void CheckFileSize(IOR_test_t *test, char * testFilename, IOR_offset_t da
|
|||
|
||||
/* get the size of the file */
|
||||
IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum;
|
||||
aggFileSizeFromStat = backend->get_file_size(params->backend_options, testComm, testFilename);
|
||||
aggFileSizeFromStat = backend->get_file_size(params->backend_options, testFilename);
|
||||
|
||||
if (params->hints.filePerProc == TRUE) {
|
||||
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1,
|
||||
|
|
|
@ -39,6 +39,12 @@
|
|||
#include "iordef.h"
|
||||
#include "aiori.h"
|
||||
|
||||
#include <mpi.h>
|
||||
|
||||
#ifndef MPI_FILE_NULL
|
||||
# include <mpio.h>
|
||||
#endif /* not MPI_FILE_NULL */
|
||||
|
||||
#define ISPOWEROFTWO(x) ((x != 0) && !(x & (x - 1)))
|
||||
/******************** DATA Packet Type ***************************************/
|
||||
/* Holds the types of data packets: generic, offset, timestamp, incompressible */
|
||||
|
|
Loading…
Reference in New Issue