vitalif
/
mdtest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Extracted file size routine that is replicated in each module into IOR removing the MPI dependency from AIORI.

master
Julian M. Kunkel 2020-06-30 12:33:56 +01:00
parent 3395fc621c
commit 82417128cd
9 changed files with 25 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/aiori-DFS.c
View File

@ -114,7 +114,7 @@ static void DFS_Delete(char *, aiori_mod_opt_t *);
static char* DFS_GetVersion();
static void DFS_Fsync(aiori_fd_t *, aiori_mod_opt_t *);
static void DFS_Sync(aiori_mod_opt_t *);
static IOR_offset_t DFS_GetFileSize(aiori_mod_opt_t *, MPI_Comm, char *);
static IOR_offset_t DFS_GetFileSize(aiori_mod_opt_t *, char *);
static int DFS_Statfs (const char *, ior_aiori_statfs_t *, aiori_mod_opt_t *);
static int DFS_Stat (const char *, struct stat *, aiori_mod_opt_t *);
static int DFS_Mkdir (const char *, mode_t, aiori_mod_opt_t *);
@ -774,7 +774,7 @@ static char* DFS_GetVersion()
* Use DFS stat() to return aggregate file size.
*/
static IOR_offset_t
DFS_GetFileSize(aiori_mod_opt_t * test, MPI_Comm comm, char *testFileName)
DFS_GetFileSize(aiori_mod_opt_t * test, char *testFileName)
{
dfs_obj_t *obj;
daos_size_t fsize, tmpMin, tmpMax, tmpSum;
@ -792,27 +792,6 @@ DFS_GetFileSize(aiori_mod_opt_t * test, MPI_Comm comm, char *testFileName)
dfs_release(obj);
if (hints->filePerProc == TRUE) {
MPI_CHECK(MPI_Allreduce(&fsize, &tmpSum, 1,
MPI_LONG_LONG_INT, MPI_SUM, comm),
"cannot total data moved");
fsize = tmpSum;
} else {
MPI_CHECK(MPI_Allreduce(&fsize, &tmpMin, 1,
MPI_LONG_LONG_INT, MPI_MIN, comm),
"cannot total data moved");
MPI_CHECK(MPI_Allreduce(&fsize, &tmpMax, 1,
MPI_LONG_LONG_INT, MPI_MAX, comm),
"cannot total data moved");
if (tmpMin != tmpMax) {
if (rank == 0) {
WARN("inconsistent file size by different tasks");
}
/* incorrect, but now consistent across tasks */
fsize = tmpMin;
}
}
return (fsize);
}

2
src/aiori-DUMMY.c
View File

@ -108,7 +108,7 @@ static char * DUMMY_getVersion()
return "0.5";
}
static IOR_offset_t DUMMY_GetFileSize(aiori_mod_opt_t * options, MPI_Comm testComm, char *testFileName)
static IOR_offset_t DUMMY_GetFileSize(aiori_mod_opt_t * options, char *testFileName)
{
if(verbose > 4){
fprintf(out_logfile, "DUMMY getFileSize: %s\n", testFileName);

6
src/aiori-HDF5.c
View File

@ -91,7 +91,7 @@ static void HDF5_Close(aiori_fd_t *, aiori_mod_opt_t *);
static void HDF5_Delete(char *, aiori_mod_opt_t *);
static char* HDF5_GetVersion();
static void HDF5_Fsync(aiori_fd_t *, aiori_mod_opt_t *);
static IOR_offset_t HDF5_GetFileSize(aiori_mod_opt_t *, MPI_Comm, char *);
static IOR_offset_t HDF5_GetFileSize(aiori_mod_opt_t *, char *);
static int HDF5_Access(const char *, int, aiori_mod_opt_t *);
static void HDF5_init_xfer_options(aiori_xfer_hint_t * params);
static int HDF5_check_params(aiori_mod_opt_t * options);
@ -660,11 +660,11 @@ static void SetupDataSet(void *fd, int flags, aiori_mod_opt_t * param)
* Use MPIIO call to get file size.
*/
static IOR_offset_t
HDF5_GetFileSize(aiori_mod_opt_t * test, MPI_Comm testComm, char *testFileName)
HDF5_GetFileSize(aiori_mod_opt_t * test, char *testFileName)
{
if(hints->dryRun)
return 0;
return(MPIIO_GetFileSize(test, testComm, testFileName));
return(MPIIO_GetFileSize(test, testFileName));
}
/*

24
src/aiori-MPIIO.c
View File

@ -562,8 +562,7 @@ static IOR_offset_t SeekOffset(MPI_File fd, IOR_offset_t offset,
* Use MPI_File_get_size() to return aggregate file size.
* NOTE: This function is used by the HDF5 and NCMPI backends.
*/
IOR_offset_t MPIIO_GetFileSize(aiori_mod_opt_t * module_options, MPI_Comm testComm,
char *testFileName)
IOR_offset_t MPIIO_GetFileSize(aiori_mod_opt_t * module_options, char *testFileName)
{
mpiio_options_t * test = (mpiio_options_t*) module_options;
if(hints->dryRun)
@ -589,26 +588,5 @@ IOR_offset_t MPIIO_GetFileSize(aiori_mod_opt_t * module_options, MPI_Comm testCo
if (mpiHints != MPI_INFO_NULL)
MPI_CHECK(MPI_Info_free(&mpiHints), "MPI_Info_free failed");
if (hints->filePerProc == TRUE) {
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1,
MPI_LONG_LONG_INT, MPI_SUM, testComm),
"cannot total data moved");
aggFileSizeFromStat = tmpSum;
} else {
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1,
MPI_LONG_LONG_INT, MPI_MIN, testComm),
"cannot total data moved");
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1,
MPI_LONG_LONG_INT, MPI_MAX, testComm),
"cannot total data moved");
if (tmpMin != tmpMax) {
if (rank == 0) {
WARN("inconsistent file size by different tasks");
}
/* incorrect, but now consistent across tasks */
aggFileSizeFromStat = tmpMin;
}
}
return (aggFileSizeFromStat);
}

24
src/aiori-POSIX.c
View File

@ -676,8 +676,7 @@ void POSIX_Delete(char *testFileName, aiori_mod_opt_t * param)
/*
* Use POSIX stat() to return aggregate file size.
*/
IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t * test, MPI_Comm testComm,
char *testFileName)
IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t * test, char *testFileName)
{
if(hints->dryRun)
return 0;
@ -689,26 +688,5 @@ IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t * test, MPI_Comm testComm,
}
aggFileSizeFromStat = stat_buf.st_size;
if (hints->filePerProc == TRUE) {
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1,
MPI_LONG_LONG_INT, MPI_SUM, testComm),
"cannot total data moved");
aggFileSizeFromStat = tmpSum;
} else {
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1,
MPI_LONG_LONG_INT, MPI_MIN, testComm),
"cannot total data moved");
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1,
MPI_LONG_LONG_INT, MPI_MAX, testComm),
"cannot total data moved");
if (tmpMin != tmpMax) {
if (rank == 0) {
WARN("inconsistent file size by different tasks");
}
/* incorrect, but now consistent across tasks */
aggFileSizeFromStat = tmpMin;
}
}
return (aggFileSizeFromStat);
}

1
src/aiori-debug.h
View File

@ -4,6 +4,7 @@
/* This file contains only debug relevant helpers */
#include <stdio.h>
#include <mpi.h>
extern FILE * out_logfile;
extern int verbose; /* verbose output */

22
src/aiori.h
View File

@ -15,12 +15,6 @@
#ifndef _AIORI_H
#define _AIORI_H
#include <mpi.h>
#ifndef MPI_FILE_NULL
# include <mpio.h>
#endif /* not MPI_FILE_NULL */
#include <sys/stat.h>
#include <stdbool.h>
@ -101,12 +95,12 @@ typedef struct ior_aiori {
*/
void (*xfer_hints)(aiori_xfer_hint_t * params);
IOR_offset_t (*xfer)(int access, aiori_fd_t *, IOR_size_t *,
IOR_offset_t size, IOR_offset_t offset, aiori_mod_opt_t *);
void (*close)(aiori_fd_t *, aiori_mod_opt_t *);
void (*delete)(char *, aiori_mod_opt_t *);
IOR_offset_t size, IOR_offset_t offset, aiori_mod_opt_t * module_options);
void (*close)(aiori_fd_t *, aiori_mod_opt_t * module_options);
void (*delete)(char *, aiori_mod_opt_t * module_options);
char* (*get_version)(void);
void (*fsync)(aiori_fd_t *, aiori_mod_opt_t *);
IOR_offset_t (*get_file_size)(aiori_mod_opt_t * module_options, MPI_Comm, char *);
void (*fsync)(aiori_fd_t *, aiori_mod_opt_t * module_options);
IOR_offset_t (*get_file_size)(aiori_mod_opt_t * module_options, char * filename);
int (*statfs) (const char *, ior_aiori_statfs_t *, aiori_mod_opt_t * module_options);
int (*mkdir) (const char *path, mode_t mode, aiori_mod_opt_t * module_options);
int (*rmdir) (const char *path, aiori_mod_opt_t * module_options);
@ -164,7 +158,7 @@ void aiori_posix_xfer_hints(aiori_xfer_hint_t * params);
aiori_fd_t *POSIX_Create(char *testFileName, int flags, aiori_mod_opt_t * module_options);
int POSIX_Mknod(char *testFileName);
aiori_fd_t *POSIX_Open(char *testFileName, int flags, aiori_mod_opt_t * module_options);
IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t * test, MPI_Comm testComm, char *testFileName);
IOR_offset_t POSIX_GetFileSize(aiori_mod_opt_t * test, char *testFileName);
void POSIX_Delete(char *testFileName, aiori_mod_opt_t * module_options);
void POSIX_Close(aiori_fd_t *fd, aiori_mod_opt_t * module_options);
option_help * POSIX_options(aiori_mod_opt_t ** init_backend_options, aiori_mod_opt_t * init_values);
@ -172,7 +166,7 @@ option_help * POSIX_options(aiori_mod_opt_t ** init_backend_options, aiori_mod_o
/* NOTE: these 3 MPI-IO functions are exported for reuse by HDF5/PNetCDF */
void MPIIO_Delete(char *testFileName, aiori_mod_opt_t * module_options);
IOR_offset_t MPIIO_GetFileSize(aiori_mod_opt_t * options, MPI_Comm testComm, char *testFileName);
int MPIIO_Access(const char *, int, aiori_mod_opt_t *);
IOR_offset_t MPIIO_GetFileSize(aiori_mod_opt_t * options, char *testFileName);
int MPIIO_Access(const char *, int, aiori_mod_opt_t * module_options);
#endif /* not _AIORI_H */

5
src/ior.c
View File

@ -313,8 +313,7 @@ CheckForOutliers(IOR_param_t *test, const double *timer, const int access)
* Check if actual file size equals expected size; if not use actual for
* calculating performance rate.
*/
static void CheckFileSize(IOR_test_t *test, char * testFilename, IOR_offset_t dataMoved, int rep,
const int access)
static void CheckFileSize(IOR_test_t *test, char * testFilename, IOR_offset_t dataMoved, int rep, const int access)
{
IOR_param_t *params = &test->params;
IOR_results_t *results = test->results;
@ -323,7 +322,7 @@ static void CheckFileSize(IOR_test_t *test, char * testFilename, IOR_offset_t da
/* get the size of the file */
IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum;
aggFileSizeFromStat = backend->get_file_size(params->backend_options, testComm, testFilename);
aggFileSizeFromStat = backend->get_file_size(params->backend_options, testFilename);
if (params->hints.filePerProc == TRUE) {
MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1,

6
src/ior.h
View File

@ -39,6 +39,12 @@
#include "iordef.h"
#include "aiori.h"
#include <mpi.h>
#ifndef MPI_FILE_NULL
# include <mpio.h>
#endif /* not MPI_FILE_NULL */
#define ISPOWEROFTWO(x) ((x != 0) && !(x & (x - 1)))
/******************** DATA Packet Type ***************************************/
/* Holds the types of data packets: generic, offset, timestamp, incompressible */