First cut at using the IOR backends for mdtest

This commit makes changes to the AIORI backends to add support for
abstacting statfs, mkdir, rmdir, stat, and access. These new
abstractions are used by a modified mdtest. Some changes:

 - Require C99. Its 2017 and most compilers now support C11. The
   benefits of using C99 include subobject naming (for aiori backend
   structs), and fixed size integers (uint64_t). There is no reason to
   use the non-standard long long type.

 - Moved some of the aiori code into aiori.c so it can be used by both
   mdtest and ior.

 - Code cleanup of mdtest. This is mostly due to the usage of the IOR
   backends rather than a mess of #if code.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
master
Nathan Hjelm 2017-10-19 15:26:52 -06:00
parent aa3fb396ea
commit 4dc0081628
14 changed files with 2663 additions and 3612 deletions

View File

@ -21,16 +21,19 @@ AM_MAINTAINER_MODE
# Checks for programs.
AX_PROG_CC_MPI
# No reason not to require modern C at this point
AC_PROG_CC_C99
# Checks for libraries.
# Checks for header files.
AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h plfs.h hdfs.h beegfs/beegfs.h])
AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h sys/param.h sys/mount.h unistd.h wchar.h plfs.h hdfs.h beegfs/beegfs.h])
# Checks for typedefs, structures, and compiler characteristics.
AC_TYPE_SIZE_T
# Checks for library functions.
AC_CHECK_FUNCS([getpagesize gettimeofday memset mkdir pow putenv realpath regcomp sqrt strcasecmp strchr strerror strncasecmp strstr uname])
AC_CHECK_FUNCS([getpagesize gettimeofday memset mkdir pow putenv realpath regcomp sqrt strcasecmp strchr strerror strncasecmp strstr uname statfs statvfs])
AC_SEARCH_LIBS([sqrt], [m], [],
[AC_MSG_ERROR([Math library not found])])

View File

@ -1 +1,2 @@
dist_data_DATA = USER_GUIDE
dist_man_MANS = mdtest.1

188
doc/mdtest.1 Normal file
View File

@ -0,0 +1,188 @@
.TH mdtest 1 "2010-05-05" "mdtest-1.8.3" "mdtest"
.SH NAME
mdtest \- test file system metadata performance
.SH SYNOPSIS
.B mdtest
.I "[-options]"
.SH DESCRIPTION
.B mdtest
is a file system metadata performance test designed to run
in a cluster MPI environment against parallel file systems.
.PP
In each iteration of the test, each MPI task creates, stats, and removes
the specified number of directories and/or files and measures the performance
in ops/second. After all the iterations complete, the maximum, minimum,
mean ops/sec and the std. deviation are reported for each operation.
.SH OPTIONS
.TP
.I "-b" branching_factor
The branching factor of the hierarchical directory structure [default: 1].
.TP
.I "-B"
No barriers will be taken between the phases (create/stat/remove) of the tests.
.TP
.I "-c"
Use ``collective creates'', meaning task 0 does all the creates.
.TP
.I "-C"
Only perform the create phase of the tests.
.TP
.I "-d" testdir[@testdir2]
The directory in which the tests will run. For multiple pathes, must use fully-qualified pathnames.
[default: working directory of mdtest].
.TP
.I "-D"
Perform test on directories only (no files).
.TP
.I "-e" bytes
Set the number of Bytes to read from each file [default: 0].
.TP
.I "-E"
Only perform the read phase of the tests.
.TP
.I "-f" first
The first number of tasks on which the test will run
[default: 0].
.TP
.I "-F"
Perform test on files only (no directories).
.TP
.I "-h"
Display help message.
.TP
.I "-i" iterations
The number of iterations the test will run
[default: 1].
.TP
.I "-I" items_per_directory
The number of items per directory in the tree [default: 0].
.TP
.I "-l" last
The last number of tasks on which the test will run
[default: 0].
.TP
.I "-L"
Files/directories only created at the leaf level of the tree.
.TP
.I "-n" number_of_items
Every process will creat/stat/remove # directories and files
[default: 0].
.TP
.I "-N" stride
Stride # between neighbor tasks for file/dir stat, 0 = local
[default: 0].
.TP
.I "-p" seconds
Pre-iteration delay (in seconds).
.TP
.I "-r"
Only perform the remove phase of the tests.
.TP
.I "-R[seed]"
Randomly stat files. There is an optional argument that provides a seed
to the random number generator. (Note: There is no space between the
.I "-R"
and
the seed if one is provided.)
.TP
.I "-s" stride
Stride between the number of tasks for each test
[default: 1].
.TP
.I "-S"
Shared file access (file only, no directories).
.TP
.I "-t"
Include unique working directory management overhead in the results
(presumes
.I "-u"
option).
.TP
.I "-T"
Only perform the stat phase of the tests.
.TP
.I "-u"
Create a unique working directory for each task
(presumes
.I "-d"
option).
.TP
.I "-v"
Increase verbosity (each instance of option increments by one).
.TP
.I "-V" value
Set verbosity value
[default: 0].
.TP
.I "-w" bytes
Set the number of Bytes to write to each file after it is created
[default: 0].
.TP
.I "-z" tree_depth
The depth of the hierarchical directory tree [default: 0].
.SH EXAMPLES
.SS "Example 1"
.nf
$ mpirun -n 2 ./mdtest -d /tmp/z -n 100 -i 2
-- started at 11/23/2009 09:05:29 --
mdtest-1.8.1 was launched with 2 total task(s) on 1 nodes
Command line used: ./mdtest -d /tmp/z -n 100 -i 2
Path: /tmp
FS: 28.8 GiB Used FS: 8.6% 8.6%Inodes: 1.8 Mi Used Inodes: 5.1%
time to create tree: 0.000078 sec
tree creation rate: 12826.617737 ops/sec
2 tasks, 200 files/directories
SUMMARY: (of 2 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
Directory creation: 21489.415 17447.551 19468.483 2020.932
Directory stat : 154657.227 28731.061 91694.144 62963.083
Directory removal : 146756.613 21489.415 84123.014 62633.599
File creation : 42024.989 28731.061 35378.025 6646.964
File stat : 146756.613 17447.551 82102.082 64654.531
File removal : 156884.384 42024.989 99454.686 57429.698
time to remove tree: 0.001031 sec
tree removal rate: 970.005550 ops/sec
-- finished at 11/23/2009 09:05:29 --
.fi
.SS "Example 2"
.nf
$ mpirun -np 2 -H pc6 ./mdtest -d /tmp/z -b 2 -z 3 -I 10
-- started at 11/23/2009 09:09:23 --
mdtest-1.8.1 was launched with 2 total task(s) on 1 nodes
Command line used: ./mdtest -d /tmp/z -b 2 -z 3 -I 10
Path: /tmp
FS: 28.8 GiB Used FS: 8.6% 8.6%Inodes: 1.8 Mi Used Inodes: 5.1%
time to create tree: 0.000765 sec
tree creation rate: 19605.659084 ops/sec
2 tasks, 300 files/directories
SUMMARY: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
Directory creation: 29365.707 29365.707 29365.707 0.000
Directory stat : 123701.455 123701.455 123701.455 0.000
Directory removal : 25623.459 25623.459 25623.459 0.000
File creation : 38704.743 38704.743 38704.743 0.000
File stat : 125477.782 125477.782 125477.782 0.000
File removal : 51911.845 51911.845 51911.845 0.000
time to remove tree: 0.000940 sec
tree removal rate: 15960.060883 ops/sec
-- finished at 11/23/2009 09:09:23 --
.fi
.SH "SEE ALSO"
\fBhttp://sourceforge.net/projects/mdtest\fR

View File

@ -1,40 +1,48 @@
bin_PROGRAMS = ior
bin_PROGRAMS = ior mdtest
if USE_CAPS
bin_PROGRAMS += IOR
bin_PROGRAMS += IOR MDTEST
endif
ior_SOURCES =
noinst_HEADERS = ior.h utilities.h parse_options.h aiori.h iordef.h
extraSOURCES = aiori.c
extraLDADD =
extraLDFLAGS =
extraCPPFLAGS =
ior_SOURCES = ior.c utilities.c parse_options.c
ior_LDFLAGS =
ior_LDADD =
ior_CPPFLAGS =
ior_LDFLAGS =
ior_LDADD =
ior_SOURCES += ior.c utilities.c parse_options.c
ior_SOURCES += ior.h utilities.h parse_options.h aiori.h iordef.h
mdtest_SOURCES = mdtest.c utilities.c
mdtest_LDFLAGS =
mdtest_LDADD =
mdtest_CPPFLAGS =
if USE_HDFS_AIORI
# TBD: figure out how to find the appropriate -I and -L dirs. Maybe we can
# get them from the corresponding bin/ dir in $PATH, or pick an
# environment var to use (and set it in modulefiles), or provide a
# config-flag, to set a variable we use here.
ior_SOURCES += aiori-HDFS.c
ior_CPPFLAGS += -I/opt/hadoop-2.2.0/include
ior_LDFLAGS += -L/opt/hadoop-2.2.0/lib/native
ior_LDADD += -lhdfs
extraSOURCES += aiori-HDFS.c
extraCPPFLAGS += -I/opt/hadoop-2.2.0/include
extraLDFLAGS += -L/opt/hadoop-2.2.0/lib/native
extraLDADD += -lhdfs
endif
if USE_HDF5_AIORI
ior_SOURCES += aiori-HDF5.c
ior_LDADD += -lhdf5 -lz
extraSOURCES += aiori-HDF5.c
extraLDADD += -lhdf5 -lz
endif
if USE_MPIIO_AIORI
ior_SOURCES += aiori-MPIIO.c
extraSOURCES += aiori-MPIIO.c
endif
if USE_NCMPI_AIORI
ior_SOURCES += aiori-NCMPI.c
ior_LDADD += -lpnetcdf
extraSOURCES += aiori-NCMPI.c
extraLDADD += -lpnetcdf
endif
if USE_PLFS_AIORI
@ -42,29 +50,42 @@ if USE_PLFS_AIORI
# $PLFS_HOME. configure.ac should extract this for us. Then we should
# add -D... to ior_CPPFLAGS, here. Then aiori-PLFS.c can conditionalize
# compilation of calls that vary according to PLFS version.
ior_SOURCES += aiori-PLFS.c
ior_LDADD += -lplfs
extraSOURCES += aiori-PLFS.c
extraLDADD += -lplfs
endif
if USE_POSIX_AIORI
ior_SOURCES += aiori-POSIX.c
extraSOURCES += aiori-POSIX.c
endif
if USE_S3_AIORI
ior_SOURCES += aiori-S3.c
extraSOURCES += aiori-S3.c
if AWS4C_DIR
ior_CPPFLAGS += $(AWS4C_CPPFLAGS)
ior_LDFLAGS += $(AWS4C_LDFLAGS)
extraCPPFLAGS += $(AWS4C_CPPFLAGS)
extraLDFLAGS += $(AWS4C_LDFLAGS)
endif
ior_LDADD += -lcurl
ior_LDADD += -lxml2
ior_LDADD += -laws4c
ior_LDADD += -laws4c_extra
extraLDADD += -lcurl
extraLDADD += -lxml2
extraLDADD += -laws4c
extraLDADD += -laws4c_extra
endif
ior_SOURCES += $(extraSOURCES)
ior_LDFLAGS += $(extraLDFLAGS)
ior_LDADD += $(extraLDADD)
ior_CPPFLAGS += $(extraCPPFLAGS)
mdtest_SOURCES += $(extraSOURCES)
mdtest_LDFLAGS += $(extraLDFLAGS)
mdtest_LDADD += $(extraLDADD)
mdtest_CPPFLAGS += $(extraCPPFLAGS)
IOR_SOURCES = $(ior_SOURCES)
IOR_LDADD = $(ior_LDADD)
IOT_CPPFLAGS = $(ior_CPPFLAGS)
MDTEST_SOURCES = $(mdtest_SOURCES)
MDTEST_LDADD = $(mdtest_LDADD)
MDTEST_CPPFLAGS = $(mdtest_CPPFLAGS)

View File

@ -96,15 +96,15 @@ static IOR_offset_t HDF5_GetFileSize(IOR_param_t *, MPI_Comm, char *);
/************************** D E C L A R A T I O N S ***************************/
ior_aiori_t hdf5_aiori = {
"HDF5",
HDF5_Create,
HDF5_Open,
HDF5_Xfer,
HDF5_Close,
HDF5_Delete,
HDF5_SetVersion,
HDF5_Fsync,
HDF5_GetFileSize
.name = "HDF5",
.create = HDF5_Create,
.open = HDF5_Open,
.xfer = HDF5_Xfer,
.close = HDF5_Close,
.delete = HDF5_Delete,
.set_version = HDF5_SetVersion,
.fsync = HDF5_Fsync,
.get_file_size = HDF5_GetFileSize,
};
static hid_t xferPropList; /* xfer property list */

View File

@ -114,15 +114,15 @@ static IOR_offset_t HDFS_GetFileSize(IOR_param_t *, MPI_Comm, char *);
/************************** D E C L A R A T I O N S ***************************/
ior_aiori_t hdfs_aiori = {
"HDFS",
HDFS_Create,
HDFS_Open,
HDFS_Xfer,
HDFS_Close,
HDFS_Delete,
HDFS_SetVersion,
HDFS_Fsync,
HDFS_GetFileSize
.name = "HDFS",
.create = HDFS_Create,
.open = HDFS_Open,
.xfer = HDFS_Xfer,
.close = HDFS_Close,
.delete = HDFS_Delete,
.set_version = HDFS_SetVersion,
.fsync = HDFS_Fsync,
.get_file_size = HDFS_GetFileSize,
};
/***************************** F U N C T I O N S ******************************/

View File

@ -60,15 +60,15 @@ static IOR_offset_t NCMPI_GetFileSize(IOR_param_t *, MPI_Comm, char *);
/************************** D E C L A R A T I O N S ***************************/
ior_aiori_t ncmpi_aiori = {
"NCMPI",
NCMPI_Create,
NCMPI_Open,
NCMPI_Xfer,
NCMPI_Close,
NCMPI_Delete,
NCMPI_SetVersion,
NCMPI_Fsync,
NCMPI_GetFileSize
.name = "NCMPI",
.create = NCMPI_Create,
.open = NCMPI_Open,
.xfer = NCMPI_Xfer,
.close = NCMPI_Close,
.delete = NCMPI_Delete,
.set_version = NCMPI_SetVersion,
.fsync = NCMPI_Fsync,
.get_file_size = NCMPI_GetFileSize,
};
/***************************** F U N C T I O N S ******************************/

View File

@ -103,19 +103,27 @@ static void PLFS_Delete(char *, IOR_param_t *);
static void PLFS_SetVersion(IOR_param_t *);
static void PLFS_Fsync(void *, IOR_param_t *);
static IOR_offset_t PLFS_GetFileSize(IOR_param_t *, MPI_Comm, char *);
static int PLFS_statfs (const char *path, ior_aiori_statfs_t *stat_buf, IOR_param_t * param);
static int PLFS_mkdir (const char *path, mode_t mode, IOR_param_t * param);
static int PLFS_rmdir (const char *path, IOR_param_t * param);
static int PLFS_access (const char *path, int mode, IOR_param_t * param);
/************************** D E C L A R A T I O N S ***************************/
ior_aiori_t plfs_aiori = {
"PLFS",
PLFS_Create,
PLFS_Open,
PLFS_Xfer,
PLFS_Close,
PLFS_Delete,
PLFS_SetVersion,
PLFS_Fsync,
PLFS_GetFileSize
.name = "PLFS",
.create = PLFS_Create,
.open = PLFS_Open,
.xfer = PLFS_Xfer,
.close = PLFS_Close,
.delete = PLFS_Delete,
.set_version = PLFS_SetVersion,
.fsync = PLFS_Fsync,
.get_file_size = PLFS_GetFileSize,
.statfs = PLFS_statfs,
.mkdir = PLFS_mkdir,
.rmdir = PLFS_rmdir,
.access = PLFS_access,
};
/***************************** F U N C T I O N S ******************************/
@ -471,3 +479,71 @@ static IOR_offset_t PLFS_GetFileSize(
return ( aggFileSizeFromStat );
}
static int PLFS_statfs (const char *path, ior_aiori_statfs_t *stat_buf, IOR_param_t * param)
{
struct statvfs stbuf;
plfs_ret = plfs_statvfs( file_system, &stbuf );
if ( plfs_ret != PLFS_SUCCESS ) {
return -1;
}
stat_buf->f_bsize = stbuf.f_bsize;
stat_buf->f_blocks = stbuf.f_blocks;
stat_buf->f_bfree = stbuf.f_bfree;
stat_buf->f_bavail = stbuf.f_bavail;
stat_buf->f_files = stbuf.f_files;
stat_buf->f_ffree = stbuf.f_ffree;
return 0;
}
static int PLFS_mkdir (const char *path, mode_t mode, IOR_param_t * param)
{
int ret;
ret = plfs_mkdir (path, mode);
if (PLFS_SUCCESS != ret) {
return -1;
}
return 0;
}
static int PLFS_rmdir (const char *path, IOR_param_t * param)
{
int ret;
ret = plfs_rmdir (path);
if (PLFS_SUCCESS != ret) {
return -1;
}
return 0;
}
static int PLFS_access (const char *path, int mode, IOR_param_t * param)
{
int ret;
ret = plfs_access (path, mode);
if (PLFS_SUCCESS != ret) {
return -1;
}
return 0;
}
static int PLFS_stat (const char *path, struct stat *stat_buf, IOR_param_t * param)
{
int ret;
ret = plfs_getattr (NULL, path, stat_buf, 0);
if ( plfs_ret != PLFS_SUCCESS ) {
ERR("Unable to plfs_getattr directory");
return -1;
}
return 0;
}

View File

@ -78,15 +78,15 @@ static IOR_offset_t POSIX_GetFileSize(IOR_param_t *, MPI_Comm, char *);
/************************** D E C L A R A T I O N S ***************************/
ior_aiori_t posix_aiori = {
"POSIX",
POSIX_Create,
POSIX_Open,
POSIX_Xfer,
POSIX_Close,
POSIX_Delete,
POSIX_SetVersion,
POSIX_Fsync,
POSIX_GetFileSize
.name = "POSIX",
.create = POSIX_Create,
.open = POSIX_Open,
.xfer = POSIX_Xfer,
.close = POSIX_Close,
.delete = POSIX_Delete,
.set_version = POSIX_SetVersion,
.fsync = POSIX_Fsync,
.get_file_size = POSIX_GetFileSize,
};
/***************************** F U N C T I O N S ******************************/

View File

@ -132,45 +132,45 @@ static IOR_offset_t S3_GetFileSize(IOR_param_t*, MPI_Comm, char*);
// N:1 writes use multi-part upload
// N:N fails if "transfer-size" != "block-size" (because that requires "append")
ior_aiori_t s3_aiori = {
"S3",
S3_Create,
S3_Open,
S3_Xfer,
S3_Close,
S3_Delete,
S3_SetVersion,
S3_Fsync,
S3_GetFileSize
.name = "S3",
.create = S3_Create,
.open = S3_Open,
.xfer = S3_Xfer,
.close = S3_Close,
.delete = S3_Delete,
.set_version = S3_SetVersion,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
};
// "S3", plus EMC-extensions enabled
// N:1 writes use multi-part upload
// N:N succeeds (because EMC-extensions support "append")
ior_aiori_t s3_plus_aiori = {
"S3_plus",
S3_Create,
S3_Open,
S3_Xfer,
S3_Close,
S3_Delete,
S3_SetVersion,
S3_Fsync,
S3_GetFileSize
.name = "S3_plus",
.create = S3_Create,
.open = S3_Open,
.xfer = S3_Xfer,
.close = S3_Close,
.delete = S3_Delete,
.set_version = S3_SetVersion,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
};
// Use EMC-extensions for N:1 write, as well
// N:1 writes use EMC byte-range
// N:N succeeds because EMC-extensions support "append"
ior_aiori_t s3_emc_aiori = {
"S3_EMC",
EMC_Create,
EMC_Open,
EMC_Xfer,
EMC_Close,
S3_Delete,
S3_SetVersion,
S3_Fsync,
S3_GetFileSize
.name = "S3_EMC",
.create = EMC_Create,
.open = EMC_Open,
.xfer = EMC_Xfer,
.close = EMC_Close,
.delete = S3_Delete,
.set_version = S3_SetVersion,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
};

150
src/aiori.c Normal file
View File

@ -0,0 +1,150 @@
/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
* vim:expandtab:shiftwidth=8:tabstop=8:
*/
/******************************************************************************\
* *
* Copyright (c) 2003, The Regents of the University of California *
* See the file COPYRIGHT for a complete copyright notice and license. *
* *
********************************************************************************
*
* Definitions and prototypes of abstract I/O interface
*
\******************************************************************************/
#include "aiori.h"
#if defined(HAVE_SYS_STATVFS_H)
#include <sys/statvfs.h>
#endif
#if defined(HAVE_SYS_STATFS_H)
#include <sys/statfs.h>
#endif
/*
* Bind the global "backend" pointer to the requested backend AIORI's
* function table.
*/
ior_aiori_t *available_aiori[] = {
#ifdef USE_HDF5_AIORI
&hdf5_aiori,
#endif
#ifdef USE_HDFS_AIORI
&hdfs_aiori,
#endif
#ifdef USE_MPIIO_AIORI
&mpiio_aiori,
#endif
#ifdef USE_NCMPI_AIORI
&ncmpi_aiori,
#endif
#ifdef USE_POSIX_AIORI
&posix_aiori,
#endif
#ifdef USE_PLFS_AIORI
&plfs_aiori,
#endif
#ifdef USE_S3_AIORI
&s3_aiori,
&s3_plus_aiori,
&s3_emc_aiori,
#endif
NULL
};
/**
* Default statfs implementation.
*
* @param[in] path Path to run statfs on
* @param[out] statfs_buf AIORI statfs buffer
*
* This function provides a AIORI statfs for POSIX-compliant filesystems. It
* uses statvfs is available and falls back on statfs.
*/
static int aiori_statfs (const char *path, ior_aiori_statfs_t *stat_buf, IOR_param_t * param)
{
int ret;
#if defined(HAVE_STATVFS)
struct statvfs statfs_buf;
ret = statvfs (path, &statfs_buf);
#else
struct statfs statfs_buf;
ret = statfs (path, &statfs_buf);
#endif
if (-1 == ret) {
return -1;
}
stat_buf->f_bsize = statfs_buf.f_bsize;
stat_buf->f_blocks = statfs_buf.f_blocks;
stat_buf->f_bfree = statfs_buf.f_bfree;
stat_buf->f_files = statfs_buf.f_files;
stat_buf->f_ffree = statfs_buf.f_ffree;
return 0;
}
static int aiori_mkdir (const char *path, mode_t mode, IOR_param_t * param)
{
return mkdir (path, mode);
}
static int aiori_rmdir (const char *path, IOR_param_t * param)
{
return rmdir (path);
}
static int aiori_access (const char *path, int mode, IOR_param_t * param)
{
return access (path, mode);
}
static int aiori_stat (const char *path, struct stat *buf, IOR_param_t * param)
{
return stat (path, buf);
}
const ior_aiori_t *aiori_select (const char *api)
{
for (ior_aiori_t **tmp = available_aiori ; *tmp != NULL; ++tmp) {
if (NULL == api || strcmp(api, (*tmp)->name) == 0) {
if (NULL == (*tmp)->statfs) {
(*tmp)->statfs = aiori_statfs;
}
if (NULL == (*tmp)->mkdir) {
(*tmp)->mkdir = aiori_mkdir;
}
if (NULL == (*tmp)->rmdir) {
(*tmp)->rmdir = aiori_rmdir;
}
if (NULL == (*tmp)->access) {
(*tmp)->access = aiori_access;
}
if (NULL == (*tmp)->stat) {
(*tmp)->stat = aiori_stat;
}
return *tmp;
}
}
return NULL;
}
int aiori_count (void)
{
return sizeof (available_aiori)/sizeof(available_aiori[0]) - 1;
}
const char *aiori_default (void)
{
if (aiori_count () > 0) {
return available_aiori[0]->name;
}
return NULL;
}

View File

@ -21,6 +21,8 @@
# include <mpio.h>
#endif /* not MPI_FILE_NULL */
#include <sys/stat.h>
#include "ior.h"
#include "iordef.h" /* IOR Definitions */
@ -50,6 +52,15 @@
#define IOR_IWOTH 0x0400 /* write permission: other */
#define IOR_IXOTH 0x0800 /* execute permission: other */
typedef struct ior_aiori_statfs {
uint64_t f_bsize;
uint64_t f_blocks;
uint64_t f_bfree;
uint64_t f_bavail;
uint64_t f_files;
uint64_t f_ffree;
} ior_aiori_statfs_t;
typedef struct ior_aiori {
char *name;
void *(*create)(char *, IOR_param_t *);
@ -61,6 +72,11 @@ typedef struct ior_aiori {
void (*set_version)(IOR_param_t *);
void (*fsync)(void *, IOR_param_t *);
IOR_offset_t (*get_file_size)(IOR_param_t *, MPI_Comm, char *);
int (*statfs) (const char *, ior_aiori_statfs_t *, IOR_param_t * param);
int (*mkdir) (const char *path, mode_t mode, IOR_param_t * param);
int (*rmdir) (const char *path, IOR_param_t * param);
int (*access) (const char *path, int mode, IOR_param_t * param);
int (*stat) (const char *path, struct stat *buf, IOR_param_t * param);
} ior_aiori_t;
extern ior_aiori_t hdf5_aiori;
@ -73,6 +89,9 @@ extern ior_aiori_t s3_aiori;
extern ior_aiori_t s3_plus_aiori;
extern ior_aiori_t s3_emc_aiori;
const ior_aiori_t *aiori_select (const char *api);
int aiori_count (void);
const char *aiori_default (void);
IOR_offset_t MPIIO_GetFileSize(IOR_param_t * test, MPI_Comm testComm,
char *testFileName);

View File

@ -50,34 +50,7 @@ int totalErrorCount = 0;
double wall_clock_delta = 0;
double wall_clock_deviation;
ior_aiori_t *backend;
ior_aiori_t *available_aiori[] = {
#ifdef USE_HDF5_AIORI
&hdf5_aiori,
#endif
#ifdef USE_HDFS_AIORI
&hdfs_aiori,
#endif
#ifdef USE_MPIIO_AIORI
&mpiio_aiori,
#endif
#ifdef USE_NCMPI_AIORI
&ncmpi_aiori,
#endif
#ifdef USE_POSIX_AIORI
&posix_aiori,
#endif
#ifdef USE_PLFS_AIORI
&plfs_aiori,
#endif
#ifdef USE_S3_AIORI
&s3_aiori,
&s3_plus_aiori,
&s3_emc_aiori,
#endif
NULL
};
const ior_aiori_t *backend;
static void DestroyTests(IOR_test_t *tests_head);
static void DisplayUsage(char **);
@ -135,7 +108,7 @@ int main(int argc, char **argv)
"cannot set errhandler"); */
/* Sanity check, we were compiled with SOME backend, right? */
if (available_aiori[0] == NULL) {
if (0 == aiori_count ()) {
ERR("No IO backends compiled into ior. "
"Run 'configure --with-<backend>', and recompile.");
}
@ -204,13 +177,16 @@ int main(int argc, char **argv)
*/
void init_IOR_Param_t(IOR_param_t * p)
{
assert(available_aiori[0] != NULL);
const char *default_aiori = aiori_default ();
assert (NULL != default_aiori);
memset(p, 0, sizeof(IOR_param_t));
p->mode = IOR_IRUSR | IOR_IWUSR | IOR_IRGRP | IOR_IWGRP;
p->openFlags = IOR_RDWR | IOR_CREAT;
strncpy(p->api, available_aiori[0]->name, MAX_STR);
strncpy(p->api, default_aiori, MAX_STR);
strncpy(p->platform, "HOST(OSTYPE)", MAX_STR);
strncpy(p->testFileName, "testFile", MAXPATHLEN);
@ -247,32 +223,20 @@ void init_IOR_Param_t(IOR_param_t * p)
p->beegfs_chunkSize = -1;
}
/*
* Bind the global "backend" pointer to the requested backend AIORI's
* function table.
*/
static void AioriBind(char* api, IOR_param_t* param)
{
ior_aiori_t **tmp;
backend = NULL;
for (tmp = available_aiori; *tmp != NULL; tmp++) {
if (strcmp(api, (*tmp)->name) == 0) {
backend = *tmp;
break;
backend = aiori_select (api);
if (NULL != backend) {
if (! strncmp(api, "S3", 2)) {
if (! strcmp(api, "S3_EMC")) {
param->curl_flags |= IOR_CURL_S3_EMC_EXT;
} else {
param->curl_flags &= ~(IOR_CURL_S3_EMC_EXT);
}
}
}
if (backend == NULL) {
} else {
ERR("unrecognized IO API");
}
else if (! strncmp(api, "S3", 2)) {
if (! strcmp(api, "S3_EMC"))
param->curl_flags |= IOR_CURL_S3_EMC_EXT;
else
param->curl_flags &= ~(IOR_CURL_S3_EMC_EXT);
}
}
static void

File diff suppressed because it is too large Load Diff