2012-01-09 00:51:04 +04:00
/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
* vim : expandtab : shiftwidth = 8 : tabstop = 8 :
*/
2011-06-17 23:20:43 +04:00
/******************************************************************************\
* *
* Copyright ( c ) 2003 , The Regents of the University of California *
* See the file COPYRIGHT for a complete copyright notice and license . *
* *
\ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2011-11-12 04:40:45 +04:00
# ifdef HAVE_CONFIG_H
2014-07-31 03:17:21 +04:00
# include "config.h"
2011-11-12 04:40:45 +04:00
# endif
# include <stdio.h>
# include <stdlib.h>
2012-01-07 05:29:45 +04:00
# include <unistd.h>
2011-11-12 02:22:17 +04:00
# include <ctype.h> /* tolower() */
2011-11-12 04:40:45 +04:00
# include <errno.h>
2011-06-17 23:20:43 +04:00
# include <math.h>
# include <mpi.h>
# include <string.h>
2011-11-12 02:22:17 +04:00
# include <sys/stat.h> /* struct stat */
2011-06-17 23:20:43 +04:00
# include <time.h>
2014-07-31 03:17:21 +04:00
2011-06-17 23:20:43 +04:00
# ifndef _WIN32
2014-07-31 03:17:21 +04:00
# include <sys / time.h> /* gettimeofday() */
# include <sys / utsname.h> /* uname() */
2011-06-17 23:20:43 +04:00
# endif
2014-07-31 03:17:21 +04:00
2011-11-10 04:34:16 +04:00
# include <assert.h>
2011-06-17 23:20:43 +04:00
2011-11-12 04:40:45 +04:00
# include "ior.h"
2018-07-08 15:38:05 +03:00
# include "ior-internal.h"
2011-11-12 04:40:45 +04:00
# include "aiori.h"
# include "utilities.h"
# include "parse_options.h"
2011-06-17 23:20:43 +04:00
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
2011-11-12 04:40:45 +04:00
/* file scope globals */
extern char * * environ ;
2018-07-07 13:42:21 +03:00
static int totalErrorCount ;
static const ior_aiori_t * backend ;
2011-10-28 03:50:05 +04:00
2011-12-13 09:00:18 +04:00
static void DestroyTests ( IOR_test_t * tests_head ) ;
2011-11-12 04:40:45 +04:00
static char * PrependDir ( IOR_param_t * , char * ) ;
static char * * ParseFileName ( char * , int * ) ;
2018-07-14 10:41:35 +03:00
static void InitTests ( IOR_test_t * , MPI_Comm ) ;
2011-12-13 09:00:18 +04:00
static void TestIoSys ( IOR_test_t * ) ;
2015-05-19 18:36:28 +03:00
static void ValidateTests ( IOR_param_t * ) ;
2017-10-20 19:02:24 +03:00
static IOR_offset_t WriteOrRead ( IOR_param_t * test , IOR_results_t * results , void * fd , int access , IOR_io_buffers * ioBuffers ) ;
2011-11-12 04:40:45 +04:00
static void WriteTimes ( IOR_param_t * , double * * , int , int ) ;
2018-07-07 13:42:21 +03:00
IOR_test_t * ior_run ( int argc , char * * argv , MPI_Comm world_com , FILE * world_out ) {
2011-12-13 09:00:18 +04:00
IOR_test_t * tests_head ;
IOR_test_t * tptr ;
2018-07-07 13:42:21 +03:00
out_logfile = world_out ;
2018-07-08 15:47:55 +03:00
out_resultfile = world_out ;
2018-07-07 13:42:21 +03:00
mpi_comm_world = world_com ;
2011-06-17 23:20:43 +04:00
2018-07-07 13:42:21 +03:00
MPI_CHECK ( MPI_Comm_size ( mpi_comm_world , & numTasksWorld ) , " cannot get number of tasks " ) ;
MPI_CHECK ( MPI_Comm_rank ( mpi_comm_world , & rank ) , " cannot get rank " ) ;
2012-09-10 21:50:18 +04:00
PrintEarlyHeader ( ) ;
2015-05-19 18:36:28 +03:00
/* setup tests, and validate parameters */
2018-07-14 10:41:35 +03:00
tests_head = ParseCommandLine ( argc , argv ) ;
InitTests ( tests_head , world_com ) ;
2011-12-13 09:00:18 +04:00
verbose = tests_head - > params . verbose ;
2011-11-12 02:22:17 +04:00
2014-08-14 02:53:24 +04:00
PrintHeader ( argc , argv ) ;
2011-12-11 13:50:19 +04:00
2011-11-12 02:22:17 +04:00
/* perform each test */
2014-08-14 02:53:24 +04:00
for ( tptr = tests_head ; tptr ! = NULL ; tptr = tptr - > next ) {
2018-07-07 13:42:21 +03:00
totalErrorCount = 0 ;
2011-12-13 09:00:18 +04:00
verbose = tptr - > params . verbose ;
2011-11-12 02:22:17 +04:00
if ( rank = = 0 & & verbose > = VERBOSE_0 ) {
2018-07-08 18:47:23 +03:00
ShowTestStart ( & tptr - > params ) ;
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
}
2011-12-13 09:00:18 +04:00
TestIoSys ( tptr ) ;
2018-07-07 13:42:21 +03:00
tptr - > results - > errors = totalErrorCount ;
2018-07-08 16:59:54 +03:00
ShowTestEnd ( tptr ) ;
2011-11-12 02:22:17 +04:00
}
2014-08-14 02:53:24 +04:00
PrintLongSummaryAllTests ( tests_head ) ;
2011-12-13 09:00:18 +04:00
2011-11-12 02:22:17 +04:00
/* display finish time */
2018-07-08 16:59:54 +03:00
PrintTestEnds ( ) ;
2018-07-07 13:42:21 +03:00
return tests_head ;
}
2011-11-12 02:22:17 +04:00
2011-12-13 09:00:18 +04:00
2011-11-12 02:22:17 +04:00
2018-07-07 13:42:21 +03:00
int ior_main ( int argc , char * * argv )
{
IOR_test_t * tests_head ;
IOR_test_t * tptr ;
2018-07-08 15:07:32 +03:00
2018-07-07 13:42:21 +03:00
out_logfile = stdout ;
2018-07-08 15:47:55 +03:00
out_resultfile = stdout ;
2018-07-07 13:42:21 +03:00
/*
* check - h option from commandline without starting MPI ;
*/
2018-07-14 10:41:35 +03:00
tests_head = ParseCommandLine ( argc , argv ) ;
2018-07-07 13:42:21 +03:00
/* start the MPI code */
MPI_CHECK ( MPI_Init ( & argc , & argv ) , " cannot initialize MPI " ) ;
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
2018-07-07 13:42:21 +03:00
mpi_comm_world = MPI_COMM_WORLD ;
MPI_CHECK ( MPI_Comm_size ( mpi_comm_world , & numTasksWorld ) ,
" cannot get number of tasks " ) ;
MPI_CHECK ( MPI_Comm_rank ( mpi_comm_world , & rank ) , " cannot get rank " ) ;
2018-07-08 15:07:32 +03:00
2018-07-07 13:42:21 +03:00
PrintEarlyHeader ( ) ;
/* set error-handling */
/*MPI_CHECK(MPI_Errhandler_set(mpi_comm_world, MPI_ERRORS_RETURN),
" cannot set errhandler " ) ; */
/* setup tests, and validate parameters */
2018-07-14 10:41:35 +03:00
InitTests ( tests_head , mpi_comm_world ) ;
2018-07-07 13:42:21 +03:00
verbose = tests_head - > params . verbose ;
PrintHeader ( argc , argv ) ;
/* perform each test */
for ( tptr = tests_head ; tptr ! = NULL ; tptr = tptr - > next ) {
verbose = tptr - > params . verbose ;
if ( rank = = 0 & & verbose > = VERBOSE_0 ) {
2018-07-08 18:47:23 +03:00
ShowTestStart ( & tptr - > params ) ;
2018-07-07 13:42:21 +03:00
}
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
2018-07-07 13:42:21 +03:00
// This is useful for trapping a running MPI process. While
// this is sleeping, run the script 'testing/hdfs/gdb.attach'
if ( verbose > = VERBOSE_4 ) {
fprintf ( out_logfile , " \t rank %d: sleeping \n " , rank ) ;
sleep ( 5 ) ;
fprintf ( out_logfile , " \t rank %d: awake. \n " , rank ) ;
}
2018-08-30 00:14:19 +03:00
2018-07-07 13:42:21 +03:00
TestIoSys ( tptr ) ;
2018-07-08 16:59:54 +03:00
ShowTestEnd ( tptr ) ;
2018-07-07 13:42:21 +03:00
}
if ( verbose < 0 )
/* always print final summary */
verbose = 0 ;
PrintLongSummaryAllTests ( tests_head ) ;
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
2018-07-07 13:42:21 +03:00
/* display finish time */
2018-07-08 16:59:54 +03:00
PrintTestEnds ( ) ;
2018-07-07 13:42:21 +03:00
DestroyTests ( tests_head ) ;
MPI_CHECK ( MPI_Finalize ( ) , " cannot finalize MPI " ) ;
return totalErrorCount ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/***************************** F U N C T I O N S ******************************/
2011-11-10 04:13:44 +04:00
/*
* Initialize an IOR_param_t structure to the defaults
*/
2011-11-12 02:22:17 +04:00
void init_IOR_Param_t ( IOR_param_t * p )
2011-11-10 04:13:44 +04:00
{
2017-10-20 00:26:52 +03:00
const char * default_aiori = aiori_default ( ) ;
2018-04-23 23:10:04 +03:00
char * hdfs_user ;
2017-10-20 00:26:52 +03:00
assert ( NULL ! = default_aiori ) ;
2011-11-10 04:13:44 +04:00
memset ( p , 0 , sizeof ( IOR_param_t ) ) ;
2011-11-12 02:22:17 +04:00
p - > mode = IOR_IRUSR | IOR_IWUSR | IOR_IRGRP | IOR_IWGRP ;
p - > openFlags = IOR_RDWR | IOR_CREAT ;
2014-08-14 02:53:24 +04:00
2018-07-14 10:41:35 +03:00
p - > api = strdup ( default_aiori ) ;
p - > platform = strdup ( " HOST(OSTYPE) " ) ;
p - > testFileName = strdup ( " testFile " ) ;
p - > writeFile = p - > readFile = FALSE ;
p - > checkWrite = p - > checkRead = FALSE ;
2014-08-14 02:53:24 +04:00
2011-11-10 04:13:44 +04:00
p - > nodes = 1 ;
p - > tasksPerNode = 1 ;
p - > repetitions = 1 ;
p - > repCounter = - 1 ;
p - > open = WRITE ;
p - > taskPerNodeOffset = 1 ;
p - > segmentCount = 1 ;
p - > blockSize = 1048576 ;
p - > transferSize = 262144 ;
p - > randomSeed = - 1 ;
2015-05-21 21:05:56 +03:00
p - > incompressibleSeed = 573 ;
2018-07-07 13:42:21 +03:00
p - > testComm = mpi_comm_world ;
2011-11-10 04:13:44 +04:00
p - > setAlignment = 1 ;
p - > lustre_start_ost = - 1 ;
2014-08-29 01:39:44 +04:00
2018-04-23 23:10:04 +03:00
hdfs_user = getenv ( " USER " ) ;
if ( ! hdfs_user )
2018-07-14 10:41:35 +03:00
hdfs_user = " " ;
p - > hdfs_user = strdup ( hdfs_user ) ;
2014-08-14 02:53:24 +04:00
p - > hdfs_name_node = " default " ;
p - > hdfs_name_node_port = 0 ; /* ??? */
p - > hdfs_fs = NULL ;
2014-08-29 01:39:44 +04:00
p - > hdfs_replicas = 0 ; /* invokes the default */
2014-08-14 02:53:24 +04:00
p - > hdfs_block_size = 0 ;
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
p - > URI = NULL ;
p - > part_number = 0 ;
2017-09-21 18:12:31 +03:00
p - > beegfs_numTargets = - 1 ;
p - > beegfs_chunkSize = - 1 ;
2018-05-08 14:08:29 +03:00
p - > mmap_ptr = NULL ;
2011-11-10 04:13:44 +04:00
}
2011-11-12 04:40:45 +04:00
static void
2011-11-12 02:22:17 +04:00
DisplayOutliers ( int numTasks ,
2011-06-17 23:20:43 +04:00
double timerVal ,
2011-11-12 02:22:17 +04:00
char * timeString , int access , int outlierThreshold )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
char accessString [ MAX_STR ] ;
double sum , mean , sqrDiff , var , sd ;
/* for local timerVal, don't compensate for wall clock delta */
timerVal + = wall_clock_delta ;
MPI_CHECK ( MPI_Allreduce
( & timerVal , & sum , 1 , MPI_DOUBLE , MPI_SUM , testComm ) ,
" MPI_Allreduce() " ) ;
mean = sum / numTasks ;
sqrDiff = pow ( ( mean - timerVal ) , 2 ) ;
MPI_CHECK ( MPI_Allreduce
( & sqrDiff , & var , 1 , MPI_DOUBLE , MPI_SUM , testComm ) ,
" MPI_Allreduce() " ) ;
var = var / numTasks ;
sd = sqrt ( var ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
if ( access = = WRITE ) {
strcpy ( accessString , " write " ) ;
} else { /* READ */
strcpy ( accessString , " read " ) ;
}
if ( fabs ( timerVal - mean ) > ( double ) outlierThreshold ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " WARNING: for task %d, %s %s is %f \n " ,
2011-11-12 02:22:17 +04:00
rank , accessString , timeString , timerVal ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " (mean=%f, stddev=%f) \n " , mean , sd ) ;
fflush ( out_logfile ) ;
2011-11-12 02:22:17 +04:00
}
2011-11-12 04:40:45 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Check for outliers in start / end times and elapsed create / xfer / close times .
*/
2011-11-12 04:40:45 +04:00
static void CheckForOutliers ( IOR_param_t * test , double * * timer , int rep ,
int access )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
int shift ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
if ( access = = WRITE ) {
shift = 0 ;
} else { /* READ */
shift = 6 ;
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
DisplayOutliers ( test - > numTasks , timer [ shift + 0 ] [ rep ] ,
" start time " , access , test - > outlierThreshold ) ;
DisplayOutliers ( test - > numTasks ,
timer [ shift + 1 ] [ rep ] - timer [ shift + 0 ] [ rep ] ,
" elapsed create time " , access , test - > outlierThreshold ) ;
DisplayOutliers ( test - > numTasks ,
timer [ shift + 3 ] [ rep ] - timer [ shift + 2 ] [ rep ] ,
" elapsed transfer time " , access ,
test - > outlierThreshold ) ;
DisplayOutliers ( test - > numTasks ,
timer [ shift + 5 ] [ rep ] - timer [ shift + 4 ] [ rep ] ,
" elapsed close time " , access , test - > outlierThreshold ) ;
DisplayOutliers ( test - > numTasks , timer [ shift + 5 ] [ rep ] , " end time " ,
access , test - > outlierThreshold ) ;
2011-11-12 04:40:45 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Check if actual file size equals expected size ; if not use actual for
* calculating performance rate .
*/
2011-12-13 09:00:18 +04:00
static void CheckFileSize ( IOR_test_t * test , IOR_offset_t dataMoved , int rep )
2011-06-17 23:20:43 +04:00
{
2014-08-14 02:53:24 +04:00
IOR_param_t * params = & test - > params ;
IOR_results_t * results = test - > results ;
2011-12-13 09:00:18 +04:00
2018-07-15 21:38:17 +03:00
MPI_CHECK ( MPI_Allreduce ( & dataMoved , & results [ rep ] . aggFileSizeFromXfer ,
2011-11-12 02:22:17 +04:00
1 , MPI_LONG_LONG_INT , MPI_SUM , testComm ) ,
" cannot total data moved " ) ;
2018-08-27 20:22:38 +03:00
if ( strcasecmp ( params - > api , " HDF5 " ) ! = 0 & & strcasecmp ( params - > api , " NCMPI " ) ! = 0 & &
strcasecmp ( params - > api , " DAOS " ) ! = 0 ) {
2011-11-12 02:22:17 +04:00
if ( verbose > = VERBOSE_0 & & rank = = 0 ) {
2011-12-13 09:00:18 +04:00
if ( ( params - > expectedAggFileSize
2018-07-15 21:38:17 +03:00
! = results [ rep ] . aggFileSizeFromXfer )
| | ( results [ rep ] . aggFileSizeFromStat
! = results [ rep ] . aggFileSizeFromXfer ) ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" WARNING: Expected aggregate file size = %lld. \n " ,
2011-12-14 10:04:27 +04:00
( long long ) params - > expectedAggFileSize ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" WARNING: Stat() of aggregate file size = %lld. \n " ,
2018-07-15 21:38:17 +03:00
( long long ) results [ rep ] . aggFileSizeFromStat ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" WARNING: Using actual aggregate bytes moved = %lld. \n " ,
2018-07-15 21:38:17 +03:00
( long long ) results [ rep ] . aggFileSizeFromXfer ) ;
2017-10-20 19:02:24 +03:00
if ( params - > deadlineForStonewalling ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2017-10-20 19:02:24 +03:00
" WARNING: maybe caused by deadlineForStonewalling \n " ) ;
}
2011-11-12 02:22:17 +04:00
}
}
2011-06-17 23:20:43 +04:00
}
2018-07-15 21:38:17 +03:00
results [ rep ] . aggFileSizeForBW = results [ rep ] . aggFileSizeFromXfer ;
2011-11-12 04:40:45 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Compare buffers after reading / writing each transfer . Displays only first
* difference in buffers and returns total errors counted .
*/
2011-11-12 04:40:45 +04:00
static size_t
2011-11-12 02:22:17 +04:00
CompareBuffers ( void * expectedBuffer ,
void * unknownBuffer ,
size_t size ,
2011-12-13 09:00:18 +04:00
IOR_offset_t transferCount , IOR_param_t * test , int access )
2011-06-17 23:20:43 +04:00
{
2018-07-08 00:39:14 +03:00
char testFileName [ MAX_PATHLEN ] ;
2011-11-12 03:11:28 +04:00
char bufferLabel1 [ MAX_STR ] ;
char bufferLabel2 [ MAX_STR ] ;
2011-11-12 02:22:17 +04:00
size_t i , j , length , first , last ;
size_t errorCount = 0 ;
int inError = 0 ;
unsigned long long * goodbuf = ( unsigned long long * ) expectedBuffer ;
unsigned long long * testbuf = ( unsigned long long * ) unknownBuffer ;
2017-09-27 19:45:47 +03:00
if ( access = = WRITECHECK | | access = = READCHECK ) {
2011-11-12 02:22:17 +04:00
strcpy ( bufferLabel1 , " Expected: " ) ;
strcpy ( bufferLabel2 , " Actual: " ) ;
} else {
ERR ( " incorrect argument for CompareBuffers() " ) ;
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
length = size / sizeof ( IOR_size_t ) ;
first = - 1 ;
if ( verbose > = VERBOSE_3 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" [%d] At file byte offset %lld, comparing %llu-byte transfer \n " ,
rank , test - > offset , ( long long ) size ) ;
}
for ( i = 0 ; i < length ; i + + ) {
if ( testbuf [ i ] ! = goodbuf [ i ] ) {
errorCount + + ;
if ( verbose > = VERBOSE_2 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" [%d] At transfer buffer #%lld, index #%lld (file byte offset %lld): \n " ,
rank , transferCount - 1 , ( long long ) i ,
test - > offset +
( IOR_size_t ) ( i * sizeof ( IOR_size_t ) ) ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " [%d] %s0x " , rank , bufferLabel1 ) ;
fprintf ( out_logfile , " %016llx \n " , goodbuf [ i ] ) ;
fprintf ( out_logfile , " [%d] %s0x " , rank , bufferLabel2 ) ;
fprintf ( out_logfile , " %016llx \n " , testbuf [ i ] ) ;
2011-11-12 02:22:17 +04:00
}
if ( ! inError ) {
inError = 1 ;
first = i ;
last = i ;
} else {
last = i ;
}
} else if ( verbose > = VERBOSE_5 & & i % 4 = = 0 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" [%d] PASSED offset = %lld bytes, transfer %lld \n " ,
rank ,
( ( i * sizeof ( unsigned long long ) ) +
test - > offset ) , transferCount ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " [%d] GOOD %s0x " , rank , bufferLabel1 ) ;
2011-11-12 02:22:17 +04:00
for ( j = 0 ; j < 4 ; j + + )
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " %016llx " , goodbuf [ i + j ] ) ;
fprintf ( out_logfile , " \n [%d] GOOD %s0x " , rank , bufferLabel2 ) ;
2011-11-12 02:22:17 +04:00
for ( j = 0 ; j < 4 ; j + + )
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " %016llx " , testbuf [ i + j ] ) ;
fprintf ( out_logfile , " \n " ) ;
2011-11-12 02:22:17 +04:00
}
}
if ( inError ) {
inError = 0 ;
GetTestFileName ( testFileName , test ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" [%d] FAILED comparison of buffer containing %d-byte ints: \n " ,
rank , ( int ) sizeof ( unsigned long long int ) ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " [%d] File name = %s \n " , rank , testFileName ) ;
fprintf ( out_logfile , " [%d] In transfer %lld, " , rank ,
2011-11-12 02:22:17 +04:00
transferCount ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" %lld errors between buffer indices %lld and %lld. \n " ,
( long long ) errorCount , ( long long ) first ,
( long long ) last ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " [%d] File byte offset = %lld: \n " , rank ,
2011-11-12 02:22:17 +04:00
( ( first * sizeof ( unsigned long long ) ) + test - > offset ) ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " [%d] %s0x " , rank , bufferLabel1 ) ;
2011-11-12 02:22:17 +04:00
for ( j = first ; j < length & & j < first + 4 ; j + + )
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " %016llx " , goodbuf [ j ] ) ;
2011-11-12 02:22:17 +04:00
if ( j = = length )
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " [end of buffer] " ) ;
fprintf ( out_logfile , " \n [%d] %s0x " , rank , bufferLabel2 ) ;
2011-11-12 02:22:17 +04:00
for ( j = first ; j < length & & j < first + 4 ; j + + )
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " %016llx " , testbuf [ j ] ) ;
2011-11-12 02:22:17 +04:00
if ( j = = length )
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " [end of buffer] " ) ;
fprintf ( out_logfile , " \n " ) ;
2011-11-12 02:22:17 +04:00
if ( test - > quitOnError = = TRUE )
ERR ( " data check error, aborting execution " ) ;
2011-06-17 23:20:43 +04:00
}
2011-11-12 02:22:17 +04:00
return ( errorCount ) ;
2011-11-12 03:11:28 +04:00
}
2011-11-12 02:22:17 +04:00
2011-11-12 03:11:28 +04:00
/*
* Count all errors across all tasks ; report errors found .
*/
2011-11-12 04:40:45 +04:00
static int CountErrors ( IOR_param_t * test , int access , int errors )
2011-11-12 02:22:17 +04:00
{
int allErrors = 0 ;
if ( test - > checkWrite | | test - > checkRead ) {
MPI_CHECK ( MPI_Reduce ( & errors , & allErrors , 1 , MPI_INT , MPI_SUM ,
0 , testComm ) , " cannot reduce errors " ) ;
MPI_CHECK ( MPI_Bcast ( & allErrors , 1 , MPI_INT , 0 , testComm ) ,
" cannot broadcast allErrors value " ) ;
if ( allErrors ! = 0 ) {
totalErrorCount + = allErrors ;
test - > errorFound = TRUE ;
}
if ( rank = = 0 & & allErrors ! = 0 ) {
if ( allErrors < 0 ) {
WARN ( " overflow in errors counted " ) ;
allErrors = - 1 ;
}
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " WARNING: incorrect data on %s (%d errors found). \n " ,
2014-08-14 02:53:24 +04:00
access = = WRITECHECK ? " write " : " read " , allErrors ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" Used Time Stamp %u (0x%x) for Data Signature \n " ,
test - > timeStampSignatureValue ,
test - > timeStampSignatureValue ) ;
}
}
return ( allErrors ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Allocate a page - aligned ( required by O_DIRECT ) buffer .
*/
2012-01-13 08:34:40 +04:00
static void * aligned_buffer_alloc ( size_t size )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
size_t pageSize ;
size_t pageMask ;
char * buf , * tmp ;
char * aligned ;
pageSize = getpagesize ( ) ;
pageMask = pageSize - 1 ;
buf = malloc ( size + pageSize + sizeof ( void * ) ) ;
if ( buf = = NULL )
ERR ( " out of memory " ) ;
/* find the alinged buffer */
tmp = buf + sizeof ( char * ) ;
aligned = tmp + pageSize - ( ( size_t ) tmp & pageMask ) ;
/* write a pointer to the original malloc()ed buffer into the bytes
preceding " aligned " , so that the aligned buffer can later be free ( ) ed */
tmp = aligned - sizeof ( void * ) ;
* ( void * * ) tmp = buf ;
2011-12-13 09:00:18 +04:00
return ( void * ) aligned ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2012-01-13 08:34:40 +04:00
/*
* Free a buffer allocated by aligned_buffer_alloc ( ) .
*/
static void aligned_buffer_free ( void * buf )
{
free ( * ( void * * ) ( ( char * ) buf - sizeof ( char * ) ) ) ;
}
2018-07-15 21:38:17 +03:00
static void * safeMalloc ( uint64_t size ) {
void * d = malloc ( size ) ;
if ( d = = NULL ) {
ERR ( " Could not malloc an array " ) ;
}
memset ( d , 0 , size ) ;
return d ;
}
2011-12-13 09:00:18 +04:00
2018-07-14 10:41:35 +03:00
static void AllocResults ( IOR_test_t * test )
2011-12-13 09:00:18 +04:00
{
2018-07-15 21:38:17 +03:00
int reps ;
if ( test - > results ! = NULL )
return ;
2011-12-13 09:00:18 +04:00
2018-07-15 21:38:17 +03:00
reps = test - > params . repetitions ;
test - > results = ( IOR_results_t * ) safeMalloc ( sizeof ( IOR_results_t ) * reps ) ;
2011-12-13 09:00:18 +04:00
}
void FreeResults ( IOR_test_t * test )
{
2018-07-15 21:38:17 +03:00
if ( test - > results ! = NULL ) {
free ( test - > results ) ;
}
2011-12-13 09:00:18 +04:00
}
2017-07-25 13:51:54 +03:00
/**
2011-06-17 23:20:43 +04:00
* Create new test for list of tests .
*/
2011-12-13 09:00:18 +04:00
IOR_test_t * CreateTest ( IOR_param_t * init_params , int test_num )
2011-06-17 23:20:43 +04:00
{
2011-12-13 09:00:18 +04:00
IOR_test_t * newTest = NULL ;
2011-11-12 02:22:17 +04:00
2011-12-13 09:00:18 +04:00
newTest = ( IOR_test_t * ) malloc ( sizeof ( IOR_test_t ) ) ;
2011-11-12 02:22:17 +04:00
if ( newTest = = NULL )
2011-12-13 09:00:18 +04:00
ERR ( " malloc() of IOR_test_t failed " ) ;
newTest - > params = * init_params ;
2018-07-14 10:41:35 +03:00
newTest - > params . platform = GetPlatformName ( ) ;
2011-12-13 09:00:18 +04:00
newTest - > params . id = test_num ;
newTest - > next = NULL ;
2014-08-14 02:53:24 +04:00
newTest - > results = NULL ;
2018-07-14 10:41:35 +03:00
AllocResults ( newTest ) ;
2011-12-13 09:00:18 +04:00
return newTest ;
}
static void DestroyTest ( IOR_test_t * test )
{
2014-08-14 02:53:24 +04:00
FreeResults ( test ) ;
free ( test ) ;
2011-12-13 09:00:18 +04:00
}
static void DestroyTests ( IOR_test_t * tests_head )
{
2014-08-14 02:53:24 +04:00
IOR_test_t * tptr , * next ;
2011-12-13 09:00:18 +04:00
2014-08-14 02:53:24 +04:00
for ( tptr = tests_head ; tptr ! = NULL ; tptr = next ) {
next = tptr - > next ;
DestroyTest ( tptr ) ;
}
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Distribute IOR_HINTs to all tasks ' environments .
*/
2011-11-12 02:22:17 +04:00
void DistributeHints ( void )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
char hint [ MAX_HINTS ] [ MAX_STR ] , fullHint [ MAX_STR ] , hintVariable [ MAX_STR ] ;
int hintCount = 0 , i ;
if ( rank = = 0 ) {
for ( i = 0 ; environ [ i ] ! = NULL ; i + + ) {
if ( strncmp ( environ [ i ] , " IOR_HINT " , strlen ( " IOR_HINT " ) )
= = 0 ) {
hintCount + + ;
if ( hintCount = = MAX_HINTS ) {
WARN ( " exceeded max hints; reset MAX_HINTS and recompile " ) ;
hintCount = MAX_HINTS ;
break ;
}
/* assume no IOR_HINT is greater than MAX_STR in length */
strncpy ( hint [ hintCount - 1 ] , environ [ i ] ,
MAX_STR - 1 ) ;
}
2011-06-17 23:20:43 +04:00
}
}
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Bcast ( & hintCount , sizeof ( hintCount ) , MPI_BYTE ,
2011-06-17 23:20:43 +04:00
0 , MPI_COMM_WORLD ) , " cannot broadcast hints " ) ;
2011-11-12 02:22:17 +04:00
for ( i = 0 ; i < hintCount ; i + + ) {
MPI_CHECK ( MPI_Bcast ( & hint [ i ] , MAX_STR , MPI_BYTE ,
0 , MPI_COMM_WORLD ) ,
" cannot broadcast hints " ) ;
strcpy ( fullHint , hint [ i ] ) ;
strcpy ( hintVariable , strtok ( fullHint , " = " ) ) ;
if ( getenv ( hintVariable ) = = NULL ) {
/* doesn't exist in this task's environment; better set it */
if ( putenv ( hint [ i ] ) ! = 0 )
WARN ( " cannot set environment variable " ) ;
}
2011-06-17 23:20:43 +04:00
}
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Fill buffer , which is transfer size bytes long , with known 8 - byte long long
* int values . In even - numbered 8 - byte long long ints , store MPI task in high
* bits and timestamp signature in low bits . In odd - numbered 8 - byte long long
* ints , store transfer offset . If storeFileOffset option is used , the file
* ( not transfer ) offset is stored instead .
*/
2015-05-21 21:05:56 +03:00
static void
FillIncompressibleBuffer ( void * buffer , IOR_param_t * test )
{
size_t i ;
unsigned long long hi , lo ;
unsigned long long * buf = ( unsigned long long * ) buffer ;
2017-09-27 19:45:47 +03:00
2015-05-21 21:05:56 +03:00
for ( i = 0 ; i < test - > transferSize / sizeof ( unsigned long long ) ; i + + ) {
hi = ( ( unsigned long long ) rand_r ( & test - > incompressibleSeed ) < < 32 ) ;
lo = ( unsigned long long ) rand_r ( & test - > incompressibleSeed ) ;
2017-09-27 19:45:47 +03:00
buf [ i ] = hi | lo ;
2015-05-21 21:05:56 +03:00
}
}
unsigned int reseed_incompressible_prng = TRUE ;
2011-11-12 04:40:45 +04:00
static void
2011-11-12 02:22:17 +04:00
FillBuffer ( void * buffer ,
IOR_param_t * test , unsigned long long offset , int fillrank )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
size_t i ;
unsigned long long hi , lo ;
unsigned long long * buf = ( unsigned long long * ) buffer ;
2017-09-27 19:45:47 +03:00
2015-05-21 21:05:56 +03:00
if ( test - > dataPacketType = = incompressible ) { /* Make for some non compressable buffers with randomish data */
2011-11-12 02:22:17 +04:00
2015-05-21 21:05:56 +03:00
/* In order for write checks to work, we have to restart the psuedo random sequence */
if ( reseed_incompressible_prng = = TRUE ) {
2017-09-27 19:45:47 +03:00
test - > incompressibleSeed = test - > setTimeStampSignature + rank ; /* We copied seed into timestampSignature at initialization, also add the rank to add randomness between processes */
2015-05-21 21:05:56 +03:00
reseed_incompressible_prng = FALSE ;
}
FillIncompressibleBuffer ( buffer , test ) ;
}
2017-09-27 19:45:47 +03:00
2015-05-21 21:05:56 +03:00
else {
hi = ( ( unsigned long long ) fillrank ) < < 32 ;
lo = ( unsigned long long ) test - > timeStampSignatureValue ;
for ( i = 0 ; i < test - > transferSize / sizeof ( unsigned long long ) ; i + + ) {
if ( ( i % 2 ) = = 0 ) {
/* evens contain MPI rank and time in seconds */
buf [ i ] = hi | lo ;
} else {
/* odds contain offset */
buf [ i ] = offset + ( i * sizeof ( unsigned long long ) ) ;
}
2011-11-12 02:22:17 +04:00
}
2011-06-17 23:20:43 +04:00
}
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Return string describing machine name and type .
*/
2018-07-14 10:41:35 +03:00
char * GetPlatformName ( )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
char nodeName [ MAX_STR ] , * p , * start , sysName [ MAX_STR ] ;
2018-07-14 10:41:35 +03:00
char platformName [ MAX_STR ] ;
2011-11-12 02:22:17 +04:00
struct utsname name ;
if ( uname ( & name ) ! = 0 ) {
2011-12-15 01:40:25 +04:00
EWARN ( " cannot get platform name " ) ;
2011-11-12 02:22:17 +04:00
sprintf ( sysName , " %s " , " Unknown " ) ;
sprintf ( nodeName , " %s " , " Unknown " ) ;
2011-06-17 23:20:43 +04:00
} else {
2011-11-12 02:22:17 +04:00
sprintf ( sysName , " %s " , name . sysname ) ;
sprintf ( nodeName , " %s " , name . nodename ) ;
2011-06-17 23:20:43 +04:00
}
2011-11-12 02:22:17 +04:00
start = nodeName ;
if ( strlen ( nodeName ) = = 0 ) {
p = start ;
} else {
/* point to one character back from '\0' */
p = start + strlen ( nodeName ) - 1 ;
}
/*
* to cut off trailing node number , search backwards
* for the first non - numeric character
*/
while ( p ! = start ) {
if ( * p < ' 0 ' | | * p > ' 9 ' ) {
* ( p + 1 ) = ' \0 ' ;
break ;
} else {
p - - ;
}
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
sprintf ( platformName , " %s(%s) " , nodeName , sysName ) ;
2018-07-14 10:41:35 +03:00
return strdup ( platformName ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
2011-06-17 23:20:43 +04:00
/*
* Parse file name .
*/
2011-11-12 04:40:45 +04:00
static char * * ParseFileName ( char * name , int * count )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
char * * fileNames , * tmp , * token ;
char delimiterString [ 3 ] = { FILENAME_DELIMITER , ' \n ' , ' \0 ' } ;
int i = 0 ;
* count = 0 ;
tmp = name ;
/* pass one */
/* if something there, count the first item */
if ( * tmp ! = ' \0 ' ) {
( * count ) + + ;
}
/* count the rest of the filenames */
while ( * tmp ! = ' \0 ' ) {
if ( * tmp = = FILENAME_DELIMITER ) {
( * count ) + + ;
}
tmp + + ;
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
fileNames = ( char * * ) malloc ( ( * count ) * sizeof ( char * * ) ) ;
if ( fileNames = = NULL )
ERR ( " out of memory " ) ;
/* pass two */
token = strtok ( name , delimiterString ) ;
while ( token ! = NULL ) {
fileNames [ i ] = token ;
token = strtok ( NULL , delimiterString ) ;
i + + ;
}
return ( fileNames ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2018-07-08 15:38:05 +03:00
2011-06-17 23:20:43 +04:00
/*
2018-07-08 15:38:05 +03:00
* Return test file name to access .
* for single shared file , fileNames [ 0 ] is returned in testFileName
2011-06-17 23:20:43 +04:00
*/
2018-07-08 15:38:05 +03:00
void GetTestFileName ( char * testFileName , IOR_param_t * test )
2011-06-17 23:20:43 +04:00
{
2018-07-08 15:38:05 +03:00
char * * fileNames ;
char initialTestFileName [ MAX_PATHLEN ] ;
char testFileNameRoot [ MAX_STR ] ;
char tmpString [ MAX_STR ] ;
int count ;
2012-01-09 03:46:43 +04:00
2018-07-08 15:38:05 +03:00
/* parse filename for multiple file systems */
strcpy ( initialTestFileName , test - > testFileName ) ;
fileNames = ParseFileName ( initialTestFileName , & count ) ;
if ( count > 1 & & test - > uniqueDir = = TRUE )
ERR ( " cannot use multiple file names with unique directories " ) ;
if ( test - > filePerProc ) {
strcpy ( testFileNameRoot ,
fileNames [ ( ( rank +
rankOffset ) % test - > numTasks ) % count ] ) ;
} else {
strcpy ( testFileNameRoot , fileNames [ 0 ] ) ;
2011-06-17 23:20:43 +04:00
}
2012-01-09 03:46:43 +04:00
2018-07-08 15:38:05 +03:00
/* give unique name if using multiple files */
if ( test - > filePerProc ) {
/*
* prepend rank subdirectory before filename
* e . g . , / dir / file = > / dir / < rank > / file
*/
if ( test - > uniqueDir = = TRUE ) {
strcpy ( testFileNameRoot ,
PrependDir ( test , testFileNameRoot ) ) ;
}
sprintf ( testFileName , " %s.%08d " , testFileNameRoot ,
( rank + rankOffset ) % test - > numTasks ) ;
} else {
strcpy ( testFileName , testFileNameRoot ) ;
}
2012-01-09 03:46:43 +04:00
2018-07-08 15:38:05 +03:00
/* add suffix for multiple files */
if ( test - > repCounter > - 1 ) {
sprintf ( tmpString , " .%d " , test - > repCounter ) ;
strcat ( testFileName , tmpString ) ;
}
free ( fileNames ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* From absolute directory , insert rank as subdirectory . Allows each task
* to write to its own directory . E . g . , / dir / file = > / dir / < rank > / file .
*/
2011-11-12 04:40:45 +04:00
static char * PrependDir ( IOR_param_t * test , char * rootDir )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
char * dir ;
char fname [ MAX_STR + 1 ] ;
char * p ;
int i ;
dir = ( char * ) malloc ( MAX_STR + 1 ) ;
if ( dir = = NULL )
ERR ( " out of memory " ) ;
/* get dir name */
strcpy ( dir , rootDir ) ;
i = strlen ( dir ) - 1 ;
while ( i > 0 ) {
if ( dir [ i ] = = ' \0 ' | | dir [ i ] = = ' / ' ) {
dir [ i ] = ' / ' ;
dir [ i + 1 ] = ' \0 ' ;
break ;
}
i - - ;
}
/* get file name */
strcpy ( fname , rootDir ) ;
p = fname ;
while ( i > 0 ) {
if ( fname [ i ] = = ' \0 ' | | fname [ i ] = = ' / ' ) {
p = fname + ( i + 1 ) ;
break ;
}
i - - ;
}
/* create directory with rank as subdirectory */
sprintf ( dir , " %s%d " , dir , ( rank + rankOffset ) % test - > numTasks ) ;
/* dir doesn't exist, so create */
if ( access ( dir , F_OK ) ! = 0 ) {
if ( mkdir ( dir , S_IRWXU ) < 0 ) {
ERR ( " cannot create directory " ) ;
}
/* check if correct permissions */
} else if ( access ( dir , R_OK ) ! = 0 | | access ( dir , W_OK ) ! = 0 | |
access ( dir , X_OK ) ! = 0 ) {
ERR ( " invalid directory permissions " ) ;
}
/* concatenate dir and file names */
strcat ( dir , " / " ) ;
strcat ( dir , p ) ;
return dir ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/******************************************************************************/
/*
* Reduce test results , and show if verbose set .
*/
2011-12-13 09:00:18 +04:00
static void ReduceIterResults ( IOR_test_t * test , double * * timer , int rep ,
2011-11-12 04:40:45 +04:00
int access )
2011-06-17 23:20:43 +04:00
{
2018-07-08 15:38:05 +03:00
double reduced [ 12 ] = { 0 } ;
2014-08-14 02:53:24 +04:00
double diff [ 6 ] ;
double * diff_subset ;
double totalTime ;
double bw ;
2018-07-08 15:38:05 +03:00
int i ;
MPI_Op op ;
2011-11-12 02:22:17 +04:00
2014-08-14 02:53:24 +04:00
assert ( access = = WRITE | | access = = READ ) ;
2011-12-11 08:45:19 +04:00
2011-11-12 02:22:17 +04:00
/* Find the minimum start time of the even numbered timers, and the
maximum finish time for the odd numbered timers */
for ( i = 0 ; i < 12 ; i + + ) {
op = i % 2 ? MPI_MAX : MPI_MIN ;
MPI_CHECK ( MPI_Reduce ( & timer [ i ] [ rep ] , & reduced [ i ] , 1 , MPI_DOUBLE ,
op , 0 , testComm ) , " MPI_Reduce() " ) ;
2011-06-17 23:20:43 +04:00
}
2011-12-11 08:45:19 +04:00
if ( rank ! = 0 ) {
2014-08-14 02:53:24 +04:00
/* Only rank 0 tallies and prints the results. */
return ;
}
/* Calculate elapsed times and throughput numbers */
for ( i = 0 ; i < 6 ; i + + ) {
diff [ i ] = reduced [ 2 * i + 1 ] - reduced [ 2 * i ] ;
}
if ( access = = WRITE ) {
totalTime = reduced [ 5 ] - reduced [ 0 ] ;
2018-07-15 21:38:17 +03:00
test - > results [ rep ] . writeTime = totalTime ;
2014-08-14 02:53:24 +04:00
diff_subset = & diff [ 0 ] ;
} else { /* READ */
totalTime = reduced [ 11 ] - reduced [ 6 ] ;
2018-07-15 21:38:17 +03:00
test - > results [ rep ] . readTime = totalTime ;
2014-08-14 02:53:24 +04:00
diff_subset = & diff [ 3 ] ;
}
2011-11-12 02:22:17 +04:00
2011-12-11 08:45:19 +04:00
if ( verbose < VERBOSE_0 ) {
2014-08-14 02:53:24 +04:00
return ;
}
2018-07-15 21:38:17 +03:00
bw = ( double ) test - > results [ rep ] . aggFileSizeForBW / totalTime ;
2012-01-09 06:55:46 +04:00
2018-07-08 15:38:05 +03:00
PrintReducedResult ( test , access , bw , diff_subset , totalTime , rep ) ;
2012-01-09 06:55:46 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Check for file ( s ) , then remove all files if file - per - proc , else single file .
Algorithms 'S3', 'S3_plus', and 'S3_EMC' all available.
These are variants on S3. S3 uses the "pure" S3 interface, e.g. using
Multi-Part-Upload. The "plus" variant enables EMC-extensions in the aws4c
library. This allows the N:N case to use "append", in the case where
"transfer_size" != "block_size" for IOR. In pure S3, the N:N case will
fail, because the EMC-extensions won't be enabled, and appending (which
attempts to use the EMC byte-range tricks to do this) will throw an error.
In the S3_EMC alg, N:1 uses EMCs other byte-range tricks to write different
parts of an N:1 file, and also uses append to write the parts of an N:N
file. Preliminary tests show these EMC extensions look to improve BW by
~20%.
I put all three algs in aiori-S3.c, because it seemed some code was getting
reused. Not sure if that's still going to make sense after the TBD, below.
TBD: Recently realized that the "pure' S3 shouldn't be trying to use
appends for anything. In the N:N case, it should just use MPU, within each
file. Then, there's no need for S3_plus. We just have S3, which does MPU
for all writes where transfer_size != block_size, and uses (standard)
byte-range reads for reading. Then S3_EMC uses "appends for N:N writes,
and byte-range writes for N:1 writes. This separates the code for the two
algs a little more, but we might still want them in the same file.
2014-10-30 01:04:30 +03:00
*
2011-06-17 23:20:43 +04:00
*/
2011-11-12 04:40:45 +04:00
static void RemoveFile ( char * testFileName , int filePerProc , IOR_param_t * test )
2011-11-12 02:22:17 +04:00
{
2018-07-07 12:29:27 +03:00
int tmpRankOffset = 0 ;
2011-11-12 02:22:17 +04:00
if ( filePerProc ) {
/* in random tasks, delete own file */
if ( test - > reorderTasksRandom = = TRUE ) {
tmpRankOffset = rankOffset ;
rankOffset = 0 ;
GetTestFileName ( testFileName , test ) ;
}
2018-08-27 20:22:38 +03:00
if ( backend - > access ( testFileName , F_OK , test ) = = 0 | |
strcasecmp ( test - > api , " DAOS " ) = = 0 ) {
2011-11-12 02:22:17 +04:00
backend - > delete ( testFileName , test ) ;
}
if ( test - > reorderTasksRandom = = TRUE ) {
rankOffset = tmpRankOffset ;
GetTestFileName ( testFileName , test ) ;
}
} else {
2018-08-27 20:22:38 +03:00
if ( ( rank = = 0 ) & & ( backend - > access ( testFileName , F_OK , test ) = = 0 | |
2018-09-13 01:53:42 +03:00
strcasecmp ( test - > api , " DAOS " ) = = 0 ) ) {
2011-11-12 02:22:17 +04:00
backend - > delete ( testFileName , test ) ;
}
}
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Setup tests by parsing commandline and creating test script .
2015-05-19 18:36:28 +03:00
* Perform a sanity - check on the configured parameters .
2011-06-17 23:20:43 +04:00
*/
2018-07-14 10:41:35 +03:00
static void InitTests ( IOR_test_t * tests , MPI_Comm com )
2011-06-17 23:20:43 +04:00
{
2018-07-14 10:41:35 +03:00
int size ;
MPI_CHECK ( MPI_Comm_size ( com , & size ) , " MPI_Comm_size() error " ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/* count the tasks per node */
2018-07-14 10:41:35 +03:00
tasksPerNode = CountTasksPerNode ( com ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/*
* Since there is no guarantee that anyone other than
* task 0 has the environment settings for the hints , pass
2018-07-07 13:42:21 +03:00
* the hint = value pair to everyone else in mpi_comm_world
2011-11-12 02:22:17 +04:00
*/
DistributeHints ( ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/* check validity of tests and create test queue */
while ( tests ! = NULL ) {
2018-07-14 10:41:35 +03:00
IOR_param_t * params = & tests - > params ;
params - > testComm = com ;
params - > nodes = params - > numTasks / tasksPerNode ;
params - > tasksPerNode = tasksPerNode ;
if ( params - > numTasks = = 0 ) {
params - > numTasks = size ;
}
params - > expectedAggFileSize =
params - > blockSize * params - > segmentCount * params - > numTasks ;
2015-05-19 18:36:28 +03:00
ValidateTests ( & tests - > params ) ;
2011-12-13 09:00:18 +04:00
tests = tests - > next ;
2011-11-12 02:22:17 +04:00
}
2011-06-17 23:20:43 +04:00
2018-07-07 13:42:21 +03:00
init_clock ( ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/* seed random number generator */
2018-07-07 13:42:21 +03:00
SeedRandGen ( mpi_comm_world ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2011-11-12 03:11:28 +04:00
/*
* Setup transfer buffers , creating and filling as needed .
*/
2017-09-27 19:45:47 +03:00
static void XferBuffersSetup ( IOR_io_buffers * ioBuffers , IOR_param_t * test ,
2015-05-27 19:24:52 +03:00
int pretendRank )
2011-06-17 23:20:43 +04:00
{
2015-05-27 19:24:52 +03:00
ioBuffers - > buffer = aligned_buffer_alloc ( test - > transferSize ) ;
if ( test - > checkWrite | | test - > checkRead ) {
ioBuffers - > checkBuffer = aligned_buffer_alloc ( test - > transferSize ) ;
2012-01-13 08:34:40 +04:00
}
2017-11-29 12:17:02 +03:00
if ( test - > checkRead | | test - > checkWrite ) {
2015-05-27 19:24:52 +03:00
ioBuffers - > readCheckBuffer = aligned_buffer_alloc ( test - > transferSize ) ;
2012-01-13 08:34:40 +04:00
}
return ;
}
/*
* Free transfer buffers .
*/
2015-05-27 19:24:52 +03:00
static void XferBuffersFree ( IOR_io_buffers * ioBuffers , IOR_param_t * test )
2012-01-13 08:34:40 +04:00
{
2015-05-27 19:24:52 +03:00
aligned_buffer_free ( ioBuffers - > buffer ) ;
if ( test - > checkWrite | | test - > checkRead ) {
aligned_buffer_free ( ioBuffers - > checkBuffer ) ;
2012-01-13 08:34:40 +04:00
}
2015-05-27 19:24:52 +03:00
if ( test - > checkRead ) {
aligned_buffer_free ( ioBuffers - > readCheckBuffer ) ;
2011-06-17 23:20:43 +04:00
}
2012-01-13 08:34:40 +04:00
2011-11-12 02:22:17 +04:00
return ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2012-09-10 21:50:18 +04:00
2011-12-13 09:00:18 +04:00
2012-01-07 05:29:45 +04:00
/*
* malloc a buffer , touching every page in an attempt to defeat lazy allocation .
*/
static void * malloc_and_touch ( size_t size )
{
size_t page_size ;
char * buf ;
char * ptr ;
if ( size = = 0 )
return NULL ;
2014-08-14 02:53:24 +04:00
page_size = sysconf ( _SC_PAGESIZE ) ;
2012-01-07 05:29:45 +04:00
buf = ( char * ) malloc ( size ) ;
if ( buf = = NULL )
2012-01-09 06:41:30 +04:00
return NULL ;
2012-01-07 05:29:45 +04:00
for ( ptr = buf ; ptr < buf + size ; ptr + = page_size ) {
* ptr = ( char ) 1 ;
}
return ( void * ) buf ;
}
2011-12-14 10:48:14 +04:00
static void file_hits_histogram ( IOR_param_t * params )
{
2018-07-07 12:29:27 +03:00
int * rankoffs = NULL ;
int * filecont = NULL ;
int * filehits = NULL ;
2014-08-14 02:53:24 +04:00
int ifile ;
int jfile ;
if ( rank = = 0 ) {
rankoffs = ( int * ) malloc ( params - > numTasks * sizeof ( int ) ) ;
filecont = ( int * ) malloc ( params - > numTasks * sizeof ( int ) ) ;
filehits = ( int * ) malloc ( params - > numTasks * sizeof ( int ) ) ;
}
MPI_CHECK ( MPI_Gather ( & rankOffset , 1 , MPI_INT , rankoffs ,
2018-07-07 13:42:21 +03:00
1 , MPI_INT , 0 , mpi_comm_world ) ,
2014-08-14 02:53:24 +04:00
" MPI_Gather error " ) ;
if ( rank ! = 0 )
return ;
memset ( ( void * ) filecont , 0 , params - > numTasks * sizeof ( int ) ) ;
for ( ifile = 0 ; ifile < params - > numTasks ; ifile + + ) {
filecont [ ( ifile + rankoffs [ ifile ] ) % params - > numTasks ] + + ;
}
memset ( ( void * ) filehits , 0 , params - > numTasks * sizeof ( int ) ) ;
for ( ifile = 0 ; ifile < params - > numTasks ; ifile + + )
for ( jfile = 0 ; jfile < params - > numTasks ; jfile + + ) {
if ( ifile = = filecont [ jfile ] )
filehits [ ifile ] + + ;
}
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " #File Hits Dist: " ) ;
2014-08-14 02:53:24 +04:00
jfile = 0 ;
ifile = 0 ;
while ( jfile < params - > numTasks & & ifile < params - > numTasks ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " %d " , filehits [ ifile ] ) ;
2014-08-14 02:53:24 +04:00
jfile + = filehits [ ifile ] , ifile + + ;
}
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " \n " ) ;
2014-08-14 02:53:24 +04:00
free ( rankoffs ) ;
free ( filecont ) ;
free ( filehits ) ;
2011-12-14 10:48:14 +04:00
}
2012-01-09 00:30:05 +04:00
int test_time_elapsed ( IOR_param_t * params , double startTime )
{
2014-08-14 02:53:24 +04:00
double endTime ;
2012-01-09 00:30:05 +04:00
2014-08-14 02:53:24 +04:00
if ( params - > maxTimeDuration = = 0 )
return 0 ;
2012-01-09 00:30:05 +04:00
2014-08-14 02:53:24 +04:00
endTime = startTime + ( params - > maxTimeDuration * 60 ) ;
2012-01-09 00:30:05 +04:00
2014-08-14 02:53:24 +04:00
return GetTimeStamp ( ) > = endTime ;
2012-01-09 00:30:05 +04:00
}
2012-01-09 06:41:30 +04:00
/*
* hog some memory as a rough simulation of a real application ' s memory use
*/
static void * HogMemory ( IOR_param_t * params )
{
size_t size ;
void * buf ;
if ( params - > memoryPerTask ! = 0 ) {
size = params - > memoryPerTask ;
} else if ( params - > memoryPerNode ! = 0 ) {
if ( verbose > = VERBOSE_3 )
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " This node hogging %ld bytes of memory \n " ,
2012-01-09 06:41:30 +04:00
params - > memoryPerNode ) ;
size = params - > memoryPerNode / params - > tasksPerNode ;
} else {
return NULL ;
}
if ( verbose > = VERBOSE_3 )
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " This task hogging %ld bytes of memory \n " , size ) ;
2012-01-09 06:41:30 +04:00
buf = malloc_and_touch ( size ) ;
if ( buf = = NULL )
ERR ( " malloc of simulated applciation buffer failed " ) ;
return buf ;
}
2011-06-17 23:20:43 +04:00
/*
* Using the test parameters , run iteration ( s ) of single test .
*/
2011-12-13 09:00:18 +04:00
static void TestIoSys ( IOR_test_t * test )
2011-06-17 23:20:43 +04:00
{
2014-08-14 02:53:24 +04:00
IOR_param_t * params = & test - > params ;
IOR_results_t * results = test - > results ;
2011-11-12 02:22:17 +04:00
char testFileName [ MAX_STR ] ;
double * timer [ 12 ] ;
double startTime ;
2015-05-27 19:24:52 +03:00
int pretendRank ;
2012-01-09 00:30:05 +04:00
int i , rep ;
2011-11-12 02:22:17 +04:00
void * fd ;
MPI_Group orig_group , new_group ;
int range [ 3 ] ;
IOR_offset_t dataMoved ; /* for data rate calculation */
2012-01-07 05:29:45 +04:00
void * hog_buf ;
2015-05-27 19:24:52 +03:00
IOR_io_buffers ioBuffers ;
2011-11-12 02:22:17 +04:00
/* set up communicator for test */
2011-12-13 09:00:18 +04:00
if ( params - > numTasks > numTasksWorld ) {
2011-11-12 02:22:17 +04:00
if ( rank = = 0 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" WARNING: More tasks requested (%d) than available (%d), " ,
2011-12-13 09:00:18 +04:00
params - > numTasks , numTasksWorld ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " running on %d tasks. \n " ,
2011-11-12 02:22:17 +04:00
numTasksWorld ) ;
2011-06-17 23:20:43 +04:00
}
2011-12-13 09:00:18 +04:00
params - > numTasks = numTasksWorld ;
2011-11-12 02:22:17 +04:00
}
2018-07-07 13:42:21 +03:00
MPI_CHECK ( MPI_Comm_group ( mpi_comm_world , & orig_group ) ,
2011-11-12 02:22:17 +04:00
" MPI_Comm_group() error " ) ;
2014-08-14 02:53:24 +04:00
range [ 0 ] = 0 ; /* first rank */
2011-12-13 09:00:18 +04:00
range [ 1 ] = params - > numTasks - 1 ; /* last rank */
2014-08-14 02:53:24 +04:00
range [ 2 ] = 1 ; /* stride */
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Group_range_incl ( orig_group , 1 , & range , & new_group ) ,
" MPI_Group_range_incl() error " ) ;
2018-07-07 13:42:21 +03:00
MPI_CHECK ( MPI_Comm_create ( mpi_comm_world , new_group , & testComm ) ,
2011-11-12 02:22:17 +04:00
" MPI_Comm_create() error " ) ;
2013-10-07 18:12:20 +04:00
MPI_CHECK ( MPI_Group_free ( & orig_group ) , " MPI_Group_Free() error " ) ;
MPI_CHECK ( MPI_Group_free ( & new_group ) , " MPI_Group_Free() error " ) ;
2011-12-13 09:00:18 +04:00
params - > testComm = testComm ;
2011-11-12 02:22:17 +04:00
if ( testComm = = MPI_COMM_NULL ) {
/* tasks not in the group do not participate in this test */
2018-07-07 13:42:21 +03:00
MPI_CHECK ( MPI_Barrier ( mpi_comm_world ) , " barrier error " ) ;
2011-11-12 02:22:17 +04:00
return ;
}
if ( rank = = 0 & & verbose > = VERBOSE_1 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " Participating tasks: %d \n " , params - > numTasks ) ;
fflush ( out_logfile ) ;
2011-11-12 02:22:17 +04:00
}
2011-12-13 09:00:18 +04:00
if ( rank = = 0 & & params - > reorderTasks = = TRUE & & verbose > = VERBOSE_1 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" Using reorderTasks '-C' (expecting block, not cyclic, task assignment) \n " ) ;
2018-07-07 13:42:21 +03:00
fflush ( out_logfile ) ;
2011-06-17 23:20:43 +04:00
}
2018-07-08 00:19:42 +03:00
params - > tasksPerNode = CountTasksPerNode ( testComm ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/* setup timers */
for ( i = 0 ; i < 12 ; i + + ) {
2011-12-13 09:00:18 +04:00
timer [ i ] = ( double * ) malloc ( params - > repetitions * sizeof ( double ) ) ;
2011-11-12 02:22:17 +04:00
if ( timer [ i ] = = NULL )
2011-12-11 08:45:19 +04:00
ERR ( " malloc failed " ) ;
2011-11-12 02:22:17 +04:00
}
2011-12-11 08:45:19 +04:00
2011-11-12 02:22:17 +04:00
/* bind I/O calls to specific API */
2018-07-14 12:27:31 +03:00
backend = aiori_select ( params - > api ) ;
2011-06-17 23:20:43 +04:00
2018-08-30 00:14:19 +03:00
if ( backend - > initialize )
2018-08-30 01:25:48 +03:00
backend - > initialize ( params ) ;
2018-06-13 21:37:37 +03:00
2011-11-12 02:22:17 +04:00
/* show test setup */
if ( rank = = 0 & & verbose > = VERBOSE_0 )
2011-12-13 09:00:18 +04:00
ShowSetup ( params ) ;
2011-06-17 23:20:43 +04:00
2012-01-09 06:41:30 +04:00
hog_buf = HogMemory ( params ) ;
2012-01-07 05:29:45 +04:00
2015-05-27 19:24:52 +03:00
pretendRank = ( rank + rankOffset ) % params - > numTasks ;
/* IO Buffer Setup */
2017-09-27 19:45:47 +03:00
if ( params - > setTimeStampSignature ) { // initialize the buffer properly
params - > timeStampSignatureValue = ( unsigned int ) params - > setTimeStampSignature ;
}
2015-05-27 19:24:52 +03:00
XferBuffersSetup ( & ioBuffers , params , pretendRank ) ;
2017-09-27 19:45:47 +03:00
reseed_incompressible_prng = TRUE ; // reset pseudo random generator, necessary to guarantee the next call to FillBuffer produces the same value as it is right now
2015-05-27 19:24:52 +03:00
/* Initial time stamp */
2011-11-12 02:22:17 +04:00
startTime = GetTimeStamp ( ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/* loop over test iterations */
2018-07-15 21:38:17 +03:00
uint64_t params_saved_wearout = params - > stoneWallingWearOutIterations ;
2011-12-13 09:00:18 +04:00
for ( rep = 0 ; rep < params - > repetitions ; rep + + ) {
2018-07-08 18:47:23 +03:00
PrintRepeatStart ( ) ;
2011-11-12 02:22:17 +04:00
/* Get iteration start time in seconds in task 0 and broadcast to
all tasks */
if ( rank = = 0 ) {
2017-09-27 19:45:47 +03:00
if ( ! params - > setTimeStampSignature ) {
2011-11-12 02:22:17 +04:00
time_t currentTime ;
if ( ( currentTime = time ( NULL ) ) = = - 1 ) {
ERR ( " cannot get current time " ) ;
}
2011-12-13 09:00:18 +04:00
params - > timeStampSignatureValue =
2014-08-14 02:53:24 +04:00
( unsigned int ) currentTime ;
2017-09-27 19:45:47 +03:00
if ( verbose > = VERBOSE_2 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2017-09-27 19:45:47 +03:00
" Using Time Stamp %u (0x%x) for Data Signature \n " ,
params - > timeStampSignatureValue ,
params - > timeStampSignatureValue ) ;
}
2011-11-12 02:22:17 +04:00
}
2014-08-14 02:53:24 +04:00
if ( rep = = 0 & & verbose > = VERBOSE_0 ) {
2018-07-08 15:47:55 +03:00
PrintTableHeader ( ) ;
2014-08-14 02:53:24 +04:00
}
2011-11-12 02:22:17 +04:00
}
MPI_CHECK ( MPI_Bcast
2011-12-13 09:00:18 +04:00
( & params - > timeStampSignatureValue , 1 , MPI_UNSIGNED , 0 ,
2011-11-12 02:22:17 +04:00
testComm ) , " cannot broadcast start time value " ) ;
2017-12-09 13:52:13 +03:00
FillBuffer ( ioBuffers . buffer , params , 0 , pretendRank ) ;
2011-11-12 02:22:17 +04:00
/* use repetition count for number of multiple files */
2011-12-13 09:00:18 +04:00
if ( params - > multiFile )
params - > repCounter = rep ;
2011-11-12 02:22:17 +04:00
/*
* write the file ( s ) , getting timing between I / O calls
*/
2012-01-09 00:30:05 +04:00
if ( params - > writeFile & & ! test_time_elapsed ( params , startTime ) ) {
2011-12-13 09:00:18 +04:00
GetTestFileName ( testFileName , params ) ;
2011-11-12 02:22:17 +04:00
if ( verbose > = VERBOSE_3 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " task %d writing %s \n " , rank ,
2011-11-12 02:22:17 +04:00
testFileName ) ;
}
2011-12-13 09:00:18 +04:00
DelaySecs ( params - > interTestDelay ) ;
if ( params - > useExistingTestFile = = FALSE ) {
RemoveFile ( testFileName , params - > filePerProc ,
params ) ;
2011-11-12 02:22:17 +04:00
}
2018-07-15 21:38:17 +03:00
params - > stoneWallingWearOutIterations = params_saved_wearout ;
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
2011-12-13 09:00:18 +04:00
params - > open = WRITE ;
2011-11-12 02:22:17 +04:00
timer [ 0 ] [ rep ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
fd = backend - > create ( testFileName , params ) ;
2011-11-12 02:22:17 +04:00
timer [ 1 ] [ rep ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
if ( params - > intraTestBarriers )
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) ,
" barrier error " ) ;
if ( rank = = 0 & & verbose > = VERBOSE_1 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-12-13 13:07:55 +04:00
" Commencing write performance test: %s " ,
2014-08-14 02:53:24 +04:00
CurrentTimeString ( ) ) ;
2011-11-12 02:22:17 +04:00
}
timer [ 2 ] [ rep ] = GetTimeStamp ( ) ;
2018-07-15 21:38:17 +03:00
dataMoved = WriteOrRead ( params , & results [ rep ] , fd , WRITE , & ioBuffers ) ;
2014-08-29 01:39:44 +04:00
if ( params - > verbose > = VERBOSE_4 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " * data moved = %llu \n " , dataMoved ) ;
fflush ( out_logfile ) ;
2014-08-29 01:39:44 +04:00
}
2011-11-12 02:22:17 +04:00
timer [ 3 ] [ rep ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
if ( params - > intraTestBarriers )
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) ,
" barrier error " ) ;
timer [ 4 ] [ rep ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
backend - > close ( fd , params ) ;
2011-11-12 02:22:17 +04:00
timer [ 5 ] [ rep ] = GetTimeStamp ( ) ;
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
2014-08-14 02:53:24 +04:00
/* get the size of the file just written */
2018-07-15 21:38:17 +03:00
results [ rep ] . aggFileSizeFromStat =
2014-08-14 02:53:24 +04:00
backend - > get_file_size ( params , testComm , testFileName ) ;
2011-11-12 02:22:17 +04:00
2014-08-14 02:53:24 +04:00
/* check if stat() of file doesn't equal expected file size,
use actual amount of byte moved */
CheckFileSize ( test , dataMoved , rep ) ;
2011-11-12 02:22:17 +04:00
if ( verbose > = VERBOSE_3 )
2011-12-13 09:00:18 +04:00
WriteTimes ( params , timer , rep , WRITE ) ;
2011-11-12 02:22:17 +04:00
ReduceIterResults ( test , timer , rep , WRITE ) ;
2011-12-13 09:00:18 +04:00
if ( params - > outlierThreshold ) {
CheckForOutliers ( params , timer , rep , WRITE ) ;
2011-11-12 02:22:17 +04:00
}
2018-07-15 21:38:17 +03:00
/* check if in this round we run write with stonewalling */
if ( params - > deadlineForStonewalling > 0 ) {
params - > stoneWallingWearOutIterations = results [ rep ] . pairs_accessed ;
}
2011-11-12 02:22:17 +04:00
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/*
* perform a check of data , reading back data and comparing
* against what was expected to be written
*/
2012-01-09 00:30:05 +04:00
if ( params - > checkWrite & & ! test_time_elapsed ( params , startTime ) ) {
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
if ( rank = = 0 & & verbose > = VERBOSE_1 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" Verifying contents of the file(s) just written. \n " ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " %s \n " , CurrentTimeString ( ) ) ;
2011-11-12 02:22:17 +04:00
}
2011-12-13 09:00:18 +04:00
if ( params - > reorderTasks ) {
2011-11-12 02:22:17 +04:00
/* move two nodes away from writing node */
2018-07-14 10:41:35 +03:00
rankOffset = ( 2 * params - > tasksPerNode ) % params - > numTasks ;
2011-11-12 02:22:17 +04:00
}
2015-05-21 21:05:56 +03:00
2017-11-29 12:17:02 +03:00
// update the check buffer
FillBuffer ( ioBuffers . readCheckBuffer , params , 0 , ( rank + rankOffset ) % params - > numTasks ) ;
2015-05-21 21:05:56 +03:00
reseed_incompressible_prng = TRUE ; /* Re-Seed the PRNG to get same sequence back, if random */
2011-12-13 09:00:18 +04:00
GetTestFileName ( testFileName , params ) ;
params - > open = WRITECHECK ;
fd = backend - > open ( testFileName , params ) ;
2018-07-15 21:38:17 +03:00
dataMoved = WriteOrRead ( params , & results [ rep ] , fd , WRITECHECK , & ioBuffers ) ;
2011-12-13 09:00:18 +04:00
backend - > close ( fd , params ) ;
2011-11-12 02:22:17 +04:00
rankOffset = 0 ;
}
/*
* read the file ( s ) , getting timing between I / O calls
*/
2017-09-27 19:45:47 +03:00
if ( ( params - > readFile | | params - > checkRead ) & & ! test_time_elapsed ( params , startTime ) ) {
2018-07-15 21:38:17 +03:00
/* check for stonewall */
if ( params - > stoneWallingStatusFile ) {
params - > stoneWallingWearOutIterations = ReadStoneWallingIterations ( params - > stoneWallingStatusFile ) ;
if ( params - > stoneWallingWearOutIterations = = - 1 & & rank = = 0 ) {
fprintf ( out_logfile , " WARNING: Could not read back the stonewalling status from the file! " ) ;
params - > stoneWallingWearOutIterations = 0 ;
}
}
2017-09-27 19:45:47 +03:00
int operation_flag = READ ;
if ( params - > checkRead ) {
// actually read and then compare the buffer
operation_flag = READCHECK ;
}
2011-11-12 02:22:17 +04:00
/* Get rankOffset [file offset] for this process to read, based on -C,-Z,-Q,-X options */
/* Constant process offset reading */
2011-12-13 09:00:18 +04:00
if ( params - > reorderTasks ) {
2011-11-12 02:22:17 +04:00
/* move taskPerNodeOffset nodes[1==default] away from writing node */
2018-07-14 10:41:35 +03:00
rankOffset = ( params - > taskPerNodeOffset *
params - > tasksPerNode ) % params - > numTasks ;
2011-11-12 02:22:17 +04:00
}
/* random process offset reading */
2011-12-13 09:00:18 +04:00
if ( params - > reorderTasksRandom ) {
2011-11-12 02:22:17 +04:00
/* this should not intefere with randomOffset within a file because GetOffsetArrayRandom */
/* seeds every random() call */
2014-08-14 02:53:24 +04:00
int nodeoffset ;
2011-11-12 02:22:17 +04:00
unsigned int iseed0 ;
2011-12-13 09:00:18 +04:00
nodeoffset = params - > taskPerNodeOffset ;
2011-12-14 10:48:14 +04:00
nodeoffset = ( nodeoffset < params - > nodes ) ? nodeoffset : params - > nodes - 1 ;
if ( params - > reorderTasksRandomSeed < 0 )
2014-08-14 02:53:24 +04:00
iseed0 = - 1 * params - > reorderTasksRandomSeed + rep ;
else
iseed0 = params - > reorderTasksRandomSeed ;
2011-11-12 02:22:17 +04:00
srand ( rank + iseed0 ) ;
{
2011-12-13 09:00:18 +04:00
rankOffset = rand ( ) % params - > numTasks ;
2011-11-12 02:22:17 +04:00
}
while ( rankOffset <
2011-12-13 09:00:18 +04:00
( nodeoffset * params - > tasksPerNode ) ) {
rankOffset = rand ( ) % params - > numTasks ;
2011-11-12 02:22:17 +04:00
}
/* Get more detailed stats if requested by verbose level */
if ( verbose > = VERBOSE_2 ) {
2014-08-14 02:53:24 +04:00
file_hits_histogram ( params ) ;
2011-11-12 02:22:17 +04:00
}
}
2017-09-27 19:45:47 +03:00
if ( operation_flag = = READCHECK ) {
2017-10-25 16:57:50 +03:00
FillBuffer ( ioBuffers . readCheckBuffer , params , 0 , ( rank + rankOffset ) % params - > numTasks ) ;
2017-09-27 19:45:47 +03:00
}
2011-11-12 02:22:17 +04:00
/* Using globally passed rankOffset, following function generates testFileName to read */
2011-12-13 09:00:18 +04:00
GetTestFileName ( testFileName , params ) ;
2011-11-12 02:22:17 +04:00
if ( verbose > = VERBOSE_3 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " task %d reading %s \n " , rank ,
2011-11-12 02:22:17 +04:00
testFileName ) ;
}
2011-12-13 09:00:18 +04:00
DelaySecs ( params - > interTestDelay ) ;
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
2011-12-13 09:00:18 +04:00
params - > open = READ ;
2011-11-12 02:22:17 +04:00
timer [ 6 ] [ rep ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
fd = backend - > open ( testFileName , params ) ;
2011-11-12 02:22:17 +04:00
timer [ 7 ] [ rep ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
if ( params - > intraTestBarriers )
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) ,
" barrier error " ) ;
if ( rank = = 0 & & verbose > = VERBOSE_1 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-12-13 13:07:55 +04:00
" Commencing read performance test: %s " ,
2014-08-14 02:53:24 +04:00
CurrentTimeString ( ) ) ;
2011-11-12 02:22:17 +04:00
}
timer [ 8 ] [ rep ] = GetTimeStamp ( ) ;
2018-07-15 21:38:17 +03:00
dataMoved = WriteOrRead ( params , & results [ rep ] , fd , operation_flag , & ioBuffers ) ;
2011-11-12 02:22:17 +04:00
timer [ 9 ] [ rep ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
if ( params - > intraTestBarriers )
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) ,
" barrier error " ) ;
timer [ 10 ] [ rep ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
backend - > close ( fd , params ) ;
2011-11-12 02:22:17 +04:00
timer [ 11 ] [ rep ] = GetTimeStamp ( ) ;
/* get the size of the file just read */
2018-07-15 21:38:17 +03:00
results [ rep ] . aggFileSizeFromStat =
2014-08-14 02:53:24 +04:00
backend - > get_file_size ( params , testComm ,
testFileName ) ;
2011-11-12 02:22:17 +04:00
/* check if stat() of file doesn't equal expected file size,
use actual amount of byte moved */
CheckFileSize ( test , dataMoved , rep ) ;
if ( verbose > = VERBOSE_3 )
2011-12-13 09:00:18 +04:00
WriteTimes ( params , timer , rep , READ ) ;
2011-11-12 02:22:17 +04:00
ReduceIterResults ( test , timer , rep , READ ) ;
2011-12-13 09:00:18 +04:00
if ( params - > outlierThreshold ) {
CheckForOutliers ( params , timer , rep , READ ) ;
2011-11-12 02:22:17 +04:00
}
}
2011-12-13 09:00:18 +04:00
if ( ! params - > keepFile
2012-01-09 06:55:46 +04:00
& & ! ( params - > errorFound & & params - > keepFileWithError ) ) {
double start , finish ;
start = GetTimeStamp ( ) ;
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
2011-12-13 09:00:18 +04:00
RemoveFile ( testFileName , params - > filePerProc , params ) ;
2012-01-09 06:55:46 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
finish = GetTimeStamp ( ) ;
PrintRemoveTiming ( start , finish , rep ) ;
} else {
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
2011-11-12 02:22:17 +04:00
}
2011-12-13 09:00:18 +04:00
params - > errorFound = FALSE ;
2011-11-12 02:22:17 +04:00
rankOffset = 0 ;
2018-07-08 18:47:23 +03:00
PrintRepeatEnd ( ) ;
2011-11-12 02:22:17 +04:00
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Comm_free ( & testComm ) , " MPI_Comm_free() error " ) ;
2011-12-13 09:00:18 +04:00
2012-01-14 01:27:55 +04:00
if ( params - > summary_every_test ) {
PrintLongSummaryHeader ( ) ;
PrintLongSummaryOneTest ( test ) ;
} else {
PrintShortSummary ( test ) ;
}
2011-12-13 09:00:18 +04:00
2015-05-27 19:24:52 +03:00
XferBuffersFree ( & ioBuffers , params ) ;
2014-08-14 02:53:24 +04:00
if ( hog_buf ! = NULL )
free ( hog_buf ) ;
2011-11-12 02:22:17 +04:00
for ( i = 0 ; i < 12 ; i + + ) {
free ( timer [ i ] ) ;
2011-06-17 23:20:43 +04:00
}
2012-01-07 05:29:45 +04:00
2018-08-30 00:14:19 +03:00
if ( backend - > finalize )
backend - > finalize ( NULL ) ;
2011-11-12 02:22:17 +04:00
/* Sync with the tasks that did not participate in this test */
2018-07-07 13:42:21 +03:00
MPI_CHECK ( MPI_Barrier ( mpi_comm_world ) , " barrier error " ) ;
2012-01-07 05:29:45 +04:00
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Determine if valid tests from parameters .
*/
2015-05-19 18:36:28 +03:00
static void ValidateTests ( IOR_param_t * test )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
IOR_param_t defaults ;
init_IOR_Param_t ( & defaults ) ;
2015-05-19 18:36:28 +03:00
2011-11-12 02:22:17 +04:00
if ( test - > repetitions < = 0 )
WARN_RESET ( " too few test repetitions " ,
test , & defaults , repetitions ) ;
if ( test - > numTasks < = 0 )
ERR ( " too few tasks for testing " ) ;
if ( test - > interTestDelay < 0 )
WARN_RESET ( " inter-test delay must be nonnegative value " ,
test , & defaults , interTestDelay ) ;
if ( test - > readFile ! = TRUE & & test - > writeFile ! = TRUE
& & test - > checkRead ! = TRUE & & test - > checkWrite ! = TRUE )
2017-12-09 19:22:58 +03:00
ERR ( " test must write, read, or check read/write file " ) ;
if ( ! test - > setTimeStampSignature & & test - > writeFile ! = TRUE & & test - > checkRead = = TRUE )
2017-12-09 19:24:31 +03:00
ERR ( " using readCheck only requires to write a timeStampSignature -- use -G " ) ;
2011-11-12 02:22:17 +04:00
if ( test - > segmentCount < 0 )
ERR ( " segment count must be positive value " ) ;
if ( ( test - > blockSize % sizeof ( IOR_size_t ) ) ! = 0 )
ERR ( " block size must be a multiple of access size " ) ;
if ( test - > blockSize < 0 )
ERR ( " block size must be non-negative integer " ) ;
if ( ( test - > transferSize % sizeof ( IOR_size_t ) ) ! = 0 )
ERR ( " transfer size must be a multiple of access size " ) ;
if ( test - > setAlignment < 0 )
ERR ( " alignment must be non-negative integer " ) ;
if ( test - > transferSize < 0 )
ERR ( " transfer size must be non-negative integer " ) ;
if ( test - > transferSize = = 0 ) {
ERR ( " test will not complete with zero transfer size " ) ;
2011-06-17 23:20:43 +04:00
} else {
2011-11-12 02:22:17 +04:00
if ( ( test - > blockSize % test - > transferSize ) ! = 0 )
ERR ( " block size must be a multiple of transfer size " ) ;
}
if ( test - > blockSize < test - > transferSize )
ERR ( " block size must not be smaller than transfer size " ) ;
2015-05-19 18:36:28 +03:00
/* specific APIs */
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " MPIIO " ) = = 0 )
2011-11-12 02:22:17 +04:00
& & ( test - > blockSize < sizeof ( IOR_size_t )
| | test - > transferSize < sizeof ( IOR_size_t ) ) )
ERR ( " block/transfer size may not be smaller than IOR_size_t for MPIIO " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " HDF5 " ) = = 0 )
2011-11-12 02:22:17 +04:00
& & ( test - > blockSize < sizeof ( IOR_size_t )
| | test - > transferSize < sizeof ( IOR_size_t ) ) )
ERR ( " block/transfer size may not be smaller than IOR_size_t for HDF5 " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " NCMPI " ) = = 0 )
2011-11-12 02:22:17 +04:00
& & ( test - > blockSize < sizeof ( IOR_size_t )
| | test - > transferSize < sizeof ( IOR_size_t ) ) )
ERR ( " block/transfer size may not be smaller than IOR_size_t for NCMPI " ) ;
if ( ( test - > useFileView = = TRUE )
& & ( sizeof ( MPI_Aint ) < 8 ) /* used for 64-bit datatypes */
& & ( ( test - > numTasks * test - > blockSize ) >
( 2 * ( IOR_offset_t ) GIBIBYTE ) ) )
ERR ( " segment size must be < 2GiB " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " POSIX " ) ! = 0 ) & & test - > singleXferAttempt )
2011-11-12 02:22:17 +04:00
WARN_RESET ( " retry only available in POSIX " ,
test , & defaults , singleXferAttempt ) ;
2018-04-27 20:44:20 +03:00
if ( ( ( strcasecmp ( test - > api , " POSIX " ) ! = 0 )
& & ( strcasecmp ( test - > api , " MPIIO " ) ! = 0 )
2018-07-12 18:13:40 +03:00
& & ( strcasecmp ( test - > api , " MMAP " ) ! = 0 )
2018-08-15 01:08:04 +03:00
& & ( strcasecmp ( test - > api , " HDFS " ) ! = 0 )
& & ( strcasecmp ( test - > api , " RADOS " ) ! = 0 ) ) & & test - > fsync )
2018-04-27 20:44:20 +03:00
WARN_RESET ( " fsync() not supported in selected backend " ,
2011-11-12 02:22:17 +04:00
test , & defaults , fsync ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " MPIIO " ) ! = 0 ) & & test - > preallocate )
2011-11-12 02:22:17 +04:00
WARN_RESET ( " preallocation only available in MPIIO " ,
test , & defaults , preallocate ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " MPIIO " ) ! = 0 ) & & test - > useFileView )
2011-11-12 02:22:17 +04:00
WARN_RESET ( " file view only available in MPIIO " ,
test , & defaults , useFileView ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " MPIIO " ) ! = 0 ) & & test - > useSharedFilePointer )
2011-11-12 02:22:17 +04:00
WARN_RESET ( " shared file pointer only available in MPIIO " ,
test , & defaults , useSharedFilePointer ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " MPIIO " ) = = 0 ) & & test - > useSharedFilePointer )
2011-11-12 02:22:17 +04:00
WARN_RESET ( " shared file pointer not implemented " ,
test , & defaults , useSharedFilePointer ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " MPIIO " ) ! = 0 ) & & test - > useStridedDatatype )
2011-11-12 02:22:17 +04:00
WARN_RESET ( " strided datatype only available in MPIIO " ,
test , & defaults , useStridedDatatype ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " MPIIO " ) = = 0 ) & & test - > useStridedDatatype )
2011-11-12 02:22:17 +04:00
WARN_RESET ( " strided datatype not implemented " ,
test , & defaults , useStridedDatatype ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " MPIIO " ) = = 0 )
2011-11-12 02:22:17 +04:00
& & test - > useStridedDatatype & & ( test - > blockSize < sizeof ( IOR_size_t )
| | test - > transferSize <
sizeof ( IOR_size_t ) ) )
ERR ( " need larger file size for strided datatype in MPIIO " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " POSIX " ) = = 0 ) & & test - > showHints )
2011-11-12 02:22:17 +04:00
WARN_RESET ( " hints not available in POSIX " ,
test , & defaults , showHints ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " POSIX " ) = = 0 ) & & test - > collective )
2011-11-12 02:22:17 +04:00
WARN_RESET ( " collective not available in POSIX " ,
test , & defaults , collective ) ;
2018-05-08 14:08:29 +03:00
if ( ( strcasecmp ( test - > api , " MMAP " ) = = 0 ) & & test - > fsyncPerWrite
& & ( test - > transferSize & ( sysconf ( _SC_PAGESIZE ) - 1 ) ) )
ERR ( " transfer size must be aligned with PAGESIZE for MMAP with fsyncPerWrite " ) ;
2015-05-19 18:36:28 +03:00
/* parameter consitency */
2011-11-12 02:22:17 +04:00
if ( test - > reorderTasks = = TRUE & & test - > reorderTasksRandom = = TRUE )
ERR ( " Both Constant and Random task re-ordering specified. Choose one and resubmit " ) ;
if ( test - > randomOffset & & test - > reorderTasksRandom
& & test - > filePerProc = = FALSE )
ERR ( " random offset and random reorder tasks specified with single-shared-file. Choose one and resubmit " ) ;
if ( test - > randomOffset & & test - > reorderTasks
& & test - > filePerProc = = FALSE )
ERR ( " random offset and constant reorder tasks specified with single-shared-file. Choose one and resubmit " ) ;
if ( test - > randomOffset & & test - > checkRead )
ERR ( " random offset not available with read check option (use write check) " ) ;
if ( test - > randomOffset & & test - > storeFileOffset )
ERR ( " random offset not available with store file offset option) " ) ;
2015-05-19 18:36:28 +03:00
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " MPIIO " ) = = 0 ) & & test - > randomOffset
2011-11-12 02:22:17 +04:00
& & test - > collective )
ERR ( " random offset not available with collective MPIIO " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " MPIIO " ) = = 0 ) & & test - > randomOffset
2011-11-12 02:22:17 +04:00
& & test - > useFileView )
ERR ( " random offset not available with MPIIO fileviews " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " HDF5 " ) = = 0 ) & & test - > randomOffset )
2011-11-12 02:22:17 +04:00
ERR ( " random offset not available with HDF5 " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " NCMPI " ) = = 0 ) & & test - > randomOffset )
2011-11-12 02:22:17 +04:00
ERR ( " random offset not available with NCMPI " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " HDF5 " ) ! = 0 ) & & test - > individualDataSets )
2011-11-12 02:22:17 +04:00
WARN_RESET ( " individual datasets only available in HDF5 " ,
test , & defaults , individualDataSets ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " HDF5 " ) = = 0 ) & & test - > individualDataSets )
2011-11-12 02:22:17 +04:00
WARN_RESET ( " individual data sets not implemented " ,
test , & defaults , individualDataSets ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " NCMPI " ) = = 0 ) & & test - > filePerProc )
2011-11-12 02:22:17 +04:00
ERR ( " file-per-proc not available in current NCMPI " ) ;
if ( test - > noFill ) {
2017-12-05 02:20:31 +03:00
if ( strcasecmp ( test - > api , " HDF5 " ) ! = 0 ) {
2011-11-12 02:22:17 +04:00
ERR ( " 'no fill' option only available in HDF5 " ) ;
} else {
/* check if hdf5 available */
# if defined (H5_VERS_MAJOR) && defined (H5_VERS_MINOR)
/* no-fill option not available until hdf5-1.6.x */
# if (H5_VERS_MAJOR > 0 && H5_VERS_MINOR > 5)
;
# else
char errorString [ MAX_STR ] ;
sprintf ( errorString ,
" 'no fill' option not available in %s " ,
test - > apiVersion ) ;
ERR ( errorString ) ;
# endif
# else
WARN ( " unable to determine HDF5 version for 'no fill' usage " ) ;
# endif
}
}
if ( test - > useExistingTestFile & & test - > lustre_set_striping )
ERR ( " Lustre stripe options are incompatible with useExistingTestFile " ) ;
2015-05-19 18:36:28 +03:00
/* N:1 and N:N */
IOR_offset_t NtoN = test - > filePerProc ;
IOR_offset_t Nto1 = ! NtoN ;
IOR_offset_t s = test - > segmentCount ;
IOR_offset_t t = test - > transferSize ;
IOR_offset_t b = test - > blockSize ;
if ( Nto1 & & ( s ! = 1 ) & & ( b ! = t ) ) {
ERR ( " N:1 (strided) requires xfer-size == block-size " ) ;
}
2011-11-12 03:11:28 +04:00
}
2011-11-12 02:22:17 +04:00
2018-02-28 20:50:30 +03:00
/**
* Returns a precomputed array of IOR_offset_t for the inner benchmark loop .
* They are sequential and the last element is set to - 1 as end marker .
* @ param test IOR_param_t for getting transferSize , blocksize and SegmentCount
* @ param pretendRank int pretended Rank for shifting the offsest corectly
* @ return IOR_offset_t
*/
2011-11-12 04:40:45 +04:00
static IOR_offset_t * GetOffsetArraySequential ( IOR_param_t * test ,
int pretendRank )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
IOR_offset_t i , j , k = 0 ;
IOR_offset_t offsets ;
IOR_offset_t * offsetArray ;
/* count needed offsets */
offsets = ( test - > blockSize / test - > transferSize ) * test - > segmentCount ;
/* setup empty array */
offsetArray =
2014-08-14 02:53:24 +04:00
( IOR_offset_t * ) malloc ( ( offsets + 1 ) * sizeof ( IOR_offset_t ) ) ;
2011-11-12 02:22:17 +04:00
if ( offsetArray = = NULL )
ERR ( " malloc() failed " ) ;
offsetArray [ offsets ] = - 1 ; /* set last offset with -1 */
/* fill with offsets */
for ( i = 0 ; i < test - > segmentCount ; i + + ) {
for ( j = 0 ; j < ( test - > blockSize / test - > transferSize ) ; j + + ) {
offsetArray [ k ] = j * test - > transferSize ;
if ( test - > filePerProc ) {
offsetArray [ k ] + = i * test - > blockSize ;
} else {
offsetArray [ k ] + =
2014-08-14 02:53:24 +04:00
( i * test - > numTasks * test - > blockSize )
+ ( pretendRank * test - > blockSize ) ;
2011-11-12 02:22:17 +04:00
}
k + + ;
}
}
return ( offsetArray ) ;
2011-11-12 03:11:28 +04:00
}
2011-11-12 02:22:17 +04:00
2018-02-28 20:50:30 +03:00
/**
2018-07-07 13:42:21 +03:00
* Returns a precomputed array of IOR_offset_t for the inner benchmark loop .
2018-02-28 20:50:30 +03:00
* They get created sequentially and mixed up in the end . The last array element
* is set to - 1 as end marker .
* It should be noted that as the seeds get synchronised across all processes
* every process computes the same random order if used with filePerProc .
* For a shared file all transfers get randomly assigned to ranks . The processes
* can also have differen ' t numbers of transfers . This might lead to a bigger
* diversion in accesse as it dose with filePerProc . This is expected but
* should be mined .
* @ param test IOR_param_t for getting transferSize , blocksize and SegmentCount
* @ param pretendRank int pretended Rank for shifting the offsest corectly
* @ return IOR_offset_t
* @ return
*/
2011-11-12 04:40:45 +04:00
static IOR_offset_t * GetOffsetArrayRandom ( IOR_param_t * test , int pretendRank ,
int access )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
int seed ;
IOR_offset_t i , value , tmp ;
2011-11-12 03:11:28 +04:00
IOR_offset_t offsets = 0 ;
IOR_offset_t offsetCnt = 0 ;
2011-11-12 02:22:17 +04:00
IOR_offset_t fileSize ;
IOR_offset_t * offsetArray ;
/* set up seed for random() */
if ( access = = WRITE | | access = = READ ) {
test - > randomSeed = seed = random ( ) ;
2011-06-17 23:20:43 +04:00
} else {
2011-11-12 02:22:17 +04:00
seed = test - > randomSeed ;
2011-06-17 23:20:43 +04:00
}
2011-11-12 02:22:17 +04:00
srandom ( seed ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
fileSize = test - > blockSize * test - > segmentCount ;
if ( test - > filePerProc = = FALSE ) {
fileSize * = test - > numTasks ;
2011-06-17 23:20:43 +04:00
}
2011-11-12 02:22:17 +04:00
/* count needed offsets (pass 1) */
2011-06-17 23:20:43 +04:00
for ( i = 0 ; i < fileSize ; i + = test - > transferSize ) {
2011-11-12 02:22:17 +04:00
if ( test - > filePerProc = = FALSE ) {
2018-02-28 20:50:30 +03:00
// this counts which process get how many transferes in
// a shared file
2011-11-12 02:22:17 +04:00
if ( ( random ( ) % test - > numTasks ) = = pretendRank ) {
offsets + + ;
}
} else {
offsets + + ;
}
2011-06-17 23:20:43 +04:00
}
2011-11-12 02:22:17 +04:00
/* setup empty array */
offsetArray =
2014-08-14 02:53:24 +04:00
( IOR_offset_t * ) malloc ( ( offsets + 1 ) * sizeof ( IOR_offset_t ) ) ;
2011-11-12 02:22:17 +04:00
if ( offsetArray = = NULL )
ERR ( " malloc() failed " ) ;
offsetArray [ offsets ] = - 1 ; /* set last offset with -1 */
if ( test - > filePerProc ) {
/* fill array */
for ( i = 0 ; i < offsets ; i + + ) {
offsetArray [ i ] = i * test - > transferSize ;
}
} else {
/* fill with offsets (pass 2) */
2018-02-28 20:50:30 +03:00
srandom ( seed ) ; /* need same seed to get same transfers as counted in the beginning*/
2011-11-12 02:22:17 +04:00
for ( i = 0 ; i < fileSize ; i + = test - > transferSize ) {
if ( ( random ( ) % test - > numTasks ) = = pretendRank ) {
offsetArray [ offsetCnt ] = i ;
offsetCnt + + ;
}
}
}
/* reorder array */
for ( i = 0 ; i < offsets ; i + + ) {
value = random ( ) % offsets ;
tmp = offsetArray [ value ] ;
offsetArray [ value ] = offsetArray [ i ] ;
offsetArray [ i ] = tmp ;
}
SeedRandGen ( test - > testComm ) ; /* synchronize seeds across tasks */
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
return ( offsetArray ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2017-10-20 19:02:24 +03:00
static IOR_offset_t WriteOrReadSingle ( IOR_offset_t pairCnt , IOR_offset_t * offsetArray , int pretendRank ,
IOR_offset_t * transferCount , int * errors , IOR_param_t * test , int * fd , IOR_io_buffers * ioBuffers , int access ) {
2018-07-07 12:29:27 +03:00
IOR_offset_t amtXferred = 0 ;
2017-10-20 19:02:24 +03:00
IOR_offset_t transfer ;
void * buffer = ioBuffers - > buffer ;
void * checkBuffer = ioBuffers - > checkBuffer ;
void * readCheckBuffer = ioBuffers - > readCheckBuffer ;
test - > offset = offsetArray [ pairCnt ] ;
transfer = test - > transferSize ;
if ( access = = WRITE ) {
2017-11-30 13:56:26 +03:00
/* fills each transfer with a unique pattern
* containing the offset into the file */
2017-10-20 19:02:24 +03:00
if ( test - > storeFileOffset = = TRUE ) {
FillBuffer ( buffer , test , test - > offset , pretendRank ) ;
}
amtXferred =
backend - > xfer ( access , fd , buffer , transfer , test ) ;
if ( amtXferred ! = transfer )
ERR ( " cannot write to file " ) ;
} else if ( access = = READ ) {
amtXferred =
backend - > xfer ( access , fd , buffer , transfer , test ) ;
if ( amtXferred ! = transfer )
ERR ( " cannot read from file " ) ;
} else if ( access = = WRITECHECK ) {
memset ( checkBuffer , ' a ' , transfer ) ;
2017-11-29 12:17:02 +03:00
2017-11-30 13:56:26 +03:00
if ( test - > storeFileOffset = = TRUE ) {
FillBuffer ( readCheckBuffer , test , test - > offset , pretendRank ) ;
}
amtXferred = backend - > xfer ( access , fd , checkBuffer , transfer , test ) ;
2017-10-20 19:02:24 +03:00
if ( amtXferred ! = transfer )
ERR ( " cannot read from file write check " ) ;
( * transferCount ) + + ;
2017-11-29 12:17:02 +03:00
* errors + = CompareBuffers ( readCheckBuffer , checkBuffer , transfer ,
2017-10-20 19:02:24 +03:00
* transferCount , test ,
WRITECHECK ) ;
} else if ( access = = READCHECK ) {
amtXferred = backend - > xfer ( access , fd , buffer , transfer , test ) ;
if ( amtXferred ! = transfer ) {
ERR ( " cannot read from file " ) ;
}
if ( test - > storeFileOffset = = TRUE ) {
FillBuffer ( readCheckBuffer , test , test - > offset , pretendRank ) ;
}
* errors + = CompareBuffers ( readCheckBuffer , buffer , transfer , * transferCount , test , READCHECK ) ;
}
return amtXferred ;
}
2011-06-17 23:20:43 +04:00
/*
* Write or Read data to file ( s ) . This loops through the strides , writing
* out the data to each block in transfer sizes , until the remainder left is 0.
*/
2017-10-20 19:02:24 +03:00
static IOR_offset_t WriteOrRead ( IOR_param_t * test , IOR_results_t * results , void * fd , int access , IOR_io_buffers * ioBuffers )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
int errors = 0 ;
2012-01-13 08:34:40 +04:00
IOR_offset_t transferCount = 0 ;
2018-07-07 13:42:21 +03:00
uint64_t pairCnt = 0 ;
2012-01-13 08:34:40 +04:00
IOR_offset_t * offsetArray ;
2011-11-12 02:22:17 +04:00
int pretendRank ;
IOR_offset_t dataMoved = 0 ; /* for data rate calculation */
double startForStonewall ;
int hitStonewall ;
/* initialize values */
pretendRank = ( rank + rankOffset ) % test - > numTasks ;
if ( test - > randomOffset ) {
offsetArray = GetOffsetArrayRandom ( test , pretendRank , access ) ;
} else {
offsetArray = GetOffsetArraySequential ( test , pretendRank ) ;
2011-06-17 23:20:43 +04:00
}
2011-11-12 02:22:17 +04:00
startForStonewall = GetTimeStamp ( ) ;
2018-07-15 21:38:17 +03:00
hitStonewall = 0 ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/* loop over offsets to access */
2017-10-20 19:13:29 +03:00
while ( ( offsetArray [ pairCnt ] ! = - 1 ) & & ! hitStonewall ) {
2017-10-20 19:02:24 +03:00
dataMoved + = WriteOrReadSingle ( pairCnt , offsetArray , pretendRank , & transferCount , & errors , test , fd , ioBuffers , access ) ;
2011-11-12 02:22:17 +04:00
pairCnt + + ;
2018-07-14 14:22:36 +03:00
hitStonewall = ( ( test - > deadlineForStonewalling ! = 0
& & ( GetTimeStamp ( ) - startForStonewall )
2017-10-20 19:13:29 +03:00
> test - > deadlineForStonewalling ) ) | | ( test - > stoneWallingWearOutIterations ! = 0 & & pairCnt = = test - > stoneWallingWearOutIterations ) ;
2011-11-12 02:22:17 +04:00
}
2017-10-20 19:02:24 +03:00
if ( test - > stoneWallingWearOut ) {
2018-07-07 13:42:21 +03:00
if ( verbose > = VERBOSE_1 ) {
fprintf ( out_logfile , " %d: stonewalling pairs accessed: %lld \n " , rank , ( long long ) pairCnt ) ;
}
long long data_moved_ll = ( long long ) dataMoved ;
long long pairs_accessed_min = 0 ;
2017-10-20 19:02:24 +03:00
MPI_CHECK ( MPI_Allreduce ( & pairCnt , & results - > pairs_accessed ,
1 , MPI_LONG_LONG_INT , MPI_MAX , testComm ) , " cannot reduce pairs moved " ) ;
2018-07-07 13:42:21 +03:00
double stonewall_runtime = GetTimeStamp ( ) - startForStonewall ;
results - > stonewall_time = stonewall_runtime ;
MPI_CHECK ( MPI_Reduce ( & pairCnt , & pairs_accessed_min ,
1 , MPI_LONG_LONG_INT , MPI_MIN , 0 , testComm ) , " cannot reduce pairs moved " ) ;
MPI_CHECK ( MPI_Reduce ( & data_moved_ll , & results - > stonewall_min_data_accessed ,
1 , MPI_LONG_LONG_INT , MPI_MIN , 0 , testComm ) , " cannot reduce pairs moved " ) ;
MPI_CHECK ( MPI_Reduce ( & data_moved_ll , & results - > stonewall_avg_data_accessed ,
1 , MPI_LONG_LONG_INT , MPI_SUM , 0 , testComm ) , " cannot reduce pairs moved " ) ;
if ( rank = = 0 ) {
fprintf ( out_logfile , " stonewalling pairs accessed min: %lld max: %zu -- min data: %.1f GiB mean data: %.1f GiB time: %.1fs \n " ,
pairs_accessed_min , results - > pairs_accessed ,
results - > stonewall_min_data_accessed / 1024.0 / 1024 / 1024 , results - > stonewall_avg_data_accessed / 1024.0 / 1024 / 1024 / test - > numTasks , results - > stonewall_time ) ;
results - > stonewall_min_data_accessed * = test - > numTasks ;
}
if ( pairs_accessed_min = = pairCnt ) {
results - > stonewall_min_data_accessed = 0 ;
results - > stonewall_avg_data_accessed = 0 ;
2017-10-20 19:02:24 +03:00
}
if ( pairCnt ! = results - > pairs_accessed ) {
// some work needs still to be done !
for ( ; pairCnt < results - > pairs_accessed ; pairCnt + + ) {
dataMoved + = WriteOrReadSingle ( pairCnt , offsetArray , pretendRank , & transferCount , & errors , test , fd , ioBuffers , access ) ;
}
}
} else {
results - > pairs_accessed = pairCnt ;
2011-11-12 02:22:17 +04:00
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
totalErrorCount + = CountErrors ( test , access , errors ) ;
2011-06-17 23:20:43 +04:00
2012-01-13 08:34:40 +04:00
free ( offsetArray ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
if ( access = = WRITE & & test - > fsync = = TRUE ) {
backend - > fsync ( fd , test ) ; /*fsync after all accesses */
}
return ( dataMoved ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Write times taken during each iteration of the test .
*/
2011-11-12 04:40:45 +04:00
static void
2011-11-12 02:22:17 +04:00
WriteTimes ( IOR_param_t * test , double * * timer , int iteration , int writeOrRead )
2011-06-17 23:20:43 +04:00
{
2011-11-12 03:11:28 +04:00
char accessType [ MAX_STR ] ;
char timerName [ MAX_STR ] ;
2018-07-07 13:42:21 +03:00
int i , start = 0 , stop = 0 ;
2011-11-12 02:22:17 +04:00
if ( writeOrRead = = WRITE ) {
start = 0 ;
stop = 6 ;
strcpy ( accessType , " WRITE " ) ;
} else if ( writeOrRead = = READ ) {
start = 6 ;
stop = 12 ;
strcpy ( accessType , " READ " ) ;
} else {
ERR ( " incorrect WRITE/READ option " ) ;
}
2011-11-10 04:13:44 +04:00
2011-11-12 02:22:17 +04:00
for ( i = start ; i < stop ; i + + ) {
switch ( i ) {
case 0 :
strcpy ( timerName , " write open start " ) ;
break ;
case 1 :
strcpy ( timerName , " write open stop " ) ;
break ;
case 2 :
strcpy ( timerName , " write start " ) ;
break ;
case 3 :
strcpy ( timerName , " write stop " ) ;
break ;
case 4 :
strcpy ( timerName , " write close start " ) ;
break ;
case 5 :
strcpy ( timerName , " write close stop " ) ;
break ;
case 6 :
strcpy ( timerName , " read open start " ) ;
break ;
case 7 :
strcpy ( timerName , " read open stop " ) ;
break ;
case 8 :
strcpy ( timerName , " read start " ) ;
break ;
case 9 :
strcpy ( timerName , " read stop " ) ;
break ;
case 10 :
strcpy ( timerName , " read close start " ) ;
break ;
case 11 :
strcpy ( timerName , " read close stop " ) ;
break ;
default :
strcpy ( timerName , " invalid timer " ) ;
break ;
}
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " Test %d: Iter=%d, Task=%d, Time=%f, %s \n " ,
2011-11-12 02:22:17 +04:00
test - > id , iteration , ( int ) rank , timer [ i ] [ iteration ] ,
timerName ) ;
}
2011-11-12 03:11:28 +04:00
}