2012-01-09 00:51:04 +04:00
/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
* vim : expandtab : shiftwidth = 8 : tabstop = 8 :
*/
2011-06-17 23:20:43 +04:00
/******************************************************************************\
* *
* Copyright ( c ) 2003 , The Regents of the University of California *
* See the file COPYRIGHT for a complete copyright notice and license . *
* *
\ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2011-11-12 04:40:45 +04:00
# ifdef HAVE_CONFIG_H
2014-07-31 03:17:21 +04:00
# include "config.h"
2011-11-12 04:40:45 +04:00
# endif
# include <stdio.h>
# include <stdlib.h>
2012-01-07 05:29:45 +04:00
# include <unistd.h>
2011-11-12 02:22:17 +04:00
# include <ctype.h> /* tolower() */
2011-11-12 04:40:45 +04:00
# include <errno.h>
2011-06-17 23:20:43 +04:00
# include <math.h>
# include <mpi.h>
# include <string.h>
2018-10-08 23:47:28 +03:00
# if defined(HAVE_STRINGS_H)
# include <strings.h>
# endif
2011-11-12 02:22:17 +04:00
# include <sys/stat.h> /* struct stat */
2011-06-17 23:20:43 +04:00
# include <time.h>
2014-07-31 03:17:21 +04:00
2011-06-17 23:20:43 +04:00
# ifndef _WIN32
2014-07-31 03:17:21 +04:00
# include <sys / time.h> /* gettimeofday() */
# include <sys / utsname.h> /* uname() */
2011-06-17 23:20:43 +04:00
# endif
2014-07-31 03:17:21 +04:00
2011-11-10 04:34:16 +04:00
# include <assert.h>
2011-06-17 23:20:43 +04:00
2011-11-12 04:40:45 +04:00
# include "ior.h"
2018-07-08 15:38:05 +03:00
# include "ior-internal.h"
2011-11-12 04:40:45 +04:00
# include "aiori.h"
# include "utilities.h"
# include "parse_options.h"
2011-06-17 23:20:43 +04:00
2018-09-19 20:06:05 +03:00
# define IOR_NB_TIMERS 6
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
2011-11-12 04:40:45 +04:00
/* file scope globals */
extern char * * environ ;
2018-07-07 13:42:21 +03:00
static int totalErrorCount ;
static const ior_aiori_t * backend ;
2011-10-28 03:50:05 +04:00
2011-12-13 09:00:18 +04:00
static void DestroyTests ( IOR_test_t * tests_head ) ;
2011-11-12 04:40:45 +04:00
static char * PrependDir ( IOR_param_t * , char * ) ;
static char * * ParseFileName ( char * , int * ) ;
2021-01-20 18:00:33 +03:00
static void InitTests ( IOR_test_t * ) ;
2011-12-13 09:00:18 +04:00
static void TestIoSys ( IOR_test_t * ) ;
2021-01-20 17:57:21 +03:00
static void ValidateTests ( IOR_param_t * params , MPI_Comm com ) ;
2018-09-19 20:06:05 +03:00
static IOR_offset_t WriteOrRead ( IOR_param_t * test , IOR_results_t * results ,
2020-05-31 14:11:00 +03:00
aiori_fd_t * fd , const int access ,
2018-09-19 20:06:05 +03:00
IOR_io_buffers * ioBuffers ) ;
2011-11-12 04:40:45 +04:00
2020-05-31 14:50:03 +03:00
static void ior_set_xfer_hints ( IOR_param_t * p ) {
aiori_xfer_hint_t * hints = & p - > hints ;
hints - > dryRun = p - > dryRun ;
hints - > filePerProc = p - > filePerProc ;
hints - > collective = p - > collective ;
hints - > numTasks = p - > numTasks ;
hints - > numNodes = p - > numNodes ;
hints - > randomOffset = p - > randomOffset ;
hints - > fsyncPerWrite = p - > fsyncPerWrite ;
hints - > segmentCount = p - > segmentCount ;
hints - > blockSize = p - > blockSize ;
hints - > transferSize = p - > transferSize ;
hints - > expectedAggFileSize = p - > expectedAggFileSize ;
hints - > singleXferAttempt = p - > singleXferAttempt ;
if ( backend - > xfer_hints ) {
backend - > xfer_hints ( hints ) ;
}
}
2020-06-24 13:10:42 +03:00
int aiori_warning_as_errors = 0 ;
2020-05-30 20:19:48 +03:00
static void test_initialize ( IOR_test_t * test ) {
verbose = test - > params . verbose ;
backend = test - > params . backend ;
if ( backend - > initialize ) {
backend - > initialize ( test - > params . backend_options ) ;
}
2020-05-31 14:50:03 +03:00
ior_set_xfer_hints ( & test - > params ) ;
2020-06-24 13:10:42 +03:00
aiori_warning_as_errors = test - > params . warningAsErrors ;
2020-05-31 15:30:31 +03:00
if ( rank = = 0 & & verbose > = VERBOSE_0 ) {
ShowTestStart ( & test - > params ) ;
}
2020-05-30 20:19:48 +03:00
}
static void test_finalize ( IOR_test_t * test ) {
backend = test - > params . backend ;
if ( backend - > finalize ) {
backend - > finalize ( test - > params . backend_options ) ;
}
}
2018-07-07 13:42:21 +03:00
IOR_test_t * ior_run ( int argc , char * * argv , MPI_Comm world_com , FILE * world_out ) {
2011-12-13 09:00:18 +04:00
IOR_test_t * tests_head ;
IOR_test_t * tptr ;
2018-07-07 13:42:21 +03:00
out_logfile = world_out ;
2018-07-08 15:47:55 +03:00
out_resultfile = world_out ;
2011-06-17 23:20:43 +04:00
2021-01-20 17:57:21 +03:00
MPI_CHECK ( MPI_Comm_rank ( world_com , & rank ) , " cannot get rank " ) ;
2012-09-10 21:50:18 +04:00
2015-05-19 18:36:28 +03:00
/* setup tests, and validate parameters */
2021-01-20 17:57:21 +03:00
tests_head = ParseCommandLine ( argc , argv , world_com ) ;
2021-01-20 18:00:33 +03:00
InitTests ( tests_head ) ;
2011-11-12 02:22:17 +04:00
2014-08-14 02:53:24 +04:00
PrintHeader ( argc , argv ) ;
2011-12-11 13:50:19 +04:00
2011-11-12 02:22:17 +04:00
/* perform each test */
2014-08-14 02:53:24 +04:00
for ( tptr = tests_head ; tptr ! = NULL ; tptr = tptr - > next ) {
2020-05-30 20:19:48 +03:00
test_initialize ( tptr ) ;
2018-07-07 13:42:21 +03:00
totalErrorCount = 0 ;
2011-12-13 09:00:18 +04:00
TestIoSys ( tptr ) ;
2018-07-07 13:42:21 +03:00
tptr - > results - > errors = totalErrorCount ;
2018-07-08 16:59:54 +03:00
ShowTestEnd ( tptr ) ;
2020-05-30 20:19:48 +03:00
test_finalize ( tptr ) ;
2011-11-12 02:22:17 +04:00
}
2014-08-14 02:53:24 +04:00
PrintLongSummaryAllTests ( tests_head ) ;
2011-12-13 09:00:18 +04:00
2011-11-12 02:22:17 +04:00
/* display finish time */
2018-07-08 16:59:54 +03:00
PrintTestEnds ( ) ;
2018-07-07 13:42:21 +03:00
return tests_head ;
}
2011-11-12 02:22:17 +04:00
2011-12-13 09:00:18 +04:00
2011-11-12 02:22:17 +04:00
2018-07-07 13:42:21 +03:00
int ior_main ( int argc , char * * argv )
{
IOR_test_t * tests_head ;
IOR_test_t * tptr ;
2018-07-08 15:07:32 +03:00
2018-07-07 13:42:21 +03:00
out_logfile = stdout ;
2018-07-08 15:47:55 +03:00
out_resultfile = stdout ;
2018-07-07 13:42:21 +03:00
/*
* check - h option from commandline without starting MPI ;
*/
2021-01-20 17:57:21 +03:00
tests_head = ParseCommandLine ( argc , argv , MPI_COMM_WORLD ) ;
2018-07-07 13:42:21 +03:00
/* start the MPI code */
MPI_CHECK ( MPI_Init ( & argc , & argv ) , " cannot initialize MPI " ) ;
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
2021-01-20 17:57:21 +03:00
MPI_CHECK ( MPI_Comm_rank ( MPI_COMM_WORLD , & rank ) , " cannot get rank " ) ;
2018-07-08 15:07:32 +03:00
2018-07-07 13:42:21 +03:00
/* set error-handling */
/*MPI_CHECK(MPI_Errhandler_set(mpi_comm_world, MPI_ERRORS_RETURN),
" cannot set errhandler " ) ; */
/* setup tests, and validate parameters */
2021-01-20 18:00:33 +03:00
InitTests ( tests_head ) ;
2018-07-07 13:42:21 +03:00
PrintHeader ( argc , argv ) ;
/* perform each test */
for ( tptr = tests_head ; tptr ! = NULL ; tptr = tptr - > next ) {
2020-05-30 20:19:48 +03:00
test_initialize ( tptr ) ;
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
2018-07-07 13:42:21 +03:00
// This is useful for trapping a running MPI process. While
// this is sleeping, run the script 'testing/hdfs/gdb.attach'
if ( verbose > = VERBOSE_4 ) {
fprintf ( out_logfile , " \t rank %d: sleeping \n " , rank ) ;
sleep ( 5 ) ;
fprintf ( out_logfile , " \t rank %d: awake. \n " , rank ) ;
}
2018-08-30 00:14:19 +03:00
2018-07-07 13:42:21 +03:00
TestIoSys ( tptr ) ;
2018-07-08 16:59:54 +03:00
ShowTestEnd ( tptr ) ;
2020-05-30 20:19:48 +03:00
test_finalize ( tptr ) ;
2018-07-07 13:42:21 +03:00
}
2019-10-24 01:17:37 +03:00
if ( verbose < = VERBOSE_0 )
2018-07-07 13:42:21 +03:00
/* always print final summary */
2019-10-24 01:17:37 +03:00
verbose = VERBOSE_1 ;
2018-07-07 13:42:21 +03:00
PrintLongSummaryAllTests ( tests_head ) ;
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
2018-07-07 13:42:21 +03:00
/* display finish time */
2018-07-08 16:59:54 +03:00
PrintTestEnds ( ) ;
2018-07-07 13:42:21 +03:00
MPI_CHECK ( MPI_Finalize ( ) , " cannot finalize MPI " ) ;
2018-09-17 12:38:57 +03:00
DestroyTests ( tests_head ) ;
2018-07-07 13:42:21 +03:00
return totalErrorCount ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/***************************** F U N C T I O N S ******************************/
2011-11-10 04:13:44 +04:00
/*
* Initialize an IOR_param_t structure to the defaults
*/
2021-01-20 17:57:21 +03:00
void init_IOR_Param_t ( IOR_param_t * p , MPI_Comm com )
2011-11-10 04:13:44 +04:00
{
2017-10-20 00:26:52 +03:00
const char * default_aiori = aiori_default ( ) ;
assert ( NULL ! = default_aiori ) ;
2011-11-10 04:13:44 +04:00
memset ( p , 0 , sizeof ( IOR_param_t ) ) ;
2018-07-14 10:41:35 +03:00
p - > api = strdup ( default_aiori ) ;
p - > platform = strdup ( " HOST(OSTYPE) " ) ;
p - > testFileName = strdup ( " testFile " ) ;
p - > writeFile = p - > readFile = FALSE ;
p - > checkWrite = p - > checkRead = FALSE ;
2014-08-14 02:53:24 +04:00
2019-08-31 01:45:03 +03:00
/*
* These can be overridden from the command - line but otherwise will be
* set from MPI .
*/
p - > numTasks = - 1 ;
p - > numNodes = - 1 ;
p - > numTasksOnNode0 = - 1 ;
2011-11-10 04:13:44 +04:00
p - > repetitions = 1 ;
p - > repCounter = - 1 ;
p - > open = WRITE ;
p - > taskPerNodeOffset = 1 ;
p - > segmentCount = 1 ;
p - > blockSize = 1048576 ;
p - > transferSize = 262144 ;
p - > randomSeed = - 1 ;
2015-05-21 21:05:56 +03:00
p - > incompressibleSeed = 573 ;
2021-01-20 17:57:21 +03:00
p - > testComm = com ; // this com might change for smaller tests
p - > mpi_comm_world = com ;
2014-08-29 01:39:44 +04:00
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
p - > URI = NULL ;
2011-11-10 04:13:44 +04:00
}
2011-11-12 04:40:45 +04:00
static void
2011-11-12 02:22:17 +04:00
DisplayOutliers ( int numTasks ,
2011-06-17 23:20:43 +04:00
double timerVal ,
2011-11-12 02:22:17 +04:00
char * timeString , int access , int outlierThreshold )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
char accessString [ MAX_STR ] ;
double sum , mean , sqrDiff , var , sd ;
/* for local timerVal, don't compensate for wall clock delta */
2021-01-20 22:38:54 +03:00
//timerVal += wall_clock_delta;
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Allreduce
( & timerVal , & sum , 1 , MPI_DOUBLE , MPI_SUM , testComm ) ,
" MPI_Allreduce() " ) ;
mean = sum / numTasks ;
sqrDiff = pow ( ( mean - timerVal ) , 2 ) ;
MPI_CHECK ( MPI_Allreduce
( & sqrDiff , & var , 1 , MPI_DOUBLE , MPI_SUM , testComm ) ,
" MPI_Allreduce() " ) ;
var = var / numTasks ;
sd = sqrt ( var ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
if ( access = = WRITE ) {
strcpy ( accessString , " write " ) ;
} else { /* READ */
strcpy ( accessString , " read " ) ;
}
if ( fabs ( timerVal - mean ) > ( double ) outlierThreshold ) {
2020-03-17 18:31:56 +03:00
char hostname [ MAX_STR ] ;
int ret = gethostname ( hostname , MAX_STR ) ;
if ( ret ! = 0 )
strcpy ( hostname , " unknown " ) ;
2020-06-24 13:10:42 +03:00
EWARNF ( " for %s, task %d, %s %s is %f (mean=%f, stddev=%f) \n " ,
hostname , rank , accessString , timeString , timerVal , mean , sd ) ;
2011-11-12 02:22:17 +04:00
}
2011-11-12 04:40:45 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Check for outliers in start / end times and elapsed create / xfer / close times .
*/
2018-09-19 20:06:05 +03:00
static void
CheckForOutliers ( IOR_param_t * test , const double * timer , const int access )
2011-06-17 23:20:43 +04:00
{
2018-09-19 20:06:05 +03:00
DisplayOutliers ( test - > numTasks , timer [ 0 ] ,
2011-11-12 02:22:17 +04:00
" start time " , access , test - > outlierThreshold ) ;
DisplayOutliers ( test - > numTasks ,
2018-09-19 20:06:05 +03:00
timer [ 1 ] - timer [ 0 ] ,
2011-11-12 02:22:17 +04:00
" elapsed create time " , access , test - > outlierThreshold ) ;
DisplayOutliers ( test - > numTasks ,
2018-09-19 20:06:05 +03:00
timer [ 3 ] - timer [ 2 ] ,
2011-11-12 02:22:17 +04:00
" elapsed transfer time " , access ,
test - > outlierThreshold ) ;
DisplayOutliers ( test - > numTasks ,
2018-09-19 20:06:05 +03:00
timer [ 5 ] - timer [ 4 ] ,
2011-11-12 02:22:17 +04:00
" elapsed close time " , access , test - > outlierThreshold ) ;
2018-09-19 20:06:05 +03:00
DisplayOutliers ( test - > numTasks , timer [ 5 ] , " end time " ,
2011-11-12 02:22:17 +04:00
access , test - > outlierThreshold ) ;
2011-11-12 04:40:45 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Check if actual file size equals expected size ; if not use actual for
* calculating performance rate .
*/
2020-06-30 14:33:56 +03:00
static void CheckFileSize ( IOR_test_t * test , char * testFilename , IOR_offset_t dataMoved , int rep , const int access )
2011-06-17 23:20:43 +04:00
{
2014-08-14 02:53:24 +04:00
IOR_param_t * params = & test - > params ;
IOR_results_t * results = test - > results ;
2018-09-20 00:39:25 +03:00
IOR_point_t * point = ( access = = WRITE ) ? & results [ rep ] . write :
& results [ rep ] . read ;
2011-12-13 09:00:18 +04:00
2020-06-30 14:03:05 +03:00
/* get the size of the file */
IOR_offset_t aggFileSizeFromStat , tmpMin , tmpMax , tmpSum ;
2020-06-30 14:33:56 +03:00
aggFileSizeFromStat = backend - > get_file_size ( params - > backend_options , testFilename ) ;
2020-06-30 14:03:05 +03:00
if ( params - > hints . filePerProc = = TRUE ) {
MPI_CHECK ( MPI_Allreduce ( & aggFileSizeFromStat , & tmpSum , 1 ,
MPI_LONG_LONG_INT , MPI_SUM , testComm ) ,
" cannot reduce total data moved " ) ;
aggFileSizeFromStat = tmpSum ;
} else {
MPI_CHECK ( MPI_Allreduce ( & aggFileSizeFromStat , & tmpMin , 1 ,
MPI_LONG_LONG_INT , MPI_MIN , testComm ) ,
" cannot reduce total data moved " ) ;
MPI_CHECK ( MPI_Allreduce ( & aggFileSizeFromStat , & tmpMax , 1 ,
MPI_LONG_LONG_INT , MPI_MAX , testComm ) ,
" cannot reduce total data moved " ) ;
if ( tmpMin ! = tmpMax ) {
if ( rank = = 0 ) {
WARN ( " inconsistent file size by different tasks " ) ;
}
/* incorrect, but now consistent across tasks */
aggFileSizeFromStat = tmpMin ;
}
}
point - > aggFileSizeFromStat = aggFileSizeFromStat ;
2018-09-20 00:39:25 +03:00
MPI_CHECK ( MPI_Allreduce ( & dataMoved , & point - > aggFileSizeFromXfer ,
2011-11-12 02:22:17 +04:00
1 , MPI_LONG_LONG_INT , MPI_SUM , testComm ) ,
" cannot total data moved " ) ;
2020-11-03 13:01:09 +03:00
if ( strcasecmp ( params - > api , " HDF5 " ) ! = 0 & & strcasecmp ( params - > api , " NCMPI " ) ! = 0 ) {
2011-11-12 02:22:17 +04:00
if ( verbose > = VERBOSE_0 & & rank = = 0 ) {
2011-12-13 09:00:18 +04:00
if ( ( params - > expectedAggFileSize
2018-09-20 00:39:25 +03:00
! = point - > aggFileSizeFromXfer )
| | ( point - > aggFileSizeFromStat
! = point - > aggFileSizeFromXfer ) ) {
2020-06-24 13:10:42 +03:00
EWARNF ( " Expected aggregate file size = %lld " , ( long long ) params - > expectedAggFileSize ) ;
EWARNF ( " Stat() of aggregate file size = %lld " , ( long long ) point - > aggFileSizeFromStat ) ;
EWARNF ( " Using actual aggregate bytes moved = %lld " , ( long long ) point - > aggFileSizeFromXfer ) ;
2017-10-20 19:02:24 +03:00
if ( params - > deadlineForStonewalling ) {
2020-06-24 13:10:42 +03:00
EWARN ( " Maybe caused by deadlineForStonewalling " ) ;
2017-10-20 19:02:24 +03:00
}
2011-11-12 02:22:17 +04:00
}
}
2011-06-17 23:20:43 +04:00
}
2018-09-20 00:39:25 +03:00
point - > aggFileSizeForBW = point - > aggFileSizeFromXfer ;
2011-11-12 04:40:45 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Compare buffers after reading / writing each transfer . Displays only first
* difference in buffers and returns total errors counted .
*/
2011-11-12 04:40:45 +04:00
static size_t
2020-09-28 22:09:48 +03:00
CompareData ( void * expectedBuffer , size_t size , IOR_offset_t transferCount , IOR_param_t * test , IOR_offset_t offset , int fillrank , int access )
2011-06-17 23:20:43 +04:00
{
2021-01-20 01:28:34 +03:00
assert ( access = = WRITECHECK | | access = = READCHECK ) ;
2018-07-08 00:39:14 +03:00
char testFileName [ MAX_PATHLEN ] ;
2021-01-20 01:28:34 +03:00
char * bufferLabel1 = " Expected: " ;
char * bufferLabel2 = " Actual: " ;
2020-09-28 22:09:48 +03:00
size_t i , j , length ;
2011-11-12 02:22:17 +04:00
size_t errorCount = 0 ;
2020-09-28 22:09:48 +03:00
IOR_offset_t offsetSignature = 0 ;
unsigned long long hi , lo , val ; // for data verification
hi = ( ( unsigned long long ) fillrank ) < < 32 ;
lo = ( unsigned long long ) test - > timeStampSignatureValue ;
if ( test - > storeFileOffset ) {
offsetSignature = offset ;
}
unsigned long long * testbuf = ( unsigned long long * ) expectedBuffer ;
2011-11-12 02:22:17 +04:00
length = size / sizeof ( IOR_size_t ) ;
if ( verbose > = VERBOSE_3 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" [%d] At file byte offset %lld, comparing %llu-byte transfer \n " ,
2020-06-10 19:47:07 +03:00
rank , ( long long ) offset , ( long long ) size ) ;
2011-11-12 02:22:17 +04:00
}
2020-09-28 22:09:48 +03:00
int incompressibleSeed = test - > setTimeStampSignature + fillrank ;
2011-11-12 02:22:17 +04:00
for ( i = 0 ; i < length ; i + + ) {
2020-09-28 22:09:48 +03:00
if ( test - > dataPacketType = = incompressible ) {
/* same logic as in FillIncompressibleBuffer() */
/* WARNING: make sure that both functions are changed at the same time */
hi = ( ( unsigned long long ) rand_r ( & incompressibleSeed ) < < 32 ) ;
lo = ( unsigned long long ) rand_r ( & incompressibleSeed ) ;
val = hi | lo ;
} else {
if ( ( i % 2 ) = = 0 ) {
/* evens contain MPI rank and time in seconds */
val = hi | lo ;
} else {
/* odds contain offset */
val = offsetSignature + ( i * sizeof ( unsigned long long ) ) ;
}
}
if ( testbuf [ i ] ! = val ) {
2011-11-12 02:22:17 +04:00
errorCount + + ;
if ( verbose > = VERBOSE_2 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" [%d] At transfer buffer #%lld, index #%lld (file byte offset %lld): \n " ,
rank , transferCount - 1 , ( long long ) i ,
2020-06-10 19:47:07 +03:00
( long long ) offset +
2011-11-12 02:22:17 +04:00
( IOR_size_t ) ( i * sizeof ( IOR_size_t ) ) ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " [%d] %s0x " , rank , bufferLabel1 ) ;
2020-09-28 22:09:48 +03:00
fprintf ( out_logfile , " %016llx \n " , val ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " [%d] %s0x " , rank , bufferLabel2 ) ;
fprintf ( out_logfile , " %016llx \n " , testbuf [ i ] ) ;
2011-11-12 02:22:17 +04:00
}
2020-09-28 22:09:48 +03:00
} else if ( verbose > = VERBOSE_5 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2020-09-28 22:09:48 +03:00
" [%d] PASSED offset = %llu bytes, transfer %lld \n " ,
rank , ( ( i * sizeof ( unsigned long long ) ) + offset ) , transferCount ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " [%d] GOOD %s0x " , rank , bufferLabel1 ) ;
2020-09-28 22:09:48 +03:00
fprintf ( out_logfile , " %016llx " , val ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " \n [%d] GOOD %s0x " , rank , bufferLabel2 ) ;
2020-09-28 22:09:48 +03:00
fprintf ( out_logfile , " %016llx " , testbuf [ i ] ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " \n " ) ;
2011-11-12 02:22:17 +04:00
}
}
2021-01-20 18:06:45 +03:00
if ( errorCount > 0 & & verbose > = VERBOSE_1 ) {
2011-11-12 02:22:17 +04:00
GetTestFileName ( testFileName , test ) ;
2020-09-28 22:09:48 +03:00
EWARNF ( " [%d] FAILED comparison of buffer in file %s during transfer %lld offset %lld containing %d-byte ints (%zd errors) " ,
rank , testFileName , transferCount , offset , ( int ) sizeof ( unsigned long long int ) , errorCount ) ;
} else if ( verbose > = VERBOSE_2 ) {
fprintf ( out_logfile , " [%d] comparison successful during transfer %lld offset %lld \n " , rank , transferCount , offset ) ;
2011-06-17 23:20:43 +04:00
}
2021-01-20 01:28:34 +03:00
2011-11-12 02:22:17 +04:00
return ( errorCount ) ;
2011-11-12 03:11:28 +04:00
}
2011-11-12 02:22:17 +04:00
2011-11-12 03:11:28 +04:00
/*
* Count all errors across all tasks ; report errors found .
*/
2011-11-12 04:40:45 +04:00
static int CountErrors ( IOR_param_t * test , int access , int errors )
2011-11-12 02:22:17 +04:00
{
int allErrors = 0 ;
if ( test - > checkWrite | | test - > checkRead ) {
MPI_CHECK ( MPI_Reduce ( & errors , & allErrors , 1 , MPI_INT , MPI_SUM ,
0 , testComm ) , " cannot reduce errors " ) ;
MPI_CHECK ( MPI_Bcast ( & allErrors , 1 , MPI_INT , 0 , testComm ) ,
" cannot broadcast allErrors value " ) ;
if ( allErrors ! = 0 ) {
totalErrorCount + = allErrors ;
test - > errorFound = TRUE ;
}
if ( rank = = 0 & & allErrors ! = 0 ) {
if ( allErrors < 0 ) {
WARN ( " overflow in errors counted " ) ;
allErrors = - 1 ;
}
2020-06-24 13:10:42 +03:00
EWARNF ( " Incorrect data on %s (%d errors found). \n " ,
2014-08-14 02:53:24 +04:00
access = = WRITECHECK ? " write " : " read " , allErrors ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" Used Time Stamp %u (0x%x) for Data Signature \n " ,
test - > timeStampSignatureValue ,
test - > timeStampSignatureValue ) ;
}
}
return ( allErrors ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Allocate a page - aligned ( required by O_DIRECT ) buffer .
*/
2012-01-13 08:34:40 +04:00
static void * aligned_buffer_alloc ( size_t size )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
size_t pageMask ;
char * buf , * tmp ;
char * aligned ;
2018-10-28 21:34:22 +03:00
# ifdef HAVE_SYSCONF
2018-10-08 23:47:28 +03:00
long pageSize = sysconf ( _SC_PAGESIZE ) ;
2018-10-28 21:34:22 +03:00
# else
size_t pageSize = getpagesize ( ) ;
2018-10-08 23:47:28 +03:00
# endif
2011-11-12 02:22:17 +04:00
pageMask = pageSize - 1 ;
buf = malloc ( size + pageSize + sizeof ( void * ) ) ;
if ( buf = = NULL )
ERR ( " out of memory " ) ;
/* find the alinged buffer */
tmp = buf + sizeof ( char * ) ;
aligned = tmp + pageSize - ( ( size_t ) tmp & pageMask ) ;
/* write a pointer to the original malloc()ed buffer into the bytes
preceding " aligned " , so that the aligned buffer can later be free ( ) ed */
tmp = aligned - sizeof ( void * ) ;
* ( void * * ) tmp = buf ;
2011-12-13 09:00:18 +04:00
return ( void * ) aligned ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2012-01-13 08:34:40 +04:00
/*
* Free a buffer allocated by aligned_buffer_alloc ( ) .
*/
static void aligned_buffer_free ( void * buf )
{
free ( * ( void * * ) ( ( char * ) buf - sizeof ( char * ) ) ) ;
}
2018-08-29 19:49:41 +03:00
void AllocResults ( IOR_test_t * test )
2011-12-13 09:00:18 +04:00
{
2018-07-15 21:38:17 +03:00
int reps ;
if ( test - > results ! = NULL )
2019-03-27 23:04:48 +03:00
return ;
2018-07-15 21:38:17 +03:00
reps = test - > params . repetitions ;
test - > results = ( IOR_results_t * ) safeMalloc ( sizeof ( IOR_results_t ) * reps ) ;
2011-12-13 09:00:18 +04:00
}
void FreeResults ( IOR_test_t * test )
{
2018-07-15 21:38:17 +03:00
if ( test - > results ! = NULL ) {
free ( test - > results ) ;
}
2011-12-13 09:00:18 +04:00
}
2017-07-25 13:51:54 +03:00
/**
2011-06-17 23:20:43 +04:00
* Create new test for list of tests .
*/
2011-12-13 09:00:18 +04:00
IOR_test_t * CreateTest ( IOR_param_t * init_params , int test_num )
2011-06-17 23:20:43 +04:00
{
2011-12-13 09:00:18 +04:00
IOR_test_t * newTest = NULL ;
2011-11-12 02:22:17 +04:00
2011-12-13 09:00:18 +04:00
newTest = ( IOR_test_t * ) malloc ( sizeof ( IOR_test_t ) ) ;
2011-11-12 02:22:17 +04:00
if ( newTest = = NULL )
2011-12-13 09:00:18 +04:00
ERR ( " malloc() of IOR_test_t failed " ) ;
newTest - > params = * init_params ;
2018-07-14 10:41:35 +03:00
newTest - > params . platform = GetPlatformName ( ) ;
2011-12-13 09:00:18 +04:00
newTest - > params . id = test_num ;
newTest - > next = NULL ;
2014-08-14 02:53:24 +04:00
newTest - > results = NULL ;
2018-07-14 10:41:35 +03:00
2011-12-13 09:00:18 +04:00
return newTest ;
}
static void DestroyTest ( IOR_test_t * test )
{
2014-08-14 02:53:24 +04:00
FreeResults ( test ) ;
free ( test ) ;
2011-12-13 09:00:18 +04:00
}
static void DestroyTests ( IOR_test_t * tests_head )
{
2014-08-14 02:53:24 +04:00
IOR_test_t * tptr , * next ;
2011-12-13 09:00:18 +04:00
2014-08-14 02:53:24 +04:00
for ( tptr = tests_head ; tptr ! = NULL ; tptr = next ) {
next = tptr - > next ;
DestroyTest ( tptr ) ;
}
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Distribute IOR_HINTs to all tasks ' environments .
*/
2021-01-20 17:57:21 +03:00
static void DistributeHints ( MPI_Comm com )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
char hint [ MAX_HINTS ] [ MAX_STR ] , fullHint [ MAX_STR ] , hintVariable [ MAX_STR ] ;
int hintCount = 0 , i ;
if ( rank = = 0 ) {
for ( i = 0 ; environ [ i ] ! = NULL ; i + + ) {
if ( strncmp ( environ [ i ] , " IOR_HINT " , strlen ( " IOR_HINT " ) )
= = 0 ) {
hintCount + + ;
if ( hintCount = = MAX_HINTS ) {
WARN ( " exceeded max hints; reset MAX_HINTS and recompile " ) ;
hintCount = MAX_HINTS ;
break ;
}
/* assume no IOR_HINT is greater than MAX_STR in length */
strncpy ( hint [ hintCount - 1 ] , environ [ i ] ,
MAX_STR - 1 ) ;
}
2011-06-17 23:20:43 +04:00
}
}
2021-01-20 17:57:21 +03:00
MPI_CHECK ( MPI_Bcast ( & hintCount , sizeof ( hintCount ) , MPI_BYTE , 0 , com ) , " cannot broadcast hints " ) ;
2011-11-12 02:22:17 +04:00
for ( i = 0 ; i < hintCount ; i + + ) {
2021-01-20 17:57:21 +03:00
MPI_CHECK ( MPI_Bcast ( & hint [ i ] , MAX_STR , MPI_BYTE , 0 , com ) ,
2011-11-12 02:22:17 +04:00
" cannot broadcast hints " ) ;
strcpy ( fullHint , hint [ i ] ) ;
strcpy ( hintVariable , strtok ( fullHint , " = " ) ) ;
if ( getenv ( hintVariable ) = = NULL ) {
/* doesn't exist in this task's environment; better set it */
if ( putenv ( hint [ i ] ) ! = 0 )
WARN ( " cannot set environment variable " ) ;
}
2011-06-17 23:20:43 +04:00
}
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Fill buffer , which is transfer size bytes long , with known 8 - byte long long
* int values . In even - numbered 8 - byte long long ints , store MPI task in high
* bits and timestamp signature in low bits . In odd - numbered 8 - byte long long
* ints , store transfer offset . If storeFileOffset option is used , the file
* ( not transfer ) offset is stored instead .
*/
2020-09-28 22:09:48 +03:00
static unsigned int reseed_incompressible_prng = TRUE ;
2015-05-21 21:05:56 +03:00
static void
FillIncompressibleBuffer ( void * buffer , IOR_param_t * test )
{
size_t i ;
unsigned long long hi , lo ;
unsigned long long * buf = ( unsigned long long * ) buffer ;
2017-09-27 19:45:47 +03:00
2020-09-28 22:09:48 +03:00
/* In order for write checks to work, we have to restart the pseudo random sequence */
/* This function has the same logic as CompareData() */
/* WARNING: make sure that both functions are changed at the same time */
if ( reseed_incompressible_prng = = TRUE ) {
test - > incompressibleSeed = test - > setTimeStampSignature + rank ; /* We copied seed into timestampSignature at initialization, also add the rank to add randomness between processes */
reseed_incompressible_prng = FALSE ;
}
2015-05-21 21:05:56 +03:00
for ( i = 0 ; i < test - > transferSize / sizeof ( unsigned long long ) ; i + + ) {
hi = ( ( unsigned long long ) rand_r ( & test - > incompressibleSeed ) < < 32 ) ;
lo = ( unsigned long long ) rand_r ( & test - > incompressibleSeed ) ;
2017-09-27 19:45:47 +03:00
buf [ i ] = hi | lo ;
2015-05-21 21:05:56 +03:00
}
}
2011-11-12 04:40:45 +04:00
static void
2011-11-12 02:22:17 +04:00
FillBuffer ( void * buffer ,
IOR_param_t * test , unsigned long long offset , int fillrank )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
size_t i ;
unsigned long long hi , lo ;
unsigned long long * buf = ( unsigned long long * ) buffer ;
2017-09-27 19:45:47 +03:00
2020-07-03 10:09:40 +03:00
if ( test - > dataPacketType = = incompressible ) { /* Make for some non compressible buffers with randomish data */
2015-05-21 21:05:56 +03:00
FillIncompressibleBuffer ( buffer , test ) ;
2020-09-28 22:09:48 +03:00
} else {
2015-05-21 21:05:56 +03:00
hi = ( ( unsigned long long ) fillrank ) < < 32 ;
lo = ( unsigned long long ) test - > timeStampSignatureValue ;
for ( i = 0 ; i < test - > transferSize / sizeof ( unsigned long long ) ; i + + ) {
if ( ( i % 2 ) = = 0 ) {
/* evens contain MPI rank and time in seconds */
buf [ i ] = hi | lo ;
} else {
/* odds contain offset */
buf [ i ] = offset + ( i * sizeof ( unsigned long long ) ) ;
}
2011-11-12 02:22:17 +04:00
}
2011-06-17 23:20:43 +04:00
}
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Return string describing machine name and type .
*/
2018-07-14 10:41:35 +03:00
char * GetPlatformName ( )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
char nodeName [ MAX_STR ] , * p , * start , sysName [ MAX_STR ] ;
2018-07-14 10:41:35 +03:00
char platformName [ MAX_STR ] ;
2011-11-12 02:22:17 +04:00
struct utsname name ;
if ( uname ( & name ) ! = 0 ) {
2011-12-15 01:40:25 +04:00
EWARN ( " cannot get platform name " ) ;
2011-11-12 02:22:17 +04:00
sprintf ( sysName , " %s " , " Unknown " ) ;
sprintf ( nodeName , " %s " , " Unknown " ) ;
2011-06-17 23:20:43 +04:00
} else {
2011-11-12 02:22:17 +04:00
sprintf ( sysName , " %s " , name . sysname ) ;
sprintf ( nodeName , " %s " , name . nodename ) ;
2011-06-17 23:20:43 +04:00
}
2011-11-12 02:22:17 +04:00
start = nodeName ;
if ( strlen ( nodeName ) = = 0 ) {
p = start ;
} else {
/* point to one character back from '\0' */
p = start + strlen ( nodeName ) - 1 ;
}
/*
* to cut off trailing node number , search backwards
* for the first non - numeric character
*/
while ( p ! = start ) {
if ( * p < ' 0 ' | | * p > ' 9 ' ) {
* ( p + 1 ) = ' \0 ' ;
break ;
} else {
p - - ;
}
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
sprintf ( platformName , " %s(%s) " , nodeName , sysName ) ;
2018-07-14 10:41:35 +03:00
return strdup ( platformName ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
2011-06-17 23:20:43 +04:00
/*
* Parse file name .
*/
2011-11-12 04:40:45 +04:00
static char * * ParseFileName ( char * name , int * count )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
char * * fileNames , * tmp , * token ;
char delimiterString [ 3 ] = { FILENAME_DELIMITER , ' \n ' , ' \0 ' } ;
int i = 0 ;
* count = 0 ;
tmp = name ;
/* pass one */
/* if something there, count the first item */
if ( * tmp ! = ' \0 ' ) {
( * count ) + + ;
}
/* count the rest of the filenames */
while ( * tmp ! = ' \0 ' ) {
if ( * tmp = = FILENAME_DELIMITER ) {
( * count ) + + ;
}
tmp + + ;
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
fileNames = ( char * * ) malloc ( ( * count ) * sizeof ( char * * ) ) ;
if ( fileNames = = NULL )
ERR ( " out of memory " ) ;
/* pass two */
token = strtok ( name , delimiterString ) ;
while ( token ! = NULL ) {
fileNames [ i ] = token ;
token = strtok ( NULL , delimiterString ) ;
i + + ;
}
return ( fileNames ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2018-07-08 15:38:05 +03:00
2011-06-17 23:20:43 +04:00
/*
2018-07-08 15:38:05 +03:00
* Return test file name to access .
* for single shared file , fileNames [ 0 ] is returned in testFileName
2011-06-17 23:20:43 +04:00
*/
2018-07-08 15:38:05 +03:00
void GetTestFileName ( char * testFileName , IOR_param_t * test )
2011-06-17 23:20:43 +04:00
{
2018-07-08 15:38:05 +03:00
char * * fileNames ;
char initialTestFileName [ MAX_PATHLEN ] ;
char testFileNameRoot [ MAX_STR ] ;
char tmpString [ MAX_STR ] ;
2019-11-05 17:37:54 +03:00
int count ;
int socket , core ;
2012-01-09 03:46:43 +04:00
2018-07-08 15:38:05 +03:00
/* parse filename for multiple file systems */
strcpy ( initialTestFileName , test - > testFileName ) ;
2019-11-05 17:37:54 +03:00
if ( test - > dualMount ) {
GetProcessorAndCore ( & socket , & core ) ;
2020-08-03 14:30:21 +03:00
sprintf ( tmpString , " %s%d/%s " , initialTestFileName , socket , " data " ) ;
2019-11-05 17:37:54 +03:00
strcpy ( initialTestFileName , tmpString ) ;
}
2018-07-08 15:38:05 +03:00
fileNames = ParseFileName ( initialTestFileName , & count ) ;
if ( count > 1 & & test - > uniqueDir = = TRUE )
ERR ( " cannot use multiple file names with unique directories " ) ;
if ( test - > filePerProc ) {
strcpy ( testFileNameRoot ,
fileNames [ ( ( rank +
rankOffset ) % test - > numTasks ) % count ] ) ;
} else {
strcpy ( testFileNameRoot , fileNames [ 0 ] ) ;
2011-06-17 23:20:43 +04:00
}
2012-01-09 03:46:43 +04:00
2018-07-08 15:38:05 +03:00
/* give unique name if using multiple files */
if ( test - > filePerProc ) {
/*
* prepend rank subdirectory before filename
* e . g . , / dir / file = > / dir / < rank > / file
*/
if ( test - > uniqueDir = = TRUE ) {
strcpy ( testFileNameRoot ,
PrependDir ( test , testFileNameRoot ) ) ;
}
sprintf ( testFileName , " %s.%08d " , testFileNameRoot ,
( rank + rankOffset ) % test - > numTasks ) ;
} else {
strcpy ( testFileName , testFileNameRoot ) ;
}
2012-01-09 03:46:43 +04:00
2018-07-08 15:38:05 +03:00
/* add suffix for multiple files */
if ( test - > repCounter > - 1 ) {
sprintf ( tmpString , " .%d " , test - > repCounter ) ;
strcat ( testFileName , tmpString ) ;
}
free ( fileNames ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* From absolute directory , insert rank as subdirectory . Allows each task
* to write to its own directory . E . g . , / dir / file = > / dir / < rank > / file .
*/
2011-11-12 04:40:45 +04:00
static char * PrependDir ( IOR_param_t * test , char * rootDir )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
char * dir ;
2019-08-31 00:31:23 +03:00
char * fname ;
2011-11-12 02:22:17 +04:00
int i ;
dir = ( char * ) malloc ( MAX_STR + 1 ) ;
if ( dir = = NULL )
ERR ( " out of memory " ) ;
/* get dir name */
strcpy ( dir , rootDir ) ;
i = strlen ( dir ) - 1 ;
while ( i > 0 ) {
if ( dir [ i ] = = ' \0 ' | | dir [ i ] = = ' / ' ) {
dir [ i ] = ' / ' ;
dir [ i + 1 ] = ' \0 ' ;
break ;
}
i - - ;
}
/* get file name */
2019-08-31 00:31:23 +03:00
fname = rootDir + i + 1 ;
2011-11-12 02:22:17 +04:00
/* create directory with rank as subdirectory */
2019-08-31 00:31:23 +03:00
sprintf ( dir + i + 1 , " %d " , ( rank + rankOffset ) % test - > numTasks ) ;
2011-11-12 02:22:17 +04:00
/* dir doesn't exist, so create */
2020-05-31 13:50:15 +03:00
if ( backend - > access ( dir , F_OK , test - > backend_options ) ! = 0 ) {
if ( backend - > mkdir ( dir , S_IRWXU , test - > backend_options ) < 0 ) {
2019-08-31 00:11:25 +03:00
ERRF ( " cannot create directory: %s " , dir ) ;
2011-11-12 02:22:17 +04:00
}
/* check if correct permissions */
2020-05-31 13:50:15 +03:00
} else if ( backend - > access ( dir , R_OK , test - > backend_options ) ! = 0 | |
backend - > access ( dir , W_OK , test - > backend_options ) ! = 0 | |
backend - > access ( dir , X_OK , test - > backend_options ) ! = 0 ) {
2019-08-31 00:11:25 +03:00
ERRF ( " invalid directory permissions: %s " , dir ) ;
2011-11-12 02:22:17 +04:00
}
/* concatenate dir and file names */
strcat ( dir , " / " ) ;
2019-08-31 00:31:23 +03:00
strcat ( dir , fname ) ;
2011-11-12 02:22:17 +04:00
return dir ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/******************************************************************************/
/*
* Reduce test results , and show if verbose set .
*/
2018-09-19 20:06:05 +03:00
static void
ReduceIterResults ( IOR_test_t * test , double * timer , const int rep , const int access )
2011-06-17 23:20:43 +04:00
{
2018-09-19 20:06:05 +03:00
double reduced [ IOR_NB_TIMERS ] = { 0 } ;
double diff [ IOR_NB_TIMERS / 2 + 1 ] ;
2019-09-10 21:39:31 +03:00
double totalTime , accessTime ;
IOR_param_t * params = & test - > params ;
double bw , iops , latency , minlatency ;
2018-09-19 20:06:05 +03:00
int i ;
MPI_Op op ;
2011-11-12 02:22:17 +04:00
2018-09-19 20:06:05 +03:00
assert ( access = = WRITE | | access = = READ ) ;
2011-12-11 08:45:19 +04:00
2011-11-12 02:22:17 +04:00
/* Find the minimum start time of the even numbered timers, and the
maximum finish time for the odd numbered timers */
2018-09-19 20:06:05 +03:00
for ( i = 0 ; i < IOR_NB_TIMERS ; i + + ) {
2011-11-12 02:22:17 +04:00
op = i % 2 ? MPI_MAX : MPI_MIN ;
2018-09-19 20:06:05 +03:00
MPI_CHECK ( MPI_Reduce ( & timer [ i ] , & reduced [ i ] , 1 , MPI_DOUBLE ,
2011-11-12 02:22:17 +04:00
op , 0 , testComm ) , " MPI_Reduce() " ) ;
2011-06-17 23:20:43 +04:00
}
2018-09-19 20:06:05 +03:00
/* Calculate elapsed times and throughput numbers */
for ( i = 0 ; i < IOR_NB_TIMERS / 2 ; i + + )
diff [ i ] = reduced [ 2 * i + 1 ] - reduced [ 2 * i ] ;
2011-11-12 02:22:17 +04:00
2018-09-19 20:06:05 +03:00
totalTime = reduced [ 5 ] - reduced [ 0 ] ;
2019-09-10 21:39:31 +03:00
accessTime = reduced [ 3 ] - reduced [ 2 ] ;
2014-08-14 02:53:24 +04:00
2018-09-20 00:39:25 +03:00
IOR_point_t * point = ( access = = WRITE ) ? & test - > results [ rep ] . write :
& test - > results [ rep ] . read ;
2014-08-14 02:53:24 +04:00
2018-09-20 00:39:25 +03:00
point - > time = totalTime ;
2012-01-09 06:55:46 +04:00
2018-09-19 20:06:05 +03:00
if ( verbose < VERBOSE_0 )
return ;
2018-09-20 00:39:25 +03:00
bw = ( double ) point - > aggFileSizeForBW / totalTime ;
2019-09-10 21:39:31 +03:00
/* For IOPS in this iteration, we divide the total amount of IOs from
* all ranks over the entire access time ( first start - > last end ) . */
iops = ( point - > aggFileSizeForBW / params - > transferSize ) / accessTime ;
/* For Latency, we divide the total access time for each task over the
* number of I / Os issued from that task ; then reduce and display the
* minimum ( best ) latency achieved . So what is reported is the average
* latency of all ops from a single task , then taking the minimum of
2019-10-30 17:43:43 +03:00
* that between all tasks . */
2019-09-10 21:39:31 +03:00
latency = ( timer [ 3 ] - timer [ 2 ] ) / ( params - > blockSize / params - > transferSize ) ;
MPI_CHECK ( MPI_Reduce ( & latency , & minlatency , 1 , MPI_DOUBLE ,
MPI_MIN , 0 , testComm ) , " MPI_Reduce() " ) ;
/* Only rank 0 tallies and prints the results. */
if ( rank ! = 0 )
return ;
PrintReducedResult ( test , access , bw , iops , latency , diff , totalTime , rep ) ;
2012-01-09 06:55:46 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Check for file ( s ) , then remove all files if file - per - proc , else single file .
Algorithms 'S3', 'S3_plus', and 'S3_EMC' all available.
These are variants on S3. S3 uses the "pure" S3 interface, e.g. using
Multi-Part-Upload. The "plus" variant enables EMC-extensions in the aws4c
library. This allows the N:N case to use "append", in the case where
"transfer_size" != "block_size" for IOR. In pure S3, the N:N case will
fail, because the EMC-extensions won't be enabled, and appending (which
attempts to use the EMC byte-range tricks to do this) will throw an error.
In the S3_EMC alg, N:1 uses EMCs other byte-range tricks to write different
parts of an N:1 file, and also uses append to write the parts of an N:N
file. Preliminary tests show these EMC extensions look to improve BW by
~20%.
I put all three algs in aiori-S3.c, because it seemed some code was getting
reused. Not sure if that's still going to make sense after the TBD, below.
TBD: Recently realized that the "pure' S3 shouldn't be trying to use
appends for anything. In the N:N case, it should just use MPU, within each
file. Then, there's no need for S3_plus. We just have S3, which does MPU
for all writes where transfer_size != block_size, and uses (standard)
byte-range reads for reading. Then S3_EMC uses "appends for N:N writes,
and byte-range writes for N:1 writes. This separates the code for the two
algs a little more, but we might still want them in the same file.
2014-10-30 01:04:30 +03:00
*
2011-06-17 23:20:43 +04:00
*/
2011-11-12 04:40:45 +04:00
static void RemoveFile ( char * testFileName , int filePerProc , IOR_param_t * test )
2011-11-12 02:22:17 +04:00
{
2018-07-07 12:29:27 +03:00
int tmpRankOffset = 0 ;
2011-11-12 02:22:17 +04:00
if ( filePerProc ) {
/* in random tasks, delete own file */
if ( test - > reorderTasksRandom = = TRUE ) {
tmpRankOffset = rankOffset ;
rankOffset = 0 ;
GetTestFileName ( testFileName , test ) ;
}
2020-05-31 13:50:15 +03:00
if ( backend - > access ( testFileName , F_OK , test - > backend_options ) = = 0 ) {
2019-08-31 00:11:25 +03:00
if ( verbose > = VERBOSE_3 ) {
fprintf ( out_logfile , " task %d removing %s \n " , rank ,
testFileName ) ;
}
2020-05-31 13:50:15 +03:00
backend - > delete ( testFileName , test - > backend_options ) ;
2011-11-12 02:22:17 +04:00
}
if ( test - > reorderTasksRandom = = TRUE ) {
rankOffset = tmpRankOffset ;
GetTestFileName ( testFileName , test ) ;
}
} else {
2020-05-31 13:50:15 +03:00
if ( ( rank = = 0 ) & & ( backend - > access ( testFileName , F_OK , test - > backend_options ) = = 0 ) ) {
2019-08-31 00:11:25 +03:00
if ( verbose > = VERBOSE_3 ) {
fprintf ( out_logfile , " task %d removing %s \n " , rank ,
testFileName ) ;
}
2020-05-31 13:50:15 +03:00
backend - > delete ( testFileName , test - > backend_options ) ;
2011-11-12 02:22:17 +04:00
}
}
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Setup tests by parsing commandline and creating test script .
2015-05-19 18:36:28 +03:00
* Perform a sanity - check on the configured parameters .
2011-06-17 23:20:43 +04:00
*/
2021-01-20 18:00:33 +03:00
static void InitTests ( IOR_test_t * tests )
2011-06-17 23:20:43 +04:00
{
2021-01-20 18:00:33 +03:00
if ( tests = = NULL ) {
return ;
}
MPI_Comm com = tests - > params . mpi_comm_world ;
2019-08-31 01:45:03 +03:00
int mpiNumNodes = 0 ;
int mpiNumTasks = 0 ;
int mpiNumTasksOnNode0 = 0 ;
2018-07-14 10:41:35 +03:00
2020-06-24 13:10:42 +03:00
verbose = tests - > params . verbose ;
aiori_warning_as_errors = tests - > params . warningAsErrors ;
2019-08-31 01:45:03 +03:00
/*
* These default values are the same for every test and expensive to
* retrieve so just do it once .
*/
mpiNumNodes = GetNumNodes ( com ) ;
mpiNumTasks = GetNumTasks ( com ) ;
mpiNumTasksOnNode0 = GetNumTasksOnNode0 ( com ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/*
* Since there is no guarantee that anyone other than
* task 0 has the environment settings for the hints , pass
2018-07-07 13:42:21 +03:00
* the hint = value pair to everyone else in mpi_comm_world
2011-11-12 02:22:17 +04:00
*/
2021-01-20 17:57:21 +03:00
DistributeHints ( com ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/* check validity of tests and create test queue */
while ( tests ! = NULL ) {
2018-07-14 10:41:35 +03:00
IOR_param_t * params = & tests - > params ;
params - > testComm = com ;
2019-08-31 01:45:03 +03:00
/* use MPI values if not overridden on command-line */
if ( params - > numNodes = = - 1 ) {
params - > numNodes = mpiNumNodes ;
}
if ( params - > numTasks = = - 1 ) {
params - > numTasks = mpiNumTasks ;
} else if ( params - > numTasks > mpiNumTasks ) {
if ( rank = = 0 ) {
2020-06-24 13:10:42 +03:00
EWARNF ( " More tasks requested (%d) than available (%d), " ,
2019-08-31 01:45:03 +03:00
params - > numTasks , mpiNumTasks ) ;
2020-06-24 13:10:42 +03:00
EWARNF ( " running with %d tasks. \n " , mpiNumTasks ) ;
2019-08-31 01:45:03 +03:00
}
params - > numTasks = mpiNumTasks ;
}
if ( params - > numTasksOnNode0 = = - 1 ) {
params - > numTasksOnNode0 = mpiNumTasksOnNode0 ;
2018-07-14 10:41:35 +03:00
}
2019-08-31 01:45:03 +03:00
params - > tasksBlockMapping = QueryNodeMapping ( com , false ) ;
2018-07-14 10:41:35 +03:00
params - > expectedAggFileSize =
params - > blockSize * params - > segmentCount * params - > numTasks ;
2021-01-20 17:57:21 +03:00
ValidateTests ( & tests - > params , com ) ;
2011-12-13 09:00:18 +04:00
tests = tests - > next ;
2011-11-12 02:22:17 +04:00
}
2011-06-17 23:20:43 +04:00
2021-01-20 17:06:05 +03:00
init_clock ( com ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2011-11-12 03:11:28 +04:00
/*
* Setup transfer buffers , creating and filling as needed .
*/
2017-09-27 19:45:47 +03:00
static void XferBuffersSetup ( IOR_io_buffers * ioBuffers , IOR_param_t * test ,
2015-05-27 19:24:52 +03:00
int pretendRank )
2011-06-17 23:20:43 +04:00
{
2015-05-27 19:24:52 +03:00
ioBuffers - > buffer = aligned_buffer_alloc ( test - > transferSize ) ;
2012-01-13 08:34:40 +04:00
}
/*
* Free transfer buffers .
*/
2015-05-27 19:24:52 +03:00
static void XferBuffersFree ( IOR_io_buffers * ioBuffers , IOR_param_t * test )
2012-01-13 08:34:40 +04:00
{
2015-05-27 19:24:52 +03:00
aligned_buffer_free ( ioBuffers - > buffer ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2012-09-10 21:50:18 +04:00
2011-12-13 09:00:18 +04:00
2012-01-07 05:29:45 +04:00
/*
* malloc a buffer , touching every page in an attempt to defeat lazy allocation .
*/
static void * malloc_and_touch ( size_t size )
{
size_t page_size ;
char * buf ;
char * ptr ;
if ( size = = 0 )
return NULL ;
2014-08-14 02:53:24 +04:00
page_size = sysconf ( _SC_PAGESIZE ) ;
2012-01-07 05:29:45 +04:00
buf = ( char * ) malloc ( size ) ;
if ( buf = = NULL )
2012-01-09 06:41:30 +04:00
return NULL ;
2012-01-07 05:29:45 +04:00
for ( ptr = buf ; ptr < buf + size ; ptr + = page_size ) {
* ptr = ( char ) 1 ;
}
return ( void * ) buf ;
}
2011-12-14 10:48:14 +04:00
static void file_hits_histogram ( IOR_param_t * params )
{
2018-07-07 12:29:27 +03:00
int * rankoffs = NULL ;
int * filecont = NULL ;
int * filehits = NULL ;
2014-08-14 02:53:24 +04:00
int ifile ;
int jfile ;
if ( rank = = 0 ) {
rankoffs = ( int * ) malloc ( params - > numTasks * sizeof ( int ) ) ;
filecont = ( int * ) malloc ( params - > numTasks * sizeof ( int ) ) ;
filehits = ( int * ) malloc ( params - > numTasks * sizeof ( int ) ) ;
}
MPI_CHECK ( MPI_Gather ( & rankOffset , 1 , MPI_INT , rankoffs ,
2021-01-20 17:57:21 +03:00
1 , MPI_INT , 0 , params - > testComm ) ,
2014-08-14 02:53:24 +04:00
" MPI_Gather error " ) ;
if ( rank ! = 0 )
return ;
memset ( ( void * ) filecont , 0 , params - > numTasks * sizeof ( int ) ) ;
for ( ifile = 0 ; ifile < params - > numTasks ; ifile + + ) {
filecont [ ( ifile + rankoffs [ ifile ] ) % params - > numTasks ] + + ;
}
memset ( ( void * ) filehits , 0 , params - > numTasks * sizeof ( int ) ) ;
for ( ifile = 0 ; ifile < params - > numTasks ; ifile + + )
for ( jfile = 0 ; jfile < params - > numTasks ; jfile + + ) {
if ( ifile = = filecont [ jfile ] )
filehits [ ifile ] + + ;
}
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " #File Hits Dist: " ) ;
2014-08-14 02:53:24 +04:00
jfile = 0 ;
ifile = 0 ;
while ( jfile < params - > numTasks & & ifile < params - > numTasks ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " %d " , filehits [ ifile ] ) ;
2014-08-14 02:53:24 +04:00
jfile + = filehits [ ifile ] , ifile + + ;
}
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " \n " ) ;
2014-08-14 02:53:24 +04:00
free ( rankoffs ) ;
free ( filecont ) ;
free ( filehits ) ;
2011-12-14 10:48:14 +04:00
}
2012-01-09 00:30:05 +04:00
int test_time_elapsed ( IOR_param_t * params , double startTime )
{
2014-08-14 02:53:24 +04:00
double endTime ;
2012-01-09 00:30:05 +04:00
2014-08-14 02:53:24 +04:00
if ( params - > maxTimeDuration = = 0 )
return 0 ;
2012-01-09 00:30:05 +04:00
2014-08-14 02:53:24 +04:00
endTime = startTime + ( params - > maxTimeDuration * 60 ) ;
2012-01-09 00:30:05 +04:00
2014-08-14 02:53:24 +04:00
return GetTimeStamp ( ) > = endTime ;
2012-01-09 00:30:05 +04:00
}
2012-01-09 06:41:30 +04:00
/*
* hog some memory as a rough simulation of a real application ' s memory use
*/
static void * HogMemory ( IOR_param_t * params )
{
size_t size ;
void * buf ;
if ( params - > memoryPerTask ! = 0 ) {
size = params - > memoryPerTask ;
} else if ( params - > memoryPerNode ! = 0 ) {
if ( verbose > = VERBOSE_3 )
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " This node hogging %ld bytes of memory \n " ,
2012-01-09 06:41:30 +04:00
params - > memoryPerNode ) ;
2019-08-31 01:45:03 +03:00
size = params - > memoryPerNode / params - > numTasksOnNode0 ;
2012-01-09 06:41:30 +04:00
} else {
return NULL ;
}
if ( verbose > = VERBOSE_3 )
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " This task hogging %ld bytes of memory \n " , size ) ;
2012-01-09 06:41:30 +04:00
buf = malloc_and_touch ( size ) ;
if ( buf = = NULL )
ERR ( " malloc of simulated applciation buffer failed " ) ;
return buf ;
}
2018-09-19 20:06:05 +03:00
/*
* Write times taken during each iteration of the test .
*/
static void
WriteTimes ( IOR_param_t * test , const double * timer , const int iteration ,
const int access )
{
char timerName [ MAX_STR ] ;
2012-01-09 06:41:30 +04:00
2018-09-19 20:06:05 +03:00
for ( int i = 0 ; i < IOR_NB_TIMERS ; i + + ) {
if ( access = = WRITE ) {
switch ( i ) {
case 0 :
strcpy ( timerName , " write open start " ) ;
break ;
case 1 :
strcpy ( timerName , " write open stop " ) ;
break ;
case 2 :
strcpy ( timerName , " write start " ) ;
break ;
case 3 :
strcpy ( timerName , " write stop " ) ;
break ;
case 4 :
strcpy ( timerName , " write close start " ) ;
break ;
case 5 :
strcpy ( timerName , " write close stop " ) ;
break ;
default :
strcpy ( timerName , " invalid timer " ) ;
break ;
}
}
else {
switch ( i ) {
case 0 :
strcpy ( timerName , " read open start " ) ;
break ;
case 1 :
strcpy ( timerName , " read open stop " ) ;
break ;
case 2 :
strcpy ( timerName , " read start " ) ;
break ;
case 3 :
strcpy ( timerName , " read stop " ) ;
break ;
case 4 :
strcpy ( timerName , " read close start " ) ;
break ;
case 5 :
strcpy ( timerName , " read close stop " ) ;
break ;
default :
strcpy ( timerName , " invalid timer " ) ;
break ;
}
}
fprintf ( out_logfile , " Test %d: Iter=%d, Task=%d, Time=%f, %s \n " ,
test - > id , iteration , ( int ) rank , timer [ i ] ,
timerName ) ;
}
}
2011-06-17 23:20:43 +04:00
/*
* Using the test parameters , run iteration ( s ) of single test .
*/
2011-12-13 09:00:18 +04:00
static void TestIoSys ( IOR_test_t * test )
2011-06-17 23:20:43 +04:00
{
2014-08-14 02:53:24 +04:00
IOR_param_t * params = & test - > params ;
IOR_results_t * results = test - > results ;
2011-11-12 02:22:17 +04:00
char testFileName [ MAX_STR ] ;
2018-09-19 20:06:05 +03:00
double timer [ IOR_NB_TIMERS ] ;
2011-11-12 02:22:17 +04:00
double startTime ;
2015-05-27 19:24:52 +03:00
int pretendRank ;
2018-09-19 20:06:05 +03:00
int rep ;
2020-05-31 14:11:00 +03:00
aiori_fd_t * fd ;
2011-11-12 02:22:17 +04:00
MPI_Group orig_group , new_group ;
int range [ 3 ] ;
IOR_offset_t dataMoved ; /* for data rate calculation */
2012-01-07 05:29:45 +04:00
void * hog_buf ;
2015-05-27 19:24:52 +03:00
IOR_io_buffers ioBuffers ;
2011-11-12 02:22:17 +04:00
/* set up communicator for test */
2021-01-20 17:57:21 +03:00
MPI_CHECK ( MPI_Comm_group ( params - > mpi_comm_world , & orig_group ) ,
2011-11-12 02:22:17 +04:00
" MPI_Comm_group() error " ) ;
2014-08-14 02:53:24 +04:00
range [ 0 ] = 0 ; /* first rank */
2011-12-13 09:00:18 +04:00
range [ 1 ] = params - > numTasks - 1 ; /* last rank */
2014-08-14 02:53:24 +04:00
range [ 2 ] = 1 ; /* stride */
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Group_range_incl ( orig_group , 1 , & range , & new_group ) ,
" MPI_Group_range_incl() error " ) ;
2021-01-20 17:57:21 +03:00
MPI_CHECK ( MPI_Comm_create ( params - > mpi_comm_world , new_group , & testComm ) ,
2011-11-12 02:22:17 +04:00
" MPI_Comm_create() error " ) ;
2013-10-07 18:12:20 +04:00
MPI_CHECK ( MPI_Group_free ( & orig_group ) , " MPI_Group_Free() error " ) ;
MPI_CHECK ( MPI_Group_free ( & new_group ) , " MPI_Group_Free() error " ) ;
2011-12-13 09:00:18 +04:00
params - > testComm = testComm ;
2011-11-12 02:22:17 +04:00
if ( testComm = = MPI_COMM_NULL ) {
/* tasks not in the group do not participate in this test */
2021-01-20 17:57:21 +03:00
MPI_CHECK ( MPI_Barrier ( params - > mpi_comm_world ) , " barrier error " ) ;
2011-11-12 02:22:17 +04:00
return ;
}
if ( rank = = 0 & & verbose > = VERBOSE_1 ) {
2019-10-24 01:17:37 +03:00
fprintf ( out_logfile , " Participating tasks : %d \n " , params - > numTasks ) ;
2018-07-07 13:42:21 +03:00
fflush ( out_logfile ) ;
2011-11-12 02:22:17 +04:00
}
2011-12-13 09:00:18 +04:00
if ( rank = = 0 & & params - > reorderTasks = = TRUE & & verbose > = VERBOSE_1 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2019-08-01 03:42:03 +03:00
" Using reorderTasks '-C' (useful to avoid read cache in client) \n " ) ;
2018-07-07 13:42:21 +03:00
fflush ( out_logfile ) ;
2011-06-17 23:20:43 +04:00
}
2011-11-12 02:22:17 +04:00
/* show test setup */
if ( rank = = 0 & & verbose > = VERBOSE_0 )
2011-12-13 09:00:18 +04:00
ShowSetup ( params ) ;
2011-06-17 23:20:43 +04:00
2012-01-09 06:41:30 +04:00
hog_buf = HogMemory ( params ) ;
2012-01-07 05:29:45 +04:00
2015-05-27 19:24:52 +03:00
pretendRank = ( rank + rankOffset ) % params - > numTasks ;
/* IO Buffer Setup */
2017-09-27 19:45:47 +03:00
if ( params - > setTimeStampSignature ) { // initialize the buffer properly
2019-12-22 14:21:40 +03:00
params - > timeStampSignatureValue = ( unsigned int ) params - > setTimeStampSignature ;
2017-09-27 19:45:47 +03:00
}
2015-05-27 19:24:52 +03:00
XferBuffersSetup ( & ioBuffers , params , pretendRank ) ;
2017-09-27 19:45:47 +03:00
reseed_incompressible_prng = TRUE ; // reset pseudo random generator, necessary to guarantee the next call to FillBuffer produces the same value as it is right now
2015-05-27 19:24:52 +03:00
/* Initial time stamp */
2011-11-12 02:22:17 +04:00
startTime = GetTimeStamp ( ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/* loop over test iterations */
2018-07-15 21:38:17 +03:00
uint64_t params_saved_wearout = params - > stoneWallingWearOutIterations ;
2020-12-01 21:24:09 +03:00
/* Check if the file exists and warn users */
2020-12-03 23:54:51 +03:00
if ( ( params - > writeFile | | params - > checkWrite ) & & ( params - > hints . filePerProc | | rank = = 0 ) ) {
2020-12-02 13:14:28 +03:00
struct stat sb ;
GetTestFileName ( testFileName , params ) ;
int ret = backend - > stat ( testFileName , & sb , params - > backend_options ) ;
if ( ret = = 0 ) {
EWARNF ( " The file \" %s \" exists already and will be overwritten " , testFileName ) ;
}
2020-12-01 21:24:09 +03:00
}
2011-12-13 09:00:18 +04:00
for ( rep = 0 ; rep < params - > repetitions ; rep + + ) {
2011-11-12 02:22:17 +04:00
/* Get iteration start time in seconds in task 0 and broadcast to
all tasks */
if ( rank = = 0 ) {
2017-09-27 19:45:47 +03:00
if ( ! params - > setTimeStampSignature ) {
2011-11-12 02:22:17 +04:00
time_t currentTime ;
if ( ( currentTime = time ( NULL ) ) = = - 1 ) {
ERR ( " cannot get current time " ) ;
}
2011-12-13 09:00:18 +04:00
params - > timeStampSignatureValue =
2014-08-14 02:53:24 +04:00
( unsigned int ) currentTime ;
2019-10-24 01:17:37 +03:00
}
if ( verbose > = VERBOSE_2 ) {
fprintf ( out_logfile ,
" Using Time Stamp %u (0x%x) for Data Signature \n " ,
params - > timeStampSignatureValue ,
params - > timeStampSignatureValue ) ;
2011-11-12 02:22:17 +04:00
}
2014-08-14 02:53:24 +04:00
if ( rep = = 0 & & verbose > = VERBOSE_0 ) {
2018-07-08 15:47:55 +03:00
PrintTableHeader ( ) ;
2014-08-14 02:53:24 +04:00
}
2011-11-12 02:22:17 +04:00
}
MPI_CHECK ( MPI_Bcast
2011-12-13 09:00:18 +04:00
( & params - > timeStampSignatureValue , 1 , MPI_UNSIGNED , 0 ,
2011-11-12 02:22:17 +04:00
testComm ) , " cannot broadcast start time value " ) ;
2017-12-09 13:52:13 +03:00
FillBuffer ( ioBuffers . buffer , params , 0 , pretendRank ) ;
2011-11-12 02:22:17 +04:00
/* use repetition count for number of multiple files */
2011-12-13 09:00:18 +04:00
if ( params - > multiFile )
params - > repCounter = rep ;
2011-11-12 02:22:17 +04:00
/*
* write the file ( s ) , getting timing between I / O calls
*/
2012-01-09 00:30:05 +04:00
if ( params - > writeFile & & ! test_time_elapsed ( params , startTime ) ) {
2011-12-13 09:00:18 +04:00
GetTestFileName ( testFileName , params ) ;
2011-11-12 02:22:17 +04:00
if ( verbose > = VERBOSE_3 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " task %d writing %s \n " , rank ,
2011-11-12 02:22:17 +04:00
testFileName ) ;
}
2011-12-13 09:00:18 +04:00
DelaySecs ( params - > interTestDelay ) ;
if ( params - > useExistingTestFile = = FALSE ) {
RemoveFile ( testFileName , params - > filePerProc ,
params ) ;
2011-11-12 02:22:17 +04:00
}
2018-07-15 21:38:17 +03:00
params - > stoneWallingWearOutIterations = params_saved_wearout ;
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
2011-12-13 09:00:18 +04:00
params - > open = WRITE ;
2018-09-19 20:06:05 +03:00
timer [ 0 ] = GetTimeStamp ( ) ;
2020-05-30 21:01:20 +03:00
fd = backend - > create ( testFileName , IOR_WRONLY | IOR_CREAT | IOR_TRUNC , params - > backend_options ) ;
2020-06-30 21:41:49 +03:00
if ( fd = = NULL ) FAIL ( " Cannot create file " ) ;
2018-09-19 20:06:05 +03:00
timer [ 1 ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
if ( params - > intraTestBarriers )
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) ,
" barrier error " ) ;
if ( rank = = 0 & & verbose > = VERBOSE_1 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-12-13 13:07:55 +04:00
" Commencing write performance test: %s " ,
2014-08-14 02:53:24 +04:00
CurrentTimeString ( ) ) ;
2011-11-12 02:22:17 +04:00
}
2018-09-19 20:06:05 +03:00
timer [ 2 ] = GetTimeStamp ( ) ;
2018-09-20 00:39:25 +03:00
dataMoved = WriteOrRead ( params , & results [ rep ] , fd , WRITE , & ioBuffers ) ;
2014-08-29 01:39:44 +04:00
if ( params - > verbose > = VERBOSE_4 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " * data moved = %llu \n " , dataMoved ) ;
fflush ( out_logfile ) ;
2014-08-29 01:39:44 +04:00
}
2018-09-19 20:06:05 +03:00
timer [ 3 ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
if ( params - > intraTestBarriers )
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) ,
" barrier error " ) ;
2018-09-19 20:06:05 +03:00
timer [ 4 ] = GetTimeStamp ( ) ;
2020-05-31 13:50:15 +03:00
backend - > close ( fd , params - > backend_options ) ;
2011-11-12 02:22:17 +04:00
2018-09-19 20:06:05 +03:00
timer [ 5 ] = GetTimeStamp ( ) ;
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
2014-08-14 02:53:24 +04:00
/* check if stat() of file doesn't equal expected file size,
use actual amount of byte moved */
2020-06-30 14:03:05 +03:00
CheckFileSize ( test , testFileName , dataMoved , rep , WRITE ) ;
2011-11-12 02:22:17 +04:00
if ( verbose > = VERBOSE_3 )
2011-12-13 09:00:18 +04:00
WriteTimes ( params , timer , rep , WRITE ) ;
2011-11-12 02:22:17 +04:00
ReduceIterResults ( test , timer , rep , WRITE ) ;
2011-12-13 09:00:18 +04:00
if ( params - > outlierThreshold ) {
2018-09-19 20:06:05 +03:00
CheckForOutliers ( params , timer , WRITE ) ;
2011-11-12 02:22:17 +04:00
}
2018-07-15 21:38:17 +03:00
/* check if in this round we run write with stonewalling */
if ( params - > deadlineForStonewalling > 0 ) {
2018-09-20 00:39:25 +03:00
params - > stoneWallingWearOutIterations = results [ rep ] . write . pairs_accessed ;
2018-07-15 21:38:17 +03:00
}
2011-11-12 02:22:17 +04:00
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
/*
* perform a check of data , reading back data and comparing
* against what was expected to be written
*/
2012-01-09 00:30:05 +04:00
if ( params - > checkWrite & & ! test_time_elapsed ( params , startTime ) ) {
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
if ( rank = = 0 & & verbose > = VERBOSE_1 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2011-11-12 02:22:17 +04:00
" Verifying contents of the file(s) just written. \n " ) ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " %s \n " , CurrentTimeString ( ) ) ;
2011-11-12 02:22:17 +04:00
}
2011-12-13 09:00:18 +04:00
if ( params - > reorderTasks ) {
2011-11-12 02:22:17 +04:00
/* move two nodes away from writing node */
2019-07-28 19:25:42 +03:00
int shift = 1 ; /* assume a by-node (round-robin) mapping of tasks to nodes */
if ( params - > tasksBlockMapping ) {
2019-08-31 01:45:03 +03:00
shift = params - > numTasksOnNode0 ; /* switch to by-slot (contiguous block) mapping */
2019-07-28 00:27:20 +03:00
}
rankOffset = ( 2 * shift ) % params - > numTasks ;
2011-11-12 02:22:17 +04:00
}
2015-05-21 21:05:56 +03:00
reseed_incompressible_prng = TRUE ; /* Re-Seed the PRNG to get same sequence back, if random */
2011-12-13 09:00:18 +04:00
GetTestFileName ( testFileName , params ) ;
params - > open = WRITECHECK ;
2020-05-31 13:50:15 +03:00
fd = backend - > open ( testFileName , IOR_RDONLY , params - > backend_options ) ;
2020-06-30 21:41:49 +03:00
if ( fd = = NULL ) FAIL ( " Cannot open file " ) ;
2018-09-20 00:39:25 +03:00
dataMoved = WriteOrRead ( params , & results [ rep ] , fd , WRITECHECK , & ioBuffers ) ;
2020-05-31 13:50:15 +03:00
backend - > close ( fd , params - > backend_options ) ;
2011-11-12 02:22:17 +04:00
rankOffset = 0 ;
}
/*
* read the file ( s ) , getting timing between I / O calls
*/
2017-09-27 19:45:47 +03:00
if ( ( params - > readFile | | params - > checkRead ) & & ! test_time_elapsed ( params , startTime ) ) {
2018-07-15 21:38:17 +03:00
/* check for stonewall */
if ( params - > stoneWallingStatusFile ) {
2021-01-20 17:06:05 +03:00
params - > stoneWallingWearOutIterations = ReadStoneWallingIterations ( params - > stoneWallingStatusFile , params - > testComm ) ;
2018-07-15 21:38:17 +03:00
if ( params - > stoneWallingWearOutIterations = = - 1 & & rank = = 0 ) {
2020-06-24 13:10:42 +03:00
WARN ( " Could not read back the stonewalling status from the file! " ) ;
2018-07-15 21:38:17 +03:00
params - > stoneWallingWearOutIterations = 0 ;
}
}
2017-09-27 19:45:47 +03:00
int operation_flag = READ ;
if ( params - > checkRead ) {
// actually read and then compare the buffer
operation_flag = READCHECK ;
}
2011-11-12 02:22:17 +04:00
/* Get rankOffset [file offset] for this process to read, based on -C,-Z,-Q,-X options */
/* Constant process offset reading */
2011-12-13 09:00:18 +04:00
if ( params - > reorderTasks ) {
2019-08-01 19:20:01 +03:00
/* move one node away from writing node */
2019-07-28 19:25:42 +03:00
int shift = 1 ; /* assume a by-node (round-robin) mapping of tasks to nodes */
if ( params - > tasksBlockMapping ) {
2019-08-31 01:45:03 +03:00
shift = params - > numTasksOnNode0 ; /* switch to a by-slot (contiguous block) mapping */
2019-07-28 00:27:20 +03:00
}
rankOffset = ( params - > taskPerNodeOffset * shift ) % params - > numTasks ;
2011-11-12 02:22:17 +04:00
}
/* random process offset reading */
2011-12-13 09:00:18 +04:00
if ( params - > reorderTasksRandom ) {
2011-11-12 02:22:17 +04:00
/* this should not intefere with randomOffset within a file because GetOffsetArrayRandom */
2018-10-08 23:47:28 +03:00
/* seeds every rand() call */
2014-08-14 02:53:24 +04:00
int nodeoffset ;
2011-11-12 02:22:17 +04:00
unsigned int iseed0 ;
2011-12-13 09:00:18 +04:00
nodeoffset = params - > taskPerNodeOffset ;
2019-08-31 01:45:03 +03:00
nodeoffset = ( nodeoffset < params - > numNodes ) ? nodeoffset : params - > numNodes - 1 ;
2011-12-14 10:48:14 +04:00
if ( params - > reorderTasksRandomSeed < 0 )
2014-08-14 02:53:24 +04:00
iseed0 = - 1 * params - > reorderTasksRandomSeed + rep ;
else
iseed0 = params - > reorderTasksRandomSeed ;
2011-11-12 02:22:17 +04:00
srand ( rank + iseed0 ) ;
{
2011-12-13 09:00:18 +04:00
rankOffset = rand ( ) % params - > numTasks ;
2011-11-12 02:22:17 +04:00
}
while ( rankOffset <
2019-08-31 01:45:03 +03:00
( nodeoffset * params - > numTasksOnNode0 ) ) {
2011-12-13 09:00:18 +04:00
rankOffset = rand ( ) % params - > numTasks ;
2011-11-12 02:22:17 +04:00
}
/* Get more detailed stats if requested by verbose level */
if ( verbose > = VERBOSE_2 ) {
2014-08-14 02:53:24 +04:00
file_hits_histogram ( params ) ;
2011-11-12 02:22:17 +04:00
}
}
/* Using globally passed rankOffset, following function generates testFileName to read */
2011-12-13 09:00:18 +04:00
GetTestFileName ( testFileName , params ) ;
2011-11-12 02:22:17 +04:00
if ( verbose > = VERBOSE_3 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " task %d reading %s \n " , rank ,
2011-11-12 02:22:17 +04:00
testFileName ) ;
}
2011-12-13 09:00:18 +04:00
DelaySecs ( params - > interTestDelay ) ;
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
2011-12-13 09:00:18 +04:00
params - > open = READ ;
2018-09-19 20:06:05 +03:00
timer [ 0 ] = GetTimeStamp ( ) ;
2020-05-31 13:50:15 +03:00
fd = backend - > open ( testFileName , IOR_RDONLY , params - > backend_options ) ;
2020-06-30 21:41:49 +03:00
if ( fd = = NULL ) FAIL ( " Cannot open file " ) ;
2018-09-19 20:06:05 +03:00
timer [ 1 ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
if ( params - > intraTestBarriers )
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) ,
" barrier error " ) ;
if ( rank = = 0 & & verbose > = VERBOSE_1 ) {
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile ,
2019-08-31 01:45:03 +03:00
" Commencing read performance test: %s \n " ,
2014-08-14 02:53:24 +04:00
CurrentTimeString ( ) ) ;
2011-11-12 02:22:17 +04:00
}
2018-09-19 20:06:05 +03:00
timer [ 2 ] = GetTimeStamp ( ) ;
2018-09-20 00:39:25 +03:00
dataMoved = WriteOrRead ( params , & results [ rep ] , fd , operation_flag , & ioBuffers ) ;
2018-09-19 20:06:05 +03:00
timer [ 3 ] = GetTimeStamp ( ) ;
2011-12-13 09:00:18 +04:00
if ( params - > intraTestBarriers )
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) ,
" barrier error " ) ;
2018-09-19 20:06:05 +03:00
timer [ 4 ] = GetTimeStamp ( ) ;
2020-05-31 13:50:15 +03:00
backend - > close ( fd , params - > backend_options ) ;
2018-09-19 20:06:05 +03:00
timer [ 5 ] = GetTimeStamp ( ) ;
2011-11-12 02:22:17 +04:00
/* check if stat() of file doesn't equal expected file size,
use actual amount of byte moved */
2020-06-30 14:03:05 +03:00
CheckFileSize ( test , testFileName , dataMoved , rep , READ ) ;
2011-11-12 02:22:17 +04:00
if ( verbose > = VERBOSE_3 )
2011-12-13 09:00:18 +04:00
WriteTimes ( params , timer , rep , READ ) ;
2011-11-12 02:22:17 +04:00
ReduceIterResults ( test , timer , rep , READ ) ;
2011-12-13 09:00:18 +04:00
if ( params - > outlierThreshold ) {
2018-09-19 20:06:05 +03:00
CheckForOutliers ( params , timer , READ ) ;
2011-11-12 02:22:17 +04:00
}
}
2011-12-13 09:00:18 +04:00
if ( ! params - > keepFile
2012-01-09 06:55:46 +04:00
& & ! ( params - > errorFound & & params - > keepFileWithError ) ) {
double start , finish ;
start = GetTimeStamp ( ) ;
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
2011-12-13 09:00:18 +04:00
RemoveFile ( testFileName , params - > filePerProc , params ) ;
2012-01-09 06:55:46 +04:00
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
finish = GetTimeStamp ( ) ;
PrintRemoveTiming ( start , finish , rep ) ;
} else {
MPI_CHECK ( MPI_Barrier ( testComm ) , " barrier error " ) ;
2011-11-12 02:22:17 +04:00
}
2011-12-13 09:00:18 +04:00
params - > errorFound = FALSE ;
2011-11-12 02:22:17 +04:00
rankOffset = 0 ;
2018-07-08 18:47:23 +03:00
2011-11-12 02:22:17 +04:00
}
2020-10-28 12:38:17 +03:00
PrintRepeatEnd ( ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
MPI_CHECK ( MPI_Comm_free ( & testComm ) , " MPI_Comm_free() error " ) ;
2011-12-13 09:00:18 +04:00
2012-01-14 01:27:55 +04:00
if ( params - > summary_every_test ) {
PrintLongSummaryHeader ( ) ;
PrintLongSummaryOneTest ( test ) ;
} else {
PrintShortSummary ( test ) ;
}
2011-12-13 09:00:18 +04:00
2015-05-27 19:24:52 +03:00
XferBuffersFree ( & ioBuffers , params ) ;
2014-08-14 02:53:24 +04:00
if ( hog_buf ! = NULL )
free ( hog_buf ) ;
2012-01-07 05:29:45 +04:00
2011-11-12 02:22:17 +04:00
/* Sync with the tasks that did not participate in this test */
2021-01-20 17:57:21 +03:00
MPI_CHECK ( MPI_Barrier ( params - > mpi_comm_world ) , " barrier error " ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
/*
* Determine if valid tests from parameters .
*/
2021-01-20 17:57:21 +03:00
static void ValidateTests ( IOR_param_t * test , MPI_Comm com )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
IOR_param_t defaults ;
2021-01-20 17:57:21 +03:00
init_IOR_Param_t ( & defaults , com ) ;
2015-05-19 18:36:28 +03:00
2011-11-12 02:22:17 +04:00
if ( test - > repetitions < = 0 )
WARN_RESET ( " too few test repetitions " ,
test , & defaults , repetitions ) ;
if ( test - > numTasks < = 0 )
ERR ( " too few tasks for testing " ) ;
if ( test - > interTestDelay < 0 )
WARN_RESET ( " inter-test delay must be nonnegative value " ,
test , & defaults , interTestDelay ) ;
if ( test - > readFile ! = TRUE & & test - > writeFile ! = TRUE
& & test - > checkRead ! = TRUE & & test - > checkWrite ! = TRUE )
2017-12-09 19:22:58 +03:00
ERR ( " test must write, read, or check read/write file " ) ;
if ( ! test - > setTimeStampSignature & & test - > writeFile ! = TRUE & & test - > checkRead = = TRUE )
2017-12-09 19:24:31 +03:00
ERR ( " using readCheck only requires to write a timeStampSignature -- use -G " ) ;
2011-11-12 02:22:17 +04:00
if ( test - > segmentCount < 0 )
ERR ( " segment count must be positive value " ) ;
if ( ( test - > blockSize % sizeof ( IOR_size_t ) ) ! = 0 )
ERR ( " block size must be a multiple of access size " ) ;
if ( test - > blockSize < 0 )
ERR ( " block size must be non-negative integer " ) ;
if ( ( test - > transferSize % sizeof ( IOR_size_t ) ) ! = 0 )
ERR ( " transfer size must be a multiple of access size " ) ;
if ( test - > transferSize < 0 )
ERR ( " transfer size must be non-negative integer " ) ;
if ( test - > transferSize = = 0 ) {
ERR ( " test will not complete with zero transfer size " ) ;
2011-06-17 23:20:43 +04:00
} else {
2011-11-12 02:22:17 +04:00
if ( ( test - > blockSize % test - > transferSize ) ! = 0 )
ERR ( " block size must be a multiple of transfer size " ) ;
}
if ( test - > blockSize < test - > transferSize )
ERR ( " block size must not be smaller than transfer size " ) ;
2020-11-06 13:38:50 +03:00
if ( test - > randomOffset & & test - > blockSize = = test - > transferSize )
ERR ( " IOR will randomize access within a block and repeats the same pattern for all segments, therefore choose blocksize > transferSize " ) ;
2020-12-01 16:52:29 +03:00
if ( ! test - > randomOffset & & test - > randomPrefillBlocksize )
ERR ( " Setting the randomPrefill option without using random is not useful " ) ;
if ( test - > randomPrefillBlocksize & & ( test - > blockSize % test - > randomPrefillBlocksize ! = 0 ) )
ERR ( " The randomPrefill option must divide the blockSize " ) ;
2015-05-19 18:36:28 +03:00
/* specific APIs */
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " MPIIO " ) = = 0 )
2011-11-12 02:22:17 +04:00
& & ( test - > blockSize < sizeof ( IOR_size_t )
| | test - > transferSize < sizeof ( IOR_size_t ) ) )
ERR ( " block/transfer size may not be smaller than IOR_size_t for MPIIO " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " HDF5 " ) = = 0 )
2011-11-12 02:22:17 +04:00
& & ( test - > blockSize < sizeof ( IOR_size_t )
| | test - > transferSize < sizeof ( IOR_size_t ) ) )
ERR ( " block/transfer size may not be smaller than IOR_size_t for HDF5 " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " NCMPI " ) = = 0 )
2011-11-12 02:22:17 +04:00
& & ( test - > blockSize < sizeof ( IOR_size_t )
| | test - > transferSize < sizeof ( IOR_size_t ) ) )
ERR ( " block/transfer size may not be smaller than IOR_size_t for NCMPI " ) ;
2018-04-27 20:44:20 +03:00
if ( ( ( strcasecmp ( test - > api , " POSIX " ) ! = 0 )
& & ( strcasecmp ( test - > api , " MPIIO " ) ! = 0 )
2018-07-12 18:13:40 +03:00
& & ( strcasecmp ( test - > api , " MMAP " ) ! = 0 )
2018-08-15 01:08:04 +03:00
& & ( strcasecmp ( test - > api , " HDFS " ) ! = 0 )
2020-05-31 19:20:58 +03:00
& & ( strcasecmp ( test - > api , " DFS " ) ! = 0 )
2019-08-02 07:03:59 +03:00
& & ( strcasecmp ( test - > api , " Gfarm " ) ! = 0 )
2020-03-05 04:40:05 +03:00
& & ( strcasecmp ( test - > api , " RADOS " ) ! = 0 )
& & ( strcasecmp ( test - > api , " CEPHFS " ) ! = 0 ) ) & & test - > fsync )
2018-04-27 20:44:20 +03:00
WARN_RESET ( " fsync() not supported in selected backend " ,
2011-11-12 02:22:17 +04:00
test , & defaults , fsync ) ;
2020-07-03 10:09:40 +03:00
/* parameter consistency */
2011-11-12 02:22:17 +04:00
if ( test - > reorderTasks = = TRUE & & test - > reorderTasksRandom = = TRUE )
ERR ( " Both Constant and Random task re-ordering specified. Choose one and resubmit " ) ;
if ( test - > randomOffset & & test - > reorderTasksRandom
& & test - > filePerProc = = FALSE )
ERR ( " random offset and random reorder tasks specified with single-shared-file. Choose one and resubmit " ) ;
if ( test - > randomOffset & & test - > reorderTasks
& & test - > filePerProc = = FALSE )
ERR ( " random offset and constant reorder tasks specified with single-shared-file. Choose one and resubmit " ) ;
2020-11-04 16:47:35 +03:00
if ( test - > randomOffset & & test - > checkRead & & test - > randomSeed = = - 1 )
ERR ( " random offset with read check option requires to set the random seed " ) ;
2011-11-12 02:22:17 +04:00
if ( test - > randomOffset & & test - > storeFileOffset )
ERR ( " random offset not available with store file offset option) " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " HDF5 " ) = = 0 ) & & test - > randomOffset )
2011-11-12 02:22:17 +04:00
ERR ( " random offset not available with HDF5 " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " NCMPI " ) = = 0 ) & & test - > randomOffset )
2011-11-12 02:22:17 +04:00
ERR ( " random offset not available with NCMPI " ) ;
2017-12-05 02:20:31 +03:00
if ( ( strcasecmp ( test - > api , " NCMPI " ) = = 0 ) & & test - > filePerProc )
2011-11-12 02:22:17 +04:00
ERR ( " file-per-proc not available in current NCMPI " ) ;
2015-05-19 18:36:28 +03:00
2020-05-31 14:50:03 +03:00
backend = test - > backend ;
ior_set_xfer_hints ( test ) ;
2019-09-01 17:59:52 +03:00
/* allow the backend to validate the options */
if ( test - > backend - > check_params ) {
2020-05-30 22:09:37 +03:00
int check = test - > backend - > check_params ( test - > backend_options ) ;
2020-05-30 20:19:48 +03:00
if ( check ) {
2019-09-01 17:59:52 +03:00
ERR ( " The backend returned that the test parameters are invalid. " ) ;
}
2015-05-19 18:36:28 +03:00
}
2011-11-12 03:11:28 +04:00
}
2011-11-12 02:22:17 +04:00
2018-02-28 20:50:30 +03:00
/**
* Returns a precomputed array of IOR_offset_t for the inner benchmark loop .
2020-11-06 12:52:35 +03:00
* They get created sequentially and mixed up in the end .
* It should be noted that as the seeds get synchronised across all processes if not FilePerProcess is set
* every process computes the same random order .
2018-02-28 20:50:30 +03:00
* For a shared file all transfers get randomly assigned to ranks . The processes
* can also have differen ' t numbers of transfers . This might lead to a bigger
* diversion in accesse as it dose with filePerProc . This is expected but
* should be mined .
* @ param test IOR_param_t for getting transferSize , blocksize and SegmentCount
2020-07-03 10:09:40 +03:00
* @ param pretendRank int pretended Rank for shifting the offsets correctly
2018-02-28 20:50:30 +03:00
* @ return IOR_offset_t
*/
2020-11-06 13:27:11 +03:00
IOR_offset_t * GetOffsetArrayRandom ( IOR_param_t * test , int pretendRank , IOR_offset_t * out_count )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
int seed ;
2020-11-06 12:52:35 +03:00
IOR_offset_t i ;
IOR_offset_t offsets ;
2011-11-12 03:11:28 +04:00
IOR_offset_t offsetCnt = 0 ;
2011-11-12 02:22:17 +04:00
IOR_offset_t * offsetArray ;
2020-11-06 12:52:35 +03:00
if ( test - > filePerProc ) {
/* set up seed, each process can determine which regions to access individually */
if ( test - > randomSeed = = - 1 ) {
seed = time ( NULL ) ;
test - > randomSeed = seed ;
} else {
seed = test - > randomSeed + pretendRank ;
}
} else {
/* Shared file requires that the seed is synchronized */
if ( test - > randomSeed = = - 1 ) {
// all processes need to have the same seed.
if ( rank = = 0 ) {
seed = time ( NULL ) ;
}
MPI_CHECK ( MPI_Bcast ( & seed , 1 , MPI_INT , 0 , test - > testComm ) , " cannot broadcast random seed value " ) ;
test - > randomSeed = seed ;
} else {
seed = test - > randomSeed ;
}
2011-06-17 23:20:43 +04:00
}
2020-11-06 12:52:35 +03:00
srandom ( seed ) ;
2011-11-12 02:22:17 +04:00
/* count needed offsets (pass 1) */
2020-11-06 13:27:11 +03:00
if ( test - > filePerProc ) {
offsets = test - > blockSize / test - > transferSize ;
} else {
2020-11-06 12:52:35 +03:00
offsets = 0 ;
2020-11-06 13:27:11 +03:00
for ( i = 0 ; i < test - > blockSize * test - > numTasks ; i + = test - > transferSize ) {
2020-11-06 12:52:35 +03:00
// this counts which process get how many transferes in the shared file
if ( ( rand ( ) % test - > numTasks ) = = pretendRank ) {
offsets + + ;
}
2020-11-04 16:47:35 +03:00
}
2011-06-17 23:20:43 +04:00
}
2011-11-12 02:22:17 +04:00
/* setup empty array */
2020-11-06 13:27:11 +03:00
offsetArray = ( IOR_offset_t * ) safeMalloc ( offsets * sizeof ( IOR_offset_t ) ) ;
2020-11-06 12:52:35 +03:00
2020-11-06 13:27:11 +03:00
* out_count = offsets ;
2011-11-12 02:22:17 +04:00
if ( test - > filePerProc ) {
2020-11-06 12:52:35 +03:00
/* fill array */
for ( i = 0 ; i < offsets ; i + + ) {
2020-11-06 13:27:11 +03:00
offsetArray [ i ] = i * test - > transferSize ;
2020-11-06 12:52:35 +03:00
}
2011-11-12 02:22:17 +04:00
} else {
2020-11-06 12:52:35 +03:00
/* fill with offsets (pass 2) */
srandom ( seed ) ; /* need same seed to get same transfers as counted in the beginning*/
2020-11-06 13:27:11 +03:00
for ( i = 0 ; i < test - > blockSize * test - > numTasks ; i + = test - > transferSize ) {
2020-11-06 12:52:35 +03:00
if ( ( rand ( ) % test - > numTasks ) = = pretendRank ) {
offsetArray [ offsetCnt ] = i ;
offsetCnt + + ;
2011-11-12 02:22:17 +04:00
}
2020-11-06 12:52:35 +03:00
}
2011-11-12 02:22:17 +04:00
}
/* reorder array */
for ( i = 0 ; i < offsets ; i + + ) {
2020-11-06 12:52:35 +03:00
IOR_offset_t value , tmp ;
2018-10-08 23:47:28 +03:00
value = rand ( ) % offsets ;
2011-11-12 02:22:17 +04:00
tmp = offsetArray [ value ] ;
offsetArray [ value ] = offsetArray [ i ] ;
offsetArray [ i ] = tmp ;
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
return ( offsetArray ) ;
2011-11-12 03:11:28 +04:00
}
2011-06-17 23:20:43 +04:00
2020-12-01 16:52:29 +03:00
static IOR_offset_t WriteOrReadSingle ( IOR_offset_t offset , int pretendRank , IOR_offset_t transfer , IOR_offset_t * transferCount , int * errors , IOR_param_t * test , aiori_fd_t * fd , IOR_io_buffers * ioBuffers , int access ) {
2018-07-07 12:29:27 +03:00
IOR_offset_t amtXferred = 0 ;
2017-10-20 19:02:24 +03:00
void * buffer = ioBuffers - > buffer ;
if ( access = = WRITE ) {
2017-11-30 13:56:26 +03:00
/* fills each transfer with a unique pattern
* containing the offset into the file */
2017-10-20 19:02:24 +03:00
if ( test - > storeFileOffset = = TRUE ) {
2020-06-10 19:47:07 +03:00
FillBuffer ( buffer , test , offset , pretendRank ) ;
2017-10-20 19:02:24 +03:00
}
2020-06-10 19:47:07 +03:00
amtXferred = backend - > xfer ( access , fd , buffer , transfer , offset , test - > backend_options ) ;
2017-10-20 19:02:24 +03:00
if ( amtXferred ! = transfer )
ERR ( " cannot write to file " ) ;
2020-05-30 20:19:48 +03:00
if ( test - > fsyncPerWrite )
backend - > fsync ( fd , test - > backend_options ) ;
2018-09-30 12:01:21 +03:00
if ( test - > interIODelay > 0 ) {
2018-10-06 19:30:00 +03:00
struct timespec wait = { test - > interIODelay / 1000 / 1000 , 1000l * ( test - > interIODelay % 1000000 ) } ;
nanosleep ( & wait , NULL ) ;
2018-09-30 12:01:21 +03:00
}
2017-10-20 19:02:24 +03:00
} else if ( access = = READ ) {
2020-06-10 19:47:07 +03:00
amtXferred = backend - > xfer ( access , fd , buffer , transfer , offset , test - > backend_options ) ;
2017-10-20 19:02:24 +03:00
if ( amtXferred ! = transfer )
ERR ( " cannot read from file " ) ;
2018-09-30 12:01:21 +03:00
if ( test - > interIODelay > 0 ) {
2018-10-06 19:30:00 +03:00
struct timespec wait = { test - > interIODelay / 1000 / 1000 , 1000l * ( test - > interIODelay % 1000000 ) } ;
nanosleep ( & wait , NULL ) ;
2018-09-30 12:01:21 +03:00
}
2017-10-20 19:02:24 +03:00
} else if ( access = = WRITECHECK ) {
2020-09-28 22:09:48 +03:00
( ( long long int * ) buffer ) [ 0 ] = ~ ( ( long long int * ) buffer ) [ 0 ] ; // changes the buffer, no memset to reduce the memory pressure
amtXferred = backend - > xfer ( access , fd , buffer , transfer , offset , test - > backend_options ) ;
2017-10-20 19:02:24 +03:00
if ( amtXferred ! = transfer )
ERR ( " cannot read from file write check " ) ;
( * transferCount ) + + ;
2020-09-28 22:09:48 +03:00
* errors + = CompareData ( buffer , transfer , * transferCount , test , offset , pretendRank , WRITECHECK ) ;
2017-10-20 19:02:24 +03:00
} else if ( access = = READCHECK ) {
2020-09-28 22:09:48 +03:00
( ( long long int * ) buffer ) [ 0 ] = ~ ( ( long long int * ) buffer ) [ 0 ] ; // changes the buffer, no memset to reduce the memory pressure
amtXferred = backend - > xfer ( access , fd , buffer , transfer , offset , test - > backend_options ) ;
2017-10-20 19:02:24 +03:00
if ( amtXferred ! = transfer ) {
ERR ( " cannot read from file " ) ;
}
2020-09-28 22:09:48 +03:00
* errors + = CompareData ( buffer , transfer , * transferCount , test , offset , pretendRank , READCHECK ) ;
2017-10-20 19:02:24 +03:00
}
return amtXferred ;
}
2020-12-01 17:45:07 +03:00
static void prefillSegment ( IOR_param_t * test , void * randomPrefillBuffer , int pretendRank , aiori_fd_t * fd , IOR_io_buffers * ioBuffers , int startSegment , int endSegment ) {
// prefill the whole file already with an invalid pattern
int offsets = test - > blockSize / test - > randomPrefillBlocksize ;
void * oldBuffer = ioBuffers - > buffer ;
IOR_offset_t transferCount ;
int errors ;
ioBuffers - > buffer = randomPrefillBuffer ;
for ( int i = startSegment ; i < endSegment ; i + + ) {
for ( int j = 0 ; j < offsets ; j + + ) {
IOR_offset_t offset = j * test - > randomPrefillBlocksize ;
if ( test - > filePerProc ) {
offset + = i * test - > blockSize ;
} else {
offset + = ( i * test - > numTasks * test - > blockSize ) + ( pretendRank * test - > blockSize ) ;
}
WriteOrReadSingle ( offset , pretendRank , test - > randomPrefillBlocksize , & transferCount , & errors , test , fd , ioBuffers , WRITE ) ;
}
}
ioBuffers - > buffer = oldBuffer ;
}
2011-06-17 23:20:43 +04:00
/*
* Write or Read data to file ( s ) . This loops through the strides , writing
* out the data to each block in transfer sizes , until the remainder left is 0.
*/
2018-09-19 20:06:05 +03:00
static IOR_offset_t WriteOrRead ( IOR_param_t * test , IOR_results_t * results ,
2020-05-31 14:11:00 +03:00
aiori_fd_t * fd , const int access , IOR_io_buffers * ioBuffers )
2011-06-17 23:20:43 +04:00
{
2011-11-12 02:22:17 +04:00
int errors = 0 ;
2012-01-13 08:34:40 +04:00
IOR_offset_t transferCount = 0 ;
2018-07-07 13:42:21 +03:00
uint64_t pairCnt = 0 ;
2011-11-12 02:22:17 +04:00
int pretendRank ;
IOR_offset_t dataMoved = 0 ; /* for data rate calculation */
double startForStonewall ;
int hitStonewall ;
2020-11-05 17:00:09 +03:00
int i , j ;
2018-09-20 00:39:25 +03:00
IOR_point_t * point = ( ( access = = WRITE ) | | ( access = = WRITECHECK ) ) ?
& results - > write : & results - > read ;
2011-11-12 02:22:17 +04:00
/* initialize values */
pretendRank = ( rank + rankOffset ) % test - > numTasks ;
2020-11-05 17:00:09 +03:00
// offsetArray = GetOffsetArraySequential(test, pretendRank);
2011-06-17 23:20:43 +04:00
2020-11-06 13:27:11 +03:00
IOR_offset_t offsets ;
IOR_offset_t * offsets_rnd ;
2011-11-12 02:22:17 +04:00
if ( test - > randomOffset ) {
2020-11-06 13:27:11 +03:00
offsets_rnd = GetOffsetArrayRandom ( test , pretendRank , & offsets ) ;
} else {
offsets = ( test - > blockSize / test - > transferSize ) ;
}
2020-12-01 17:45:07 +03:00
void * randomPrefillBuffer = NULL ;
if ( test - > randomPrefillBlocksize & & ( access = = WRITE | | access = = WRITECHECK ) ) {
randomPrefillBuffer = aligned_buffer_alloc ( test - > randomPrefillBlocksize ) ;
// store invalid data into the buffer
memset ( randomPrefillBuffer , - 1 , test - > randomPrefillBlocksize ) ;
2011-06-17 23:20:43 +04:00
}
2020-12-01 16:52:29 +03:00
// start timer after random offset was generated
2011-11-12 02:22:17 +04:00
startForStonewall = GetTimeStamp ( ) ;
2018-07-15 21:38:17 +03:00
hitStonewall = 0 ;
2011-06-17 23:20:43 +04:00
2020-12-01 17:45:07 +03:00
if ( randomPrefillBuffer & & test - > deadlineForStonewalling = = 0 ) {
double t_start = GetTimeStamp ( ) ;
prefillSegment ( test , randomPrefillBuffer , pretendRank , fd , ioBuffers , 0 , test - > segmentCount ) ;
if ( rank = = 0 & & verbose > VERBOSE_1 ) {
fprintf ( out_logfile , " Random prefill took: %fs \n " , GetTimeStamp ( ) - t_start ) ;
2020-12-01 16:52:29 +03:00
}
2020-12-01 17:47:57 +03:00
// must synchronize processes to ensure they are not running ahead
MPI_Barrier ( test - > testComm ) ;
2020-12-01 16:52:29 +03:00
}
2011-11-12 02:22:17 +04:00
2020-11-05 17:00:09 +03:00
for ( i = 0 ; i < test - > segmentCount & & ! hitStonewall ; i + + ) {
2020-12-01 17:45:07 +03:00
if ( randomPrefillBuffer & & test - > deadlineForStonewalling ! = 0 ) {
// prefill the whole segment with data, this needs to be done collectively
double t_start = GetTimeStamp ( ) ;
prefillSegment ( test , randomPrefillBuffer , pretendRank , fd , ioBuffers , i , i + 1 ) ;
2020-12-01 17:47:57 +03:00
MPI_Barrier ( test - > testComm ) ;
2020-12-01 17:45:07 +03:00
if ( rank = = 0 & & verbose > VERBOSE_1 ) {
fprintf ( out_logfile , " Random: synchronizing segment count with barrier and prefill took: %fs \n " , GetTimeStamp ( ) - t_start ) ;
}
2020-12-01 16:52:29 +03:00
}
2020-11-06 13:27:11 +03:00
for ( j = 0 ; j < offsets & & ! hitStonewall ; j + + ) {
2020-11-05 17:00:09 +03:00
IOR_offset_t offset ;
if ( test - > randomOffset ) {
2020-11-06 13:27:11 +03:00
if ( test - > filePerProc ) {
offset = offsets_rnd [ j ] + ( i * test - > blockSize ) ;
} else {
offset = offsets_rnd [ j ] + ( i * test - > numTasks * test - > blockSize ) ;
}
2020-11-05 17:00:09 +03:00
} else {
offset = j * test - > transferSize ;
if ( test - > filePerProc ) {
offset + = i * test - > blockSize ;
} else {
offset + = ( i * test - > numTasks * test - > blockSize ) + ( pretendRank * test - > blockSize ) ;
}
}
2020-12-01 16:52:29 +03:00
dataMoved + = WriteOrReadSingle ( offset , pretendRank , test - > transferSize , & transferCount , & errors , test , fd , ioBuffers , access ) ;
2020-11-05 17:00:09 +03:00
pairCnt + + ;
2019-12-19 16:42:34 +03:00
2020-11-05 17:00:09 +03:00
hitStonewall = ( ( test - > deadlineForStonewalling ! = 0
& & ( GetTimeStamp ( ) - startForStonewall ) > test - > deadlineForStonewalling ) )
| | ( test - > stoneWallingWearOutIterations ! = 0 & & pairCnt = = test - > stoneWallingWearOutIterations ) ;
2019-12-19 16:42:34 +03:00
2020-11-05 17:00:09 +03:00
if ( test - > collective & & test - > deadlineForStonewalling ) {
// if collective-mode, you'll get a HANG, if some rank 'accidentally' leave this loop
// it absolutely must be an 'all or none':
2021-01-20 17:35:06 +03:00
MPI_CHECK ( MPI_Bcast ( & hitStonewall , 1 , MPI_INT , 0 , testComm ) , " hitStonewall broadcast failed " ) ;
2020-11-05 17:00:09 +03:00
}
}
2011-11-12 02:22:17 +04:00
}
2017-10-20 19:02:24 +03:00
if ( test - > stoneWallingWearOut ) {
2018-07-07 13:42:21 +03:00
if ( verbose > = VERBOSE_1 ) {
fprintf ( out_logfile , " %d: stonewalling pairs accessed: %lld \n " , rank , ( long long ) pairCnt ) ;
}
long long data_moved_ll = ( long long ) dataMoved ;
long long pairs_accessed_min = 0 ;
2018-09-20 00:39:25 +03:00
MPI_CHECK ( MPI_Allreduce ( & pairCnt , & point - > pairs_accessed ,
2017-10-20 19:02:24 +03:00
1 , MPI_LONG_LONG_INT , MPI_MAX , testComm ) , " cannot reduce pairs moved " ) ;
2018-07-07 13:42:21 +03:00
double stonewall_runtime = GetTimeStamp ( ) - startForStonewall ;
2018-09-20 00:39:25 +03:00
point - > stonewall_time = stonewall_runtime ;
2018-07-07 13:42:21 +03:00
MPI_CHECK ( MPI_Reduce ( & pairCnt , & pairs_accessed_min ,
1 , MPI_LONG_LONG_INT , MPI_MIN , 0 , testComm ) , " cannot reduce pairs moved " ) ;
2018-09-20 00:39:25 +03:00
MPI_CHECK ( MPI_Reduce ( & data_moved_ll , & point - > stonewall_min_data_accessed ,
2018-07-07 13:42:21 +03:00
1 , MPI_LONG_LONG_INT , MPI_MIN , 0 , testComm ) , " cannot reduce pairs moved " ) ;
2020-12-04 00:07:45 +03:00
MPI_CHECK ( MPI_Reduce ( & data_moved_ll , & point - > stonewall_total_data_accessed ,
2018-07-07 13:42:21 +03:00
1 , MPI_LONG_LONG_INT , MPI_SUM , 0 , testComm ) , " cannot reduce pairs moved " ) ;
if ( rank = = 0 ) {
2020-12-04 00:07:45 +03:00
point - > stonewall_avg_data_accessed = point - > stonewall_total_data_accessed / test - > numTasks ;
2018-07-07 13:42:21 +03:00
fprintf ( out_logfile , " stonewalling pairs accessed min: %lld max: %zu -- min data: %.1f GiB mean data: %.1f GiB time: %.1fs \n " ,
2018-09-20 00:39:25 +03:00
pairs_accessed_min , point - > pairs_accessed ,
2020-12-04 00:07:45 +03:00
point - > stonewall_min_data_accessed / 1024.0 / 1024 / 1024 , point - > stonewall_avg_data_accessed / 1024.0 / 1024 / 1024 , point - > stonewall_time ) ;
2018-07-07 13:42:21 +03:00
}
2018-09-20 00:39:25 +03:00
if ( pairCnt ! = point - > pairs_accessed ) {
2021-01-20 01:28:34 +03:00
// some work needs still to be done, complete the current block !
i - - ;
if ( j = = offsets ) {
j = 0 ; // current block is completed
2021-01-20 02:19:01 +03:00
i + + ;
2021-01-20 01:28:34 +03:00
}
2020-11-05 17:00:09 +03:00
for ( ; pairCnt < point - > pairs_accessed ; i + + ) {
2021-01-20 01:28:34 +03:00
for ( ; j < offsets & & pairCnt < point - > pairs_accessed ; j + + ) {
2020-11-05 17:00:09 +03:00
IOR_offset_t offset ;
if ( test - > randomOffset ) {
2020-11-06 13:27:11 +03:00
if ( test - > filePerProc ) {
offset = offsets_rnd [ j ] + ( i * test - > blockSize ) ;
} else {
offset = offsets_rnd [ j ] + ( i * test - > numTasks * test - > blockSize ) ;
}
2020-11-05 17:00:09 +03:00
} else {
offset = j * test - > transferSize ;
if ( test - > filePerProc ) {
offset + = i * test - > blockSize ;
} else {
offset + = ( i * test - > numTasks * test - > blockSize ) + ( pretendRank * test - > blockSize ) ;
}
}
2020-12-01 16:52:29 +03:00
dataMoved + = WriteOrReadSingle ( offset , pretendRank , test - > transferSize , & transferCount , & errors , test , fd , ioBuffers , access ) ;
2020-11-05 17:00:09 +03:00
pairCnt + + ;
}
2021-01-20 01:28:34 +03:00
j = 0 ;
2017-10-20 19:02:24 +03:00
}
}
} else {
2018-09-20 00:39:25 +03:00
point - > pairs_accessed = pairCnt ;
2011-11-12 02:22:17 +04:00
}
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
totalErrorCount + = CountErrors ( test , access , errors ) ;
2011-06-17 23:20:43 +04:00
2011-11-12 02:22:17 +04:00
if ( access = = WRITE & & test - > fsync = = TRUE ) {
2020-05-31 13:50:15 +03:00
backend - > fsync ( fd , test - > backend_options ) ; /*fsync after all accesses */
2011-11-12 02:22:17 +04:00
}
2020-12-01 17:45:07 +03:00
if ( randomPrefillBuffer ) {
aligned_buffer_free ( randomPrefillBuffer ) ;
}
2011-11-12 02:22:17 +04:00
return ( dataMoved ) ;
2011-11-12 03:11:28 +04:00
}