2017-06-28 19:42:20 +03:00
/*
* Copyright ( C ) 2003 , The Regents of the University of California .
* Produced at the Lawrence Livermore National Laboratory .
* Written by Christopher J . Morrone < morrone @ llnl . gov > ,
* Bill Loewe < loewe @ loewe . net > , Tyce McLarty < mclarty @ llnl . gov > ,
* and Ryan Kroiss < rrkroiss @ lanl . gov > .
* All rights reserved .
* UCRL - CODE - 155800
*
* Please read the COPYRIGHT file .
*
* This program is free software ; you can redistribute it and / or modify
* it under the terms of the GNU General Public License ( as published by
* the Free Software Foundation ) version 2 , dated June 1991.
*
* This program is distributed in the hope that it will be useful ,
* but WITHOUT ANY WARRANTY ; without even the IMPLIED WARRANTY OF
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
* terms and conditions of the GNU General Public License for more details .
*
* You should have received a copy of the GNU General Public License
* along with this program ; if not , write to the Free Software
* Foundation , Inc . , 59 Temple Place , Suite 330 , Boston , MA 02111 - 1307 USA
*
* CVS info :
* $ RCSfile : mdtest . c , v $
* $ Revision : 1.4 $
* $ Date : 2013 / 11 / 27 17 : 05 : 31 $
* $ Author : brettkettering $
*/
# include <limits.h>
# include <math.h>
# include <stdio.h>
# include <stdlib.h>
2017-10-20 00:26:52 +03:00
# include <stdbool.h>
# include <inttypes.h>
2017-06-28 19:42:20 +03:00
# include <sys/types.h>
# include <sys/stat.h>
2019-07-27 22:22:15 +03:00
# include <stdarg.h>
2017-06-28 19:42:20 +03:00
2018-07-12 20:09:13 +03:00
# include "option.h"
2018-07-07 12:29:27 +03:00
# include "utilities.h"
2018-07-07 10:41:33 +03:00
2017-10-20 00:26:52 +03:00
# if HAVE_SYS_PARAM_H
2017-06-28 19:42:20 +03:00
# include <sys/param.h>
2017-10-20 00:26:52 +03:00
# endif
# if HAVE_SYS_MOUNT_H
2017-06-28 19:42:20 +03:00
# include <sys/mount.h>
2017-10-20 00:26:52 +03:00
# endif
# if HAVE_SYS_STATFS_H
2017-06-28 19:42:20 +03:00
# include <sys/statfs.h>
# endif
2017-10-20 00:26:52 +03:00
# if HAVE_SYS_STATVFS_H
2017-06-28 19:42:20 +03:00
# include <sys/statvfs.h>
# endif
# include <fcntl.h>
# include <string.h>
2018-10-08 23:47:28 +03:00
# if HAVE_STRINGS_H
# include <strings.h>
# endif
2017-06-28 19:42:20 +03:00
# include <unistd.h>
# include <dirent.h>
# include <errno.h>
# include <time.h>
# include <sys/time.h>
2017-10-20 00:26:52 +03:00
# include "aiori.h"
# include "ior.h"
2018-07-07 12:29:27 +03:00
# include "mdtest.h"
2017-10-20 00:26:52 +03:00
# include <mpi.h>
2017-06-28 19:42:20 +03:00
# define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH
# define DIRMODE S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH
2019-05-16 23:25:34 +03:00
# define RELEASE_VERS META_VERSION
2019-08-03 11:12:48 +03:00
# define TEST_DIR "test-dir"
2017-06-28 19:42:20 +03:00
# define ITEM_COUNT 25000
2018-07-07 12:29:27 +03:00
# define LLU "%lu"
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
static int size ;
static uint64_t * rand_array ;
2018-07-08 00:39:14 +03:00
static char testdir [ MAX_PATHLEN ] ;
static char testdirpath [ MAX_PATHLEN ] ;
static char base_tree_name [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
static char * * filenames ;
2018-07-08 00:39:14 +03:00
static char hostname [ MAX_PATHLEN ] ;
static char mk_name [ MAX_PATHLEN ] ;
static char stat_name [ MAX_PATHLEN ] ;
static char read_name [ MAX_PATHLEN ] ;
static char rm_name [ MAX_PATHLEN ] ;
static char unique_mk_dir [ MAX_PATHLEN ] ;
static char unique_chdir_dir [ MAX_PATHLEN ] ;
static char unique_stat_dir [ MAX_PATHLEN ] ;
static char unique_read_dir [ MAX_PATHLEN ] ;
static char unique_rm_dir [ MAX_PATHLEN ] ;
static char unique_rm_uni_dir [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
static char * write_buffer ;
static char * read_buffer ;
2019-08-01 20:29:32 +03:00
static char * verify_read_buffer ;
2018-07-08 01:56:39 +03:00
static char * stoneWallingStatusFile ;
2018-07-07 12:29:27 +03:00
static int barriers ;
2017-10-20 00:26:52 +03:00
static int create_only ;
static int stat_only ;
static int read_only ;
2019-08-01 20:29:32 +03:00
static int verify_read ;
static int verification_error ;
2017-10-20 00:26:52 +03:00
static int remove_only ;
static int leaf_only ;
2018-07-07 12:29:27 +03:00
static unsigned branch_factor ;
2017-10-20 00:26:52 +03:00
static int depth ;
2017-06-28 19:42:20 +03:00
/*
* This is likely a small value , but it ' s sometimes computed by
* branch_factor ^ ( depth + 1 ) , so we ' ll make it a larger variable ,
* just in case .
*/
2017-10-20 00:26:52 +03:00
static uint64_t num_dirs_in_tree ;
2017-06-28 19:42:20 +03:00
/*
* As we start moving towards Exascale , we could have billions
* of files in a directory . Make room for that possibility with
* a larger variable .
*/
2018-07-14 18:46:24 +03:00
static uint64_t items ;
2017-10-20 00:26:52 +03:00
static uint64_t items_per_dir ;
2018-10-17 18:04:49 +03:00
static uint64_t num_dirs_in_tree_calc ; /* this is a workaround until the overal code is refactored */
2018-07-26 12:37:01 +03:00
static int directory_loops ;
2018-07-07 12:29:27 +03:00
static int print_time ;
2019-08-01 19:54:11 +03:00
static int print_rate_and_time ;
2017-10-20 00:26:52 +03:00
static int random_seed ;
static int shared_file ;
static int files_only ;
static int dirs_only ;
static int pre_delay ;
static int unique_dir_per_task ;
static int time_unique_dir_overhead ;
2018-07-07 12:29:27 +03:00
static int throttle ;
2017-10-20 00:26:52 +03:00
static int collective_creates ;
static size_t write_bytes ;
2018-07-07 12:29:27 +03:00
static int stone_wall_timer_seconds ;
2017-10-20 00:26:52 +03:00
static size_t read_bytes ;
static int sync_file ;
2019-08-26 20:57:14 +03:00
static int call_sync ;
2017-10-20 00:26:52 +03:00
static int path_count ;
static int nstride ; /* neighbor stride */
2018-10-08 15:30:47 +03:00
static int make_node = 0 ;
2018-07-07 12:29:27 +03:00
static mdtest_results_t * summary_table ;
2017-10-20 00:26:52 +03:00
static pid_t pid ;
static uid_t uid ;
2019-02-11 16:49:14 +03:00
/* Use the POSIX backend by default */
2017-10-20 00:26:52 +03:00
static const ior_aiori_t * backend ;
static IOR_param_t param ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
/* This structure describes the processing status for stonewalling */
typedef struct {
double start_time ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
int stone_wall_timer_seconds ;
2017-06-28 19:42:20 +03:00
2018-07-14 18:46:24 +03:00
uint64_t items_start ;
uint64_t items_done ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
uint64_t items_per_dir ;
} rank_progress_t ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
# define CHECK_STONE_WALL(p) (((p)->stone_wall_timer_seconds != 0) && ((GetTimeStamp() - (p)->start_time) > (p)->stone_wall_timer_seconds))
2017-06-28 19:42:20 +03:00
/* for making/removing unique directory && stating/deleting subdirectory */
enum { MK_UNI_DIR , STAT_SUB_DIR , READ_SUB_DIR , RM_SUB_DIR , RM_UNI_DIR } ;
2019-07-27 22:22:15 +03:00
/* a helper function for passing debug and verbose messages.
use the MACRO as it will insert __LINE__ for you .
Pass the verbose level for root to print , then the verbose level for anyone to print .
Pass - 1 to suppress the print for anyone .
Then do the standard printf stuff . This function adds the newline for you .
*/
# define VERBOSE(root,any,...) VerboseMessage(root,any,__LINE__,__VA_ARGS__)
void VerboseMessage ( int root_level , int any_level , int line , char * format , . . . ) {
if ( ( rank = = 0 & & verbose > = root_level ) | | ( any_level > 0 & & verbose > = any_level ) ) {
char buffer [ 1024 ] ;
va_list args ;
va_start ( args , format ) ;
vsnprintf ( buffer , 1024 , format , args ) ;
va_end ( args ) ;
2019-07-27 23:31:49 +03:00
if ( root_level = = 0 & & any_level = = - 1 ) {
/* No header when it is just the standard output */
fprintf ( out_logfile , " %s \n " , buffer ) ;
} else {
/* add a header when the verbose is greater than 0 */
fprintf ( out_logfile , " V-%d: Rank %3d Line %5d %s \n " , root_level , rank , line , buffer ) ;
}
fflush ( out_logfile ) ;
2019-07-27 22:22:15 +03:00
}
}
2017-06-28 19:42:20 +03:00
2019-08-01 20:29:32 +03:00
void generate_memory_pattern ( char * buffer , size_t bytes ) {
for ( int i = 0 ; i < bytes ; i + + ) {
buffer [ i ] = i + 1 ;
}
}
2017-06-28 19:42:20 +03:00
void offset_timers ( double * t , int tcount ) {
2017-10-20 00:26:52 +03:00
double toffset ;
int i ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " V-1: Entering offset_timers... " ) ;
2017-06-28 19:42:20 +03:00
2018-10-17 17:38:35 +03:00
toffset = GetTimeStamp ( ) - t [ tcount ] ;
2017-10-20 00:26:52 +03:00
for ( i = 0 ; i < tcount + 1 ; i + + ) {
t [ i ] + = toffset ;
}
2017-06-28 19:42:20 +03:00
}
void parse_dirpath ( char * dirpath_arg ) {
2017-10-20 00:26:52 +03:00
char * tmp , * token ;
char delimiter_string [ 3 ] = { ' @ ' , ' \n ' , ' \0 ' } ;
int i = 0 ;
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering parse_dirpath on %s... " , dirpath_arg ) ;
2017-10-20 00:26:52 +03:00
tmp = dirpath_arg ;
if ( * tmp ! = ' \0 ' ) path_count + + ;
while ( * tmp ! = ' \0 ' ) {
if ( * tmp = = ' @ ' ) {
path_count + + ;
}
tmp + + ;
}
2019-01-24 03:02:58 +03:00
// prevent changes to the original dirpath_arg
dirpath_arg = strdup ( dirpath_arg ) ;
2017-10-20 00:26:52 +03:00
filenames = ( char * * ) malloc ( path_count * sizeof ( char * * ) ) ;
2019-01-24 03:02:58 +03:00
if ( filenames = = NULL | | dirpath_arg = = NULL ) {
2017-10-20 00:26:52 +03:00
FAIL ( " out of memory " ) ;
}
token = strtok ( dirpath_arg , delimiter_string ) ;
while ( token ! = NULL ) {
filenames [ i ] = token ;
token = strtok ( NULL , delimiter_string ) ;
i + + ;
}
2017-06-28 19:42:20 +03:00
}
2018-07-26 12:37:01 +03:00
static void prep_testdir ( int j , int dir_iter ) {
int pos = sprintf ( testdir , " %s " , testdirpath ) ;
if ( testdir [ strlen ( testdir ) - 1 ] ! = ' / ' ) {
pos + = sprintf ( & testdir [ pos ] , " / " ) ;
}
pos + = sprintf ( & testdir [ pos ] , " %s " , TEST_DIR ) ;
pos + = sprintf ( & testdir [ pos ] , " .%d-%d " , j , dir_iter ) ;
}
2019-08-26 20:57:14 +03:00
static void phase_end ( ) {
if ( call_sync ) {
2019-09-01 17:47:42 +03:00
if ( ! backend - > sync ) {
FAIL ( " Error, backend does not provide the sync method, but your requested to use sync. " ) ;
}
backend - > sync ( & param ) ;
2019-08-26 20:57:14 +03:00
}
if ( barriers ) {
MPI_Barrier ( testComm ) ;
}
}
2017-06-28 19:42:20 +03:00
/*
* This function copies the unique directory name for a given option to
* the " to " parameter . Some memory must be allocated to the " to " parameter .
*/
void unique_dir_access ( int opt , char * to ) {
2017-10-20 00:26:52 +03:00
if ( opt = = MK_UNI_DIR ) {
MPI_Barrier ( testComm ) ;
2018-07-26 12:37:01 +03:00
sprintf ( to , " %s/%s " , testdir , unique_chdir_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = STAT_SUB_DIR ) {
2018-07-26 12:37:01 +03:00
sprintf ( to , " %s/%s " , testdir , unique_stat_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = READ_SUB_DIR ) {
2018-07-26 12:37:01 +03:00
sprintf ( to , " %s/%s " , testdir , unique_read_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = RM_SUB_DIR ) {
2018-07-26 12:37:01 +03:00
sprintf ( to , " %s/%s " , testdir , unique_rm_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = RM_UNI_DIR ) {
2018-07-26 12:37:01 +03:00
sprintf ( to , " %s/%s " , testdir , unique_rm_uni_dir ) ;
2017-10-20 00:26:52 +03:00
}
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering unique_dir_access, set it to %s " , to ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
static void create_remove_dirs ( const char * path , bool create , uint64_t itemNum ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
const char * operation = create ? " create " : " remove " ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
if ( ( itemNum % ITEM_COUNT = = 0 & & ( itemNum ! = 0 ) ) ) {
VERBOSE ( 3 , 5 , " dir: " LLU " " , operation , itemNum ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//create dirs
sprintf ( curr_item , " %s/dir.%s% " PRIu64 , path , create ? mk_name : rm_name , itemNum ) ;
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper (dirs %s): curr_item is '%s' " , operation , curr_item ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( create ) {
if ( backend - > mkdir ( curr_item , DIRMODE , & param ) = = - 1 ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to create directory %s " , curr_item ) ;
2017-10-20 00:26:52 +03:00
}
} else {
if ( backend - > rmdir ( curr_item , & param ) = = - 1 ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to remove directory %s " , curr_item ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
static void remove_file ( const char * path , uint64_t itemNum ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
if ( ( itemNum % ITEM_COUNT = = 0 & & ( itemNum ! = 0 ) ) ) {
VERBOSE ( 3 , 5 , " remove file: " LLU " \n " , itemNum ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//remove files
2018-07-07 12:29:27 +03:00
sprintf ( curr_item , " %s/file.%s " LLU " " , path , rm_name , itemNum ) ;
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper (non-dirs remove): curr_item is '%s' " , curr_item ) ;
2017-10-20 00:26:52 +03:00
if ( ! ( shared_file & & rank ! = 0 ) ) {
backend - > delete ( curr_item , & param ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
static void create_file ( const char * path , uint64_t itemNum ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
void * aiori_fh ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
if ( ( itemNum % ITEM_COUNT = = 0 & & ( itemNum ! = 0 ) ) ) {
VERBOSE ( 3 , 5 , " create file: " LLU " " , itemNum ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//create files
2018-07-07 12:29:27 +03:00
sprintf ( curr_item , " %s/file.%s " LLU " " , path , mk_name , itemNum ) ;
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper (non-dirs create): curr_item is '%s' " , curr_item ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( collective_creates ) {
param . openFlags = IOR_WRONLY ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper (collective): open... " ) ;
2017-06-28 19:42:20 +03:00
2018-10-08 15:30:47 +03:00
if ( make_node )
aiori_fh = backend - > mknod ( curr_item ) ;
else
aiori_fh = backend - > open ( curr_item , & param ) ;
2017-10-20 00:26:52 +03:00
if ( NULL = = aiori_fh ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to open file %s " , curr_item ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/*
* ! collective_creates
*/
} else {
param . openFlags = IOR_CREAT | IOR_WRONLY ;
param . filePerProc = ! shared_file ;
2019-01-24 03:31:12 +03:00
param . mode = FILEMODE ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper (non-collective, shared): open... " ) ;
2017-06-28 19:42:20 +03:00
2018-10-08 15:30:47 +03:00
if ( make_node )
aiori_fh = backend - > mknod ( curr_item ) ;
else
aiori_fh = backend - > create ( curr_item , & param ) ;
2017-10-20 00:26:52 +03:00
if ( NULL = = aiori_fh ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to create file %s " , curr_item ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( write_bytes > 0 ) {
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper: write... " ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/*
* According to Bill Loewe , writes are only done one time , so they are always at
* offset 0 ( zero ) .
*/
param . offset = 0 ;
param . fsyncPerWrite = sync_file ;
2018-07-07 12:29:27 +03:00
if ( write_bytes ! = ( size_t ) backend - > xfer ( WRITE , aiori_fh , ( IOR_size_t * ) write_buffer , write_bytes , & param ) ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to write file %s " , curr_item ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper: close... " ) ;
2017-06-28 19:42:20 +03:00
2018-10-08 15:30:47 +03:00
if ( ! make_node )
backend - > close ( aiori_fh , & param ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* helper for creating/removing items */
void create_remove_items_helper ( const int dirs , const int create , const char * path ,
2018-07-07 12:29:27 +03:00
uint64_t itemNum , rank_progress_t * progress ) {
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering create_remove_items_helper on %s " , path ) ;
2017-06-28 19:42:20 +03:00
2018-07-08 01:56:39 +03:00
for ( uint64_t i = progress - > items_start ; i < progress - > items_per_dir ; + + i ) {
2017-10-20 00:26:52 +03:00
if ( ! dirs ) {
if ( create ) {
create_file ( path , itemNum + i ) ;
} else {
remove_file ( path , itemNum + i ) ;
}
2017-06-28 19:42:20 +03:00
} else {
2017-10-20 00:26:52 +03:00
create_remove_dirs ( path , create , itemNum + i ) ;
2017-06-28 19:42:20 +03:00
}
2018-07-07 12:29:27 +03:00
if ( CHECK_STONE_WALL ( progress ) ) {
2018-12-20 00:37:37 +03:00
if ( progress - > items_done = = 0 ) {
progress - > items_done = i + 1 ;
2018-12-19 23:29:47 +03:00
}
2018-07-07 12:29:27 +03:00
return ;
}
2017-10-20 00:26:52 +03:00
}
2018-07-08 01:56:39 +03:00
progress - > items_done = progress - > items_per_dir ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* helper function to do collective operations */
2018-07-07 12:29:27 +03:00
void collective_helper ( const int dirs , const int create , const char * path , uint64_t itemNum , rank_progress_t * progress ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering collective_helper on %s " , path ) ;
2018-07-08 01:56:39 +03:00
for ( uint64_t i = progress - > items_start ; i < progress - > items_per_dir ; + + i ) {
2017-10-20 00:26:52 +03:00
if ( dirs ) {
create_remove_dirs ( path , create , itemNum + i ) ;
continue ;
2017-06-28 19:42:20 +03:00
}
2018-07-07 12:29:27 +03:00
sprintf ( curr_item , " %s/file.%s " LLU " " , path , create ? mk_name : rm_name , itemNum + i ) ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " create file: %s " , curr_item ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( create ) {
void * aiori_fh ;
//create files
param . openFlags = IOR_WRONLY | IOR_CREAT ;
2019-01-24 03:31:12 +03:00
param . mode = FILEMODE ;
2017-10-20 00:26:52 +03:00
aiori_fh = backend - > create ( curr_item , & param ) ;
if ( NULL = = aiori_fh ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to create file %s " , curr_item ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
backend - > close ( aiori_fh , & param ) ;
} else if ( ! ( shared_file & & rank ! = 0 ) ) {
//remove files
backend - > delete ( curr_item , & param ) ;
2017-06-28 19:42:20 +03:00
}
2018-07-07 12:29:27 +03:00
if ( CHECK_STONE_WALL ( progress ) ) {
progress - > items_done = i + 1 ;
return ;
}
2017-06-28 19:42:20 +03:00
}
2018-07-08 01:56:39 +03:00
progress - > items_done = progress - > items_per_dir ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* recusive function to create and remove files/directories from the
directory tree */
2018-07-07 12:29:27 +03:00
void create_remove_items ( int currDepth , const int dirs , const int create , const int collective , const char * path , uint64_t dirNum , rank_progress_t * progress ) {
unsigned i ;
2018-07-08 00:39:14 +03:00
char dir [ MAX_PATHLEN ] ;
char temp_path [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
unsigned long long currDir = dirNum ;
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering create_remove_items on %s, currDepth = %d... " , path , currDepth ) ;
2017-06-28 19:42:20 +03:00
2018-07-08 00:39:14 +03:00
memset ( dir , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
strcpy ( temp_path , path ) ;
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " create_remove_items (start): temp_path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( currDepth = = 0 ) {
/* create items at this depth */
if ( ! leaf_only | | ( depth = = 0 & & leaf_only ) ) {
if ( collective ) {
2018-07-07 12:29:27 +03:00
collective_helper ( dirs , create , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-07 12:29:27 +03:00
create_remove_items_helper ( dirs , create , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( depth > 0 ) {
create_remove_items ( + + currDepth , dirs , create ,
2018-07-07 12:29:27 +03:00
collective , temp_path , + + dirNum , progress ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
} else if ( currDepth < = depth ) {
/* iterate through the branches */
for ( i = 0 ; i < branch_factor ; i + + ) {
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* determine the current branch and append it to the path */
sprintf ( dir , " %s.%llu/ " , base_tree_name , currDir ) ;
strcat ( temp_path , " / " ) ;
strcat ( temp_path , dir ) ;
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " create_remove_items (for loop): temp_path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create the items in this branch */
if ( ! leaf_only | | ( leaf_only & & currDepth = = depth ) ) {
if ( collective ) {
2018-07-07 12:29:27 +03:00
collective_helper ( dirs , create , temp_path , currDir * items_per_dir , progress ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-07 12:29:27 +03:00
create_remove_items_helper ( dirs , create , temp_path , currDir * items_per_dir , progress ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* make the recursive call for the next level below this branch */
create_remove_items (
+ + currDepth ,
dirs ,
create ,
collective ,
temp_path ,
2018-07-07 12:29:27 +03:00
( currDir * ( unsigned long long ) branch_factor ) + 1 ,
progress
) ;
2017-10-20 00:26:52 +03:00
currDepth - - ;
/* reset the path */
strcpy ( temp_path , path ) ;
currDir + + ;
}
}
2017-06-28 19:42:20 +03:00
}
/* stats all of the items created as specified by the input parameters */
2018-07-26 12:37:01 +03:00
void mdtest_stat ( const int random , const int dirs , const long dir_iter , const char * path , rank_progress_t * progress ) {
2017-10-20 00:26:52 +03:00
struct stat buf ;
uint64_t parent_dir , item_num = 0 ;
2018-07-08 00:39:14 +03:00
char item [ MAX_PATHLEN ] , temp [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering mdtest_stat on %s " , path ) ;
2017-06-28 19:42:20 +03:00
2018-07-26 12:37:01 +03:00
uint64_t stop_items = items ;
2019-05-14 16:55:11 +03:00
if ( directory_loops ! = 1 ) {
2018-07-26 12:37:01 +03:00
stop_items = items_per_dir ;
2017-10-20 00:26:52 +03:00
}
/* iterate over all of the item IDs */
2018-07-26 12:37:01 +03:00
for ( uint64_t i = 0 ; i < stop_items ; + + i ) {
2017-10-20 00:26:52 +03:00
/*
* It doesn ' t make sense to pass the address of the array because that would
* be like passing char * * . Tested it on a Cray and it seems to work either
* way , but it seems that it is correct without the " & " .
*
2018-07-08 00:39:14 +03:00
memset ( & item , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
*/
2018-07-08 00:39:14 +03:00
memset ( item , 0 , MAX_PATHLEN ) ;
memset ( temp , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
/* determine the item number to stat */
if ( random ) {
item_num = rand_array [ i ] ;
} else {
item_num = i ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* make adjustments if in leaf only mode*/
if ( leaf_only ) {
item_num + = items_per_dir *
2018-07-14 18:46:24 +03:00
( num_dirs_in_tree - ( uint64_t ) pow ( branch_factor , depth ) ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create name of file/dir to stat */
if ( dirs ) {
2019-07-28 18:55:00 +03:00
if ( ( i % ITEM_COUNT = = 0 ) & & ( i ! = 0 ) ) {
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " stat dir: " LLU " " , i ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-07 12:29:27 +03:00
sprintf ( item , " dir.%s " LLU " " , stat_name , item_num ) ;
2017-10-20 00:26:52 +03:00
} else {
2019-07-28 18:55:00 +03:00
if ( ( i % ITEM_COUNT = = 0 ) & & ( i ! = 0 ) ) {
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " stat file: " LLU " " , i ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-07 12:29:27 +03:00
sprintf ( item , " file.%s " LLU " " , stat_name , item_num ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* determine the path to the file/dir to be stat'ed */
parent_dir = item_num / items_per_dir ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( parent_dir > 0 ) { //item is not in tree's root directory
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* prepend parent directory to item's path */
2018-07-07 12:29:27 +03:00
sprintf ( temp , " %s. " LLU " /%s " , base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//still not at the tree's root dir
while ( parent_dir > branch_factor ) {
parent_dir = ( uint64_t ) ( ( parent_dir - 1 ) / branch_factor ) ;
2018-07-07 12:29:27 +03:00
sprintf ( temp , " %s. " LLU " /%s " , base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* Now get item to have the full path */
sprintf ( temp , " %s/%s " , path , item ) ;
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* below temp used to be hiername */
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " mdtest_stat %4s: %s " , ( dirs ? " dir " : " file " ) , item ) ;
2017-10-20 00:26:52 +03:00
if ( - 1 = = backend - > stat ( item , & buf , & param ) ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to stat %s %s " , dirs ? " directory " : " file " , item ) ;
2017-06-28 19:42:20 +03:00
}
}
}
/* reads all of the items created as specified by the input parameters */
2018-07-26 12:37:01 +03:00
void mdtest_read ( int random , int dirs , const long dir_iter , char * path ) {
2018-07-14 18:46:24 +03:00
uint64_t parent_dir , item_num = 0 ;
2018-07-08 00:39:14 +03:00
char item [ MAX_PATHLEN ] , temp [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
void * aiori_fh ;
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering mdtest_read on %s " , path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* allocate read buffer */
if ( read_bytes > 0 ) {
read_buffer = ( char * ) malloc ( read_bytes ) ;
if ( read_buffer = = NULL ) {
FAIL ( " out of memory " ) ;
}
2019-08-01 20:29:32 +03:00
if ( verify_read > 0 ) {
verify_read_buffer = ( char * ) malloc ( read_bytes ) ;
if ( verify_read_buffer = = NULL ) {
FAIL ( " out of memory " ) ;
}
generate_memory_pattern ( verify_read_buffer , read_bytes ) ;
}
2017-06-28 19:42:20 +03:00
}
2018-07-26 12:37:01 +03:00
uint64_t stop_items = items ;
if ( directory_loops ! = 1 ) {
stop_items = items_per_dir ;
2017-10-20 00:26:52 +03:00
}
/* iterate over all of the item IDs */
2018-07-26 12:37:01 +03:00
for ( uint64_t i = 0 ; i < stop_items ; + + i ) {
2017-10-20 00:26:52 +03:00
/*
* It doesn ' t make sense to pass the address of the array because that would
* be like passing char * * . Tested it on a Cray and it seems to work either
* way , but it seems that it is correct without the " & " .
*
* NTH : Both are technically correct in C .
*
2018-07-08 00:39:14 +03:00
* memset ( & item , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
*/
2018-07-08 00:39:14 +03:00
memset ( item , 0 , MAX_PATHLEN ) ;
memset ( temp , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
/* determine the item number to read */
if ( random ) {
item_num = rand_array [ i ] ;
} else {
item_num = i ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* make adjustments if in leaf only mode*/
if ( leaf_only ) {
item_num + = items_per_dir *
( num_dirs_in_tree - ( uint64_t ) pow ( branch_factor , depth ) ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create name of file to read */
if ( ! dirs ) {
2019-07-27 22:22:15 +03:00
if ( ( i % ITEM_COUNT = = 0 ) & & ( i ! = 0 ) ) {
VERBOSE ( 3 , 5 , " read file: " LLU " " , i ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-07 12:29:27 +03:00
sprintf ( item , " file.%s " LLU " " , read_name , item_num ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* determine the path to the file/dir to be read'ed */
parent_dir = item_num / items_per_dir ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( parent_dir > 0 ) { //item is not in tree's root directory
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* prepend parent directory to item's path */
2018-07-07 12:29:27 +03:00
sprintf ( temp , " %s. " LLU " /%s " , base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* still not at the tree's root dir */
while ( parent_dir > branch_factor ) {
parent_dir = ( unsigned long long ) ( ( parent_dir - 1 ) / branch_factor ) ;
2018-07-07 12:29:27 +03:00
sprintf ( temp , " %s. " LLU " /%s " , base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* Now get item to have the full path */
sprintf ( temp , " %s/%s " , path , item ) ;
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* below temp used to be hiername */
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " mdtest_read file: %s " , item ) ;
2017-10-20 00:26:52 +03:00
/* open file for reading */
param . openFlags = O_RDONLY ;
aiori_fh = backend - > open ( item , & param ) ;
if ( NULL = = aiori_fh ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to open file %s " , item ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read file */
if ( read_bytes > 0 ) {
2019-08-15 18:21:30 +03:00
read_buffer [ 0 ] = 42 ; /* use a random value to ensure that the read_buffer is now different from the expected buffer and read isn't sometimes NOOP */
2018-07-07 12:29:27 +03:00
if ( read_bytes ! = ( size_t ) backend - > xfer ( READ , aiori_fh , ( IOR_size_t * ) read_buffer , read_bytes , & param ) ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to read file %s " , item ) ;
2017-10-20 00:26:52 +03:00
}
2019-08-01 20:29:32 +03:00
if ( verify_read ) {
if ( memcmp ( read_buffer , verify_read_buffer , read_bytes ) ! = 0 ) {
VERBOSE ( 2 , - 1 , " Error verifying %s " , item ) ;
verification_error + + ;
}
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* close file */
backend - > close ( aiori_fh , & param ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* This method should be called by rank 0. It subsequently does all of
the creates and removes for the other ranks */
2018-07-07 12:29:27 +03:00
void collective_create_remove ( const int create , const int dirs , const int ntasks , const char * path , rank_progress_t * progress ) {
2018-07-08 00:39:14 +03:00
char temp [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " Entering collective_create_remove on %s " , path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* rank 0 does all of the creates and removes for all of the ranks */
for ( int i = 0 ; i < ntasks ; + + i ) {
2018-07-08 00:39:14 +03:00
memset ( temp , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
strcpy ( temp , testdir ) ;
strcat ( temp , " / " ) ;
/* set the base tree name appropriately */
if ( unique_dir_per_task ) {
sprintf ( base_tree_name , " mdtest_tree.%d " , i ) ;
} else {
sprintf ( base_tree_name , " mdtest_tree " ) ;
}
/* Setup to do I/O to the appropriate test dir */
strcat ( temp , base_tree_name ) ;
strcat ( temp , " .0 " ) ;
/* set all item names appropriately */
if ( ! shared_file ) {
sprintf ( mk_name , " mdtest.%d. " , ( i + ( 0 * nstride ) ) % ntasks ) ;
sprintf ( stat_name , " mdtest.%d. " , ( i + ( 1 * nstride ) ) % ntasks ) ;
sprintf ( read_name , " mdtest.%d. " , ( i + ( 2 * nstride ) ) % ntasks ) ;
sprintf ( rm_name , " mdtest.%d. " , ( i + ( 3 * nstride ) ) % ntasks ) ;
}
if ( unique_dir_per_task ) {
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " i %d nstride %d ntasks %d " , i , nstride , ntasks ) ;
2017-10-20 00:26:52 +03:00
sprintf ( unique_mk_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( i + ( 0 * nstride ) ) % ntasks ) ;
sprintf ( unique_chdir_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( i + ( 1 * nstride ) ) % ntasks ) ;
sprintf ( unique_stat_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( i + ( 2 * nstride ) ) % ntasks ) ;
sprintf ( unique_read_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( i + ( 3 * nstride ) ) % ntasks ) ;
sprintf ( unique_rm_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( i + ( 4 * nstride ) ) % ntasks ) ;
sprintf ( unique_rm_uni_dir , " %s " , testdir ) ;
}
/* Now that everything is set up as it should be, do the create or remove */
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " collective_create_remove (create_remove_items): temp is '%s' " , temp ) ;
2017-10-20 00:26:52 +03:00
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , dirs , create , 1 , temp , 0 , progress ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* reset all of the item names */
if ( unique_dir_per_task ) {
sprintf ( base_tree_name , " mdtest_tree.0 " ) ;
2017-06-28 19:42:20 +03:00
} else {
2017-10-20 00:26:52 +03:00
sprintf ( base_tree_name , " mdtest_tree " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( ! shared_file ) {
sprintf ( mk_name , " mdtest.%d. " , ( 0 + ( 0 * nstride ) ) % ntasks ) ;
sprintf ( stat_name , " mdtest.%d. " , ( 0 + ( 1 * nstride ) ) % ntasks ) ;
sprintf ( read_name , " mdtest.%d. " , ( 0 + ( 2 * nstride ) ) % ntasks ) ;
sprintf ( rm_name , " mdtest.%d. " , ( 0 + ( 3 * nstride ) ) % ntasks ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
sprintf ( unique_mk_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( 0 + ( 0 * nstride ) ) % ntasks ) ;
sprintf ( unique_chdir_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( 0 + ( 1 * nstride ) ) % ntasks ) ;
sprintf ( unique_stat_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( 0 + ( 2 * nstride ) ) % ntasks ) ;
sprintf ( unique_read_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( 0 + ( 3 * nstride ) ) % ntasks ) ;
sprintf ( unique_rm_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( 0 + ( 4 * nstride ) ) % ntasks ) ;
sprintf ( unique_rm_uni_dir , " %s " , testdir ) ;
2017-06-28 19:42:20 +03:00
}
}
2018-07-07 12:29:27 +03:00
void directory_test ( const int iteration , const int ntasks , const char * path , rank_progress_t * progress ) {
2017-10-20 00:26:52 +03:00
int size ;
double t [ 5 ] = { 0 } ;
2018-07-08 00:39:14 +03:00
char temp_path [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
MPI_Comm_size ( testComm , & size ) ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " Entering directory_test on %s " , path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
MPI_Barrier ( testComm ) ;
2018-10-17 17:38:35 +03:00
t [ 0 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create phase */
if ( create_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( MK_UNI_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 0 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , - 1 , " directory_test: create path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* "touch" the files */
if ( collective_creates ) {
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
collective_create_remove ( 1 , 1 , ntasks , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
} else {
/* create directories */
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , 1 , 1 , 0 , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 1 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat phase */
if ( stat_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( STAT_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 1 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " stat path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat directories */
if ( random_seed > 0 ) {
2018-07-26 12:37:01 +03:00
mdtest_stat ( 1 , 1 , dir_iter , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-26 12:37:01 +03:00
mdtest_stat ( 0 , 1 , dir_iter , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 2 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read phase */
if ( read_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( READ_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 2 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " directory_test: read path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read directories */
if ( random_seed > 0 ) {
; /* N/A */
} else {
; /* N/A */
}
2018-07-26 12:37:01 +03:00
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 3 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( remove_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( RM_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 3 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " directory_test: remove directories path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* remove directories */
if ( collective_creates ) {
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
collective_create_remove ( 0 , 1 , ntasks , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
} else {
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , 1 , 0 , 0 , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 4 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( remove_only ) {
if ( unique_dir_per_task ) {
unique_dir_access ( RM_UNI_DIR , temp_path ) ;
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " directory_test: remove unique directories path is '%s' \n " , temp_path ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task & & ! time_unique_dir_overhead ) {
offset_timers ( t , 4 ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* calculate times */
if ( create_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 0 ] = items * size / ( t [ 1 ] - t [ 0 ] ) ;
summary_table [ iteration ] . time [ 0 ] = t [ 1 ] - t [ 0 ] ;
summary_table [ iteration ] . items [ 0 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 0 ] = items ;
2017-10-20 00:26:52 +03:00
}
if ( stat_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 1 ] = items * size / ( t [ 2 ] - t [ 1 ] ) ;
summary_table [ iteration ] . time [ 1 ] = t [ 2 ] - t [ 1 ] ;
summary_table [ iteration ] . items [ 1 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 1 ] = items ;
2017-10-20 00:26:52 +03:00
}
if ( read_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 2 ] = items * size / ( t [ 3 ] - t [ 2 ] ) ;
summary_table [ iteration ] . time [ 2 ] = t [ 3 ] - t [ 2 ] ;
summary_table [ iteration ] . items [ 2 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 2 ] = items ;
2017-10-20 00:26:52 +03:00
}
if ( remove_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 3 ] = items * size / ( t [ 4 ] - t [ 3 ] ) ;
summary_table [ iteration ] . time [ 3 ] = t [ 4 ] - t [ 3 ] ;
summary_table [ iteration ] . items [ 3 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 3 ] = items ;
2017-06-28 19:42:20 +03:00
}
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Directory creation: %14.3f sec, %14.3f ops/sec " , t [ 1 ] - t [ 0 ] , summary_table [ iteration ] . rate [ 0 ] ) ;
VERBOSE ( 1 , - 1 , " Directory stat : %14.3f sec, %14.3f ops/sec " , t [ 2 ] - t [ 1 ] , summary_table [ iteration ] . rate [ 1 ] ) ;
2019-07-27 22:22:15 +03:00
/* N/A
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Directory read : %14.3f sec, %14.3f ops/sec " , t [ 3 ] - t [ 2 ] , summary_table [ iteration ] . rate [ 2 ] ) ;
2019-07-27 22:22:15 +03:00
*/
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Directory removal : %14.3f sec, %14.3f ops/sec " , t [ 4 ] - t [ 3 ] , summary_table [ iteration ] . rate [ 3 ] ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2018-07-08 01:56:39 +03:00
/* Returns if the stonewall was hit */
int updateStoneWallIterations ( int iteration , rank_progress_t * progress , double tstart ) {
int hit = 0 ;
2018-07-15 12:02:45 +03:00
uint64_t done = progress - > items_done ;
2018-07-08 01:56:39 +03:00
long long unsigned max_iter = 0 ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , 1 , " stonewall hit with %lld items " , ( long long ) progress - > items_done ) ;
2018-07-15 12:02:45 +03:00
MPI_Allreduce ( & progress - > items_done , & max_iter , 1 , MPI_LONG_LONG_INT , MPI_MAX , testComm ) ;
2018-10-17 17:38:35 +03:00
summary_table [ iteration ] . stonewall_time [ MDTEST_FILE_CREATE_NUM ] = GetTimeStamp ( ) - tstart ;
2018-07-08 01:56:39 +03:00
// continue to the maximum...
long long min_accessed = 0 ;
MPI_Reduce ( & progress - > items_done , & min_accessed , 1 , MPI_LONG_LONG_INT , MPI_MIN , 0 , testComm ) ;
long long sum_accessed = 0 ;
MPI_Reduce ( & progress - > items_done , & sum_accessed , 1 , MPI_LONG_LONG_INT , MPI_SUM , 0 , testComm ) ;
2019-08-01 19:57:45 +03:00
summary_table [ iteration ] . stonewall_item_sum [ MDTEST_FILE_CREATE_NUM ] = sum_accessed ;
summary_table [ iteration ] . stonewall_item_min [ MDTEST_FILE_CREATE_NUM ] = min_accessed * size ;
2018-07-08 01:56:39 +03:00
2018-12-19 23:29:47 +03:00
if ( items ! = ( sum_accessed / size ) ) {
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " Continue stonewall hit min: %lld max: %lld avg: %.1f \n " , min_accessed , max_iter , ( ( double ) sum_accessed ) / size ) ;
2018-07-08 01:56:39 +03:00
hit = 1 ;
}
2018-07-15 12:02:45 +03:00
progress - > items_start = done ;
2018-07-08 01:56:39 +03:00
progress - > items_per_dir = max_iter ;
return hit ;
}
2018-07-07 12:29:27 +03:00
void file_test ( const int iteration , const int ntasks , const char * path , rank_progress_t * progress ) {
2017-10-20 00:26:52 +03:00
int size ;
double t [ 5 ] = { 0 } ;
2018-07-08 00:39:14 +03:00
char temp_path [ MAX_PATHLEN ] ;
2018-07-07 12:29:27 +03:00
MPI_Comm_size ( testComm , & size ) ;
2017-10-20 00:26:52 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " Entering file_test on %s " , path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
MPI_Barrier ( testComm ) ;
2018-10-17 17:38:35 +03:00
t [ 0 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create phase */
2018-07-08 00:19:42 +03:00
if ( create_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( MK_UNI_DIR , temp_path ) ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 5 , 5 , " operating on %s " , temp_path ) ;
2017-10-20 00:26:52 +03:00
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 0 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-27 20:30:07 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , - 1 , " file_test: create path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* "touch" the files */
if ( collective_creates ) {
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
collective_create_remove ( 1 , 0 , ntasks , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
MPI_Barrier ( testComm ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create files */
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , 0 , 1 , 0 , temp_path , 0 , progress ) ;
if ( stone_wall_timer_seconds ) {
2018-07-08 01:56:39 +03:00
int hit = updateStoneWallIterations ( iteration , progress , t [ 0 ] ) ;
if ( hit ) {
progress - > stone_wall_timer_seconds = 0 ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , 1 , " stonewall: %lld of %lld " , ( long long ) progress - > items_start , ( long long ) progress - > items_per_dir ) ;
2018-07-08 01:56:39 +03:00
create_remove_items ( 0 , 0 , 1 , 0 , temp_path , 0 , progress ) ;
// now reset the values
progress - > stone_wall_timer_seconds = stone_wall_timer_seconds ;
2018-12-19 23:29:47 +03:00
items = progress - > items_done ;
2018-07-07 12:29:27 +03:00
}
2018-07-08 01:56:39 +03:00
if ( stoneWallingStatusFile ) {
2018-07-14 18:46:24 +03:00
StoreStoneWallingIterations ( stoneWallingStatusFile , progress - > items_done ) ;
2018-07-07 12:29:27 +03:00
}
2018-12-20 00:37:37 +03:00
// reset stone wall timer to allow proper cleanup
progress - > stone_wall_timer_seconds = 0 ;
2018-07-07 12:29:27 +03:00
}
2018-07-26 12:37:01 +03:00
}
2018-07-08 01:56:39 +03:00
} else {
if ( stoneWallingStatusFile ) {
2018-07-14 14:22:36 +03:00
int64_t expected_items ;
2018-07-08 01:56:39 +03:00
/* The number of items depends on the stonewalling file */
2018-07-14 14:22:36 +03:00
expected_items = ReadStoneWallingIterations ( stoneWallingStatusFile ) ;
if ( expected_items > = 0 ) {
items = expected_items ;
2018-07-14 18:46:24 +03:00
progress - > items_per_dir = items ;
2018-07-14 14:22:36 +03:00
}
if ( rank = = 0 ) {
if ( expected_items = = - 1 ) {
fprintf ( out_logfile , " WARNING: could not read stonewall status file \n " ) ;
2019-07-27 22:22:15 +03:00
} else {
VERBOSE ( 1 , 1 , " Read stonewall status; items: " LLU " \n " , items ) ;
2018-07-14 14:22:36 +03:00
}
2018-07-08 01:56:39 +03:00
}
}
2017-06-28 19:42:20 +03:00
}
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 1 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat phase */
2018-07-08 00:19:42 +03:00
if ( stat_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( STAT_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 1 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " file_test: stat path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat files */
2018-07-26 12:37:01 +03:00
mdtest_stat ( ( random_seed > 0 ? 1 : 0 ) , 0 , dir_iter , temp_path , progress ) ;
}
2017-06-28 19:42:20 +03:00
}
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 2 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read phase */
2018-07-08 00:19:42 +03:00
if ( read_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( READ_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 2 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " file_test: read path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read files */
if ( random_seed > 0 ) {
2018-07-26 12:37:01 +03:00
mdtest_read ( 1 , 0 , dir_iter , temp_path ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-26 12:37:01 +03:00
mdtest_read ( 0 , 0 , dir_iter , temp_path ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 3 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( remove_only ) {
2018-12-20 00:37:37 +03:00
progress - > items_start = 0 ;
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( RM_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 3 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " file_test: rm directories path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( collective_creates ) {
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
collective_create_remove ( 0 , 0 , ntasks , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
} else {
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " gonna create %s " , temp_path ) ;
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , 0 , 0 , 0 , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 4 ] = GetTimeStamp ( ) ;
2017-10-20 00:26:52 +03:00
if ( remove_only ) {
if ( unique_dir_per_task ) {
unique_dir_access ( RM_UNI_DIR , temp_path ) ;
} else {
strcpy ( temp_path , path ) ;
}
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " file_test: rm unique directories path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task & & ! time_unique_dir_overhead ) {
offset_timers ( t , 4 ) ;
2017-06-28 19:42:20 +03:00
}
2018-10-17 18:04:49 +03:00
if ( num_dirs_in_tree_calc ) { /* this is temporary fix needed when using -n and -i together */
items * = num_dirs_in_tree_calc ;
}
2017-10-20 00:26:52 +03:00
/* calculate times */
if ( create_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 4 ] = items * size / ( t [ 1 ] - t [ 0 ] ) ;
summary_table [ iteration ] . time [ 4 ] = t [ 1 ] - t [ 0 ] ;
summary_table [ iteration ] . items [ 4 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 4 ] = items ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( stat_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 5 ] = items * size / ( t [ 2 ] - t [ 1 ] ) ;
summary_table [ iteration ] . time [ 5 ] = t [ 2 ] - t [ 1 ] ;
summary_table [ iteration ] . items [ 5 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 5 ] = items ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( read_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 6 ] = items * size / ( t [ 3 ] - t [ 2 ] ) ;
summary_table [ iteration ] . time [ 6 ] = t [ 3 ] - t [ 2 ] ;
summary_table [ iteration ] . items [ 6 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 6 ] = items ;
2017-10-20 00:26:52 +03:00
}
if ( remove_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 7 ] = items * size / ( t [ 4 ] - t [ 3 ] ) ;
summary_table [ iteration ] . time [ 7 ] = t [ 4 ] - t [ 3 ] ;
summary_table [ iteration ] . items [ 7 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 7 ] = items ;
2017-10-20 00:26:52 +03:00
}
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " File creation : %14.3f sec, %14.3f ops/sec " , t [ 1 ] - t [ 0 ] , summary_table [ iteration ] . rate [ 4 ] ) ;
2019-08-01 19:57:45 +03:00
if ( summary_table [ iteration ] . stonewall_time [ MDTEST_FILE_CREATE_NUM ] ) {
VERBOSE ( 1 , - 1 , " File creation (stonewall): %14.3f sec, %14.3f ops/sec " , summary_table [ iteration ] . stonewall_time [ MDTEST_FILE_CREATE_NUM ] , summary_table [ iteration ] . stonewall_item_sum [ MDTEST_FILE_CREATE_NUM ] ) ;
2017-06-28 19:42:20 +03:00
}
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " File stat : %14.3f sec, %14.3f ops/sec " , t [ 2 ] - t [ 1 ] , summary_table [ iteration ] . rate [ 5 ] ) ;
VERBOSE ( 1 , - 1 , " File read : %14.3f sec, %14.3f ops/sec " , t [ 3 ] - t [ 2 ] , summary_table [ iteration ] . rate [ 6 ] ) ;
VERBOSE ( 1 , - 1 , " File removal : %14.3f sec, %14.3f ops/sec " , t [ 4 ] - t [ 3 ] , summary_table [ iteration ] . rate [ 7 ] ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-08-01 19:54:11 +03:00
void summarize_results ( int iterations , int print_time ) {
2018-07-08 00:39:14 +03:00
char access [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
int i , j , k ;
2018-07-07 12:29:27 +03:00
int start , stop , tableSize = MDTEST_LAST_NUM ;
2017-10-20 00:26:52 +03:00
double min , max , mean , sd , sum = 0 , var = 0 , curr = 0 ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
double all [ iterations * size * tableSize ] ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " Entering summarize_results... " ) ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
MPI_Barrier ( testComm ) ;
for ( int i = 0 ; i < iterations ; i + + ) {
if ( print_time ) {
MPI_Gather ( & summary_table [ i ] . time [ 0 ] , tableSize , MPI_DOUBLE , & all [ i * tableSize * size ] , tableSize , MPI_DOUBLE , 0 , testComm ) ;
} else {
MPI_Gather ( & summary_table [ i ] . rate [ 0 ] , tableSize , MPI_DOUBLE , & all [ i * tableSize * size ] , tableSize , MPI_DOUBLE , 0 , testComm ) ;
}
}
2017-06-28 19:42:20 +03:00
2019-08-01 19:57:45 +03:00
if ( rank ! = 0 ) {
return ;
}
2017-06-28 19:42:20 +03:00
2019-08-01 19:57:45 +03:00
VERBOSE ( 0 , - 1 , " \n SUMMARY %s: (of %d iterations) " , print_time ? " time " : " rate " , iterations ) ;
VERBOSE ( 0 , - 1 , " Operation Max Min Mean Std Dev " ) ;
VERBOSE ( 0 , - 1 , " --------- --- --- ---- ------- " ) ;
2017-06-28 19:42:20 +03:00
2019-08-01 19:57:45 +03:00
/* if files only access, skip entries 0-3 (the dir tests) */
if ( files_only & & ! dirs_only ) {
start = 4 ;
} else {
start = 0 ;
}
2017-06-28 19:42:20 +03:00
2019-08-01 19:57:45 +03:00
/* if directories only access, skip entries 4-7 (the file tests) */
if ( dirs_only & & ! files_only ) {
stop = 4 ;
} else {
stop = 8 ;
}
2017-06-28 19:42:20 +03:00
2019-08-01 19:57:45 +03:00
/* special case: if no directory or file tests, skip all */
if ( ! dirs_only & & ! files_only ) {
start = stop = 0 ;
}
2017-06-28 19:42:20 +03:00
2019-08-01 19:57:45 +03:00
for ( i = start ; i < stop ; i + + ) {
2017-10-20 00:26:52 +03:00
min = max = all [ i ] ;
2019-08-01 19:57:45 +03:00
for ( k = 0 ; k < size ; k + + ) {
for ( j = 0 ; j < iterations ; j + + ) {
curr = all [ ( k * tableSize * iterations )
+ ( j * tableSize ) + i ] ;
if ( min > curr ) {
min = curr ;
2017-10-20 00:26:52 +03:00
}
2019-08-01 19:57:45 +03:00
if ( max < curr ) {
max = curr ;
2017-10-20 00:26:52 +03:00
}
2019-08-01 19:57:45 +03:00
sum + = curr ;
2017-10-20 00:26:52 +03:00
}
}
2019-08-01 19:57:45 +03:00
mean = sum / ( iterations * size ) ;
for ( k = 0 ; k < size ; k + + ) {
for ( j = 0 ; j < iterations ; j + + ) {
var + = pow ( ( mean - all [ ( k * tableSize * iterations )
+ ( j * tableSize ) + i ] ) , 2 ) ;
2018-07-07 12:29:27 +03:00
}
2017-10-20 00:26:52 +03:00
}
2019-08-01 19:57:45 +03:00
var = var / ( iterations * size ) ;
2017-10-20 00:26:52 +03:00
sd = sqrt ( var ) ;
switch ( i ) {
2019-08-01 19:57:45 +03:00
case 0 : strcpy ( access , " Directory creation : " ) ; break ;
case 1 : strcpy ( access , " Directory stat : " ) ; break ;
/* case 2: strcpy(access, "Directory read :"); break; */
case 2 : ; break ; /* N/A */
case 3 : strcpy ( access , " Directory removal : " ) ; break ;
case 4 : strcpy ( access , " File creation : " ) ; break ;
case 5 : strcpy ( access , " File stat : " ) ; break ;
case 6 : strcpy ( access , " File read : " ) ; break ;
case 7 : strcpy ( access , " File removal : " ) ; break ;
2017-10-20 00:26:52 +03:00
default : strcpy ( access , " ERR " ) ; break ;
}
2019-08-01 19:57:45 +03:00
if ( i ! = 2 ) {
fprintf ( out_logfile , " %s " , access ) ;
fprintf ( out_logfile , " %14.3f " , max ) ;
fprintf ( out_logfile , " %14.3f " , min ) ;
fprintf ( out_logfile , " %14.3f " , mean ) ;
fprintf ( out_logfile , " %14.3f \n " , sd ) ;
fflush ( out_logfile ) ;
}
2017-10-20 00:26:52 +03:00
sum = var = 0 ;
2019-08-01 19:57:45 +03:00
}
// TODO generalize once more stonewall timers are supported
double stonewall_time = 0 ;
uint64_t stonewall_items = 0 ;
for ( int i = 0 ; i < iterations ; i + + ) {
if ( summary_table [ i ] . stonewall_time [ MDTEST_FILE_CREATE_NUM ] ) {
stonewall_time + = summary_table [ i ] . stonewall_time [ MDTEST_FILE_CREATE_NUM ] ;
stonewall_items + = summary_table [ i ] . stonewall_item_sum [ MDTEST_FILE_CREATE_NUM ] ;
}
}
if ( stonewall_items ! = 0 ) {
fprintf ( out_logfile , " File create (stonewall) : " ) ;
fprintf ( out_logfile , " %14s %14s %14.3f %14s \n " , " NA " , " NA " , print_time ? stonewall_time : stonewall_items / stonewall_time , " NA " ) ;
}
/* calculate tree create/remove rates */
for ( i = 8 ; i < tableSize ; i + + ) {
min = max = all [ i ] ;
for ( j = 0 ; j < iterations ; j + + ) {
if ( print_time ) {
curr = summary_table [ j ] . time [ i ] ;
} else {
curr = summary_table [ j ] . rate [ i ] ;
}
if ( min > curr ) {
min = curr ;
}
if ( max < curr ) {
max = curr ;
}
sum + = curr ;
2017-10-20 00:26:52 +03:00
}
2019-08-01 19:57:45 +03:00
mean = sum / ( iterations ) ;
for ( j = 0 ; j < iterations ; j + + ) {
if ( print_time ) {
curr = summary_table [ j ] . time [ i ] ;
} else {
curr = summary_table [ j ] . rate [ i ] ;
}
var + = pow ( ( mean - curr ) , 2 ) ;
}
var = var / ( iterations ) ;
sd = sqrt ( var ) ;
switch ( i ) {
case 8 : strcpy ( access , " Tree creation : " ) ; break ;
case 9 : strcpy ( access , " Tree removal : " ) ; break ;
default : strcpy ( access , " ERR " ) ; break ;
}
fprintf ( out_logfile , " %s " , access ) ;
fprintf ( out_logfile , " %14.3f " , max ) ;
fprintf ( out_logfile , " %14.3f " , min ) ;
fprintf ( out_logfile , " %14.3f " , mean ) ;
fprintf ( out_logfile , " %14.3f \n " , sd ) ;
fflush ( out_logfile ) ;
sum = var = 0 ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* Checks to see if the test setup is valid. If it isn't, fail. */
void valid_tests ( ) {
2017-06-28 19:42:20 +03:00
2018-08-08 19:18:54 +03:00
if ( ( ( stone_wall_timer_seconds > 0 ) & & ( branch_factor > 1 ) ) | | ! barriers ) {
2019-07-28 20:17:11 +03:00
FAIL ( " Error, stone wall timer does only work with a branch factor <= 1 (current is %d) and with barriers \n " , branch_factor ) ;
2018-07-12 20:09:13 +03:00
}
if ( ! create_only & & ! stat_only & & ! read_only & & ! remove_only ) {
create_only = stat_only = read_only = remove_only = 1 ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " main: Setting create/stat/read/remove_only to True " ) ;
2018-07-12 20:09:13 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " Entering valid_tests... " ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* if dirs_only and files_only were both left unset, set both now */
if ( ! dirs_only & & ! files_only ) {
dirs_only = files_only = 1 ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* if shared file 'S' access, no directory tests */
if ( shared_file ) {
dirs_only = 0 ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* check for no barriers with shifting processes for different phases.
that is , one may not specify both - B and - N as it will introduce
race conditions that may cause errors stat ' ing or deleting after
creates .
*/
if ( ( barriers = = 0 ) & & ( nstride ! = 0 ) & & ( rank = = 0 ) ) {
FAIL ( " Possible race conditions will occur: -B not compatible with -N " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* check for collective_creates incompatibilities */
if ( shared_file & & collective_creates & & rank = = 0 ) {
FAIL ( " -c not compatible with -S " ) ;
}
if ( path_count > 1 & & collective_creates & & rank = = 0 ) {
FAIL ( " -c not compatible with multiple test directories " ) ;
}
if ( collective_creates & & ! barriers ) {
FAIL ( " -c not compatible with -B " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* check for shared file incompatibilities */
if ( unique_dir_per_task & & shared_file & & rank = = 0 ) {
FAIL ( " -u not compatible with -S " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check multiple directory paths and strided option */
if ( path_count > 1 & & nstride > 0 ) {
FAIL ( " cannot have multiple directory paths with -N strides between neighbor tasks " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check for shared directory and multiple directories incompatibility */
if ( path_count > 1 & & unique_dir_per_task ! = 1 ) {
FAIL ( " shared directory mode is not compatible with multiple directory paths " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check if more directory paths than ranks */
if ( path_count > size ) {
FAIL ( " cannot have more directory paths than MPI tasks " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check depth */
if ( depth < 0 ) {
FAIL ( " depth must be greater than or equal to zero " ) ;
}
/* check branch_factor */
if ( branch_factor < 1 & & depth > 0 ) {
FAIL ( " branch factor must be greater than or equal to zero " ) ;
}
/* check for valid number of items */
if ( ( items > 0 ) & & ( items_per_dir > 0 ) ) {
2018-07-26 12:37:01 +03:00
if ( unique_dir_per_task ) {
FAIL ( " only specify the number of items or the number of items per directory " ) ;
} else if ( items % items_per_dir ! = 0 ) {
FAIL ( " items must be a multiple of items per directory " ) ;
} else if ( stone_wall_timer_seconds ! = 0 ) {
FAIL ( " items + items_per_dir can only be set without stonewalling " ) ;
}
2017-10-20 00:26:52 +03:00
}
2018-10-08 15:30:47 +03:00
/* check for using mknod */
if ( write_bytes > 0 & & make_node ) {
FAIL ( " -k not compatible with -w " ) ;
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
void show_file_system_size ( char * file_system ) {
2018-07-08 00:39:14 +03:00
char real_path [ MAX_PATHLEN ] ;
char file_system_unit_str [ MAX_PATHLEN ] = " GiB " ;
char inode_unit_str [ MAX_PATHLEN ] = " Mi " ;
2017-10-20 00:26:52 +03:00
int64_t file_system_unit_val = 1024 * 1024 * 1024 ;
int64_t inode_unit_val = 1024 * 1024 ;
int64_t total_file_system_size ,
free_file_system_size ,
total_inodes ,
free_inodes ;
double total_file_system_size_hr ,
used_file_system_percentage ,
used_inode_percentage ;
ior_aiori_statfs_t stat_buf ;
int ret ;
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering show_file_system_size on %s " , file_system ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
ret = backend - > statfs ( file_system , & stat_buf , & param ) ;
if ( 0 ! = ret ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to stat file system %s " , file_system ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
total_file_system_size = stat_buf . f_blocks * stat_buf . f_bsize ;
free_file_system_size = stat_buf . f_bfree * stat_buf . f_bsize ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
used_file_system_percentage = ( 1 - ( ( double ) free_file_system_size
/ ( double ) total_file_system_size ) ) * 100 ;
total_file_system_size_hr = ( double ) total_file_system_size
/ ( double ) file_system_unit_val ;
if ( total_file_system_size_hr > 1024 ) {
total_file_system_size_hr = total_file_system_size_hr / 1024 ;
strcpy ( file_system_unit_str , " TiB " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* inodes */
total_inodes = stat_buf . f_files ;
free_inodes = stat_buf . f_ffree ;
used_inode_percentage = ( 1 - ( ( double ) free_inodes / ( double ) total_inodes ) )
* 100 ;
2017-06-28 19:42:20 +03:00
2018-01-24 21:26:53 +03:00
if ( realpath ( file_system , real_path ) = = NULL ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to use realpath() on file system %s " , file_system ) ;
2018-01-24 21:26:53 +03:00
}
2018-07-07 12:29:27 +03:00
2017-10-20 00:26:52 +03:00
/* show results */
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " Path: %s " , real_path ) ;
2019-08-01 19:57:45 +03:00
VERBOSE ( 0 , - 1 , " FS: %.1f %s Used FS: %2.1f%% Inodes: %.1f %s Used Inodes: %2.1f%% \n " ,
2019-07-27 23:31:49 +03:00
total_file_system_size_hr , file_system_unit_str , used_file_system_percentage ,
( double ) total_inodes / ( double ) inode_unit_val , inode_unit_str , used_inode_percentage ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
return ;
2017-06-28 19:42:20 +03:00
}
void display_freespace ( char * testdirpath )
{
2018-07-08 00:39:14 +03:00
char dirpath [ MAX_PATHLEN ] = { 0 } ;
2017-10-20 00:26:52 +03:00
int i ;
int directoryFound = 0 ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " Entering display_freespace on %s... " , testdirpath ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
strcpy ( dirpath , testdirpath ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* get directory for outfile */
i = strlen ( dirpath ) ;
while ( i - - > 0 ) {
if ( dirpath [ i ] = = ' / ' ) {
dirpath [ i ] = ' \0 ' ;
directoryFound = 1 ;
break ;
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* if no directory/, use '.' */
if ( directoryFound = = 0 ) {
strcpy ( dirpath , " . " ) ;
}
2017-06-28 19:42:20 +03:00
2019-06-25 00:26:15 +03:00
if ( strcasecmp ( param . api , " DFS " ) = = 0 )
2018-08-30 00:14:19 +03:00
return ;
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " Before show_file_system_size, dirpath is '%s' " , dirpath ) ;
2017-10-20 00:26:52 +03:00
show_file_system_size ( dirpath ) ;
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " After show_file_system_size, dirpath is '%s' \n " , dirpath ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
return ;
2017-06-28 19:42:20 +03:00
}
void create_remove_directory_tree ( int create ,
2018-07-07 12:29:27 +03:00
int currDepth , char * path , int dirNum , rank_progress_t * progress ) {
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
unsigned i ;
2018-07-08 00:39:14 +03:00
char dir [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , 5 , " Entering create_remove_directory_tree on %s, currDepth = %d... " , path , currDepth ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( currDepth = = 0 ) {
sprintf ( dir , " %s/%s.%d/ " , path , base_tree_name , dirNum ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( create ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 2 , 5 , " Making directory '%s' " , dir ) ;
2017-10-20 00:26:52 +03:00
if ( - 1 = = backend - > mkdir ( dir , DIRMODE , & param ) ) {
2019-07-28 19:07:03 +03:00
fprintf ( out_logfile , " error could not create directory '%s' \n " , dir ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
create_remove_directory_tree ( create , + + currDepth , dir , + + dirNum , progress ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ! create ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 2 , 5 , " Remove directory '%s' " , dir ) ;
2017-10-20 00:26:52 +03:00
if ( - 1 = = backend - > rmdir ( dir , & param ) ) {
2019-07-28 20:17:11 +03:00
FAIL ( " Unable to remove directory %s " , dir ) ;
2017-10-20 00:26:52 +03:00
}
}
} else if ( currDepth < = depth ) {
2017-06-28 19:42:20 +03:00
2018-07-08 00:39:14 +03:00
char temp_path [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
strcpy ( temp_path , path ) ;
int currDir = dirNum ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
for ( i = 0 ; i < branch_factor ; i + + ) {
sprintf ( dir , " %s.%d/ " , base_tree_name , currDir ) ;
strcat ( temp_path , dir ) ;
if ( create ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 2 , 5 , " Making directory '%s' " , temp_path ) ;
2017-10-20 00:26:52 +03:00
if ( - 1 = = backend - > mkdir ( temp_path , DIRMODE , & param ) ) {
2019-07-28 20:17:11 +03:00
FAIL ( " Unable to create directory %s " , temp_path ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
create_remove_directory_tree ( create , + + currDepth ,
2018-07-07 12:29:27 +03:00
temp_path , ( branch_factor * currDir ) + 1 , progress ) ;
2017-10-20 00:26:52 +03:00
currDepth - - ;
if ( ! create ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 2 , 5 , " Remove directory '%s' " , temp_path ) ;
2017-10-20 00:26:52 +03:00
if ( - 1 = = backend - > rmdir ( temp_path , & param ) ) {
2019-07-28 20:17:11 +03:00
FAIL ( " Unable to remove directory %s " , temp_path ) ;
2017-10-20 00:26:52 +03:00
}
}
strcpy ( temp_path , path ) ;
currDir + + ;
}
2017-06-28 19:42:20 +03:00
}
}
2018-07-08 01:56:39 +03:00
static void mdtest_iteration ( int i , int j , MPI_Group testgroup , mdtest_results_t * summary_table ) {
rank_progress_t progress_o ;
memset ( & progress_o , 0 , sizeof ( progress_o ) ) ;
progress_o . start_time = GetTimeStamp ( ) ;
progress_o . stone_wall_timer_seconds = stone_wall_timer_seconds ;
progress_o . items_per_dir = items_per_dir ;
rank_progress_t * progress = & progress_o ;
2018-07-07 12:29:27 +03:00
/* start and end times of directory tree create/remove */
double startCreate , endCreate ;
2018-08-08 19:18:54 +03:00
int k ;
2018-07-07 12:29:27 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " main: * iteration %d * " , j + 1 ) ;
2018-07-07 12:29:27 +03:00
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( j , dir_iter ) ;
2018-07-07 12:29:27 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 2 , 5 , " main (for j loop): making testdir, '%s' " , testdir ) ;
2018-07-26 12:37:01 +03:00
if ( ( rank < path_count ) & & backend - > access ( testdir , F_OK , & param ) ! = 0 ) {
if ( backend - > mkdir ( testdir , DIRMODE , & param ) ! = 0 ) {
2019-07-28 20:17:11 +03:00
FAIL ( " Unable to create test directory %s " , testdir ) ;
2018-08-24 00:58:53 +03:00
}
2018-07-26 12:37:01 +03:00
}
2018-07-07 12:29:27 +03:00
}
2018-08-24 00:58:53 +03:00
2018-07-07 12:29:27 +03:00
if ( create_only ) {
2018-07-26 12:37:01 +03:00
/* create hierarchical directory structure */
MPI_Barrier ( testComm ) ;
2018-10-17 17:38:35 +03:00
startCreate = GetTimeStamp ( ) ;
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( j , dir_iter ) ;
if ( unique_dir_per_task ) {
if ( collective_creates & & ( rank = = 0 ) ) {
/*
* This is inside two loops , one of which already uses " i " and the other uses " j " .
* I don ' t know how this ever worked . I ' m changing this loop to use " k " .
*/
for ( k = 0 ; k < size ; k + + ) {
sprintf ( base_tree_name , " mdtest_tree.%d " , k ) ;
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " main (create hierarchical directory loop-collective): Calling create_remove_directory_tree with '%s' " , testdir ) ;
2018-07-26 12:37:01 +03:00
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 1 , 0 , testdir , 0 , progress ) ;
if ( CHECK_STONE_WALL ( progress ) ) {
size = k ;
break ;
}
}
} else if ( ! collective_creates ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with '%s' " , testdir ) ;
2018-07-26 12:37:01 +03:00
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 1 , 0 , testdir , 0 , progress ) ;
}
} else {
if ( rank = = 0 ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with '%s' " , testdir ) ;
2018-07-26 12:37:01 +03:00
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 1 , 0 , testdir , 0 , progress ) ;
}
}
2018-07-07 12:29:27 +03:00
}
MPI_Barrier ( testComm ) ;
2018-10-17 17:38:35 +03:00
endCreate = GetTimeStamp ( ) ;
2018-07-07 12:29:27 +03:00
summary_table - > rate [ 8 ] =
num_dirs_in_tree / ( endCreate - startCreate ) ;
summary_table - > time [ 8 ] = ( endCreate - startCreate ) ;
summary_table - > items [ 8 ] = num_dirs_in_tree ;
summary_table - > stonewall_last_item [ 8 ] = num_dirs_in_tree ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " V-1: main: Tree creation : %14.3f sec, %14.3f ops/sec " , ( endCreate - startCreate ) , summary_table - > rate [ 8 ] ) ;
2018-07-07 12:29:27 +03:00
}
2018-07-26 12:37:01 +03:00
sprintf ( unique_mk_dir , " %s.0 " , base_tree_name ) ;
sprintf ( unique_chdir_dir , " %s.0 " , base_tree_name ) ;
sprintf ( unique_stat_dir , " %s.0 " , base_tree_name ) ;
sprintf ( unique_read_dir , " %s.0 " , base_tree_name ) ;
sprintf ( unique_rm_dir , " %s.0 " , base_tree_name ) ;
unique_rm_uni_dir [ 0 ] = 0 ;
2018-07-07 12:29:27 +03:00
if ( ! unique_dir_per_task ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , - 1 , " V-3: main: Using unique_mk_dir, '%s' " , unique_mk_dir ) ;
2018-07-07 12:29:27 +03:00
}
if ( rank < i ) {
if ( ! shared_file ) {
sprintf ( mk_name , " mdtest.%d. " , ( rank + ( 0 * nstride ) ) % i ) ;
sprintf ( stat_name , " mdtest.%d. " , ( rank + ( 1 * nstride ) ) % i ) ;
sprintf ( read_name , " mdtest.%d. " , ( rank + ( 2 * nstride ) ) % i ) ;
sprintf ( rm_name , " mdtest.%d. " , ( rank + ( 3 * nstride ) ) % i ) ;
}
if ( unique_dir_per_task ) {
2019-07-27 23:31:49 +03:00
VERBOSE ( 3 , 5 , " i %d nstride %d " , i , nstride ) ;
2018-07-26 12:37:01 +03:00
sprintf ( unique_mk_dir , " mdtest_tree.%d.0 " , ( rank + ( 0 * nstride ) ) % i ) ;
sprintf ( unique_chdir_dir , " mdtest_tree.%d.0 " , ( rank + ( 1 * nstride ) ) % i ) ;
sprintf ( unique_stat_dir , " mdtest_tree.%d.0 " , ( rank + ( 2 * nstride ) ) % i ) ;
sprintf ( unique_read_dir , " mdtest_tree.%d.0 " , ( rank + ( 3 * nstride ) ) % i ) ;
sprintf ( unique_rm_dir , " mdtest_tree.%d.0 " , ( rank + ( 4 * nstride ) ) % i ) ;
unique_rm_uni_dir [ 0 ] = 0 ;
2019-07-27 23:31:49 +03:00
VERBOSE ( 5 , 5 , " mk_dir %s chdir %s stat_dir %s read_dir %s rm_dir %s \n " , unique_mk_dir , unique_chdir_dir , unique_stat_dir , unique_read_dir , unique_rm_dir ) ;
2018-07-07 12:29:27 +03:00
}
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , - 1 , " V-3: main: Copied unique_mk_dir, '%s', to topdir " , unique_mk_dir ) ;
2018-07-07 12:29:27 +03:00
if ( dirs_only & & ! shared_file ) {
if ( pre_delay ) {
2018-07-08 00:26:57 +03:00
DelaySecs ( pre_delay ) ;
2018-07-07 12:29:27 +03:00
}
directory_test ( j , i , unique_mk_dir , progress ) ;
}
if ( files_only ) {
if ( pre_delay ) {
2018-07-08 00:26:57 +03:00
DelaySecs ( pre_delay ) ;
2018-07-07 12:29:27 +03:00
}
2019-07-27 23:31:49 +03:00
VERBOSE ( 3 , 5 , " will file_test on %s " , unique_mk_dir ) ;
2018-07-07 12:29:27 +03:00
file_test ( j , i , unique_mk_dir , progress ) ;
}
}
/* remove directory structure */
if ( ! unique_dir_per_task ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , - 1 , " main: Using testdir, '%s' " , testdir ) ;
2018-07-07 12:29:27 +03:00
}
MPI_Barrier ( testComm ) ;
if ( remove_only ) {
2018-12-20 00:37:37 +03:00
progress - > items_start = 0 ;
2018-10-17 17:38:35 +03:00
startCreate = GetTimeStamp ( ) ;
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( j , dir_iter ) ;
if ( unique_dir_per_task ) {
if ( collective_creates & & ( rank = = 0 ) ) {
/*
* This is inside two loops , one of which already uses " i " and the other uses " j " .
* I don ' t know how this ever worked . I ' m changing this loop to use " k " .
*/
for ( k = 0 ; k < size ; k + + ) {
sprintf ( base_tree_name , " mdtest_tree.%d " , k ) ;
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , - 1 , " main (remove hierarchical directory loop-collective): Calling create_remove_directory_tree with '%s' " , testdir ) ;
2018-07-26 12:37:01 +03:00
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 0 , 0 , testdir , 0 , progress ) ;
if ( CHECK_STONE_WALL ( progress ) ) {
size = k ;
break ;
}
}
} else if ( ! collective_creates ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , - 1 , " main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with '%s' " , testdir ) ;
2018-07-26 12:37:01 +03:00
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 0 , 0 , testdir , 0 , progress ) ;
}
} else {
if ( rank = = 0 ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , - 1 , " V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with '%s' " , testdir ) ;
2018-08-24 00:58:53 +03:00
2018-07-26 12:37:01 +03:00
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 0 , 0 , testdir , 0 , progress ) ;
}
}
2018-07-07 12:29:27 +03:00
}
MPI_Barrier ( testComm ) ;
2018-10-17 17:38:35 +03:00
endCreate = GetTimeStamp ( ) ;
2018-07-07 12:29:27 +03:00
summary_table - > rate [ 9 ] = num_dirs_in_tree / ( endCreate - startCreate ) ;
summary_table - > time [ 9 ] = endCreate - startCreate ;
summary_table - > items [ 9 ] = num_dirs_in_tree ;
summary_table - > stonewall_last_item [ 8 ] = num_dirs_in_tree ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " main Tree removal : %14.3f sec, %14.3f ops/sec " , ( endCreate - startCreate ) , summary_table - > rate [ 9 ] ) ;
2019-07-28 19:07:03 +03:00
VERBOSE ( 2 , - 1 , " main (at end of for j loop): Removing testdir of '%s' \n " , testdir ) ;
2018-07-07 12:29:27 +03:00
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( j , dir_iter ) ;
if ( ( rank < path_count ) & & backend - > access ( testdir , F_OK , & param ) = = 0 ) {
//if (( rank == 0 ) && access(testdir, F_OK) == 0) {
if ( backend - > rmdir ( testdir , & param ) = = - 1 ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to remove directory %s " , testdir ) ;
2018-07-26 12:37:01 +03:00
}
}
2018-07-07 12:29:27 +03:00
}
} else {
summary_table - > rate [ 9 ] = 0 ;
}
}
2018-08-24 00:58:53 +03:00
2018-07-07 12:29:27 +03:00
void mdtest_init_args ( ) {
barriers = 1 ;
branch_factor = 1 ;
throttle = 1 ;
2018-07-08 01:56:39 +03:00
stoneWallingStatusFile = NULL ;
2018-07-07 12:29:27 +03:00
create_only = 0 ;
stat_only = 0 ;
read_only = 0 ;
2019-08-01 20:29:32 +03:00
verify_read = 0 ;
verification_error = 0 ;
2018-07-07 12:29:27 +03:00
remove_only = 0 ;
leaf_only = 0 ;
depth = 0 ;
num_dirs_in_tree = 0 ;
items_per_dir = 0 ;
random_seed = 0 ;
print_time = 0 ;
2019-08-01 19:54:11 +03:00
print_rate_and_time = 0 ;
2018-07-07 12:29:27 +03:00
shared_file = 0 ;
files_only = 0 ;
dirs_only = 0 ;
pre_delay = 0 ;
unique_dir_per_task = 0 ;
time_unique_dir_overhead = 0 ;
items = 0 ;
2018-10-17 18:16:46 +03:00
num_dirs_in_tree_calc = 0 ;
2018-07-07 12:29:27 +03:00
collective_creates = 0 ;
write_bytes = 0 ;
stone_wall_timer_seconds = 0 ;
read_bytes = 0 ;
sync_file = 0 ;
2019-08-26 20:57:14 +03:00
call_sync = 0 ;
2018-07-07 12:29:27 +03:00
path_count = 0 ;
nstride = 0 ;
2018-10-08 15:30:47 +03:00
make_node = 0 ;
2018-08-24 00:58:53 +03:00
}
2018-07-07 12:29:27 +03:00
mdtest_results_t * mdtest_run ( int argc , char * * argv , MPI_Comm world_com , FILE * world_out ) {
testComm = world_com ;
out_logfile = world_out ;
mpi_comm_world = world_com ;
init_clock ( ) ;
mdtest_init_args ( ) ;
2018-08-08 19:18:54 +03:00
int i , j ;
2017-10-20 00:26:52 +03:00
int nodeCount ;
MPI_Group worldgroup , testgroup ;
struct {
int first ;
int last ;
int stride ;
} range = { 0 , 0 , 1 } ;
int first = 1 ;
int last = 0 ;
int stride = 1 ;
int iterations = 1 ;
2018-07-12 20:09:13 +03:00
verbose = 0 ;
int no_barriers = 0 ;
char * path = " ./out " ;
int randomize = 0 ;
2018-08-15 19:07:17 +03:00
char APIs [ 1024 ] ;
2018-09-10 19:43:12 +03:00
char APIs_legacy [ 1024 ] ;
2019-02-11 16:49:14 +03:00
aiori_supported_apis ( APIs , APIs_legacy , MDTEST ) ;
2018-08-15 19:07:17 +03:00
char apiStr [ 1024 ] ;
sprintf ( apiStr , " API for I/O [%s] " , APIs ) ;
2018-07-12 20:09:13 +03:00
option_help options [ ] = {
2019-03-27 23:04:48 +03:00
{ ' a ' , NULL , apiStr , OPTION_OPTIONAL_ARGUMENT , ' s ' , & param . api } ,
2018-07-12 20:09:13 +03:00
{ ' b ' , NULL , " branching factor of hierarchical directory structure " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & branch_factor } ,
{ ' d ' , NULL , " the directory in which the tests will run " , OPTION_OPTIONAL_ARGUMENT , ' s ' , & path } ,
{ ' B ' , NULL , " no barriers between phases " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & no_barriers } ,
{ ' C ' , NULL , " only create files/dirs " , OPTION_FLAG , ' d ' , & create_only } ,
{ ' T ' , NULL , " only stat files/dirs " , OPTION_FLAG , ' d ' , & stat_only } ,
{ ' E ' , NULL , " only read files/dir " , OPTION_FLAG , ' d ' , & read_only } ,
{ ' r ' , NULL , " only remove files or directories left behind by previous runs " , OPTION_FLAG , ' d ' , & remove_only } ,
{ ' D ' , NULL , " perform test on directories only (no files) " , OPTION_FLAG , ' d ' , & dirs_only } ,
{ ' e ' , NULL , " bytes to read from each file " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & read_bytes } ,
{ ' f ' , NULL , " first number of tasks on which the test will run " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & first } ,
{ ' F ' , NULL , " perform test on files only (no directories) " , OPTION_FLAG , ' d ' , & files_only } ,
2018-09-05 11:14:22 +03:00
{ ' i ' , NULL , " number of iterations the test will run " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & iterations } ,
2018-07-12 20:09:13 +03:00
{ ' I ' , NULL , " number of items per directory in tree " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & items_per_dir } ,
2018-10-08 15:30:47 +03:00
{ ' k ' , NULL , " use mknod to create file " , OPTION_FLAG , ' d ' , & make_node } ,
2018-07-12 20:09:13 +03:00
{ ' l ' , NULL , " last number of tasks on which the test will run " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & last } ,
{ ' L ' , NULL , " files only at leaf level of tree " , OPTION_FLAG , ' d ' , & leaf_only } ,
{ ' n ' , NULL , " every process will creat/stat/read/remove # directories and files " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & items } ,
2019-08-01 03:42:03 +03:00
{ ' N ' , NULL , " stride # between tasks for file/dir operation (local=0; set to 1 to avoid client cache) " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & nstride } ,
2018-07-12 20:09:13 +03:00
{ ' p ' , NULL , " pre-iteration delay (in seconds) " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & pre_delay } ,
2019-08-01 19:54:11 +03:00
{ ' P ' , NULL , " print rate AND time " , OPTION_FLAG , ' d ' , & print_rate_and_time } ,
2018-07-14 18:46:24 +03:00
{ ' R ' , NULL , " random access to files (only for stat) " , OPTION_FLAG , ' d ' , & randomize } ,
2018-07-12 20:09:13 +03:00
{ 0 , " random-seed " , " random seed for -R " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & random_seed } ,
{ ' s ' , NULL , " stride between the number of tasks for each test " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & stride } ,
{ ' S ' , NULL , " shared file access (file only, no directories) " , OPTION_FLAG , ' d ' , & shared_file } ,
{ ' c ' , NULL , " collective creates: task 0 does all creates " , OPTION_FLAG , ' d ' , & collective_creates } ,
{ ' t ' , NULL , " time unique working directory overhead " , OPTION_FLAG , ' d ' , & time_unique_dir_overhead } ,
{ ' u ' , NULL , " unique working directory for each task " , OPTION_FLAG , ' d ' , & unique_dir_per_task } ,
{ ' v ' , NULL , " verbosity (each instance of option increments by one) " , OPTION_FLAG , ' d ' , & verbose } ,
{ ' V ' , NULL , " verbosity value " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & verbose } ,
{ ' w ' , NULL , " bytes to write to each file after it is created " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & write_bytes } ,
{ ' W ' , NULL , " number in seconds; stonewall timer, write as many seconds and ensure all processes did the same number of operations (currently only stops during create phase) " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & stone_wall_timer_seconds } ,
{ ' x ' , NULL , " StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs " , OPTION_OPTIONAL_ARGUMENT , ' s ' , & stoneWallingStatusFile } ,
2019-08-01 20:29:32 +03:00
{ ' X ' , " verify-read " , " Verify the data read " , OPTION_FLAG , ' d ' , & verify_read } ,
2018-07-12 20:09:13 +03:00
{ ' y ' , NULL , " sync file after writing " , OPTION_FLAG , ' d ' , & sync_file } ,
2019-08-26 20:57:14 +03:00
{ ' Y ' , NULL , " call the sync command after each phase (included in the timing; note it causes all IO to be flushed from your node) " , OPTION_FLAG , ' d ' , & call_sync } ,
2018-07-12 20:09:13 +03:00
{ ' z ' , NULL , " depth of hierarchical directory structure " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & depth } ,
{ ' Z ' , NULL , " print time instead of rate " , OPTION_FLAG , ' d ' , & print_time } ,
LAST_OPTION
} ;
2019-03-27 23:04:48 +03:00
options_all_t * global_options = airoi_create_all_module_options ( options ) ;
option_parse ( argc , argv , global_options ) ;
updateParsedOptions ( & param , global_options ) ;
2019-08-03 03:23:05 +03:00
free ( global_options - > modules ) ;
free ( global_options ) ;
2019-03-27 23:04:48 +03:00
backend = param . backend ;
2018-07-12 20:09:13 +03:00
2018-07-07 12:29:27 +03:00
MPI_Comm_rank ( testComm , & rank ) ;
MPI_Comm_size ( testComm , & size ) ;
2017-06-28 19:42:20 +03:00
2019-01-24 03:31:12 +03:00
if ( backend - > initialize )
backend - > initialize ( ) ;
2017-10-20 00:26:52 +03:00
pid = getpid ( ) ;
uid = getuid ( ) ;
2017-06-28 19:42:20 +03:00
2019-07-26 04:18:52 +03:00
tasksPerNode = CountTasksPerNode ( testComm ) ;
nodeCount = size / tasksPerNode ;
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
char cmd_buffer [ 4096 ] ;
strncpy ( cmd_buffer , argv [ 0 ] , 4096 ) ;
for ( i = 1 ; i < argc ; i + + ) {
2019-07-28 19:07:03 +03:00
snprintf ( & cmd_buffer [ strlen ( cmd_buffer ) ] , 4096 - strlen ( cmd_buffer ) , " '%s' " , argv [ i ] ) ;
2017-10-20 00:26:52 +03:00
}
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " -- started at %s -- \n " , PrintTimestamp ( ) ) ;
VERBOSE ( 0 , - 1 , " mdtest-%s was launched with %d total task(s) on %d node(s) " , RELEASE_VERS , size , nodeCount ) ;
VERBOSE ( 0 , - 1 , " Command line used: %s " , cmd_buffer ) ;
2017-06-28 19:42:20 +03:00
2018-07-12 20:09:13 +03:00
/* adjust special variables */
barriers = ! no_barriers ;
if ( path ! = NULL ) {
parse_dirpath ( path ) ;
2017-06-28 19:42:20 +03:00
}
2018-07-12 20:09:13 +03:00
if ( randomize > 0 ) {
if ( random_seed = = 0 ) {
/* Ensure all procs have the same random number */
random_seed = time ( NULL ) ;
MPI_Barrier ( testComm ) ;
MPI_Bcast ( & random_seed , 1 , MPI_INT , 0 , testComm ) ;
}
random_seed + = rank ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
if ( ( items > 0 ) & & ( items_per_dir > 0 ) & & ( ! unique_dir_per_task ) ) {
directory_loops = items / items_per_dir ;
} else {
directory_loops = 1 ;
2017-10-20 00:26:52 +03:00
}
valid_tests ( ) ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
// option_print_current(options);
VERBOSE ( 1 , - 1 , " api : %s " , param . api ) ;
VERBOSE ( 1 , - 1 , " barriers : %s " , ( barriers ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " collective_creates : %s " , ( collective_creates ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " create_only : %s " , ( create_only ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " dirpath(s): " ) ;
for ( i = 0 ; i < path_count ; i + + ) {
VERBOSE ( 1 , - 1 , " \t %s " , filenames [ i ] ) ;
}
VERBOSE ( 1 , - 1 , " dirs_only : %s " , ( dirs_only ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " read_bytes : " LLU " " , read_bytes ) ;
VERBOSE ( 1 , - 1 , " read_only : %s " , ( read_only ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " first : %d " , first ) ;
VERBOSE ( 1 , - 1 , " files_only : %s " , ( files_only ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " iterations : %d " , iterations ) ;
VERBOSE ( 1 , - 1 , " items_per_dir : " LLU " " , items_per_dir ) ;
VERBOSE ( 1 , - 1 , " last : %d " , last ) ;
VERBOSE ( 1 , - 1 , " leaf_only : %s " , ( leaf_only ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " items : " LLU " " , items ) ;
VERBOSE ( 1 , - 1 , " nstride : %d " , nstride ) ;
VERBOSE ( 1 , - 1 , " pre_delay : %d " , pre_delay ) ;
VERBOSE ( 1 , - 1 , " remove_only : %s " , ( leaf_only ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " random_seed : %d " , random_seed ) ;
VERBOSE ( 1 , - 1 , " stride : %d " , stride ) ;
VERBOSE ( 1 , - 1 , " shared_file : %s " , ( shared_file ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " time_unique_dir_overhead: %s " , ( time_unique_dir_overhead ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " stone_wall_timer_seconds: %d " , stone_wall_timer_seconds ) ;
VERBOSE ( 1 , - 1 , " stat_only : %s " , ( stat_only ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " unique_dir_per_task : %s " , ( unique_dir_per_task ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " write_bytes : " LLU " " , write_bytes ) ;
VERBOSE ( 1 , - 1 , " sync_file : %s " , ( sync_file ? " True " : " False " ) ) ;
2019-08-26 20:57:14 +03:00
VERBOSE ( 1 , - 1 , " call_sync : %s " , ( call_sync ? " True " : " False " ) ) ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " depth : %d " , depth ) ;
VERBOSE ( 1 , - 1 , " make_node : %d " , make_node ) ;
2017-10-20 00:26:52 +03:00
/* setup total number of items and number of items per dir */
if ( depth < = 0 ) {
num_dirs_in_tree = 1 ;
2017-06-28 19:42:20 +03:00
} else {
2017-10-20 00:26:52 +03:00
if ( branch_factor < 1 ) {
num_dirs_in_tree = 1 ;
} else if ( branch_factor = = 1 ) {
num_dirs_in_tree = depth + 1 ;
} else {
2018-10-17 15:50:04 +03:00
num_dirs_in_tree = ( pow ( branch_factor , depth + 1 ) - 1 ) / ( branch_factor - 1 ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( items_per_dir > 0 ) {
2018-10-17 15:50:04 +03:00
if ( items = = 0 ) {
2019-05-14 16:55:11 +03:00
if ( leaf_only ) {
items = items_per_dir * ( uint64_t ) pow ( branch_factor , depth ) ;
} else {
items = items_per_dir * num_dirs_in_tree ;
}
2018-10-17 18:04:49 +03:00
} else {
num_dirs_in_tree_calc = num_dirs_in_tree ;
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
} else {
2017-10-20 00:26:52 +03:00
if ( leaf_only ) {
if ( branch_factor < = 1 ) {
items_per_dir = items ;
} else {
2018-07-14 18:46:24 +03:00
items_per_dir = ( uint64_t ) ( items / pow ( branch_factor , depth ) ) ;
items = items_per_dir * ( uint64_t ) pow ( branch_factor , depth ) ;
2017-10-20 00:26:52 +03:00
}
} else {
items_per_dir = items / num_dirs_in_tree ;
items = items_per_dir * num_dirs_in_tree ;
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* initialize rand_array */
if ( random_seed > 0 ) {
srand ( random_seed ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
uint64_t s ;
2017-06-28 19:42:20 +03:00
2018-07-14 18:46:24 +03:00
rand_array = ( uint64_t * ) malloc ( items * sizeof ( * rand_array ) ) ;
2017-06-28 19:42:20 +03:00
2018-07-14 18:46:24 +03:00
for ( s = 0 ; s < items ; s + + ) {
2017-10-20 00:26:52 +03:00
rand_array [ s ] = s ;
}
/* shuffle list randomly */
2018-07-14 18:46:24 +03:00
uint64_t n = items ;
2017-10-20 00:26:52 +03:00
while ( n > 1 ) {
n - - ;
/*
* Generate a random number in the range 0 . . n
*
* rand ( ) returns a number from 0 . . RAND_MAX . Divide that
* by RAND_MAX and you get a floating point number in the
* range 0 . . 1. Multiply that by n and you get a number in
* the range 0 . . n .
*/
2018-07-08 00:19:42 +03:00
uint64_t k = ( uint64_t ) ( ( ( double ) rand ( ) / ( double ) RAND_MAX ) * ( double ) n ) ;
2017-10-20 00:26:52 +03:00
/*
* Now move the nth element to the kth ( randomly chosen )
* element , and the kth element to the nth element .
*/
uint64_t tmp = rand_array [ k ] ;
rand_array [ k ] = rand_array [ n ] ;
rand_array [ n ] = tmp ;
}
}
/* allocate and initialize write buffer with # */
if ( write_bytes > 0 ) {
write_buffer = ( char * ) malloc ( write_bytes ) ;
if ( write_buffer = = NULL ) {
FAIL ( " out of memory " ) ;
}
2019-08-01 20:29:32 +03:00
generate_memory_pattern ( write_buffer , write_bytes ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* setup directory path to work in */
if ( path_count = = 0 ) { /* special case where no directory path provided with '-d' option */
2018-08-08 19:18:54 +03:00
char * ret = getcwd ( testdirpath , MAX_PATHLEN ) ;
if ( ret = = NULL ) {
2019-07-28 20:17:11 +03:00
FAIL ( " Unable to get current working directory on %s " , testdirpath ) ;
2018-08-08 19:18:54 +03:00
}
2017-10-20 00:26:52 +03:00
path_count = 1 ;
} else {
strcpy ( testdirpath , filenames [ rank % path_count ] ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* if directory does not exist, create it */
if ( ( rank < path_count ) & & backend - > access ( testdirpath , F_OK , & param ) ! = 0 ) {
if ( backend - > mkdir ( testdirpath , DIRMODE , & param ) ! = 0 ) {
2019-07-28 20:17:11 +03:00
FAIL ( " Unable to create test directory path %s " , testdirpath ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* display disk usage */
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , - 1 , " main (before display_freespace): testdirpath is '%s' " , testdirpath ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) display_freespace ( testdirpath ) ;
2019-08-01 03:42:03 +03:00
int tasksBlockMapping = QueryNodeMapping ( testComm , true ) ;
2017-06-28 19:42:20 +03:00
2019-07-26 17:55:24 +03:00
/* set the shift to mimic IOR and shift by procs per node */
if ( nstride > 0 ) {
2019-07-28 19:25:42 +03:00
if ( nodeCount > 1 & & tasksBlockMapping ) {
/* the user set the stride presumably to get the consumer tasks on a different node than the producer tasks
however , if the mpirun scheduler placed the tasks by - slot ( in a contiguous block ) then we need to adjust the shift by ppn */
2019-07-26 17:55:24 +03:00
nstride * = tasksPerNode ;
}
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , 5 , " Shifting ranks by %d for each phase. " , nstride ) ;
2017-06-28 19:42:20 +03:00
}
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , - 1 , " main (after display_freespace): testdirpath is '%s' " , testdirpath ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
if ( random_seed > 0 ) {
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " random seed: %d " , random_seed ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2018-07-08 00:39:14 +03:00
if ( gethostname ( hostname , MAX_PATHLEN ) = = - 1 ) {
2017-10-20 00:26:52 +03:00
perror ( " gethostname " ) ;
2018-07-07 12:29:27 +03:00
MPI_Abort ( testComm , 2 ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( last = = 0 ) {
first = size ;
last = size ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* setup summary table for recording results */
2018-07-07 12:29:27 +03:00
summary_table = ( mdtest_results_t * ) malloc ( iterations * sizeof ( mdtest_results_t ) ) ;
2019-08-31 19:28:08 +03:00
memset ( summary_table , 0 , iterations * sizeof ( mdtest_results_t ) ) ;
2018-07-07 12:29:27 +03:00
for ( int i = 0 ; i < iterations ; i + + ) {
for ( int j = 0 ; j < MDTEST_LAST_NUM ; j + + ) {
summary_table [ i ] . rate [ j ] = 0.0 ;
summary_table [ i ] . time [ j ] = 0.0 ;
}
}
2017-10-20 00:26:52 +03:00
if ( summary_table = = NULL ) {
FAIL ( " out of memory " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
sprintf ( base_tree_name , " mdtest_tree.%d " , rank ) ;
} else {
sprintf ( base_tree_name , " mdtest_tree " ) ;
}
/* default use shared directory */
strcpy ( mk_name , " mdtest.shared. " ) ;
strcpy ( stat_name , " mdtest.shared. " ) ;
strcpy ( read_name , " mdtest.shared. " ) ;
strcpy ( rm_name , " mdtest.shared. " ) ;
2018-07-07 12:29:27 +03:00
MPI_Comm_group ( testComm , & worldgroup ) ;
2017-10-20 00:26:52 +03:00
/* Run the tests */
for ( i = first ; i < = last & & i < = size ; i + = stride ) {
range . last = i - 1 ;
MPI_Group_range_incl ( worldgroup , 1 , ( void * ) & range , & testgroup ) ;
2018-07-07 12:29:27 +03:00
MPI_Comm_create ( testComm , testgroup , & testComm ) ;
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
2018-10-17 18:04:49 +03:00
uint64_t items_all = i * items ;
if ( num_dirs_in_tree_calc ) {
items_all * = num_dirs_in_tree_calc ;
}
2017-10-20 00:26:52 +03:00
if ( files_only & & dirs_only ) {
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " %d tasks, " LLU " files/directories " , i , items_all ) ;
2017-10-20 00:26:52 +03:00
} else if ( files_only ) {
if ( ! shared_file ) {
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " %d tasks, " LLU " files " , i , items_all ) ;
2017-10-20 00:26:52 +03:00
}
else {
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " %d tasks, 1 file " , i ) ;
2017-10-20 00:26:52 +03:00
}
} else if ( dirs_only ) {
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " %d tasks, " LLU " directories " , i , items_all ) ;
2017-10-20 00:26:52 +03:00
}
}
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " " ) ;
VERBOSE ( 1 , - 1 , " Operation Duration Rate " ) ;
VERBOSE ( 1 , - 1 , " --------- -------- ---- " ) ;
2017-10-20 00:26:52 +03:00
2018-07-07 12:29:27 +03:00
for ( j = 0 ; j < iterations ; j + + ) {
2018-07-08 01:56:39 +03:00
// keep track of the current status for stonewalling
mdtest_iteration ( i , j , testgroup , & summary_table [ j ] ) ;
2017-10-20 00:26:52 +03:00
}
2019-08-01 19:54:11 +03:00
if ( print_rate_and_time ) {
summarize_results ( iterations , 0 ) ;
summarize_results ( iterations , 1 ) ;
} else {
summarize_results ( iterations , print_time ) ;
}
2017-10-20 00:26:52 +03:00
if ( i = = 1 & & stride > 1 ) {
i = 0 ;
}
2017-06-28 19:42:20 +03:00
}
2019-08-01 20:29:32 +03:00
if ( verification_error ) {
VERBOSE ( 0 , - 1 , " \n ERROR: verifying the data read! Take the performance values with care! \n " ) ;
2017-06-28 19:42:20 +03:00
}
2019-07-27 22:22:15 +03:00
VERBOSE ( 0 , - 1 , " -- finished at %s -- \n " , PrintTimestamp ( ) ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( random_seed > 0 ) {
free ( rand_array ) ;
2017-06-28 19:42:20 +03:00
}
2018-08-30 00:14:19 +03:00
if ( backend - > finalize )
backend - > finalize ( NULL ) ;
2018-06-13 21:37:37 +03:00
2018-07-07 12:29:27 +03:00
return summary_table ;
2017-06-28 19:42:20 +03:00
}