2017-06-28 19:42:20 +03:00
/*
* Copyright ( C ) 2003 , The Regents of the University of California .
* Produced at the Lawrence Livermore National Laboratory .
* Written by Christopher J . Morrone < morrone @ llnl . gov > ,
* Bill Loewe < loewe @ loewe . net > , Tyce McLarty < mclarty @ llnl . gov > ,
* and Ryan Kroiss < rrkroiss @ lanl . gov > .
* All rights reserved .
* UCRL - CODE - 155800
*
* Please read the COPYRIGHT file .
*
* This program is free software ; you can redistribute it and / or modify
* it under the terms of the GNU General Public License ( as published by
* the Free Software Foundation ) version 2 , dated June 1991.
*
* This program is distributed in the hope that it will be useful ,
* but WITHOUT ANY WARRANTY ; without even the IMPLIED WARRANTY OF
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
* terms and conditions of the GNU General Public License for more details .
*
* You should have received a copy of the GNU General Public License
* along with this program ; if not , write to the Free Software
* Foundation , Inc . , 59 Temple Place , Suite 330 , Boston , MA 02111 - 1307 USA
*
* CVS info :
* $ RCSfile : mdtest . c , v $
* $ Revision : 1.4 $
* $ Date : 2013 / 11 / 27 17 : 05 : 31 $
* $ Author : brettkettering $
*/
# include <limits.h>
# include <math.h>
# include <stdio.h>
# include <stdlib.h>
2017-10-20 00:26:52 +03:00
# include <stdbool.h>
# include <inttypes.h>
2017-06-28 19:42:20 +03:00
# include <sys/types.h>
# include <sys/stat.h>
2019-07-27 22:22:15 +03:00
# include <stdarg.h>
2017-06-28 19:42:20 +03:00
2018-07-12 20:09:13 +03:00
# include "option.h"
2018-07-07 12:29:27 +03:00
# include "utilities.h"
2018-07-07 10:41:33 +03:00
2017-10-20 00:26:52 +03:00
# if HAVE_SYS_PARAM_H
2017-06-28 19:42:20 +03:00
# include <sys/param.h>
2017-10-20 00:26:52 +03:00
# endif
# if HAVE_SYS_MOUNT_H
2017-06-28 19:42:20 +03:00
# include <sys/mount.h>
2017-10-20 00:26:52 +03:00
# endif
# if HAVE_SYS_STATFS_H
2017-06-28 19:42:20 +03:00
# include <sys/statfs.h>
# endif
2017-10-20 00:26:52 +03:00
# if HAVE_SYS_STATVFS_H
2017-06-28 19:42:20 +03:00
# include <sys/statvfs.h>
# endif
# include <fcntl.h>
# include <string.h>
2018-10-08 23:47:28 +03:00
# if HAVE_STRINGS_H
# include <strings.h>
# endif
2017-06-28 19:42:20 +03:00
# include <unistd.h>
# include <dirent.h>
# include <errno.h>
# include <time.h>
# include <sys/time.h>
2017-10-20 00:26:52 +03:00
# include "aiori.h"
# include "ior.h"
2018-07-07 12:29:27 +03:00
# include "mdtest.h"
2017-10-20 00:26:52 +03:00
# include <mpi.h>
2017-06-28 19:42:20 +03:00
2020-11-27 18:23:32 +03:00
# pragma GCC diagnostic ignored "-Wformat-overflow"
2019-09-30 14:25:59 +03:00
# ifdef HAVE_LUSTRE_LUSTREAPI
2019-09-03 08:26:12 +03:00
# include <lustre/lustreapi.h>
2019-09-30 14:25:59 +03:00
# endif /* HAVE_LUSTRE_LUSTREAPI */
2019-09-03 08:26:12 +03:00
2017-06-28 19:42:20 +03:00
# define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH
# define DIRMODE S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH
2019-05-16 23:25:34 +03:00
# define RELEASE_VERS META_VERSION
2019-08-03 11:12:48 +03:00
# define TEST_DIR "test-dir"
2017-06-28 19:42:20 +03:00
# define ITEM_COUNT 25000
2018-07-07 12:29:27 +03:00
# define LLU "%lu"
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
typedef struct {
int size ;
uint64_t * rand_array ;
char testdir [ MAX_PATHLEN ] ;
char testdirpath [ MAX_PATHLEN ] ;
char base_tree_name [ MAX_PATHLEN ] ;
char * * filenames ;
char hostname [ MAX_PATHLEN ] ;
char mk_name [ MAX_PATHLEN ] ;
char stat_name [ MAX_PATHLEN ] ;
char read_name [ MAX_PATHLEN ] ;
char rm_name [ MAX_PATHLEN ] ;
char unique_mk_dir [ MAX_PATHLEN ] ;
char unique_chdir_dir [ MAX_PATHLEN ] ;
char unique_stat_dir [ MAX_PATHLEN ] ;
char unique_read_dir [ MAX_PATHLEN ] ;
char unique_rm_dir [ MAX_PATHLEN ] ;
char unique_rm_uni_dir [ MAX_PATHLEN ] ;
char * write_buffer ;
char * stoneWallingStatusFile ;
int barriers ;
int create_only ;
int stat_only ;
int read_only ;
int verify_read ;
int verify_write ;
int verification_error ;
int remove_only ;
2021-01-21 17:10:23 +03:00
int rename_dirs ;
2020-11-27 20:49:45 +03:00
int leaf_only ;
unsigned branch_factor ;
int depth ;
2021-01-22 15:24:33 +03:00
int random_buffer_offset ; /* user settable value, otherwise random */
2020-11-27 20:49:45 +03:00
/*
* This is likely a small value , but it ' s sometimes computed by
* branch_factor ^ ( depth + 1 ) , so we ' ll make it a larger variable ,
* just in case .
*/
uint64_t num_dirs_in_tree ;
/*
* As we start moving towards Exascale , we could have billions
* of files in a directory . Make room for that possibility with
* a larger variable .
*/
uint64_t items ;
uint64_t items_per_dir ;
uint64_t num_dirs_in_tree_calc ; /* this is a workaround until the overal code is refactored */
int directory_loops ;
int print_time ;
int print_rate_and_time ;
int print_all_proc ;
int random_seed ;
int shared_file ;
int files_only ;
int dirs_only ;
int pre_delay ;
int unique_dir_per_task ;
int time_unique_dir_overhead ;
int collective_creates ;
size_t write_bytes ;
int stone_wall_timer_seconds ;
size_t read_bytes ;
int sync_file ;
int call_sync ;
int path_count ;
int nstride ; /* neighbor stride */
int make_node ;
# ifdef HAVE_LUSTRE_LUSTREAPI
int global_dir_layout ;
# endif /* HAVE_LUSTRE_LUSTREAPI */
2021-02-11 12:57:22 +03:00
char * saveRankDetailsCSV ; /* save the details about the performance to a file */
2020-11-27 20:49:45 +03:00
mdtest_results_t * summary_table ;
pid_t pid ;
uid_t uid ;
/* Use the POSIX backend by default */
const ior_aiori_t * backend ;
void * backend_options ;
aiori_xfer_hint_t hints ;
char * api ;
} mdtest_options_t ;
static mdtest_options_t o ;
2017-10-20 00:26:52 +03:00
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
/* This structure describes the processing status for stonewalling */
typedef struct {
double start_time ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
int stone_wall_timer_seconds ;
2017-06-28 19:42:20 +03:00
2018-07-14 18:46:24 +03:00
uint64_t items_start ;
uint64_t items_done ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
uint64_t items_per_dir ;
} rank_progress_t ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
# define CHECK_STONE_WALL(p) (((p)->stone_wall_timer_seconds != 0) && ((GetTimeStamp() - (p)->start_time) > (p)->stone_wall_timer_seconds))
2017-06-28 19:42:20 +03:00
/* for making/removing unique directory && stating/deleting subdirectory */
enum { MK_UNI_DIR , STAT_SUB_DIR , READ_SUB_DIR , RM_SUB_DIR , RM_UNI_DIR } ;
2019-07-27 22:22:15 +03:00
/* a helper function for passing debug and verbose messages.
use the MACRO as it will insert __LINE__ for you .
Pass the verbose level for root to print , then the verbose level for anyone to print .
Pass - 1 to suppress the print for anyone .
Then do the standard printf stuff . This function adds the newline for you .
*/
# define VERBOSE(root,any,...) VerboseMessage(root,any,__LINE__,__VA_ARGS__)
void VerboseMessage ( int root_level , int any_level , int line , char * format , . . . ) {
if ( ( rank = = 0 & & verbose > = root_level ) | | ( any_level > 0 & & verbose > = any_level ) ) {
char buffer [ 1024 ] ;
va_list args ;
va_start ( args , format ) ;
vsnprintf ( buffer , 1024 , format , args ) ;
va_end ( args ) ;
2019-07-27 23:31:49 +03:00
if ( root_level = = 0 & & any_level = = - 1 ) {
/* No header when it is just the standard output */
fprintf ( out_logfile , " %s \n " , buffer ) ;
} else {
/* add a header when the verbose is greater than 0 */
fprintf ( out_logfile , " V-%d: Rank %3d Line %5d %s \n " , root_level , rank , line , buffer ) ;
}
fflush ( out_logfile ) ;
2019-07-27 22:22:15 +03:00
}
}
2017-06-28 19:42:20 +03:00
void offset_timers ( double * t , int tcount ) {
2017-10-20 00:26:52 +03:00
double toffset ;
int i ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " V-1: Entering offset_timers... " ) ;
2017-06-28 19:42:20 +03:00
2018-10-17 17:38:35 +03:00
toffset = GetTimeStamp ( ) - t [ tcount ] ;
2017-10-20 00:26:52 +03:00
for ( i = 0 ; i < tcount + 1 ; i + + ) {
t [ i ] + = toffset ;
}
2017-06-28 19:42:20 +03:00
}
void parse_dirpath ( char * dirpath_arg ) {
2017-10-20 00:26:52 +03:00
char * tmp , * token ;
char delimiter_string [ 3 ] = { ' @ ' , ' \n ' , ' \0 ' } ;
int i = 0 ;
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering parse_dirpath on %s... " , dirpath_arg ) ;
2017-10-20 00:26:52 +03:00
tmp = dirpath_arg ;
2020-11-27 20:49:45 +03:00
if ( * tmp ! = ' \0 ' ) o . path_count + + ;
2017-10-20 00:26:52 +03:00
while ( * tmp ! = ' \0 ' ) {
if ( * tmp = = ' @ ' ) {
2020-11-27 20:49:45 +03:00
o . path_count + + ;
2017-10-20 00:26:52 +03:00
}
tmp + + ;
}
2019-01-24 03:02:58 +03:00
// prevent changes to the original dirpath_arg
dirpath_arg = strdup ( dirpath_arg ) ;
2021-01-20 18:19:13 +03:00
o . filenames = ( char * * ) safeMalloc ( o . path_count * sizeof ( char * * ) ) ;
2017-10-20 00:26:52 +03:00
token = strtok ( dirpath_arg , delimiter_string ) ;
while ( token ! = NULL ) {
2020-11-27 20:49:45 +03:00
o . filenames [ i ] = token ;
2017-10-20 00:26:52 +03:00
token = strtok ( NULL , delimiter_string ) ;
i + + ;
}
2017-06-28 19:42:20 +03:00
}
2018-07-26 12:37:01 +03:00
static void prep_testdir ( int j , int dir_iter ) {
2020-11-27 20:49:45 +03:00
int pos = sprintf ( o . testdir , " %s " , o . testdirpath ) ;
if ( o . testdir [ strlen ( o . testdir ) - 1 ] ! = ' / ' ) {
pos + = sprintf ( & o . testdir [ pos ] , " / " ) ;
2018-07-26 12:37:01 +03:00
}
2020-11-27 20:49:45 +03:00
pos + = sprintf ( & o . testdir [ pos ] , " %s " , TEST_DIR ) ;
pos + = sprintf ( & o . testdir [ pos ] , " .%d-%d " , j , dir_iter ) ;
2018-07-26 12:37:01 +03:00
}
2019-08-26 20:57:14 +03:00
static void phase_end ( ) {
2020-11-27 20:49:45 +03:00
if ( o . call_sync ) {
if ( ! o . backend - > sync ) {
2020-05-28 23:18:44 +03:00
FAIL ( " Error, backend does not provide the sync method, but you requested to use sync. \n " ) ;
2019-09-01 17:47:42 +03:00
}
2020-11-27 20:49:45 +03:00
o . backend - > sync ( o . backend_options ) ;
2019-08-26 20:57:14 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . barriers ) {
2019-08-26 20:57:14 +03:00
MPI_Barrier ( testComm ) ;
}
}
2017-06-28 19:42:20 +03:00
/*
* This function copies the unique directory name for a given option to
* the " to " parameter . Some memory must be allocated to the " to " parameter .
*/
void unique_dir_access ( int opt , char * to ) {
2017-10-20 00:26:52 +03:00
if ( opt = = MK_UNI_DIR ) {
MPI_Barrier ( testComm ) ;
2020-11-27 20:49:45 +03:00
sprintf ( to , " %s/%s " , o . testdir , o . unique_chdir_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = STAT_SUB_DIR ) {
2020-11-27 20:49:45 +03:00
sprintf ( to , " %s/%s " , o . testdir , o . unique_stat_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = READ_SUB_DIR ) {
2020-11-27 20:49:45 +03:00
sprintf ( to , " %s/%s " , o . testdir , o . unique_read_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = RM_SUB_DIR ) {
2020-11-27 20:49:45 +03:00
sprintf ( to , " %s/%s " , o . testdir , o . unique_rm_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = RM_UNI_DIR ) {
2020-11-27 20:49:45 +03:00
sprintf ( to , " %s/%s " , o . testdir , o . unique_rm_uni_dir ) ;
2017-10-20 00:26:52 +03:00
}
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering unique_dir_access, set it to %s " , to ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
static void create_remove_dirs ( const char * path , bool create , uint64_t itemNum ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
const char * operation = create ? " create " : " remove " ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
if ( ( itemNum % ITEM_COUNT = = 0 & & ( itemNum ! = 0 ) ) ) {
VERBOSE ( 3 , 5 , " dir: " LLU " " , operation , itemNum ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//create dirs
2020-11-27 20:49:45 +03:00
sprintf ( curr_item , " %s/dir.%s% " PRIu64 , path , create ? o . mk_name : o . rm_name , itemNum ) ;
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper (dirs %s): curr_item is '%s' " , operation , curr_item ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( create ) {
2020-11-27 20:49:45 +03:00
if ( o . backend - > mkdir ( curr_item , DIRMODE , o . backend_options ) = = - 1 ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " unable to create directory %s " , curr_item ) ;
2017-10-20 00:26:52 +03:00
}
} else {
2020-11-27 20:49:45 +03:00
if ( o . backend - > rmdir ( curr_item , o . backend_options ) = = - 1 ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " unable to remove directory %s " , curr_item ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
static void remove_file ( const char * path , uint64_t itemNum ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
if ( ( itemNum % ITEM_COUNT = = 0 & & ( itemNum ! = 0 ) ) ) {
VERBOSE ( 3 , 5 , " remove file: " LLU " \n " , itemNum ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//remove files
2020-11-27 20:49:45 +03:00
sprintf ( curr_item , " %s/file.%s " LLU " " , path , o . rm_name , itemNum ) ;
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper (non-dirs remove): curr_item is '%s' " , curr_item ) ;
2020-11-27 20:49:45 +03:00
if ( ! ( o . shared_file & & rank ! = 0 ) ) {
o . backend - > delete ( curr_item , o . backend_options ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2020-06-29 22:15:14 +03:00
2017-10-20 00:26:52 +03:00
static void create_file ( const char * path , uint64_t itemNum ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2020-05-31 14:11:00 +03:00
aiori_fd_t * aiori_fh = NULL ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
if ( ( itemNum % ITEM_COUNT = = 0 & & ( itemNum ! = 0 ) ) ) {
VERBOSE ( 3 , 5 , " create file: " LLU " " , itemNum ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//create files
2020-11-27 20:49:45 +03:00
sprintf ( curr_item , " %s/file.%s " LLU " " , path , o . mk_name , itemNum ) ;
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper (non-dirs create): curr_item is '%s' " , curr_item ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( o . make_node ) {
2019-11-04 19:10:36 +03:00
int ret ;
VERBOSE ( 3 , 5 , " create_remove_items_helper : mknod... " ) ;
2020-11-27 20:49:45 +03:00
ret = o . backend - > mknod ( curr_item ) ;
2019-11-04 19:10:36 +03:00
if ( ret ! = 0 )
2021-01-20 18:19:13 +03:00
EWARNF ( " unable to mknode file %s " , curr_item ) ;
2019-11-04 19:10:36 +03:00
return ;
2020-11-27 20:49:45 +03:00
} else if ( o . collective_creates ) {
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper (collective): open... " ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
aiori_fh = o . backend - > open ( curr_item , IOR_WRONLY | IOR_CREAT , o . backend_options ) ;
2021-01-20 18:19:13 +03:00
if ( NULL = = aiori_fh ) {
EWARNF ( " unable to open file %s " , curr_item ) ;
return ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/*
* ! collective_creates
*/
} else {
2020-11-27 20:49:45 +03:00
o . hints . filePerProc = ! o . shared_file ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper (non-collective, shared): open... " ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
aiori_fh = o . backend - > create ( curr_item , IOR_WRONLY | IOR_CREAT , o . backend_options ) ;
2021-01-20 18:19:13 +03:00
if ( NULL = = aiori_fh ) {
EWARNF ( " unable to create file %s " , curr_item ) ;
return ;
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( o . write_bytes > 0 ) {
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper: write... " ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/*
* According to Bill Loewe , writes are only done one time , so they are always at
* offset 0 ( zero ) .
*/
2020-11-27 20:49:45 +03:00
o . hints . fsyncPerWrite = o . sync_file ;
2021-01-22 17:05:58 +03:00
update_write_memory_pattern ( itemNum , o . write_buffer , o . write_bytes , o . random_buffer_offset , rank ) ;
2020-11-27 20:49:45 +03:00
if ( o . write_bytes ! = ( size_t ) o . backend - > xfer ( WRITE , aiori_fh , ( IOR_size_t * ) o . write_buffer , o . write_bytes , 0 , o . backend_options ) ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " unable to write file %s " , curr_item ) ;
2017-06-28 19:42:20 +03:00
}
2020-06-29 22:15:14 +03:00
2020-11-27 20:49:45 +03:00
if ( o . verify_write ) {
o . write_buffer [ 0 ] = 42 ;
if ( o . write_bytes ! = ( size_t ) o . backend - > xfer ( READ , aiori_fh , ( IOR_size_t * ) o . write_buffer , o . write_bytes , 0 , o . backend_options ) ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " unable to verify write (read/back) file %s " , curr_item ) ;
2020-06-29 22:15:14 +03:00
}
2021-01-22 17:05:58 +03:00
o . verification_error + = verify_memory_pattern ( itemNum , o . write_buffer , o . write_bytes , o . random_buffer_offset , rank ) ;
2020-06-29 22:15:14 +03:00
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " create_remove_items_helper: close... " ) ;
2020-11-27 20:49:45 +03:00
o . backend - > close ( aiori_fh , o . backend_options ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* helper for creating/removing items */
void create_remove_items_helper ( const int dirs , const int create , const char * path ,
2018-07-07 12:29:27 +03:00
uint64_t itemNum , rank_progress_t * progress ) {
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering create_remove_items_helper on %s " , path ) ;
2017-06-28 19:42:20 +03:00
2018-07-08 01:56:39 +03:00
for ( uint64_t i = progress - > items_start ; i < progress - > items_per_dir ; + + i ) {
2017-10-20 00:26:52 +03:00
if ( ! dirs ) {
if ( create ) {
create_file ( path , itemNum + i ) ;
} else {
remove_file ( path , itemNum + i ) ;
}
2017-06-28 19:42:20 +03:00
} else {
2017-10-20 00:26:52 +03:00
create_remove_dirs ( path , create , itemNum + i ) ;
2017-06-28 19:42:20 +03:00
}
2018-07-07 12:29:27 +03:00
if ( CHECK_STONE_WALL ( progress ) ) {
2018-12-20 00:37:37 +03:00
if ( progress - > items_done = = 0 ) {
progress - > items_done = i + 1 ;
2018-12-19 23:29:47 +03:00
}
2018-07-07 12:29:27 +03:00
return ;
}
2017-10-20 00:26:52 +03:00
}
2018-07-08 01:56:39 +03:00
progress - > items_done = progress - > items_per_dir ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* helper function to do collective operations */
2018-07-07 12:29:27 +03:00
void collective_helper ( const int dirs , const int create , const char * path , uint64_t itemNum , rank_progress_t * progress ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering collective_helper on %s " , path ) ;
2018-07-08 01:56:39 +03:00
for ( uint64_t i = progress - > items_start ; i < progress - > items_per_dir ; + + i ) {
2017-10-20 00:26:52 +03:00
if ( dirs ) {
create_remove_dirs ( path , create , itemNum + i ) ;
continue ;
2017-06-28 19:42:20 +03:00
}
2020-11-27 20:49:45 +03:00
sprintf ( curr_item , " %s/file.%s " LLU " " , path , create ? o . mk_name : o . rm_name , itemNum + i ) ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " create file: %s " , curr_item ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( create ) {
2020-05-31 14:11:00 +03:00
aiori_fd_t * aiori_fh ;
2017-10-20 00:26:52 +03:00
//create files
2020-11-27 20:49:45 +03:00
aiori_fh = o . backend - > create ( curr_item , IOR_WRONLY | IOR_CREAT , o . backend_options ) ;
2017-10-20 00:26:52 +03:00
if ( NULL = = aiori_fh ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " unable to create file %s " , curr_item ) ;
} else {
o . backend - > close ( aiori_fh , o . backend_options ) ;
2017-06-28 19:42:20 +03:00
}
2020-11-27 20:49:45 +03:00
} else if ( ! ( o . shared_file & & rank ! = 0 ) ) {
2017-10-20 00:26:52 +03:00
//remove files
2020-11-27 20:49:45 +03:00
o . backend - > delete ( curr_item , o . backend_options ) ;
2017-06-28 19:42:20 +03:00
}
2018-07-07 12:29:27 +03:00
if ( CHECK_STONE_WALL ( progress ) ) {
progress - > items_done = i + 1 ;
return ;
}
2017-06-28 19:42:20 +03:00
}
2018-07-08 01:56:39 +03:00
progress - > items_done = progress - > items_per_dir ;
2017-06-28 19:42:20 +03:00
}
2020-07-03 10:09:40 +03:00
/* recursive function to create and remove files/directories from the
2017-10-20 00:26:52 +03:00
directory tree */
2018-07-07 12:29:27 +03:00
void create_remove_items ( int currDepth , const int dirs , const int create , const int collective , const char * path , uint64_t dirNum , rank_progress_t * progress ) {
unsigned i ;
2018-07-08 00:39:14 +03:00
char dir [ MAX_PATHLEN ] ;
char temp_path [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
unsigned long long currDir = dirNum ;
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering create_remove_items on %s, currDepth = %d... " , path , currDepth ) ;
2017-06-28 19:42:20 +03:00
2018-07-08 00:39:14 +03:00
memset ( dir , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
strcpy ( temp_path , path ) ;
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " create_remove_items (start): temp_path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( currDepth = = 0 ) {
/* create items at this depth */
2020-11-27 20:49:45 +03:00
if ( ! o . leaf_only | | ( o . depth = = 0 & & o . leaf_only ) ) {
2017-10-20 00:26:52 +03:00
if ( collective ) {
2018-07-07 12:29:27 +03:00
collective_helper ( dirs , create , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-07 12:29:27 +03:00
create_remove_items_helper ( dirs , create , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( o . depth > 0 ) {
2017-10-20 00:26:52 +03:00
create_remove_items ( + + currDepth , dirs , create ,
2018-07-07 12:29:27 +03:00
collective , temp_path , + + dirNum , progress ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
} else if ( currDepth < = o . depth ) {
2017-10-20 00:26:52 +03:00
/* iterate through the branches */
2020-11-27 20:49:45 +03:00
for ( i = 0 ; i < o . branch_factor ; i + + ) {
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* determine the current branch and append it to the path */
2020-11-27 20:49:45 +03:00
sprintf ( dir , " %s.%llu/ " , o . base_tree_name , currDir ) ;
2017-10-20 00:26:52 +03:00
strcat ( temp_path , " / " ) ;
strcat ( temp_path , dir ) ;
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " create_remove_items (for loop): temp_path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create the items in this branch */
2020-11-27 20:49:45 +03:00
if ( ! o . leaf_only | | ( o . leaf_only & & currDepth = = o . depth ) ) {
2017-10-20 00:26:52 +03:00
if ( collective ) {
2020-11-27 20:49:45 +03:00
collective_helper ( dirs , create , temp_path , currDir * o . items_per_dir , progress ) ;
2017-10-20 00:26:52 +03:00
} else {
2020-11-27 20:49:45 +03:00
create_remove_items_helper ( dirs , create , temp_path , currDir * o . items_per_dir , progress ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* make the recursive call for the next level below this branch */
create_remove_items (
+ + currDepth ,
dirs ,
create ,
collective ,
temp_path ,
2020-11-27 20:49:45 +03:00
( currDir * ( unsigned long long ) o . branch_factor ) + 1 ,
2018-07-07 12:29:27 +03:00
progress
) ;
2017-10-20 00:26:52 +03:00
currDepth - - ;
/* reset the path */
strcpy ( temp_path , path ) ;
currDir + + ;
}
}
2017-06-28 19:42:20 +03:00
}
/* stats all of the items created as specified by the input parameters */
2018-07-26 12:37:01 +03:00
void mdtest_stat ( const int random , const int dirs , const long dir_iter , const char * path , rank_progress_t * progress ) {
2017-10-20 00:26:52 +03:00
struct stat buf ;
uint64_t parent_dir , item_num = 0 ;
2018-07-08 00:39:14 +03:00
char item [ MAX_PATHLEN ] , temp [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering mdtest_stat on %s " , path ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
uint64_t stop_items = o . items ;
2018-07-26 12:37:01 +03:00
2020-11-27 20:49:45 +03:00
if ( o . directory_loops ! = 1 ) {
stop_items = o . items_per_dir ;
2017-10-20 00:26:52 +03:00
}
/* iterate over all of the item IDs */
2018-07-26 12:37:01 +03:00
for ( uint64_t i = 0 ; i < stop_items ; + + i ) {
2017-10-20 00:26:52 +03:00
/*
* It doesn ' t make sense to pass the address of the array because that would
* be like passing char * * . Tested it on a Cray and it seems to work either
* way , but it seems that it is correct without the " & " .
*
2018-07-08 00:39:14 +03:00
memset ( & item , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
*/
2018-07-08 00:39:14 +03:00
memset ( item , 0 , MAX_PATHLEN ) ;
memset ( temp , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
/* determine the item number to stat */
if ( random ) {
2020-11-27 20:49:45 +03:00
item_num = o . rand_array [ i ] ;
2017-10-20 00:26:52 +03:00
} else {
item_num = i ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* make adjustments if in leaf only mode*/
2020-11-27 20:49:45 +03:00
if ( o . leaf_only ) {
item_num + = o . items_per_dir *
( o . num_dirs_in_tree - ( uint64_t ) pow ( o . branch_factor , o . depth ) ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create name of file/dir to stat */
if ( dirs ) {
2019-07-28 18:55:00 +03:00
if ( ( i % ITEM_COUNT = = 0 ) & & ( i ! = 0 ) ) {
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " stat dir: " LLU " " , i ) ;
2017-10-20 00:26:52 +03:00
}
2020-11-27 20:49:45 +03:00
sprintf ( item , " dir.%s " LLU " " , o . stat_name , item_num ) ;
2017-10-20 00:26:52 +03:00
} else {
2019-07-28 18:55:00 +03:00
if ( ( i % ITEM_COUNT = = 0 ) & & ( i ! = 0 ) ) {
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " stat file: " LLU " " , i ) ;
2017-10-20 00:26:52 +03:00
}
2020-11-27 20:49:45 +03:00
sprintf ( item , " file.%s " LLU " " , o . stat_name , item_num ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* determine the path to the file/dir to be stat'ed */
2020-11-27 20:49:45 +03:00
parent_dir = item_num / o . items_per_dir ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( parent_dir > 0 ) { //item is not in tree's root directory
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* prepend parent directory to item's path */
2020-11-27 20:49:45 +03:00
sprintf ( temp , " %s. " LLU " /%s " , o . base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//still not at the tree's root dir
2020-11-27 20:49:45 +03:00
while ( parent_dir > o . branch_factor ) {
parent_dir = ( uint64_t ) ( ( parent_dir - 1 ) / o . branch_factor ) ;
sprintf ( temp , " %s. " LLU " /%s " , o . base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* Now get item to have the full path */
sprintf ( temp , " %s/%s " , path , item ) ;
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* below temp used to be hiername */
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " mdtest_stat %4s: %s " , ( dirs ? " dir " : " file " ) , item ) ;
2020-11-27 20:49:45 +03:00
if ( - 1 = = o . backend - > stat ( item , & buf , o . backend_options ) ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " unable to stat %s %s " , dirs ? " directory " : " file " , item ) ;
2017-06-28 19:42:20 +03:00
}
}
}
/* reads all of the items created as specified by the input parameters */
2018-07-26 12:37:01 +03:00
void mdtest_read ( int random , int dirs , const long dir_iter , char * path ) {
2018-07-14 18:46:24 +03:00
uint64_t parent_dir , item_num = 0 ;
2018-07-08 00:39:14 +03:00
char item [ MAX_PATHLEN ] , temp [ MAX_PATHLEN ] ;
2020-05-31 14:11:00 +03:00
aiori_fd_t * aiori_fh ;
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering mdtest_read on %s " , path ) ;
2020-06-29 22:15:14 +03:00
char * read_buffer ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* allocate read buffer */
2020-11-27 20:49:45 +03:00
if ( o . read_bytes > 0 ) {
int alloc_res = posix_memalign ( ( void * * ) & read_buffer , sysconf ( _SC_PAGESIZE ) , o . read_bytes ) ;
2020-06-03 00:30:38 +03:00
if ( alloc_res ) {
2017-10-20 00:26:52 +03:00
FAIL ( " out of memory " ) ;
}
2021-01-20 00:23:30 +03:00
memset ( read_buffer , - 1 , o . read_bytes ) ;
2017-06-28 19:42:20 +03:00
}
2020-11-27 20:49:45 +03:00
uint64_t stop_items = o . items ;
2018-07-26 12:37:01 +03:00
2020-11-27 20:49:45 +03:00
if ( o . directory_loops ! = 1 ) {
stop_items = o . items_per_dir ;
2017-10-20 00:26:52 +03:00
}
/* iterate over all of the item IDs */
2018-07-26 12:37:01 +03:00
for ( uint64_t i = 0 ; i < stop_items ; + + i ) {
2017-10-20 00:26:52 +03:00
/*
* It doesn ' t make sense to pass the address of the array because that would
* be like passing char * * . Tested it on a Cray and it seems to work either
* way , but it seems that it is correct without the " & " .
*
* NTH : Both are technically correct in C .
*
2018-07-08 00:39:14 +03:00
* memset ( & item , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
*/
2018-07-08 00:39:14 +03:00
memset ( item , 0 , MAX_PATHLEN ) ;
memset ( temp , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
/* determine the item number to read */
if ( random ) {
2020-11-27 20:49:45 +03:00
item_num = o . rand_array [ i ] ;
2017-10-20 00:26:52 +03:00
} else {
item_num = i ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* make adjustments if in leaf only mode*/
2020-11-27 20:49:45 +03:00
if ( o . leaf_only ) {
item_num + = o . items_per_dir *
( o . num_dirs_in_tree - ( uint64_t ) pow ( o . branch_factor , o . depth ) ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create name of file to read */
if ( ! dirs ) {
2019-07-27 22:22:15 +03:00
if ( ( i % ITEM_COUNT = = 0 ) & & ( i ! = 0 ) ) {
VERBOSE ( 3 , 5 , " read file: " LLU " " , i ) ;
2017-10-20 00:26:52 +03:00
}
2020-11-27 20:49:45 +03:00
sprintf ( item , " file.%s " LLU " " , o . read_name , item_num ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* determine the path to the file/dir to be read'ed */
2020-11-27 20:49:45 +03:00
parent_dir = item_num / o . items_per_dir ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( parent_dir > 0 ) { //item is not in tree's root directory
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* prepend parent directory to item's path */
2020-11-27 20:49:45 +03:00
sprintf ( temp , " %s. " LLU " /%s " , o . base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* still not at the tree's root dir */
2020-11-27 20:49:45 +03:00
while ( parent_dir > o . branch_factor ) {
parent_dir = ( unsigned long long ) ( ( parent_dir - 1 ) / o . branch_factor ) ;
sprintf ( temp , " %s. " LLU " /%s " , o . base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* Now get item to have the full path */
sprintf ( temp , " %s/%s " , path , item ) ;
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* below temp used to be hiername */
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " mdtest_read file: %s " , item ) ;
2017-10-20 00:26:52 +03:00
/* open file for reading */
2020-11-27 20:49:45 +03:00
aiori_fh = o . backend - > open ( item , O_RDONLY , o . backend_options ) ;
2017-10-20 00:26:52 +03:00
if ( NULL = = aiori_fh ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " unable to open file %s " , item ) ;
continue ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read file */
2020-11-27 20:49:45 +03:00
if ( o . read_bytes > 0 ) {
2020-06-29 22:15:14 +03:00
read_buffer [ 0 ] = 42 ;
2020-11-27 20:49:45 +03:00
if ( o . read_bytes ! = ( size_t ) o . backend - > xfer ( READ , aiori_fh , ( IOR_size_t * ) read_buffer , o . read_bytes , 0 , o . backend_options ) ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " unable to read file %s " , item ) ;
continue ;
2017-10-20 00:26:52 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . verify_read ) {
2021-01-22 15:10:09 +03:00
int pretend_rank = ( 2 * o . nstride + rank ) % o . size ;
if ( o . shared_file ) {
pretend_rank = rank ;
}
2021-01-22 17:05:58 +03:00
o . verification_error + = verify_memory_pattern ( item_num , read_buffer , o . read_bytes , o . random_buffer_offset , pretend_rank ) ;
2020-11-27 20:49:45 +03:00
} else if ( ( o . read_bytes > = 8 & & ( ( uint64_t * ) read_buffer ) [ 0 ] ! = item_num ) | | ( o . read_bytes < 8 & & read_buffer [ 0 ] ! = ( char ) item_num ) ) {
2020-06-29 22:15:14 +03:00
// do a lightweight check, which cost is neglectable
2020-11-27 20:49:45 +03:00
o . verification_error + + ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* close file */
2020-11-27 20:49:45 +03:00
o . backend - > close ( aiori_fh , o . backend_options ) ;
2017-06-28 19:42:20 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . read_bytes ) {
2020-06-29 22:15:14 +03:00
free ( read_buffer ) ;
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* This method should be called by rank 0. It subsequently does all of
the creates and removes for the other ranks */
2018-07-07 12:29:27 +03:00
void collective_create_remove ( const int create , const int dirs , const int ntasks , const char * path , rank_progress_t * progress ) {
2018-07-08 00:39:14 +03:00
char temp [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " Entering collective_create_remove on %s " , path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* rank 0 does all of the creates and removes for all of the ranks */
for ( int i = 0 ; i < ntasks ; + + i ) {
2018-07-08 00:39:14 +03:00
memset ( temp , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
2020-11-27 20:49:45 +03:00
strcpy ( temp , o . testdir ) ;
2017-10-20 00:26:52 +03:00
strcat ( temp , " / " ) ;
/* set the base tree name appropriately */
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
sprintf ( o . base_tree_name , " mdtest_tree.%d " , i ) ;
2017-10-20 00:26:52 +03:00
} else {
2020-11-27 20:49:45 +03:00
sprintf ( o . base_tree_name , " mdtest_tree " ) ;
2017-10-20 00:26:52 +03:00
}
/* Setup to do I/O to the appropriate test dir */
2020-11-27 20:49:45 +03:00
strcat ( temp , o . base_tree_name ) ;
2017-10-20 00:26:52 +03:00
strcat ( temp , " .0 " ) ;
/* set all item names appropriately */
2020-11-27 20:49:45 +03:00
if ( ! o . shared_file ) {
sprintf ( o . mk_name , " mdtest.%d. " , ( i + ( 0 * o . nstride ) ) % ntasks ) ;
sprintf ( o . stat_name , " mdtest.%d. " , ( i + ( 1 * o . nstride ) ) % ntasks ) ;
sprintf ( o . read_name , " mdtest.%d. " , ( i + ( 2 * o . nstride ) ) % ntasks ) ;
sprintf ( o . rm_name , " mdtest.%d. " , ( i + ( 3 * o . nstride ) ) % ntasks ) ;
}
if ( o . unique_dir_per_task ) {
VERBOSE ( 3 , 5 , " i %d nstride %d ntasks %d " , i , o . nstride , ntasks ) ;
sprintf ( o . unique_mk_dir , " %s/mdtest_tree.%d.0 " , o . testdir ,
( i + ( 0 * o . nstride ) ) % ntasks ) ;
sprintf ( o . unique_chdir_dir , " %s/mdtest_tree.%d.0 " , o . testdir ,
( i + ( 1 * o . nstride ) ) % ntasks ) ;
sprintf ( o . unique_stat_dir , " %s/mdtest_tree.%d.0 " , o . testdir ,
( i + ( 2 * o . nstride ) ) % ntasks ) ;
sprintf ( o . unique_read_dir , " %s/mdtest_tree.%d.0 " , o . testdir ,
( i + ( 3 * o . nstride ) ) % ntasks ) ;
sprintf ( o . unique_rm_dir , " %s/mdtest_tree.%d.0 " , o . testdir ,
( i + ( 4 * o . nstride ) ) % ntasks ) ;
sprintf ( o . unique_rm_uni_dir , " %s " , o . testdir ) ;
2017-10-20 00:26:52 +03:00
}
/* Now that everything is set up as it should be, do the create or remove */
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " collective_create_remove (create_remove_items): temp is '%s' " , temp ) ;
2017-10-20 00:26:52 +03:00
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , dirs , create , 1 , temp , 0 , progress ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* reset all of the item names */
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
sprintf ( o . base_tree_name , " mdtest_tree.0 " ) ;
2017-06-28 19:42:20 +03:00
} else {
2020-11-27 20:49:45 +03:00
sprintf ( o . base_tree_name , " mdtest_tree " ) ;
}
if ( ! o . shared_file ) {
sprintf ( o . mk_name , " mdtest.%d. " , ( 0 + ( 0 * o . nstride ) ) % ntasks ) ;
sprintf ( o . stat_name , " mdtest.%d. " , ( 0 + ( 1 * o . nstride ) ) % ntasks ) ;
sprintf ( o . read_name , " mdtest.%d. " , ( 0 + ( 2 * o . nstride ) ) % ntasks ) ;
sprintf ( o . rm_name , " mdtest.%d. " , ( 0 + ( 3 * o . nstride ) ) % ntasks ) ;
}
if ( o . unique_dir_per_task ) {
sprintf ( o . unique_mk_dir , " %s/mdtest_tree.%d.0 " , o . testdir ,
( 0 + ( 0 * o . nstride ) ) % ntasks ) ;
sprintf ( o . unique_chdir_dir , " %s/mdtest_tree.%d.0 " , o . testdir ,
( 0 + ( 1 * o . nstride ) ) % ntasks ) ;
sprintf ( o . unique_stat_dir , " %s/mdtest_tree.%d.0 " , o . testdir ,
( 0 + ( 2 * o . nstride ) ) % ntasks ) ;
sprintf ( o . unique_read_dir , " %s/mdtest_tree.%d.0 " , o . testdir ,
( 0 + ( 3 * o . nstride ) ) % ntasks ) ;
sprintf ( o . unique_rm_dir , " %s/mdtest_tree.%d.0 " , o . testdir ,
( 0 + ( 4 * o . nstride ) ) % ntasks ) ;
sprintf ( o . unique_rm_uni_dir , " %s " , o . testdir ) ;
2017-06-28 19:42:20 +03:00
}
}
2021-01-21 17:10:23 +03:00
void rename_dir_test ( const int dirs , const long dir_iter , const char * path , rank_progress_t * progress ) {
uint64_t parent_dir , item_num = 0 ;
char item [ MAX_PATHLEN ] , temp [ MAX_PATHLEN ] ;
char item_last [ MAX_PATHLEN ] ;
if ( o . backend - > rename = = NULL ) {
WARN ( " Backend doesn't support rename \n " ) ;
return ;
}
VERBOSE ( 1 , - 1 , " Entering mdtest_rename on %s " , path ) ;
uint64_t stop_items = o . items ;
if ( o . directory_loops ! = 1 ) {
stop_items = o . items_per_dir ;
}
if ( stop_items = = 1 ) return ;
/* iterate over all of the item IDs */
char first_item_name [ MAX_PATHLEN ] ;
for ( uint64_t i = 0 ; i < stop_items ; + + i ) {
item_num = i ;
/* make adjustments if in leaf only mode*/
if ( o . leaf_only ) {
item_num + = o . items_per_dir * ( o . num_dirs_in_tree - ( uint64_t ) pow ( o . branch_factor , o . depth ) ) ;
}
/* create name of file/dir to stat */
if ( dirs ) {
sprintf ( item , " dir.%s " LLU " " , o . stat_name , item_num ) ;
} else {
sprintf ( item , " file.%s " LLU " " , o . stat_name , item_num ) ;
}
/* determine the path to the file/dir to be stat'ed */
parent_dir = item_num / o . items_per_dir ;
if ( parent_dir > 0 ) { //item is not in tree's root directory
/* prepend parent directory to item's path */
sprintf ( temp , " %s. " LLU " /%s " , o . base_tree_name , parent_dir , item ) ;
strcpy ( item , temp ) ;
//still not at the tree's root dir
while ( parent_dir > o . branch_factor ) {
parent_dir = ( uint64_t ) ( ( parent_dir - 1 ) / o . branch_factor ) ;
sprintf ( temp , " %s. " LLU " /%s " , o . base_tree_name , parent_dir , item ) ;
strcpy ( item , temp ) ;
}
}
/* Now get item to have the full path */
sprintf ( temp , " %s/%s " , path , item ) ;
strcpy ( item , temp ) ;
VERBOSE ( 3 , 5 , " mdtest_rename %4s: %s " , ( dirs ? " dir " : " file " ) , item ) ;
if ( i = = 0 ) {
sprintf ( first_item_name , " %s-XX " , item ) ;
strcpy ( item_last , first_item_name ) ;
} else if ( i = = stop_items - 1 ) {
strcpy ( item , first_item_name ) ;
}
if ( - 1 = = o . backend - > rename ( item , item_last , o . backend_options ) ) {
EWARNF ( " unable to rename %s %s " , dirs ? " directory " : " file " , item ) ;
}
strcpy ( item_last , item ) ;
}
}
2021-02-11 12:57:22 +03:00
static void updateResult ( mdtest_results_t * res , mdtest_test_num_t test , uint64_t item_count , int t , double * times , double * tBefore ) {
res - > time [ test ] = times [ t ] - times [ t - 1 ] ;
if ( tBefore ) {
res - > time_before_barrier [ test ] = tBefore [ t ] - times [ t - 1 ] ;
2021-02-18 12:53:07 +03:00
} else {
res - > time_before_barrier [ test ] = res - > time [ test ] ;
2021-02-11 12:57:22 +03:00
}
2021-02-18 12:53:07 +03:00
res - > rate [ test ] = item_count / res - > time [ test ] ;
res - > rate_before_barrier [ test ] = item_count / res - > time_before_barrier [ test ] ;
2021-02-11 12:57:22 +03:00
res - > items [ test ] = item_count ;
res - > stonewall_last_item [ test ] = o . items ;
}
2018-07-07 12:29:27 +03:00
void directory_test ( const int iteration , const int ntasks , const char * path , rank_progress_t * progress ) {
2017-10-20 00:26:52 +03:00
int size ;
2021-01-21 17:10:23 +03:00
double t [ 6 ] = { 0 } ;
2021-02-11 12:57:22 +03:00
double tBefore [ 6 ] = { 0 } ;
2018-07-08 00:39:14 +03:00
char temp_path [ MAX_PATHLEN ] ;
2021-01-21 17:10:23 +03:00
mdtest_results_t * res = & o . summary_table [ iteration ] ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
MPI_Comm_size ( testComm , & size ) ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " Entering directory_test on %s " , path ) ;
2017-06-28 19:42:20 +03:00
2021-02-11 12:57:22 +03:00
tBefore [ 0 ] = GetTimeStamp ( ) ;
2017-10-20 00:26:52 +03:00
MPI_Barrier ( testComm ) ;
2018-10-17 17:38:35 +03:00
t [ 0 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create phase */
2020-11-27 20:49:45 +03:00
if ( o . create_only ) {
progress - > stone_wall_timer_seconds = o . stone_wall_timer_seconds ;
2020-07-21 18:16:13 +03:00
progress - > items_done = 0 ;
progress - > start_time = GetTimeStamp ( ) ;
2020-11-27 20:49:45 +03:00
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
2018-07-26 12:37:01 +03:00
prep_testdir ( iteration , dir_iter ) ;
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
2017-10-20 00:26:52 +03:00
unique_dir_access ( MK_UNI_DIR , temp_path ) ;
2020-11-27 20:49:45 +03:00
if ( ! o . time_unique_dir_overhead ) {
2017-10-20 00:26:52 +03:00
offset_timers ( t , 0 ) ;
}
} else {
2020-11-27 20:49:45 +03:00
sprintf ( temp_path , " %s/%s " , o . testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , - 1 , " directory_test: create path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* "touch" the files */
2020-11-27 20:49:45 +03:00
if ( o . collective_creates ) {
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
collective_create_remove ( 1 , 1 , ntasks , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
} else {
/* create directories */
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , 1 , 1 , 0 , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2020-07-21 18:16:13 +03:00
progress - > stone_wall_timer_seconds = 0 ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2021-02-11 12:57:22 +03:00
tBefore [ 1 ] = GetTimeStamp ( ) ;
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 1 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat phase */
2020-11-27 20:49:45 +03:00
if ( o . stat_only ) {
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
2018-07-26 12:37:01 +03:00
prep_testdir ( iteration , dir_iter ) ;
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
2017-10-20 00:26:52 +03:00
unique_dir_access ( STAT_SUB_DIR , temp_path ) ;
2020-11-27 20:49:45 +03:00
if ( ! o . time_unique_dir_overhead ) {
2017-10-20 00:26:52 +03:00
offset_timers ( t , 1 ) ;
}
} else {
2020-11-27 20:49:45 +03:00
sprintf ( temp_path , " %s/%s " , o . testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " stat path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat directories */
2020-11-27 20:49:45 +03:00
if ( o . random_seed > 0 ) {
2018-07-26 12:37:01 +03:00
mdtest_stat ( 1 , 1 , dir_iter , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-26 12:37:01 +03:00
mdtest_stat ( 0 , 1 , dir_iter , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2021-02-11 12:57:22 +03:00
tBefore [ 2 ] = GetTimeStamp ( ) ;
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 2 ] = GetTimeStamp ( ) ;
2021-02-18 12:53:07 +03:00
if ( o . rename_dirs & & o . items > 1 ) { // moved close to execution
updateResult ( res , MDTEST_DIR_RENAME_NUM , o . items , 4 , t , tBefore ) ;
}
2017-10-20 00:26:52 +03:00
/* read phase */
2020-11-27 20:49:45 +03:00
if ( o . read_only ) {
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
2018-07-26 12:37:01 +03:00
prep_testdir ( iteration , dir_iter ) ;
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
2017-10-20 00:26:52 +03:00
unique_dir_access ( READ_SUB_DIR , temp_path ) ;
2020-11-27 20:49:45 +03:00
if ( ! o . time_unique_dir_overhead ) {
2017-10-20 00:26:52 +03:00
offset_timers ( t , 2 ) ;
}
} else {
2020-11-27 20:49:45 +03:00
sprintf ( temp_path , " %s/%s " , o . testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " directory_test: read path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read directories */
2020-11-27 20:49:45 +03:00
if ( o . random_seed > 0 ) {
2017-10-20 00:26:52 +03:00
; /* N/A */
} else {
; /* N/A */
}
2018-07-26 12:37:01 +03:00
}
2017-10-20 00:26:52 +03:00
}
2021-02-11 12:57:22 +03:00
tBefore [ 3 ] = GetTimeStamp ( ) ;
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2021-01-21 17:10:23 +03:00
2018-10-17 17:38:35 +03:00
t [ 3 ] = GetTimeStamp ( ) ;
2021-01-21 17:10:23 +03:00
if ( o . rename_dirs ) {
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
if ( o . unique_dir_per_task ) {
unique_dir_access ( STAT_SUB_DIR , temp_path ) ;
if ( ! o . time_unique_dir_overhead ) {
offset_timers ( t , 1 ) ;
}
} else {
sprintf ( temp_path , " %s/%s " , o . testdir , path ) ;
}
VERBOSE ( 3 , 5 , " rename path is '%s' " , temp_path ) ;
rename_dir_test ( 1 , dir_iter , temp_path , progress ) ;
}
}
2021-02-11 12:57:22 +03:00
tBefore [ 4 ] = GetTimeStamp ( ) ;
2021-01-21 17:10:23 +03:00
phase_end ( ) ;
t [ 4 ] = GetTimeStamp ( ) ;
if ( o . rename_dirs & & o . items > 1 ) { // moved close to execution
2021-02-18 12:53:07 +03:00
updateResult ( res , MDTEST_DIR_RENAME_NUM , o . items , 4 , t , tBefore ) ;
2021-01-21 17:10:23 +03:00
}
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( o . remove_only ) {
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
2018-07-26 12:37:01 +03:00
prep_testdir ( iteration , dir_iter ) ;
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
2017-10-20 00:26:52 +03:00
unique_dir_access ( RM_SUB_DIR , temp_path ) ;
2020-11-27 20:49:45 +03:00
if ( ! o . time_unique_dir_overhead ) {
2017-10-20 00:26:52 +03:00
offset_timers ( t , 3 ) ;
}
} else {
2020-11-27 20:49:45 +03:00
sprintf ( temp_path , " %s/%s " , o . testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " directory_test: remove directories path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* remove directories */
2020-11-27 20:49:45 +03:00
if ( o . collective_creates ) {
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
collective_create_remove ( 0 , 1 , ntasks , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
} else {
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , 1 , 0 , 0 , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2021-02-11 12:57:22 +03:00
tBefore [ 5 ] = GetTimeStamp ( ) ;
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2021-01-21 17:10:23 +03:00
t [ 5 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( o . remove_only ) {
if ( o . unique_dir_per_task ) {
2017-10-20 00:26:52 +03:00
unique_dir_access ( RM_UNI_DIR , temp_path ) ;
} else {
2020-11-27 20:49:45 +03:00
sprintf ( temp_path , " %s/%s " , o . testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " directory_test: remove unique directories path is '%s' \n " , temp_path ) ;
2017-06-28 19:42:20 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task & & ! o . time_unique_dir_overhead ) {
2021-01-21 17:10:23 +03:00
offset_timers ( t , 5 ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* calculate times */
2020-11-27 20:49:45 +03:00
if ( o . create_only ) {
2021-02-18 12:53:07 +03:00
updateResult ( res , MDTEST_DIR_CREATE_NUM , o . items , 1 , t , tBefore ) ;
2020-11-27 20:49:45 +03:00
}
if ( o . stat_only ) {
2021-02-18 12:53:07 +03:00
updateResult ( res , MDTEST_DIR_STAT_NUM , o . items , 2 , t , tBefore ) ;
2020-11-27 20:49:45 +03:00
}
if ( o . read_only ) {
2021-02-18 12:53:07 +03:00
updateResult ( res , MDTEST_DIR_READ_NUM , o . items , 3 , t , tBefore ) ;
2020-11-27 20:49:45 +03:00
}
if ( o . remove_only ) {
2021-02-18 12:53:07 +03:00
updateResult ( res , MDTEST_DIR_REMOVE_NUM , o . items , 5 , t , tBefore ) ;
2020-11-27 20:49:45 +03:00
}
VERBOSE ( 1 , - 1 , " Directory creation: %14.3f sec, %14.3f ops/sec " , t [ 1 ] - t [ 0 ] , o . summary_table [ iteration ] . rate [ 0 ] ) ;
VERBOSE ( 1 , - 1 , " Directory stat : %14.3f sec, %14.3f ops/sec " , t [ 2 ] - t [ 1 ] , o . summary_table [ iteration ] . rate [ 1 ] ) ;
2021-01-21 17:10:23 +03:00
VERBOSE ( 1 , - 1 , " Directory rename : %14.3f sec, %14.3f ops/sec " , t [ 4 ] - t [ 3 ] , o . summary_table [ iteration ] . rate [ MDTEST_DIR_RENAME_NUM ] ) ;
VERBOSE ( 1 , - 1 , " Directory removal : %14.3f sec, %14.3f ops/sec " , t [ 5 ] - t [ 4 ] , o . summary_table [ iteration ] . rate [ 4 ] ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2018-07-08 01:56:39 +03:00
/* Returns if the stonewall was hit */
2020-11-27 18:35:32 +03:00
int updateStoneWallIterations ( int iteration , uint64_t items_done , double tstart , uint64_t * out_max_iter ) {
2018-07-08 01:56:39 +03:00
int hit = 0 ;
long long unsigned max_iter = 0 ;
2019-07-27 22:22:15 +03:00
2020-11-27 18:35:32 +03:00
VERBOSE ( 1 , 1 , " stonewall hit with %lld items " , ( long long ) items_done ) ;
MPI_Allreduce ( & items_done , & max_iter , 1 , MPI_LONG_LONG_INT , MPI_MAX , testComm ) ;
2020-11-27 20:49:45 +03:00
o . summary_table [ iteration ] . stonewall_time [ MDTEST_FILE_CREATE_NUM ] = GetTimeStamp ( ) - tstart ;
2021-02-18 12:53:07 +03:00
o . summary_table [ iteration ] . stonewall_last_item [ MDTEST_FILE_CREATE_NUM ] = items_done ;
2020-11-27 18:35:32 +03:00
* out_max_iter = max_iter ;
2018-07-08 01:56:39 +03:00
// continue to the maximum...
long long min_accessed = 0 ;
2020-11-27 18:35:32 +03:00
MPI_Reduce ( & items_done , & min_accessed , 1 , MPI_LONG_LONG_INT , MPI_MIN , 0 , testComm ) ;
2018-07-08 01:56:39 +03:00
long long sum_accessed = 0 ;
2020-11-27 18:35:32 +03:00
MPI_Reduce ( & items_done , & sum_accessed , 1 , MPI_LONG_LONG_INT , MPI_SUM , 0 , testComm ) ;
2020-11-27 20:49:45 +03:00
o . summary_table [ iteration ] . stonewall_item_sum [ MDTEST_FILE_CREATE_NUM ] = sum_accessed ;
o . summary_table [ iteration ] . stonewall_item_min [ MDTEST_FILE_CREATE_NUM ] = min_accessed * o . size ;
2018-07-08 01:56:39 +03:00
2020-11-27 20:49:45 +03:00
if ( o . items ! = ( sum_accessed / o . size ) ) {
VERBOSE ( 0 , - 1 , " Continue stonewall hit min: %lld max: %lld avg: %.1f \n " , min_accessed , max_iter , ( ( double ) sum_accessed ) / o . size ) ;
2018-07-08 01:56:39 +03:00
hit = 1 ;
}
return hit ;
}
2020-11-27 20:49:45 +03:00
void file_test_create ( const int iteration , const int ntasks , const char * path , rank_progress_t * progress , double * t ) {
char temp_path [ MAX_PATHLEN ] ;
2020-11-28 13:34:20 +03:00
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
2020-11-27 20:49:45 +03:00
prep_testdir ( iteration , dir_iter ) ;
if ( o . unique_dir_per_task ) {
unique_dir_access ( MK_UNI_DIR , temp_path ) ;
VERBOSE ( 5 , 5 , " operating on %s " , temp_path ) ;
if ( ! o . time_unique_dir_overhead ) {
offset_timers ( t , 0 ) ;
}
} else {
sprintf ( temp_path , " %s/%s " , o . testdir , path ) ;
}
VERBOSE ( 3 , - 1 , " file_test: create path is '%s' " , temp_path ) ;
/* "touch" the files */
if ( o . collective_creates ) {
if ( rank = = 0 ) {
collective_create_remove ( 1 , 0 , ntasks , temp_path , progress ) ;
}
MPI_Barrier ( testComm ) ;
}
/* create files */
create_remove_items ( 0 , 0 , 1 , 0 , temp_path , 0 , progress ) ;
if ( o . stone_wall_timer_seconds ) {
// hit the stonewall
uint64_t max_iter = 0 ;
uint64_t items_done = progress - > items_done + dir_iter * o . items_per_dir ;
int hit = updateStoneWallIterations ( iteration , items_done , t [ 0 ] , & max_iter ) ;
progress - > items_start = items_done ;
progress - > items_per_dir = max_iter ;
if ( hit ) {
progress - > stone_wall_timer_seconds = 0 ;
VERBOSE ( 1 , 1 , " stonewall: %lld of %lld " , ( long long ) progress - > items_start , ( long long ) progress - > items_per_dir ) ;
create_remove_items ( 0 , 0 , 1 , 0 , temp_path , 0 , progress ) ;
// now reset the values
progress - > stone_wall_timer_seconds = o . stone_wall_timer_seconds ;
o . items = progress - > items_done ;
}
if ( o . stoneWallingStatusFile ) {
StoreStoneWallingIterations ( o . stoneWallingStatusFile , max_iter ) ;
}
// reset stone wall timer to allow proper cleanup
progress - > stone_wall_timer_seconds = 0 ;
2020-11-28 13:34:20 +03:00
// at the moment, stonewall can be done only with one directory_loop, so we can return here safely
2020-11-27 20:49:45 +03:00
break ;
}
}
}
2018-07-07 12:29:27 +03:00
void file_test ( const int iteration , const int ntasks , const char * path , rank_progress_t * progress ) {
2017-10-20 00:26:52 +03:00
int size ;
double t [ 5 ] = { 0 } ;
2021-02-11 12:57:22 +03:00
double tBefore [ 5 ] = { 0 } ;
2018-07-08 00:39:14 +03:00
char temp_path [ MAX_PATHLEN ] ;
2018-07-07 12:29:27 +03:00
MPI_Comm_size ( testComm , & size ) ;
2017-10-20 00:26:52 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " Entering file_test on %s " , path ) ;
2017-06-28 19:42:20 +03:00
2021-02-11 12:57:22 +03:00
tBefore [ 0 ] = GetTimeStamp ( ) ;
2017-10-20 00:26:52 +03:00
MPI_Barrier ( testComm ) ;
2018-10-17 17:38:35 +03:00
t [ 0 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create phase */
2020-11-27 20:49:45 +03:00
if ( o . create_only ) {
progress - > stone_wall_timer_seconds = o . stone_wall_timer_seconds ;
2020-07-21 18:16:13 +03:00
progress - > items_done = 0 ;
progress - > start_time = GetTimeStamp ( ) ;
2020-11-27 20:49:45 +03:00
file_test_create ( iteration , ntasks , path , progress , t ) ;
2018-07-08 01:56:39 +03:00
} else {
2020-11-27 20:49:45 +03:00
if ( o . stoneWallingStatusFile ) {
2018-07-14 14:22:36 +03:00
int64_t expected_items ;
2018-07-08 01:56:39 +03:00
/* The number of items depends on the stonewalling file */
2021-01-20 17:06:05 +03:00
expected_items = ReadStoneWallingIterations ( o . stoneWallingStatusFile , testComm ) ;
2018-07-14 14:22:36 +03:00
if ( expected_items > = 0 ) {
2021-01-14 20:38:11 +03:00
if ( o . directory_loops > 1 ) {
o . directory_loops = expected_items / o . items_per_dir ;
o . items = o . items_per_dir ;
} else {
o . items = expected_items ;
progress - > items_per_dir = o . items ;
}
2018-07-14 14:22:36 +03:00
}
if ( rank = = 0 ) {
if ( expected_items = = - 1 ) {
2020-06-24 13:10:42 +03:00
WARN ( " Could not read stonewall status file " ) ;
2019-07-27 22:22:15 +03:00
} else {
2020-11-27 20:49:45 +03:00
VERBOSE ( 1 , 1 , " Read stonewall status; items: " LLU " \n " , o . items ) ;
2018-07-14 14:22:36 +03:00
}
2018-07-08 01:56:39 +03:00
}
}
2017-06-28 19:42:20 +03:00
}
2021-02-11 12:57:22 +03:00
tBefore [ 1 ] = GetTimeStamp ( ) ;
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 1 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat phase */
2020-11-27 20:49:45 +03:00
if ( o . stat_only ) {
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
2018-07-26 12:37:01 +03:00
prep_testdir ( iteration , dir_iter ) ;
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
2017-10-20 00:26:52 +03:00
unique_dir_access ( STAT_SUB_DIR , temp_path ) ;
2020-11-27 20:49:45 +03:00
if ( ! o . time_unique_dir_overhead ) {
2017-10-20 00:26:52 +03:00
offset_timers ( t , 1 ) ;
}
} else {
2020-11-27 20:49:45 +03:00
sprintf ( temp_path , " %s/%s " , o . testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " file_test: stat path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat files */
2020-11-27 20:49:45 +03:00
mdtest_stat ( ( o . random_seed > 0 ? 1 : 0 ) , 0 , dir_iter , temp_path , progress ) ;
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2021-02-11 12:57:22 +03:00
tBefore [ 2 ] = GetTimeStamp ( ) ;
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 2 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read phase */
2020-11-27 20:49:45 +03:00
if ( o . read_only ) {
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
2018-07-26 12:37:01 +03:00
prep_testdir ( iteration , dir_iter ) ;
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
2017-10-20 00:26:52 +03:00
unique_dir_access ( READ_SUB_DIR , temp_path ) ;
2020-11-27 20:49:45 +03:00
if ( ! o . time_unique_dir_overhead ) {
2017-10-20 00:26:52 +03:00
offset_timers ( t , 2 ) ;
}
} else {
2020-11-27 20:49:45 +03:00
sprintf ( temp_path , " %s/%s " , o . testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " file_test: read path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read files */
2020-11-27 20:49:45 +03:00
if ( o . random_seed > 0 ) {
2018-07-26 12:37:01 +03:00
mdtest_read ( 1 , 0 , dir_iter , temp_path ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-26 12:37:01 +03:00
mdtest_read ( 0 , 0 , dir_iter , temp_path ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2021-02-11 12:57:22 +03:00
tBefore [ 3 ] = GetTimeStamp ( ) ;
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 3 ] = GetTimeStamp ( ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( o . remove_only ) {
2018-12-20 00:37:37 +03:00
progress - > items_start = 0 ;
2020-11-27 20:49:45 +03:00
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
2018-07-26 12:37:01 +03:00
prep_testdir ( iteration , dir_iter ) ;
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
2017-10-20 00:26:52 +03:00
unique_dir_access ( RM_SUB_DIR , temp_path ) ;
2020-11-27 20:49:45 +03:00
if ( ! o . time_unique_dir_overhead ) {
2017-10-20 00:26:52 +03:00
offset_timers ( t , 3 ) ;
}
} else {
2020-11-27 20:49:45 +03:00
sprintf ( temp_path , " %s/%s " , o . testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " file_test: rm directories path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( o . collective_creates ) {
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
collective_create_remove ( 0 , 0 , ntasks , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
} else {
2019-07-27 22:22:15 +03:00
VERBOSE ( 3 , 5 , " gonna create %s " , temp_path ) ;
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , 0 , 0 , 0 , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2021-02-11 12:57:22 +03:00
tBefore [ 4 ] = GetTimeStamp ( ) ;
2019-08-26 20:57:14 +03:00
phase_end ( ) ;
2018-10-17 17:38:35 +03:00
t [ 4 ] = GetTimeStamp ( ) ;
2020-11-27 20:49:45 +03:00
if ( o . remove_only ) {
if ( o . unique_dir_per_task ) {
2017-10-20 00:26:52 +03:00
unique_dir_access ( RM_UNI_DIR , temp_path ) ;
} else {
strcpy ( temp_path , path ) ;
}
2019-07-28 19:07:03 +03:00
VERBOSE ( 3 , 5 , " file_test: rm unique directories path is '%s' " , temp_path ) ;
2017-06-28 19:42:20 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task & & ! o . time_unique_dir_overhead ) {
2017-10-20 00:26:52 +03:00
offset_timers ( t , 4 ) ;
2017-06-28 19:42:20 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . num_dirs_in_tree_calc ) { /* this is temporary fix needed when using -n and -i together */
o . items * = o . num_dirs_in_tree_calc ;
2018-10-17 18:04:49 +03:00
}
2021-01-21 17:10:23 +03:00
mdtest_results_t * res = & o . summary_table [ iteration ] ;
2017-10-20 00:26:52 +03:00
/* calculate times */
2020-11-27 20:49:45 +03:00
if ( o . create_only ) {
2021-02-18 12:53:07 +03:00
updateResult ( res , MDTEST_FILE_CREATE_NUM , o . items , 1 , t , tBefore ) ;
2020-11-27 20:49:45 +03:00
}
if ( o . stat_only ) {
2021-02-18 12:53:07 +03:00
updateResult ( res , MDTEST_FILE_STAT_NUM , o . items , 2 , t , tBefore ) ;
2020-11-27 20:49:45 +03:00
}
if ( o . read_only ) {
2021-02-18 12:53:07 +03:00
updateResult ( res , MDTEST_FILE_READ_NUM , o . items , 3 , t , tBefore ) ;
2020-11-27 20:49:45 +03:00
}
if ( o . remove_only ) {
2021-02-18 12:53:07 +03:00
updateResult ( res , MDTEST_FILE_REMOVE_NUM , o . items , 4 , t , tBefore ) ;
2020-11-27 20:49:45 +03:00
}
VERBOSE ( 1 , - 1 , " File creation : %14.3f sec, %14.3f ops/sec " , t [ 1 ] - t [ 0 ] , o . summary_table [ iteration ] . rate [ 4 ] ) ;
if ( o . summary_table [ iteration ] . stonewall_time [ MDTEST_FILE_CREATE_NUM ] ) {
VERBOSE ( 1 , - 1 , " File creation (stonewall): %14.3f sec, %14.3f ops/sec " , o . summary_table [ iteration ] . stonewall_time [ MDTEST_FILE_CREATE_NUM ] , o . summary_table [ iteration ] . stonewall_item_sum [ MDTEST_FILE_CREATE_NUM ] ) ;
}
VERBOSE ( 1 , - 1 , " File stat : %14.3f sec, %14.3f ops/sec " , t [ 2 ] - t [ 1 ] , o . summary_table [ iteration ] . rate [ 5 ] ) ;
VERBOSE ( 1 , - 1 , " File read : %14.3f sec, %14.3f ops/sec " , t [ 3 ] - t [ 2 ] , o . summary_table [ iteration ] . rate [ 6 ] ) ;
VERBOSE ( 1 , - 1 , " File removal : %14.3f sec, %14.3f ops/sec " , t [ 4 ] - t [ 3 ] , o . summary_table [ iteration ] . rate [ 7 ] ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2020-11-25 12:50:26 +03:00
char const * mdtest_test_name ( int i ) {
switch ( i ) {
2021-02-11 12:57:22 +03:00
case MDTEST_DIR_CREATE_NUM : return " Directory creation " ;
case MDTEST_DIR_STAT_NUM : return " Directory stat " ;
case MDTEST_DIR_READ_NUM : return " Directory read " ;
case MDTEST_DIR_REMOVE_NUM : return " Directory removal " ;
case MDTEST_DIR_RENAME_NUM : return " Directory rename " ;
case MDTEST_FILE_CREATE_NUM : return " File creation " ;
case MDTEST_FILE_STAT_NUM : return " File stat " ;
case MDTEST_FILE_READ_NUM : return " File read " ;
case MDTEST_FILE_REMOVE_NUM : return " File removal " ;
case MDTEST_TREE_CREATE_NUM : return " Tree creation " ;
case MDTEST_TREE_REMOVE_NUM : return " Tree removal " ;
2021-01-21 17:10:23 +03:00
default : return " ERR INVALID TESTNAME : " ;
2020-11-25 12:50:26 +03:00
}
return NULL ;
}
2021-02-11 12:57:22 +03:00
/*
* Store the results of each process in a file
*/
2021-02-18 12:53:07 +03:00
static void StoreRankInformation ( int iterations , mdtest_results_t * agg ) {
2021-02-11 12:57:22 +03:00
const size_t size = sizeof ( mdtest_results_t ) * iterations ;
if ( rank = = 0 ) {
FILE * fd = fopen ( o . saveRankDetailsCSV , " a " ) ;
if ( fd = = NULL ) {
FAIL ( " Cannot open saveRankPerformanceDetails file for writes! " ) ;
}
2021-02-18 12:53:07 +03:00
mdtest_results_t * results = safeMalloc ( size * o . size ) ;
2021-02-11 12:57:22 +03:00
MPI_Gather ( o . summary_table , size / sizeof ( double ) , MPI_DOUBLE , results , size / sizeof ( double ) , MPI_DOUBLE , 0 , testComm ) ;
2021-02-18 12:53:07 +03:00
char buff [ 4096 ] ;
char * cpos = buff ;
cpos + = sprintf ( cpos , " all,%llu " , ( long long unsigned ) o . items ) ;
for ( int e = 0 ; e < MDTEST_LAST_NUM ; e + + ) {
if ( agg - > items [ e ] = = 0 ) {
cpos + = sprintf ( cpos , " ,, " ) ;
} else {
cpos + = sprintf ( cpos , " ,%.10e,%.10e " , agg - > items [ e ] / agg - > time [ e ] , agg - > time [ e ] ) ;
}
}
cpos + = sprintf ( cpos , " \n " ) ;
int ret = fwrite ( buff , cpos - buff , 1 , fd ) ;
2021-02-11 12:57:22 +03:00
for ( int iter = 0 ; iter < iterations ; iter + + ) {
for ( int i = 0 ; i < o . size ; i + + ) {
mdtest_results_t * cur = & results [ i * iterations + iter ] ;
2021-02-18 12:53:07 +03:00
cpos = buff ;
cpos + = sprintf ( cpos , " %d, " , i ) ;
for ( int e = 0 ; e < MDTEST_TREE_CREATE_NUM ; e + + ) {
2021-02-11 12:57:22 +03:00
if ( cur - > items [ e ] = = 0 ) {
cpos + = sprintf ( cpos , " ,, " ) ;
} else {
cpos + = sprintf ( cpos , " ,%.10e,%.10e " , cur - > items [ e ] / cur - > time_before_barrier [ e ] , cur - > time_before_barrier [ e ] ) ;
}
}
cpos + = sprintf ( cpos , " \n " ) ;
2021-02-18 12:53:07 +03:00
ret = fwrite ( buff , cpos - buff , 1 , fd ) ;
2021-02-11 12:57:22 +03:00
if ( ret ! = 1 ) {
WARN ( " Couln't append to saveRankPerformanceDetailsCSV file \n " ) ;
break ;
}
}
}
fclose ( fd ) ;
2021-02-18 12:53:07 +03:00
free ( results ) ;
2021-02-11 12:57:22 +03:00
} else {
/* this is a hack for now assuming all datatypes in the structure are double */
MPI_Gather ( o . summary_table , size / sizeof ( double ) , MPI_DOUBLE , NULL , size / sizeof ( double ) , MPI_DOUBLE , 0 , testComm ) ;
}
}
2021-02-18 12:53:07 +03:00
static mdtest_results_t * get_result_index ( mdtest_results_t * all_results , int proc , int iter , int interation_count ) {
return & all_results [ proc * interation_count + iter ] ;
}
2017-06-28 19:42:20 +03:00
2021-02-18 12:53:07 +03:00
static void summarize_results_rank0 ( int iterations , mdtest_results_t * all_results , int print_time ) {
int start , stop ;
double min , max , mean , sd , sum , var , curr = 0 ;
double imin , imax , imean , isum , icur ; // calculation per iteration
char const * access ;
/* if files only access, skip entries 0-3 (the dir tests) */
if ( o . files_only & & ! o . dirs_only ) {
start = MDTEST_FILE_CREATE_NUM ;
} else {
start = 0 ;
}
2017-06-28 19:42:20 +03:00
2021-02-18 12:53:07 +03:00
/* if directories only access, skip entries 4-7 (the file tests) */
if ( o . dirs_only & & ! o . files_only ) {
stop = MDTEST_FILE_CREATE_NUM ;
} else {
stop = MDTEST_TREE_CREATE_NUM ;
}
2017-06-28 19:42:20 +03:00
2021-02-18 12:53:07 +03:00
/* special case: if no directory or file tests, skip all */
if ( ! o . dirs_only & & ! o . files_only ) {
start = stop = 0 ;
}
2017-06-28 19:42:20 +03:00
2021-02-18 12:53:07 +03:00
if ( o . print_all_proc ) {
fprintf ( out_logfile , " \n Per process result (%s): \n " , print_time ? " time " : " rate " ) ;
for ( int j = 0 ; j < iterations ; j + + ) {
fprintf ( out_logfile , " iteration: %d \n " , j ) ;
for ( int i = start ; i < MDTEST_LAST_NUM ; i + + ) {
access = mdtest_test_name ( i ) ;
if ( access = = NULL ) {
continue ;
}
fprintf ( out_logfile , " Test %s " , access ) ;
for ( int k = 0 ; k < o . size ; k + + ) {
mdtest_results_t * cur = get_result_index ( all_results , k , j , iterations ) ;
if ( print_time ) {
curr = cur - > time_before_barrier [ i ] ;
} else {
curr = cur - > rate_before_barrier [ i ] ;
}
fprintf ( out_logfile , " %c%e " , ( k = = 0 ? ' ' : ' , ' ) , curr ) ;
}
fprintf ( out_logfile , " \n " ) ;
2018-07-07 12:29:27 +03:00
}
}
2021-02-18 12:53:07 +03:00
}
2017-06-28 19:42:20 +03:00
2021-02-18 12:53:07 +03:00
VERBOSE ( 0 , - 1 , " \n SUMMARY %s: (of %d iterations) " , print_time ? " time " : " rate " , iterations ) ;
VERBOSE ( 0 , - 1 ,
" Operation per Rank: Max Min Mean "
" per Iteration: Max Min Mean Std Dev " ) ;
VERBOSE ( 0 , - 1 ,
" --------- --- --- ---- "
" --- --- ---- ------- " ) ;
for ( int i = start ; i < stop ; i + + ) {
min = 1e308 ;
max = 0 ;
sum = var = 0 ;
imin = 1e308 ;
isum = imax = 0 ;
double iter_result [ iterations ] ;
for ( int j = 0 ; j < iterations ; j + + ) {
icur = print_time ? 0 : 1e308 ;
for ( int k = 0 ; k < o . size ; k + + ) {
mdtest_results_t * cur = get_result_index ( all_results , k , j , iterations ) ;
if ( print_time ) {
curr = cur - > time_before_barrier [ i ] ;
} else {
curr = cur - > rate_before_barrier [ i ] ;
}
if ( min > curr ) {
min = curr ;
}
if ( max < curr ) {
max = curr ;
}
sum + = curr ;
if ( print_time ) {
curr = cur - > time [ i ] ;
if ( icur < curr ) {
icur = curr ;
}
} else {
curr = cur - > rate [ i ] ;
if ( icur > curr ) {
icur = curr ;
2020-11-26 15:48:11 +03:00
}
}
}
2021-02-18 12:53:07 +03:00
if ( icur > imax ) {
imax = icur ;
}
if ( icur < imin ) {
imin = icur ;
}
isum + = icur ;
if ( print_time ) {
iter_result [ j ] = icur ;
} else {
iter_result [ j ] = icur * o . size ;
}
2019-08-01 19:57:45 +03:00
}
2021-02-18 12:53:07 +03:00
mean = sum / iterations / o . size ;
imean = isum / iterations ;
if ( ! print_time ) {
imax * = o . size ;
imin * = o . size ;
isum * = o . size ;
imean * = o . size ;
}
for ( int j = 0 ; j < iterations ; j + + ) {
var + = ( imean - iter_result [ j ] ) * ( imean - iter_result [ j ] ) ;
}
var = var / ( iterations - 1 ) ;
sd = sqrt ( var ) ;
access = mdtest_test_name ( i ) ;
if ( i ! = 2 ) {
fprintf ( out_logfile , " %-22s " , access ) ;
fprintf ( out_logfile , " %14.3f " , max ) ;
fprintf ( out_logfile , " %14.3f " , min ) ;
fprintf ( out_logfile , " %14.3f " , mean ) ;
fprintf ( out_logfile , " %18.3f " , imax ) ;
fprintf ( out_logfile , " %14.3f " , imin ) ;
fprintf ( out_logfile , " %14.3f " , imean ) ;
fprintf ( out_logfile , " %14.3f \n " , iterations = = 1 ? 0 : sd ) ;
fflush ( out_logfile ) ;
2019-08-01 19:57:45 +03:00
}
2021-02-18 12:53:07 +03:00
}
2017-06-28 19:42:20 +03:00
2021-02-18 12:53:07 +03:00
/* calculate tree create/remove rates, applies only to Rank 0 */
for ( int i = MDTEST_TREE_CREATE_NUM ; i < MDTEST_LAST_NUM ; i + + ) {
min = imin = 1e308 ;
max = imax = 0 ;
sum = var = 0 ;
for ( int j = 0 ; j < iterations ; j + + ) {
if ( print_time ) {
curr = o . summary_table [ j ] . time [ i ] ;
} else {
curr = o . summary_table [ j ] . rate [ i ] ;
}
if ( min > curr ) {
min = curr ;
}
if ( max < curr ) {
max = curr ;
}
sum + = curr ;
if ( curr > imax ) {
imax = curr ;
}
if ( curr < imin ) {
imin = curr ;
}
}
2017-06-28 19:42:20 +03:00
2021-02-18 12:53:07 +03:00
mean = sum / ( iterations ) ;
2017-06-28 19:42:20 +03:00
2021-02-18 12:53:07 +03:00
for ( int j = 0 ; j < iterations ; j + + ) {
if ( print_time ) {
curr = o . summary_table [ j ] . time [ i ] ;
} else {
curr = o . summary_table [ j ] . rate [ i ] ;
2020-11-25 12:50:26 +03:00
}
2021-02-18 12:53:07 +03:00
var + = ( mean - curr ) * ( mean - curr ) ;
}
var = var / ( iterations - 1 ) ;
sd = sqrt ( var ) ;
access = mdtest_test_name ( i ) ;
fprintf ( out_logfile , " %-22s " , access ) ;
fprintf ( out_logfile , " %14.3f " , max ) ;
fprintf ( out_logfile , " %14.3f " , min ) ;
fprintf ( out_logfile , " %14.3f " , mean ) ;
fprintf ( out_logfile , " %18.3f " , imax ) ;
fprintf ( out_logfile , " %14.3f " , imin ) ;
fprintf ( out_logfile , " %14.3f " , sum / iterations ) ;
fprintf ( out_logfile , " %14.3f \n " , iterations = = 1 ? 0 : sd ) ;
fflush ( out_logfile ) ;
}
}
/*
Output the results and summarize them into rank 0 ' s o . summary_table
*/
void summarize_results ( int iterations , mdtest_results_t * results ) {
const size_t size = sizeof ( mdtest_results_t ) * iterations ;
mdtest_results_t * all_results = NULL ;
if ( rank = = 0 ) {
all_results = safeMalloc ( size * o . size ) ;
memset ( all_results , 0 , size * o . size ) ;
MPI_Gather ( o . summary_table , size / sizeof ( double ) , MPI_DOUBLE , all_results , size / sizeof ( double ) , MPI_DOUBLE , 0 , testComm ) ;
// calculate the aggregated values for all processes
for ( int j = 0 ; j < iterations ; j + + ) {
for ( int i = 0 ; i < MDTEST_LAST_NUM ; i + + ) {
//double sum_rate = 0;
double max_time = 0 ;
double max_stonewall_time = 0 ;
uint64_t sum_items = 0 ;
// reduce over the processes
for ( int p = 0 ; p < o . size ; p + + ) {
mdtest_results_t * cur = get_result_index ( all_results , p , j , iterations ) ;
//sum_rate += all_results[p + j*p]->rate[i];
double t = cur - > time [ i ] ;
max_time = max_time < t ? t : max_time ;
sum_items + = cur - > items [ i ] ;
t = cur - > stonewall_time [ i ] ;
max_stonewall_time = max_stonewall_time < t ? t : max_stonewall_time ;
}
results [ j ] . items [ i ] = sum_items ;
results [ j ] . time [ i ] = max_time ;
results [ j ] . stonewall_time [ i ] = max_stonewall_time ;
if ( sum_items = = 0 ) {
results [ j ] . rate [ i ] = 0.0 ;
} else {
results [ j ] . rate [ i ] = sum_items / max_time ;
2020-11-25 12:50:26 +03:00
}
2021-02-18 12:53:07 +03:00
/* These results have already been reduced to Rank 0 */
results [ j ] . stonewall_item_sum [ i ] = o . summary_table [ j ] . stonewall_item_sum [ i ] ;
results [ j ] . stonewall_item_min [ i ] = o . summary_table [ j ] . stonewall_item_min [ i ] ;
results [ j ] . stonewall_time [ i ] = o . summary_table [ j ] . stonewall_time [ i ] ;
2020-11-25 12:50:26 +03:00
}
}
2021-02-18 12:53:07 +03:00
} else {
MPI_Gather ( o . summary_table , size / sizeof ( double ) , MPI_DOUBLE , NULL , size / sizeof ( double ) , MPI_DOUBLE , 0 , testComm ) ;
}
2020-11-25 12:50:26 +03:00
2021-02-18 12:53:07 +03:00
/* share global results across processes as these are returned by the API */
MPI_Bcast ( results , size / sizeof ( double ) , MPI_DOUBLE , 0 , testComm ) ;
2020-11-25 12:50:26 +03:00
2021-02-18 12:53:07 +03:00
/* update relevant result values with local values as these are returned by the API */
for ( int j = 0 ; j < iterations ; j + + ) {
for ( int i = 0 ; i < MDTEST_LAST_NUM ; i + + ) {
results [ j ] . time_before_barrier [ i ] = o . summary_table [ j ] . time_before_barrier [ i ] ;
results [ j ] . stonewall_last_item [ i ] = o . summary_table [ j ] . stonewall_last_item [ i ] ;
2019-08-01 19:57:45 +03:00
}
2021-02-18 12:53:07 +03:00
}
2019-08-01 19:57:45 +03:00
2021-02-18 12:53:07 +03:00
if ( rank ! = 0 ) {
return ;
}
2019-08-01 19:57:45 +03:00
2021-02-18 12:53:07 +03:00
if ( o . print_rate_and_time ) {
summarize_results_rank0 ( iterations , all_results , 0 ) ;
summarize_results_rank0 ( iterations , all_results , 1 ) ;
} else {
summarize_results_rank0 ( iterations , all_results , o . print_time ) ;
}
2019-08-01 19:57:45 +03:00
2021-02-18 12:53:07 +03:00
free ( all_results ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* Checks to see if the test setup is valid. If it isn't, fail. */
2020-06-28 19:21:24 +03:00
void md_validate_tests ( ) {
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( ( ( o . stone_wall_timer_seconds > 0 ) & & ( o . branch_factor > 1 ) ) | | ! o . barriers ) {
FAIL ( " Error, stone wall timer does only work with a branch factor <= 1 (current is %d) and with barriers \n " , o . branch_factor ) ;
2018-07-12 20:09:13 +03:00
}
2021-01-21 17:10:23 +03:00
if ( ! o . create_only & & ! o . stat_only & & ! o . read_only & & ! o . remove_only & & ! o . rename_dirs ) {
o . create_only = o . stat_only = o . read_only = o . remove_only = o . rename_dirs = 1 ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " main: Setting create/stat/read/remove_only to True " ) ;
2018-07-12 20:09:13 +03:00
}
2017-06-28 19:42:20 +03:00
2020-06-28 19:21:24 +03:00
VERBOSE ( 1 , - 1 , " Entering md_validate_tests... " ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* if dirs_only and files_only were both left unset, set both now */
2020-11-27 20:49:45 +03:00
if ( ! o . dirs_only & & ! o . files_only ) {
o . dirs_only = o . files_only = 1 ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* if shared file 'S' access, no directory tests */
2020-11-27 20:49:45 +03:00
if ( o . shared_file ) {
o . dirs_only = 0 ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* check for no barriers with shifting processes for different phases.
that is , one may not specify both - B and - N as it will introduce
race conditions that may cause errors stat ' ing or deleting after
creates .
*/
2020-11-27 20:49:45 +03:00
if ( ( o . barriers = = 0 ) & & ( o . nstride ! = 0 ) & & ( rank = = 0 ) ) {
2017-10-20 00:26:52 +03:00
FAIL ( " Possible race conditions will occur: -B not compatible with -N " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* check for collective_creates incompatibilities */
2020-11-27 20:49:45 +03:00
if ( o . shared_file & & o . collective_creates & & rank = = 0 ) {
2017-10-20 00:26:52 +03:00
FAIL ( " -c not compatible with -S " ) ;
}
2020-11-27 20:49:45 +03:00
if ( o . path_count > 1 & & o . collective_creates & & rank = = 0 ) {
2017-10-20 00:26:52 +03:00
FAIL ( " -c not compatible with multiple test directories " ) ;
}
2020-11-27 20:49:45 +03:00
if ( o . collective_creates & & ! o . barriers ) {
2017-10-20 00:26:52 +03:00
FAIL ( " -c not compatible with -B " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* check for shared file incompatibilities */
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task & & o . shared_file & & rank = = 0 ) {
2017-10-20 00:26:52 +03:00
FAIL ( " -u not compatible with -S " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check multiple directory paths and strided option */
2020-11-27 20:49:45 +03:00
if ( o . path_count > 1 & & o . nstride > 0 ) {
2017-10-20 00:26:52 +03:00
FAIL ( " cannot have multiple directory paths with -N strides between neighbor tasks " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check for shared directory and multiple directories incompatibility */
2020-11-27 20:49:45 +03:00
if ( o . path_count > 1 & & o . unique_dir_per_task ! = 1 ) {
2017-10-20 00:26:52 +03:00
FAIL ( " shared directory mode is not compatible with multiple directory paths " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check if more directory paths than ranks */
2020-11-27 20:49:45 +03:00
if ( o . path_count > o . size ) {
2017-10-20 00:26:52 +03:00
FAIL ( " cannot have more directory paths than MPI tasks " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check depth */
2020-11-27 20:49:45 +03:00
if ( o . depth < 0 ) {
2017-10-20 00:26:52 +03:00
FAIL ( " depth must be greater than or equal to zero " ) ;
}
/* check branch_factor */
2020-11-27 20:49:45 +03:00
if ( o . branch_factor < 1 & & o . depth > 0 ) {
2017-10-20 00:26:52 +03:00
FAIL ( " branch factor must be greater than or equal to zero " ) ;
}
/* check for valid number of items */
2020-11-27 20:49:45 +03:00
if ( ( o . items > 0 ) & & ( o . items_per_dir > 0 ) ) {
if ( o . unique_dir_per_task ) {
2018-07-26 12:37:01 +03:00
FAIL ( " only specify the number of items or the number of items per directory " ) ;
2020-11-27 20:49:45 +03:00
} else if ( o . items % o . items_per_dir ! = 0 ) {
2018-07-26 12:37:01 +03:00
FAIL ( " items must be a multiple of items per directory " ) ;
}
2017-10-20 00:26:52 +03:00
}
2018-10-08 15:30:47 +03:00
/* check for using mknod */
2020-11-27 20:49:45 +03:00
if ( o . write_bytes > 0 & & o . make_node ) {
2018-10-08 15:30:47 +03:00
FAIL ( " -k not compatible with -w " ) ;
}
2020-06-28 19:21:24 +03:00
2020-11-27 20:49:45 +03:00
if ( o . verify_read & & ! o . read_only )
2020-06-28 19:21:24 +03:00
FAIL ( " Verify read requires that the read test is used " ) ;
2020-11-27 20:49:45 +03:00
if ( o . verify_read & & o . read_bytes < = 0 )
2020-06-28 19:21:24 +03:00
FAIL ( " Verify read requires that read bytes is > 0 " ) ;
2020-11-27 20:49:45 +03:00
if ( o . read_only & & o . read_bytes < = 0 )
2020-06-28 19:21:24 +03:00
WARN ( " Read bytes is 0, thus, a read test will actually just open/close " ) ;
2020-06-29 22:15:14 +03:00
2020-11-27 20:49:45 +03:00
if ( o . create_only & & o . read_only & & o . read_bytes > o . write_bytes )
2020-06-29 22:15:14 +03:00
FAIL ( " When writing and reading files, read bytes must be smaller than write bytes " ) ;
2021-02-11 12:57:22 +03:00
if ( rank = = 0 & & o . saveRankDetailsCSV ) {
// check that the file is writeable, truncate it and add header
FILE * fd = fopen ( o . saveRankDetailsCSV , " w " ) ;
if ( fd = = NULL ) {
FAIL ( " Cannot open saveRankPerformanceDetails file for write! " ) ;
}
char * head = " rank,items " ;
int ret = fwrite ( head , strlen ( head ) , 1 , fd ) ;
for ( int e = 0 ; e < MDTEST_LAST_NUM ; e + + ) {
char buf [ 1024 ] ;
const char * str = mdtest_test_name ( e ) ;
sprintf ( buf , " ,rate-%s,time-%s " , str , str ) ;
ret = fwrite ( buf , strlen ( buf ) , 1 , fd ) ;
if ( ret ! = 1 ) {
FAIL ( " Cannot write header to saveRankPerformanceDetails file " ) ;
}
}
fwrite ( " \n " , 1 , 1 , fd ) ;
fclose ( fd ) ;
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
void show_file_system_size ( char * file_system ) {
2018-07-08 00:39:14 +03:00
char real_path [ MAX_PATHLEN ] ;
char file_system_unit_str [ MAX_PATHLEN ] = " GiB " ;
char inode_unit_str [ MAX_PATHLEN ] = " Mi " ;
2017-10-20 00:26:52 +03:00
int64_t file_system_unit_val = 1024 * 1024 * 1024 ;
int64_t inode_unit_val = 1024 * 1024 ;
int64_t total_file_system_size ,
free_file_system_size ,
total_inodes ,
free_inodes ;
double total_file_system_size_hr ,
used_file_system_percentage ,
used_inode_percentage ;
ior_aiori_statfs_t stat_buf ;
int ret ;
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
VERBOSE ( 1 , - 1 , " Entering show_file_system_size on %s " , file_system ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
ret = o . backend - > statfs ( file_system , & stat_buf , o . backend_options ) ;
2017-10-20 00:26:52 +03:00
if ( 0 ! = ret ) {
2019-07-28 20:17:11 +03:00
FAIL ( " unable to stat file system %s " , file_system ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
total_file_system_size = stat_buf . f_blocks * stat_buf . f_bsize ;
free_file_system_size = stat_buf . f_bfree * stat_buf . f_bsize ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
used_file_system_percentage = ( 1 - ( ( double ) free_file_system_size
/ ( double ) total_file_system_size ) ) * 100 ;
total_file_system_size_hr = ( double ) total_file_system_size
/ ( double ) file_system_unit_val ;
if ( total_file_system_size_hr > 1024 ) {
total_file_system_size_hr = total_file_system_size_hr / 1024 ;
strcpy ( file_system_unit_str , " TiB " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* inodes */
total_inodes = stat_buf . f_files ;
free_inodes = stat_buf . f_ffree ;
used_inode_percentage = ( 1 - ( ( double ) free_inodes / ( double ) total_inodes ) )
* 100 ;
2017-06-28 19:42:20 +03:00
2018-01-24 21:26:53 +03:00
if ( realpath ( file_system , real_path ) = = NULL ) {
2020-05-28 21:05:41 +03:00
WARN ( " unable to use realpath() on file system " ) ;
2018-01-24 21:26:53 +03:00
}
2018-07-07 12:29:27 +03:00
2017-10-20 00:26:52 +03:00
/* show results */
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " Path: %s " , real_path ) ;
2019-08-01 19:57:45 +03:00
VERBOSE ( 0 , - 1 , " FS: %.1f %s Used FS: %2.1f%% Inodes: %.1f %s Used Inodes: %2.1f%% \n " ,
2019-07-27 23:31:49 +03:00
total_file_system_size_hr , file_system_unit_str , used_file_system_percentage ,
( double ) total_inodes / ( double ) inode_unit_val , inode_unit_str , used_inode_percentage ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
return ;
2017-06-28 19:42:20 +03:00
}
void create_remove_directory_tree ( int create ,
2018-07-07 12:29:27 +03:00
int currDepth , char * path , int dirNum , rank_progress_t * progress ) {
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
unsigned i ;
2018-07-08 00:39:14 +03:00
char dir [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , 5 , " Entering create_remove_directory_tree on %s, currDepth = %d... " , path , currDepth ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( currDepth = = 0 ) {
2020-11-27 20:49:45 +03:00
sprintf ( dir , " %s/%s.%d/ " , path , o . base_tree_name , dirNum ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( create ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 2 , 5 , " Making directory '%s' " , dir ) ;
2020-11-27 20:49:45 +03:00
if ( - 1 = = o . backend - > mkdir ( dir , DIRMODE , o . backend_options ) ) {
2021-01-21 17:10:23 +03:00
EWARNF ( " unable to create tree directory '%s' " , dir ) ;
2017-10-20 00:26:52 +03:00
}
2019-09-30 14:25:59 +03:00
# ifdef HAVE_LUSTRE_LUSTREAPI
2019-09-03 08:26:12 +03:00
/* internal node for branching, can be non-striped for children */
2020-11-27 20:49:45 +03:00
if ( o . global_dir_layout & & \
2019-09-03 08:26:12 +03:00
llapi_dir_set_default_lmv_stripe ( dir , - 1 , 0 ,
LMV_HASH_TYPE_FNV_1A_64 ,
NULL ) = = - 1 ) {
FAIL ( " Unable to reset to global default directory layout " ) ;
}
2019-09-30 14:25:59 +03:00
# endif /* HAVE_LUSTRE_LUSTREAPI */
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
create_remove_directory_tree ( create , + + currDepth , dir , + + dirNum , progress ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ! create ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 2 , 5 , " Remove directory '%s' " , dir ) ;
2020-11-27 20:49:45 +03:00
if ( - 1 = = o . backend - > rmdir ( dir , o . backend_options ) ) {
2021-01-14 19:41:56 +03:00
EWARNF ( " Unable to remove directory %s " , dir ) ;
2017-10-20 00:26:52 +03:00
}
}
2020-11-27 20:49:45 +03:00
} else if ( currDepth < = o . depth ) {
2017-06-28 19:42:20 +03:00
2018-07-08 00:39:14 +03:00
char temp_path [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
strcpy ( temp_path , path ) ;
int currDir = dirNum ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
for ( i = 0 ; i < o . branch_factor ; i + + ) {
sprintf ( dir , " %s.%d/ " , o . base_tree_name , currDir ) ;
2017-10-20 00:26:52 +03:00
strcat ( temp_path , dir ) ;
if ( create ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 2 , 5 , " Making directory '%s' " , temp_path ) ;
2020-11-27 20:49:45 +03:00
if ( - 1 = = o . backend - > mkdir ( temp_path , DIRMODE , o . backend_options ) ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " Unable to create directory %s " , temp_path ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
create_remove_directory_tree ( create , + + currDepth ,
2020-11-27 20:49:45 +03:00
temp_path , ( o . branch_factor * currDir ) + 1 , progress ) ;
2017-10-20 00:26:52 +03:00
currDepth - - ;
if ( ! create ) {
2019-07-28 19:07:03 +03:00
VERBOSE ( 2 , 5 , " Remove directory '%s' " , temp_path ) ;
2020-11-27 20:49:45 +03:00
if ( - 1 = = o . backend - > rmdir ( temp_path , o . backend_options ) ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " Unable to remove directory %s " , temp_path ) ;
2017-10-20 00:26:52 +03:00
}
}
strcpy ( temp_path , path ) ;
currDir + + ;
}
2017-06-28 19:42:20 +03:00
}
}
2018-07-08 01:56:39 +03:00
static void mdtest_iteration ( int i , int j , MPI_Group testgroup , mdtest_results_t * summary_table ) {
rank_progress_t progress_o ;
memset ( & progress_o , 0 , sizeof ( progress_o ) ) ;
2020-07-21 18:16:13 +03:00
progress_o . stone_wall_timer_seconds = 0 ;
2020-11-27 20:49:45 +03:00
progress_o . items_per_dir = o . items_per_dir ;
2018-07-08 01:56:39 +03:00
rank_progress_t * progress = & progress_o ;
2018-07-07 12:29:27 +03:00
/* start and end times of directory tree create/remove */
double startCreate , endCreate ;
2018-08-08 19:18:54 +03:00
int k ;
2018-07-07 12:29:27 +03:00
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " main: * iteration %d * " , j + 1 ) ;
2018-07-07 12:29:27 +03:00
2021-01-21 17:10:23 +03:00
if ( o . create_only ) {
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
if ( rank > = o . path_count ) {
continue ;
}
prep_testdir ( j , dir_iter ) ;
2018-07-07 12:29:27 +03:00
2021-01-21 17:10:23 +03:00
VERBOSE ( 2 , 5 , " main (for j loop): making o.testdir, '%s' " , o . testdir ) ;
if ( o . backend - > access ( o . testdir , F_OK , o . backend_options ) ! = 0 ) {
if ( o . backend - > mkdir ( o . testdir , DIRMODE , o . backend_options ) ! = 0 ) {
EWARNF ( " Unable to create test directory %s " , o . testdir ) ;
}
2019-09-30 14:25:59 +03:00
# ifdef HAVE_LUSTRE_LUSTREAPI
2021-01-21 17:10:23 +03:00
/* internal node for branching, can be non-striped for children */
if ( o . global_dir_layout & & o . unique_dir_per_task & & llapi_dir_set_default_lmv_stripe ( o . testdir , - 1 , 0 , LMV_HASH_TYPE_FNV_1A_64 , NULL ) = = - 1 ) {
EWARNF ( " Unable to reset to global default directory layout " ) ;
}
2019-09-30 14:25:59 +03:00
# endif /* HAVE_LUSTRE_LUSTREAPI */
2021-01-21 17:10:23 +03:00
}
2018-07-26 12:37:01 +03:00
}
2018-08-24 00:58:53 +03:00
2021-01-21 17:10:23 +03:00
/* create hierarchical directory structure */
MPI_Barrier ( testComm ) ;
2018-07-26 12:37:01 +03:00
2021-01-21 17:10:23 +03:00
startCreate = GetTimeStamp ( ) ;
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
prep_testdir ( j , dir_iter ) ;
2018-07-26 12:37:01 +03:00
2021-01-21 17:10:23 +03:00
if ( o . unique_dir_per_task ) {
if ( o . collective_creates & & ( rank = = 0 ) ) {
/*
* This is inside two loops , one of which already uses " i " and the other uses " j " .
* I don ' t know how this ever worked . I ' m changing this loop to use " k " .
*/
for ( k = 0 ; k < o . size ; k + + ) {
sprintf ( o . base_tree_name , " mdtest_tree.%d " , k ) ;
VERBOSE ( 3 , 5 , " main (create hierarchical directory loop-collective): Calling create_remove_directory_tree with '%s' " , o . testdir ) ;
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 1 , 0 , o . testdir , 0 , progress ) ;
if ( CHECK_STONE_WALL ( progress ) ) {
o . size = k ;
break ;
2018-07-26 12:37:01 +03:00
}
2021-01-21 17:10:23 +03:00
}
} else if ( ! o . collective_creates ) {
VERBOSE ( 3 , 5 , " main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with '%s' " , o . testdir ) ;
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 1 , 0 , o . testdir , 0 , progress ) ;
}
} else {
if ( rank = = 0 ) {
VERBOSE ( 3 , 5 , " main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with '%s' " , o . testdir ) ;
2018-07-26 12:37:01 +03:00
2021-01-21 17:10:23 +03:00
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 1 , 0 , o . testdir , 0 , progress ) ;
2018-07-26 12:37:01 +03:00
}
2018-07-07 12:29:27 +03:00
}
2021-01-21 17:10:23 +03:00
}
MPI_Barrier ( testComm ) ;
endCreate = GetTimeStamp ( ) ;
summary_table - > rate [ MDTEST_TREE_CREATE_NUM ] = o . num_dirs_in_tree / ( endCreate - startCreate ) ;
summary_table - > time [ MDTEST_TREE_CREATE_NUM ] = ( endCreate - startCreate ) ;
summary_table - > items [ MDTEST_TREE_CREATE_NUM ] = o . num_dirs_in_tree ;
summary_table - > stonewall_last_item [ MDTEST_TREE_CREATE_NUM ] = o . num_dirs_in_tree ;
VERBOSE ( 1 , - 1 , " V-1: main: Tree creation : %14.3f sec, %14.3f ops/sec " , ( endCreate - startCreate ) , summary_table - > rate [ MDTEST_TREE_CREATE_NUM ] ) ;
2018-07-07 12:29:27 +03:00
}
2020-07-21 18:16:13 +03:00
2020-11-27 20:49:45 +03:00
sprintf ( o . unique_mk_dir , " %s.0 " , o . base_tree_name ) ;
sprintf ( o . unique_chdir_dir , " %s.0 " , o . base_tree_name ) ;
sprintf ( o . unique_stat_dir , " %s.0 " , o . base_tree_name ) ;
sprintf ( o . unique_read_dir , " %s.0 " , o . base_tree_name ) ;
sprintf ( o . unique_rm_dir , " %s.0 " , o . base_tree_name ) ;
o . unique_rm_uni_dir [ 0 ] = 0 ;
2018-07-07 12:29:27 +03:00
2020-11-27 20:49:45 +03:00
if ( ! o . unique_dir_per_task ) {
VERBOSE ( 3 , - 1 , " V-3: main: Using unique_mk_dir, '%s' " , o . unique_mk_dir ) ;
2018-07-07 12:29:27 +03:00
}
if ( rank < i ) {
2020-11-27 20:49:45 +03:00
if ( ! o . shared_file ) {
sprintf ( o . mk_name , " mdtest.%d. " , ( rank + ( 0 * o . nstride ) ) % i ) ;
sprintf ( o . stat_name , " mdtest.%d. " , ( rank + ( 1 * o . nstride ) ) % i ) ;
sprintf ( o . read_name , " mdtest.%d. " , ( rank + ( 2 * o . nstride ) ) % i ) ;
sprintf ( o . rm_name , " mdtest.%d. " , ( rank + ( 3 * o . nstride ) ) % i ) ;
2018-07-07 12:29:27 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
VERBOSE ( 3 , 5 , " i %d nstride %d " , i , o . nstride ) ;
sprintf ( o . unique_mk_dir , " mdtest_tree.%d.0 " , ( rank + ( 0 * o . nstride ) ) % i ) ;
sprintf ( o . unique_chdir_dir , " mdtest_tree.%d.0 " , ( rank + ( 1 * o . nstride ) ) % i ) ;
sprintf ( o . unique_stat_dir , " mdtest_tree.%d.0 " , ( rank + ( 2 * o . nstride ) ) % i ) ;
sprintf ( o . unique_read_dir , " mdtest_tree.%d.0 " , ( rank + ( 3 * o . nstride ) ) % i ) ;
sprintf ( o . unique_rm_dir , " mdtest_tree.%d.0 " , ( rank + ( 4 * o . nstride ) ) % i ) ;
o . unique_rm_uni_dir [ 0 ] = 0 ;
VERBOSE ( 5 , 5 , " mk_dir %s chdir %s stat_dir %s read_dir %s rm_dir %s \n " , o . unique_mk_dir , o . unique_chdir_dir , o . unique_stat_dir , o . unique_read_dir , o . unique_rm_dir ) ;
2018-07-07 12:29:27 +03:00
}
2020-11-27 20:49:45 +03:00
VERBOSE ( 3 , - 1 , " V-3: main: Copied unique_mk_dir, '%s', to topdir " , o . unique_mk_dir ) ;
2018-07-07 12:29:27 +03:00
2020-11-27 20:49:45 +03:00
if ( o . dirs_only & & ! o . shared_file ) {
if ( o . pre_delay ) {
DelaySecs ( o . pre_delay ) ;
2018-07-07 12:29:27 +03:00
}
2020-11-27 20:49:45 +03:00
directory_test ( j , i , o . unique_mk_dir , progress ) ;
2018-07-07 12:29:27 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . files_only ) {
if ( o . pre_delay ) {
DelaySecs ( o . pre_delay ) ;
2018-07-07 12:29:27 +03:00
}
2020-11-27 20:49:45 +03:00
VERBOSE ( 3 , 5 , " will file_test on %s " , o . unique_mk_dir ) ;
2020-07-21 18:16:13 +03:00
2020-11-27 20:49:45 +03:00
file_test ( j , i , o . unique_mk_dir , progress ) ;
2018-07-07 12:29:27 +03:00
}
}
/* remove directory structure */
2020-11-27 20:49:45 +03:00
if ( ! o . unique_dir_per_task ) {
VERBOSE ( 3 , - 1 , " main: Using o.testdir, '%s' " , o . testdir ) ;
2018-07-07 12:29:27 +03:00
}
MPI_Barrier ( testComm ) ;
2020-11-27 20:49:45 +03:00
if ( o . remove_only ) {
2018-12-20 00:37:37 +03:00
progress - > items_start = 0 ;
2018-10-17 17:38:35 +03:00
startCreate = GetTimeStamp ( ) ;
2020-11-27 20:49:45 +03:00
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
2018-07-26 12:37:01 +03:00
prep_testdir ( j , dir_iter ) ;
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
if ( o . collective_creates & & ( rank = = 0 ) ) {
2018-07-26 12:37:01 +03:00
/*
* This is inside two loops , one of which already uses " i " and the other uses " j " .
* I don ' t know how this ever worked . I ' m changing this loop to use " k " .
*/
2020-11-27 20:49:45 +03:00
for ( k = 0 ; k < o . size ; k + + ) {
sprintf ( o . base_tree_name , " mdtest_tree.%d " , k ) ;
2018-07-26 12:37:01 +03:00
2020-11-27 20:49:45 +03:00
VERBOSE ( 3 , - 1 , " main (remove hierarchical directory loop-collective): Calling create_remove_directory_tree with '%s' " , o . testdir ) ;
2018-07-26 12:37:01 +03:00
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
2020-11-27 20:49:45 +03:00
create_remove_directory_tree ( 0 , 0 , o . testdir , 0 , progress ) ;
2018-07-26 12:37:01 +03:00
if ( CHECK_STONE_WALL ( progress ) ) {
2020-11-27 20:49:45 +03:00
o . size = k ;
2018-07-26 12:37:01 +03:00
break ;
}
}
2020-11-27 20:49:45 +03:00
} else if ( ! o . collective_creates ) {
VERBOSE ( 3 , - 1 , " main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with '%s' " , o . testdir ) ;
2018-07-26 12:37:01 +03:00
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
2020-11-27 20:49:45 +03:00
create_remove_directory_tree ( 0 , 0 , o . testdir , 0 , progress ) ;
2018-07-26 12:37:01 +03:00
}
} else {
if ( rank = = 0 ) {
2020-11-27 20:49:45 +03:00
VERBOSE ( 3 , - 1 , " V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with '%s' " , o . testdir ) ;
2018-08-24 00:58:53 +03:00
2018-07-26 12:37:01 +03:00
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
2020-11-27 20:49:45 +03:00
create_remove_directory_tree ( 0 , 0 , o . testdir , 0 , progress ) ;
2018-07-26 12:37:01 +03:00
}
}
2018-07-07 12:29:27 +03:00
}
MPI_Barrier ( testComm ) ;
2018-10-17 17:38:35 +03:00
endCreate = GetTimeStamp ( ) ;
2021-01-21 17:10:23 +03:00
summary_table - > rate [ MDTEST_TREE_REMOVE_NUM ] = o . num_dirs_in_tree / ( endCreate - startCreate ) ;
summary_table - > time [ MDTEST_TREE_REMOVE_NUM ] = endCreate - startCreate ;
summary_table - > items [ MDTEST_TREE_REMOVE_NUM ] = o . num_dirs_in_tree ;
summary_table - > stonewall_last_item [ MDTEST_TREE_REMOVE_NUM ] = o . num_dirs_in_tree ;
VERBOSE ( 1 , - 1 , " main Tree removal : %14.3f sec, %14.3f ops/sec " , ( endCreate - startCreate ) , summary_table - > rate [ MDTEST_TREE_REMOVE_NUM ] ) ;
2020-11-27 20:49:45 +03:00
VERBOSE ( 2 , - 1 , " main (at end of for j loop): Removing o.testdir of '%s' \n " , o . testdir ) ;
2018-07-07 12:29:27 +03:00
2020-11-27 20:49:45 +03:00
for ( int dir_iter = 0 ; dir_iter < o . directory_loops ; dir_iter + + ) {
2018-07-26 12:37:01 +03:00
prep_testdir ( j , dir_iter ) ;
2020-11-27 20:49:45 +03:00
if ( ( rank < o . path_count ) & & o . backend - > access ( o . testdir , F_OK , o . backend_options ) = = 0 ) {
//if (( rank == 0 ) && access(o.testdir, F_OK) == 0) {
if ( o . backend - > rmdir ( o . testdir , o . backend_options ) = = - 1 ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " unable to remove directory %s " , o . testdir ) ;
2018-07-26 12:37:01 +03:00
}
}
2018-07-07 12:29:27 +03:00
}
} else {
2021-01-21 17:10:23 +03:00
summary_table - > rate [ MDTEST_TREE_REMOVE_NUM ] = 0 ;
2018-07-07 12:29:27 +03:00
}
}
2018-08-24 00:58:53 +03:00
2018-07-07 12:29:27 +03:00
void mdtest_init_args ( ) {
2020-11-28 13:34:20 +03:00
o = ( mdtest_options_t ) {
. barriers = 1 ,
2021-01-22 15:24:33 +03:00
. branch_factor = 1 ,
. random_buffer_offset = - 1
2020-11-28 13:34:20 +03:00
} ;
2018-08-24 00:58:53 +03:00
}
2018-07-07 12:29:27 +03:00
mdtest_results_t * mdtest_run ( int argc , char * * argv , MPI_Comm world_com , FILE * world_out ) {
testComm = world_com ;
out_logfile = world_out ;
2020-11-09 19:23:34 +03:00
out_resultfile = world_out ;
2018-07-07 12:29:27 +03:00
2021-01-20 17:06:05 +03:00
init_clock ( world_com ) ;
2018-07-07 12:29:27 +03:00
mdtest_init_args ( ) ;
2018-08-08 19:18:54 +03:00
int i , j ;
2019-08-31 01:45:03 +03:00
int numNodes ;
int numTasksOnNode0 = 0 ;
2017-10-20 00:26:52 +03:00
MPI_Group worldgroup , testgroup ;
struct {
int first ;
int last ;
int stride ;
} range = { 0 , 0 , 1 } ;
int first = 1 ;
int last = 0 ;
int stride = 1 ;
int iterations = 1 ;
2020-07-02 18:26:05 +03:00
int created_root_dir = 0 ; // was the root directory existing or newly created
2017-10-20 00:26:52 +03:00
2018-07-12 20:09:13 +03:00
verbose = 0 ;
int no_barriers = 0 ;
char * path = " ./out " ;
int randomize = 0 ;
2018-08-15 19:07:17 +03:00
char APIs [ 1024 ] ;
2018-09-10 19:43:12 +03:00
char APIs_legacy [ 1024 ] ;
2019-02-11 16:49:14 +03:00
aiori_supported_apis ( APIs , APIs_legacy , MDTEST ) ;
2018-08-15 19:07:17 +03:00
char apiStr [ 1024 ] ;
sprintf ( apiStr , " API for I/O [%s] " , APIs ) ;
2020-11-27 20:49:45 +03:00
memset ( & o . hints , 0 , sizeof ( o . hints ) ) ;
2018-08-15 19:07:17 +03:00
2018-07-12 20:09:13 +03:00
option_help options [ ] = {
2020-11-27 20:49:45 +03:00
{ ' a ' , NULL , apiStr , OPTION_OPTIONAL_ARGUMENT , ' s ' , & o . api } ,
{ ' b ' , NULL , " branching factor of hierarchical directory structure " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & o . branch_factor } ,
2018-07-12 20:09:13 +03:00
{ ' d ' , NULL , " the directory in which the tests will run " , OPTION_OPTIONAL_ARGUMENT , ' s ' , & path } ,
{ ' B ' , NULL , " no barriers between phases " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & no_barriers } ,
2020-11-27 20:49:45 +03:00
{ ' C ' , NULL , " only create files/dirs " , OPTION_FLAG , ' d ' , & o . create_only } ,
{ ' T ' , NULL , " only stat files/dirs " , OPTION_FLAG , ' d ' , & o . stat_only } ,
{ ' E ' , NULL , " only read files/dir " , OPTION_FLAG , ' d ' , & o . read_only } ,
{ ' r ' , NULL , " only remove files or directories left behind by previous runs " , OPTION_FLAG , ' d ' , & o . remove_only } ,
{ ' D ' , NULL , " perform test on directories only (no files) " , OPTION_FLAG , ' d ' , & o . dirs_only } ,
{ ' e ' , NULL , " bytes to read from each file " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & o . read_bytes } ,
2018-07-12 20:09:13 +03:00
{ ' f ' , NULL , " first number of tasks on which the test will run " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & first } ,
2020-11-27 20:49:45 +03:00
{ ' F ' , NULL , " perform test on files only (no directories) " , OPTION_FLAG , ' d ' , & o . files_only } ,
2019-09-30 14:25:59 +03:00
# ifdef HAVE_LUSTRE_LUSTREAPI
2020-11-27 20:49:45 +03:00
{ ' g ' , NULL , " global default directory layout for test subdirectories (deletes inherited striping layout) " , OPTION_FLAG , ' d ' , & o . global_dir_layout } ,
2019-09-30 14:25:59 +03:00
# endif /* HAVE_LUSTRE_LUSTREAPI */
2021-01-22 15:24:33 +03:00
{ ' G ' , NULL , " Offset for the data in the read/write buffer, if not set, a random value is used " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & o . random_buffer_offset } ,
2018-09-05 11:14:22 +03:00
{ ' i ' , NULL , " number of iterations the test will run " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & iterations } ,
2020-11-27 20:49:45 +03:00
{ ' I ' , NULL , " number of items per directory in tree " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & o . items_per_dir } ,
{ ' k ' , NULL , " use mknod to create file " , OPTION_FLAG , ' d ' , & o . make_node } ,
2018-07-12 20:09:13 +03:00
{ ' l ' , NULL , " last number of tasks on which the test will run " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & last } ,
2020-11-27 20:49:45 +03:00
{ ' L ' , NULL , " files only at leaf level of tree " , OPTION_FLAG , ' d ' , & o . leaf_only } ,
{ ' n ' , NULL , " every process will creat/stat/read/remove # directories and files " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & o . items } ,
{ ' N ' , NULL , " stride # between tasks for file/dir operation (local=0; set to 1 to avoid client cache) " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & o . nstride } ,
{ ' p ' , NULL , " pre-iteration delay (in seconds) " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & o . pre_delay } ,
{ ' P ' , NULL , " print rate AND time " , OPTION_FLAG , ' d ' , & o . print_rate_and_time } ,
{ 0 , " print-all-procs " , " all processes print an excerpt of their results " , OPTION_FLAG , ' d ' , & o . print_all_proc } ,
2018-07-14 18:46:24 +03:00
{ ' R ' , NULL , " random access to files (only for stat) " , OPTION_FLAG , ' d ' , & randomize } ,
2020-11-27 20:49:45 +03:00
{ 0 , " random-seed " , " random seed for -R " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & o . random_seed } ,
2018-07-12 20:09:13 +03:00
{ ' s ' , NULL , " stride between the number of tasks for each test " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & stride } ,
2020-11-27 20:49:45 +03:00
{ ' S ' , NULL , " shared file access (file only, no directories) " , OPTION_FLAG , ' d ' , & o . shared_file } ,
{ ' c ' , NULL , " collective creates: task 0 does all creates " , OPTION_FLAG , ' d ' , & o . collective_creates } ,
{ ' t ' , NULL , " time unique working directory overhead " , OPTION_FLAG , ' d ' , & o . time_unique_dir_overhead } ,
{ ' u ' , NULL , " unique working directory for each task " , OPTION_FLAG , ' d ' , & o . unique_dir_per_task } ,
2018-07-12 20:09:13 +03:00
{ ' v ' , NULL , " verbosity (each instance of option increments by one) " , OPTION_FLAG , ' d ' , & verbose } ,
{ ' V ' , NULL , " verbosity value " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & verbose } ,
2020-11-27 20:49:45 +03:00
{ ' w ' , NULL , " bytes to write to each file after it is created " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & o . write_bytes } ,
{ ' W ' , NULL , " number in seconds; stonewall timer, write as many seconds and ensure all processes did the same number of operations (currently only stops during create phase and files) " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & o . stone_wall_timer_seconds } ,
{ ' x ' , NULL , " StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs " , OPTION_OPTIONAL_ARGUMENT , ' s ' , & o . stoneWallingStatusFile } ,
{ ' X ' , " verify-read " , " Verify the data read " , OPTION_FLAG , ' d ' , & o . verify_read } ,
{ 0 , " verify-write " , " Verify the data after a write by reading it back immediately " , OPTION_FLAG , ' d ' , & o . verify_write } ,
{ ' y ' , NULL , " sync file after writing " , OPTION_FLAG , ' d ' , & o . sync_file } ,
{ ' Y ' , NULL , " call the sync command after each phase (included in the timing; note it causes all IO to be flushed from your node) " , OPTION_FLAG , ' d ' , & o . call_sync } ,
{ ' z ' , NULL , " depth of hierarchical directory structure " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & o . depth } ,
{ ' Z ' , NULL , " print time instead of rate " , OPTION_FLAG , ' d ' , & o . print_time } ,
2020-06-24 13:10:42 +03:00
{ 0 , " warningAsErrors " , " Any warning should lead to an error. " , OPTION_FLAG , ' d ' , & aiori_warning_as_errors } ,
2021-02-11 12:57:22 +03:00
{ 0 , " saveRankPerformanceDetails " , " Save the individual rank information into this CSV file. " , OPTION_OPTIONAL_ARGUMENT , ' s ' , & o . saveRankDetailsCSV } ,
2018-07-12 20:09:13 +03:00
LAST_OPTION
} ;
2019-03-27 23:04:48 +03:00
options_all_t * global_options = airoi_create_all_module_options ( options ) ;
option_parse ( argc , argv , global_options ) ;
2020-11-27 20:49:45 +03:00
o . backend = aiori_select ( o . api ) ;
if ( o . backend = = NULL )
2020-05-31 14:50:03 +03:00
ERR ( " Unrecognized I/O API " ) ;
2020-11-27 20:49:45 +03:00
if ( ! o . backend - > enable_mdtest )
2020-06-30 13:17:46 +03:00
ERR ( " Backend doesn't support MDTest " ) ;
2020-11-27 20:49:45 +03:00
o . backend_options = airoi_update_module_options ( o . backend , global_options ) ;
2019-08-03 03:23:05 +03:00
free ( global_options - > modules ) ;
free ( global_options ) ;
2018-07-12 20:09:13 +03:00
2018-07-07 12:29:27 +03:00
MPI_Comm_rank ( testComm , & rank ) ;
2020-11-27 20:49:45 +03:00
MPI_Comm_size ( testComm , & o . size ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( o . backend - > xfer_hints ) {
o . backend - > xfer_hints ( & o . hints ) ;
2020-05-30 22:09:37 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . backend - > check_params ) {
o . backend - > check_params ( o . backend_options ) ;
2020-05-30 22:09:37 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . backend - > initialize ) {
o . backend - > initialize ( o . backend_options ) ;
2020-11-22 14:43:42 +03:00
}
2020-11-25 12:50:26 +03:00
2020-11-27 20:49:45 +03:00
o . pid = getpid ( ) ;
o . uid = getuid ( ) ;
2017-06-28 19:42:20 +03:00
2019-08-31 01:45:03 +03:00
numNodes = GetNumNodes ( testComm ) ;
numTasksOnNode0 = GetNumTasksOnNode0 ( testComm ) ;
2017-06-28 19:42:20 +03:00
2019-07-27 23:31:49 +03:00
char cmd_buffer [ 4096 ] ;
strncpy ( cmd_buffer , argv [ 0 ] , 4096 ) ;
for ( i = 1 ; i < argc ; i + + ) {
2019-07-28 19:07:03 +03:00
snprintf ( & cmd_buffer [ strlen ( cmd_buffer ) ] , 4096 - strlen ( cmd_buffer ) , " '%s' " , argv [ i ] ) ;
2017-10-20 00:26:52 +03:00
}
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " -- started at %s -- \n " , PrintTimestamp ( ) ) ;
2020-11-27 20:49:45 +03:00
VERBOSE ( 0 , - 1 , " mdtest-%s was launched with %d total task(s) on %d node(s) " , RELEASE_VERS , o . size , numNodes ) ;
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " Command line used: %s " , cmd_buffer ) ;
2017-06-28 19:42:20 +03:00
2018-07-12 20:09:13 +03:00
/* adjust special variables */
2020-11-27 20:49:45 +03:00
o . barriers = ! no_barriers ;
2018-07-12 20:09:13 +03:00
if ( path ! = NULL ) {
parse_dirpath ( path ) ;
2017-06-28 19:42:20 +03:00
}
2018-07-12 20:09:13 +03:00
if ( randomize > 0 ) {
2020-11-27 20:49:45 +03:00
if ( o . random_seed = = 0 ) {
2018-07-12 20:09:13 +03:00
/* Ensure all procs have the same random number */
2020-11-27 20:49:45 +03:00
o . random_seed = time ( NULL ) ;
2018-07-12 20:09:13 +03:00
MPI_Barrier ( testComm ) ;
2020-11-27 20:49:45 +03:00
MPI_Bcast ( & o . random_seed , 1 , MPI_INT , 0 , testComm ) ;
2018-07-12 20:09:13 +03:00
}
2020-11-27 20:49:45 +03:00
o . random_seed + = rank ;
2017-10-20 00:26:52 +03:00
}
2021-01-22 15:24:33 +03:00
if ( o . random_buffer_offset = = - 1 ) {
o . random_buffer_offset = time ( NULL ) ;
MPI_Bcast ( & o . random_buffer_offset , 1 , MPI_INT , 0 , testComm ) ;
}
2020-11-27 20:49:45 +03:00
if ( ( o . items > 0 ) & & ( o . items_per_dir > 0 ) & & ( ! o . unique_dir_per_task ) ) {
o . directory_loops = o . items / o . items_per_dir ;
2018-07-26 12:37:01 +03:00
} else {
2020-11-27 20:49:45 +03:00
o . directory_loops = 1 ;
2017-10-20 00:26:52 +03:00
}
2020-06-28 19:21:24 +03:00
md_validate_tests ( ) ;
2017-06-28 19:42:20 +03:00
2019-07-27 22:22:15 +03:00
// option_print_current(options);
2020-11-27 20:49:45 +03:00
VERBOSE ( 1 , - 1 , " api : %s " , o . api ) ;
VERBOSE ( 1 , - 1 , " barriers : %s " , ( o . barriers ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " collective_creates : %s " , ( o . collective_creates ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " create_only : %s " , ( o . create_only ? " True " : " False " ) ) ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " dirpath(s): " ) ;
2020-11-27 20:49:45 +03:00
for ( i = 0 ; i < o . path_count ; i + + ) {
VERBOSE ( 1 , - 1 , " \t %s " , o . filenames [ i ] ) ;
2019-07-27 22:22:15 +03:00
}
2020-11-27 20:49:45 +03:00
VERBOSE ( 1 , - 1 , " dirs_only : %s " , ( o . dirs_only ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " read_bytes : " LLU " " , o . read_bytes ) ;
VERBOSE ( 1 , - 1 , " read_only : %s " , ( o . read_only ? " True " : " False " ) ) ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " first : %d " , first ) ;
2020-11-27 20:49:45 +03:00
VERBOSE ( 1 , - 1 , " files_only : %s " , ( o . files_only ? " True " : " False " ) ) ;
2019-09-30 14:25:59 +03:00
# ifdef HAVE_LUSTRE_LUSTREAPI
2020-11-27 20:49:45 +03:00
VERBOSE ( 1 , - 1 , " global_dir_layout : %s " , ( o . global_dir_layout ? " True " : " False " ) ) ;
2019-09-30 14:25:59 +03:00
# endif /* HAVE_LUSTRE_LUSTREAPI */
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " iterations : %d " , iterations ) ;
2020-11-27 20:49:45 +03:00
VERBOSE ( 1 , - 1 , " items_per_dir : " LLU " " , o . items_per_dir ) ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " last : %d " , last ) ;
2020-11-27 20:49:45 +03:00
VERBOSE ( 1 , - 1 , " leaf_only : %s " , ( o . leaf_only ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " items : " LLU " " , o . items ) ;
VERBOSE ( 1 , - 1 , " nstride : %d " , o . nstride ) ;
VERBOSE ( 1 , - 1 , " pre_delay : %d " , o . pre_delay ) ;
VERBOSE ( 1 , - 1 , " remove_only : %s " , ( o . leaf_only ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " random_seed : %d " , o . random_seed ) ;
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " stride : %d " , stride ) ;
2020-11-27 20:49:45 +03:00
VERBOSE ( 1 , - 1 , " shared_file : %s " , ( o . shared_file ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " time_unique_dir_overhead: %s " , ( o . time_unique_dir_overhead ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " stone_wall_timer_seconds: %d " , o . stone_wall_timer_seconds ) ;
VERBOSE ( 1 , - 1 , " stat_only : %s " , ( o . stat_only ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " unique_dir_per_task : %s " , ( o . unique_dir_per_task ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " write_bytes : " LLU " " , o . write_bytes ) ;
VERBOSE ( 1 , - 1 , " sync_file : %s " , ( o . sync_file ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " call_sync : %s " , ( o . call_sync ? " True " : " False " ) ) ;
VERBOSE ( 1 , - 1 , " depth : %d " , o . depth ) ;
VERBOSE ( 1 , - 1 , " make_node : %d " , o . make_node ) ;
2017-10-20 00:26:52 +03:00
/* setup total number of items and number of items per dir */
2020-11-27 20:49:45 +03:00
if ( o . depth < = 0 ) {
o . num_dirs_in_tree = 1 ;
2017-06-28 19:42:20 +03:00
} else {
2020-11-27 20:49:45 +03:00
if ( o . branch_factor < 1 ) {
o . num_dirs_in_tree = 1 ;
} else if ( o . branch_factor = = 1 ) {
o . num_dirs_in_tree = o . depth + 1 ;
2017-10-20 00:26:52 +03:00
} else {
2020-11-27 20:49:45 +03:00
o . num_dirs_in_tree = ( pow ( o . branch_factor , o . depth + 1 ) - 1 ) / ( o . branch_factor - 1 ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . items_per_dir > 0 ) {
if ( o . items = = 0 ) {
if ( o . leaf_only ) {
o . items = o . items_per_dir * ( uint64_t ) pow ( o . branch_factor , o . depth ) ;
2019-05-14 16:55:11 +03:00
} else {
2020-11-27 20:49:45 +03:00
o . items = o . items_per_dir * o . num_dirs_in_tree ;
2019-05-14 16:55:11 +03:00
}
2018-10-17 18:04:49 +03:00
} else {
2020-11-27 20:49:45 +03:00
o . num_dirs_in_tree_calc = o . num_dirs_in_tree ;
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
} else {
2020-11-27 20:49:45 +03:00
if ( o . leaf_only ) {
if ( o . branch_factor < = 1 ) {
o . items_per_dir = o . items ;
2017-10-20 00:26:52 +03:00
} else {
2020-11-27 20:49:45 +03:00
o . items_per_dir = ( uint64_t ) ( o . items / pow ( o . branch_factor , o . depth ) ) ;
o . items = o . items_per_dir * ( uint64_t ) pow ( o . branch_factor , o . depth ) ;
2017-10-20 00:26:52 +03:00
}
} else {
2020-11-27 20:49:45 +03:00
o . items_per_dir = o . items / o . num_dirs_in_tree ;
o . items = o . items_per_dir * o . num_dirs_in_tree ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* initialize rand_array */
2020-11-27 20:49:45 +03:00
if ( o . random_seed > 0 ) {
srand ( o . random_seed ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
uint64_t s ;
2017-06-28 19:42:20 +03:00
2021-01-20 18:19:13 +03:00
o . rand_array = ( uint64_t * ) safeMalloc ( o . items * sizeof ( * o . rand_array ) ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
for ( s = 0 ; s < o . items ; s + + ) {
o . rand_array [ s ] = s ;
2017-10-20 00:26:52 +03:00
}
/* shuffle list randomly */
2020-11-27 20:49:45 +03:00
uint64_t n = o . items ;
2017-10-20 00:26:52 +03:00
while ( n > 1 ) {
n - - ;
/*
* Generate a random number in the range 0 . . n
*
* rand ( ) returns a number from 0 . . RAND_MAX . Divide that
* by RAND_MAX and you get a floating point number in the
* range 0 . . 1. Multiply that by n and you get a number in
* the range 0 . . n .
*/
2018-07-08 00:19:42 +03:00
uint64_t k = ( uint64_t ) ( ( ( double ) rand ( ) / ( double ) RAND_MAX ) * ( double ) n ) ;
2017-10-20 00:26:52 +03:00
/*
* Now move the nth element to the kth ( randomly chosen )
* element , and the kth element to the nth element .
*/
2020-11-27 20:49:45 +03:00
uint64_t tmp = o . rand_array [ k ] ;
o . rand_array [ k ] = o . rand_array [ n ] ;
o . rand_array [ n ] = tmp ;
2017-10-20 00:26:52 +03:00
}
}
/* allocate and initialize write buffer with # */
2020-11-27 20:49:45 +03:00
if ( o . write_bytes > 0 ) {
int alloc_res = posix_memalign ( ( void * * ) & o . write_buffer , sysconf ( _SC_PAGESIZE ) , o . write_bytes ) ;
2020-06-03 00:30:38 +03:00
if ( alloc_res ) {
2017-10-20 00:26:52 +03:00
FAIL ( " out of memory " ) ;
}
2021-01-22 17:05:58 +03:00
generate_memory_pattern ( o . write_buffer , o . write_bytes , o . random_buffer_offset , rank ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* setup directory path to work in */
2020-11-27 20:49:45 +03:00
if ( o . path_count = = 0 ) { /* special case where no directory path provided with '-d' option */
char * ret = getcwd ( o . testdirpath , MAX_PATHLEN ) ;
2018-08-08 19:18:54 +03:00
if ( ret = = NULL ) {
2020-11-27 20:49:45 +03:00
FAIL ( " Unable to get current working directory on %s " , o . testdirpath ) ;
2018-08-08 19:18:54 +03:00
}
2020-11-27 20:49:45 +03:00
o . path_count = 1 ;
2017-10-20 00:26:52 +03:00
} else {
2020-11-27 20:49:45 +03:00
strcpy ( o . testdirpath , o . filenames [ rank % o . path_count ] ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* if directory does not exist, create it */
2020-11-27 20:49:45 +03:00
if ( ( rank < o . path_count ) & & o . backend - > access ( o . testdirpath , F_OK , o . backend_options ) ! = 0 ) {
if ( o . backend - > mkdir ( o . testdirpath , DIRMODE , o . backend_options ) ! = 0 ) {
2021-01-20 18:19:13 +03:00
EWARNF ( " Unable to create test directory path %s " , o . testdirpath ) ;
2017-10-20 00:26:52 +03:00
}
2020-07-02 18:26:05 +03:00
created_root_dir = 1 ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* display disk usage */
2020-11-27 20:49:45 +03:00
VERBOSE ( 3 , - 1 , " main (before display_freespace): o.testdirpath is '%s' " , o . testdirpath ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( rank = = 0 ) ShowFileSystemSize ( o . testdirpath , o . backend , o . backend_options ) ;
2019-08-01 03:42:03 +03:00
int tasksBlockMapping = QueryNodeMapping ( testComm , true ) ;
2017-06-28 19:42:20 +03:00
2019-07-26 17:55:24 +03:00
/* set the shift to mimic IOR and shift by procs per node */
2020-11-27 20:49:45 +03:00
if ( o . nstride > 0 ) {
2019-08-31 01:45:03 +03:00
if ( numNodes > 1 & & tasksBlockMapping ) {
2019-07-28 19:25:42 +03:00
/* the user set the stride presumably to get the consumer tasks on a different node than the producer tasks
however , if the mpirun scheduler placed the tasks by - slot ( in a contiguous block ) then we need to adjust the shift by ppn */
2020-11-27 20:49:45 +03:00
o . nstride * = numTasksOnNode0 ;
2019-07-26 17:55:24 +03:00
}
2020-11-27 20:49:45 +03:00
VERBOSE ( 0 , 5 , " Shifting ranks by %d for each phase. " , o . nstride ) ;
2017-06-28 19:42:20 +03:00
}
2020-11-27 20:49:45 +03:00
VERBOSE ( 3 , - 1 , " main (after display_freespace): o.testdirpath is '%s' " , o . testdirpath ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
2020-11-27 20:49:45 +03:00
if ( o . random_seed > 0 ) {
VERBOSE ( 0 , - 1 , " random seed: %d " , o . random_seed ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( gethostname ( o . hostname , MAX_PATHLEN ) = = - 1 ) {
2017-10-20 00:26:52 +03:00
perror ( " gethostname " ) ;
2018-07-07 12:29:27 +03:00
MPI_Abort ( testComm , 2 ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( last = = 0 ) {
2020-11-27 20:49:45 +03:00
first = o . size ;
last = o . size ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* setup summary table for recording results */
2021-01-20 18:19:13 +03:00
o . summary_table = ( mdtest_results_t * ) safeMalloc ( iterations * sizeof ( mdtest_results_t ) ) ;
2020-11-27 20:49:45 +03:00
memset ( o . summary_table , 0 , iterations * sizeof ( mdtest_results_t ) ) ;
2018-07-07 12:29:27 +03:00
2020-11-27 20:49:45 +03:00
if ( o . unique_dir_per_task ) {
sprintf ( o . base_tree_name , " mdtest_tree.%d " , rank ) ;
2017-10-20 00:26:52 +03:00
} else {
2020-11-27 20:49:45 +03:00
sprintf ( o . base_tree_name , " mdtest_tree " ) ;
2017-10-20 00:26:52 +03:00
}
2021-02-18 12:53:07 +03:00
mdtest_results_t * aggregated_results = safeMalloc ( iterations * sizeof ( mdtest_results_t ) ) ;
2017-10-20 00:26:52 +03:00
/* default use shared directory */
2020-11-27 20:49:45 +03:00
strcpy ( o . mk_name , " mdtest.shared. " ) ;
strcpy ( o . stat_name , " mdtest.shared. " ) ;
strcpy ( o . read_name , " mdtest.shared. " ) ;
strcpy ( o . rm_name , " mdtest.shared. " ) ;
2017-10-20 00:26:52 +03:00
2018-07-07 12:29:27 +03:00
MPI_Comm_group ( testComm , & worldgroup ) ;
2017-10-20 00:26:52 +03:00
/* Run the tests */
2020-11-27 20:49:45 +03:00
for ( i = first ; i < = last & & i < = o . size ; i + = stride ) {
2017-10-20 00:26:52 +03:00
range . last = i - 1 ;
MPI_Group_range_incl ( worldgroup , 1 , ( void * ) & range , & testgroup ) ;
2018-07-07 12:29:27 +03:00
MPI_Comm_create ( testComm , testgroup , & testComm ) ;
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
2020-11-27 20:49:45 +03:00
uint64_t items_all = i * o . items ;
if ( o . num_dirs_in_tree_calc ) {
items_all * = o . num_dirs_in_tree_calc ;
2018-10-17 18:04:49 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . files_only & & o . dirs_only ) {
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " %d tasks, " LLU " files/directories " , i , items_all ) ;
2020-11-27 20:49:45 +03:00
} else if ( o . files_only ) {
if ( ! o . shared_file ) {
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " %d tasks, " LLU " files " , i , items_all ) ;
2017-10-20 00:26:52 +03:00
}
else {
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " %d tasks, 1 file " , i ) ;
2017-10-20 00:26:52 +03:00
}
2020-11-27 20:49:45 +03:00
} else if ( o . dirs_only ) {
2019-07-27 23:31:49 +03:00
VERBOSE ( 0 , - 1 , " %d tasks, " LLU " directories " , i , items_all ) ;
2017-10-20 00:26:52 +03:00
}
}
2019-07-27 22:22:15 +03:00
VERBOSE ( 1 , - 1 , " " ) ;
VERBOSE ( 1 , - 1 , " Operation Duration Rate " ) ;
VERBOSE ( 1 , - 1 , " --------- -------- ---- " ) ;
2017-10-20 00:26:52 +03:00
2018-07-07 12:29:27 +03:00
for ( j = 0 ; j < iterations ; j + + ) {
2018-07-08 01:56:39 +03:00
// keep track of the current status for stonewalling
2020-11-27 20:49:45 +03:00
mdtest_iteration ( i , j , testgroup , & o . summary_table [ j ] ) ;
2017-10-20 00:26:52 +03:00
}
2021-02-18 12:53:07 +03:00
summarize_results ( iterations , aggregated_results ) ;
2021-02-11 12:57:22 +03:00
if ( o . saveRankDetailsCSV ) {
2021-02-18 12:53:07 +03:00
StoreRankInformation ( iterations , aggregated_results ) ;
2021-02-11 12:57:22 +03:00
}
2017-10-20 00:26:52 +03:00
if ( i = = 1 & & stride > 1 ) {
i = 0 ;
}
2017-06-28 19:42:20 +03:00
}
2020-11-27 20:49:45 +03:00
if ( created_root_dir & & o . remove_only & & o . backend - > rmdir ( o . testdirpath , o . backend_options ) ! = 0 ) {
FAIL ( " Unable to remove test directory path %s " , o . testdirpath ) ;
2020-07-02 18:26:05 +03:00
}
2021-02-11 18:50:48 +03:00
int total_errors = 0 ;
2021-01-22 15:10:09 +03:00
MPI_Reduce ( & o . verification_error , & total_errors , 1 , MPI_INT , MPI_SUM , 0 , testComm ) ;
2021-02-11 18:50:48 +03:00
if ( rank = = 0 & & total_errors ) {
2021-01-22 15:10:09 +03:00
VERBOSE ( 0 , - 1 , " \n ERROR: verifying the data on read (%lld errors)! Take the performance values with care! \n " , total_errors ) ;
2017-06-28 19:42:20 +03:00
}
2019-07-27 22:22:15 +03:00
VERBOSE ( 0 , - 1 , " -- finished at %s -- \n " , PrintTimestamp ( ) ) ;
2017-06-28 19:42:20 +03:00
2020-11-27 20:49:45 +03:00
if ( o . random_seed > 0 ) {
free ( o . rand_array ) ;
2017-06-28 19:42:20 +03:00
}
2020-11-27 20:49:45 +03:00
if ( o . backend - > finalize ) {
o . backend - > finalize ( o . backend_options ) ;
2020-05-30 20:30:26 +03:00
}
2018-06-13 21:37:37 +03:00
2020-11-27 20:49:45 +03:00
if ( o . write_bytes > 0 ) {
free ( o . write_buffer ) ;
2020-06-29 22:15:14 +03:00
}
2021-02-18 12:53:07 +03:00
free ( o . summary_table ) ;
2020-06-29 22:15:14 +03:00
2021-02-18 12:53:07 +03:00
return aggregated_results ;
2017-06-28 19:42:20 +03:00
}