2017-06-28 19:42:20 +03:00
/*
* Copyright ( C ) 2003 , The Regents of the University of California .
* Produced at the Lawrence Livermore National Laboratory .
* Written by Christopher J . Morrone < morrone @ llnl . gov > ,
* Bill Loewe < loewe @ loewe . net > , Tyce McLarty < mclarty @ llnl . gov > ,
* and Ryan Kroiss < rrkroiss @ lanl . gov > .
* All rights reserved .
* UCRL - CODE - 155800
*
* Please read the COPYRIGHT file .
*
* This program is free software ; you can redistribute it and / or modify
* it under the terms of the GNU General Public License ( as published by
* the Free Software Foundation ) version 2 , dated June 1991.
*
* This program is distributed in the hope that it will be useful ,
* but WITHOUT ANY WARRANTY ; without even the IMPLIED WARRANTY OF
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
* terms and conditions of the GNU General Public License for more details .
*
* You should have received a copy of the GNU General Public License
* along with this program ; if not , write to the Free Software
* Foundation , Inc . , 59 Temple Place , Suite 330 , Boston , MA 02111 - 1307 USA
*
* CVS info :
* $ RCSfile : mdtest . c , v $
* $ Revision : 1.4 $
* $ Date : 2013 / 11 / 27 17 : 05 : 31 $
* $ Author : brettkettering $
*/
# include <limits.h>
# include <math.h>
# include <stdio.h>
# include <stdlib.h>
2017-10-20 00:26:52 +03:00
# include <stdbool.h>
# include <inttypes.h>
2017-06-28 19:42:20 +03:00
# include <sys/types.h>
# include <sys/stat.h>
2018-07-12 20:09:13 +03:00
# include "option.h"
2018-07-07 12:29:27 +03:00
# include "utilities.h"
2018-07-07 10:41:33 +03:00
2017-10-20 00:26:52 +03:00
# if HAVE_SYS_PARAM_H
2017-06-28 19:42:20 +03:00
# include <sys/param.h>
2017-10-20 00:26:52 +03:00
# endif
# if HAVE_SYS_MOUNT_H
2017-06-28 19:42:20 +03:00
# include <sys/mount.h>
2017-10-20 00:26:52 +03:00
# endif
# if HAVE_SYS_STATFS_H
2017-06-28 19:42:20 +03:00
# include <sys/statfs.h>
# endif
2017-10-20 00:26:52 +03:00
# if HAVE_SYS_STATVFS_H
2017-06-28 19:42:20 +03:00
# include <sys/statvfs.h>
# endif
# include <fcntl.h>
# include <string.h>
# include <unistd.h>
# include <dirent.h>
# include <errno.h>
# include <time.h>
# include <sys/time.h>
2017-10-20 00:26:52 +03:00
# include "aiori.h"
# include "ior.h"
2018-07-07 12:29:27 +03:00
# include "mdtest.h"
2017-10-20 00:26:52 +03:00
# include <mpi.h>
2017-06-28 19:42:20 +03:00
# define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH
# define DIRMODE S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH
2017-10-20 00:26:52 +03:00
# define RELEASE_VERS "1.9.3"
# define TEST_DIR "#test-dir"
2017-06-28 19:42:20 +03:00
# define ITEM_COUNT 25000
2018-07-07 12:29:27 +03:00
# define LLU "%lu"
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
static int size ;
static uint64_t * rand_array ;
2018-07-08 00:39:14 +03:00
static char testdir [ MAX_PATHLEN ] ;
static char testdirpath [ MAX_PATHLEN ] ;
static char base_tree_name [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
static char * * filenames ;
2018-07-08 00:39:14 +03:00
static char hostname [ MAX_PATHLEN ] ;
static char mk_name [ MAX_PATHLEN ] ;
static char stat_name [ MAX_PATHLEN ] ;
static char read_name [ MAX_PATHLEN ] ;
static char rm_name [ MAX_PATHLEN ] ;
static char unique_mk_dir [ MAX_PATHLEN ] ;
static char unique_chdir_dir [ MAX_PATHLEN ] ;
static char unique_stat_dir [ MAX_PATHLEN ] ;
static char unique_read_dir [ MAX_PATHLEN ] ;
static char unique_rm_dir [ MAX_PATHLEN ] ;
static char unique_rm_uni_dir [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
static char * write_buffer ;
static char * read_buffer ;
2018-07-08 01:56:39 +03:00
static char * stoneWallingStatusFile ;
2018-07-07 12:29:27 +03:00
static int barriers ;
2017-10-20 00:26:52 +03:00
static int create_only ;
static int stat_only ;
static int read_only ;
static int remove_only ;
static int leaf_only ;
2018-07-07 12:29:27 +03:00
static unsigned branch_factor ;
2017-10-20 00:26:52 +03:00
static int depth ;
2017-06-28 19:42:20 +03:00
/*
* This is likely a small value , but it ' s sometimes computed by
* branch_factor ^ ( depth + 1 ) , so we ' ll make it a larger variable ,
* just in case .
*/
2017-10-20 00:26:52 +03:00
static uint64_t num_dirs_in_tree ;
2017-06-28 19:42:20 +03:00
/*
* As we start moving towards Exascale , we could have billions
* of files in a directory . Make room for that possibility with
* a larger variable .
*/
2018-07-14 18:46:24 +03:00
static uint64_t items ;
2017-10-20 00:26:52 +03:00
static uint64_t items_per_dir ;
2018-07-26 12:37:01 +03:00
static int directory_loops ;
2018-07-07 12:29:27 +03:00
static int print_time ;
2017-10-20 00:26:52 +03:00
static int random_seed ;
static int shared_file ;
static int files_only ;
static int dirs_only ;
static int pre_delay ;
static int unique_dir_per_task ;
static int time_unique_dir_overhead ;
2018-07-07 12:29:27 +03:00
static int throttle ;
2017-10-20 00:26:52 +03:00
static int collective_creates ;
static size_t write_bytes ;
2018-07-07 12:29:27 +03:00
static int stone_wall_timer_seconds ;
2017-10-20 00:26:52 +03:00
static size_t read_bytes ;
static int sync_file ;
static int path_count ;
static int nstride ; /* neighbor stride */
2018-07-07 12:29:27 +03:00
static mdtest_results_t * summary_table ;
2017-10-20 00:26:52 +03:00
static pid_t pid ;
static uid_t uid ;
/* just use the POSIX backend for now */
static const char * backend_name = " POSIX " ;
static const ior_aiori_t * backend ;
static IOR_param_t param ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
/* This structure describes the processing status for stonewalling */
typedef struct {
double start_time ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
int stone_wall_timer_seconds ;
2017-06-28 19:42:20 +03:00
2018-07-14 18:46:24 +03:00
uint64_t items_start ;
uint64_t items_done ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
uint64_t items_per_dir ;
} rank_progress_t ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
# define CHECK_STONE_WALL(p) (((p)->stone_wall_timer_seconds != 0) && ((GetTimeStamp() - (p)->start_time) > (p)->stone_wall_timer_seconds))
2017-06-28 19:42:20 +03:00
/* for making/removing unique directory && stating/deleting subdirectory */
enum { MK_UNI_DIR , STAT_SUB_DIR , READ_SUB_DIR , RM_SUB_DIR , RM_UNI_DIR } ;
void offset_timers ( double * t , int tcount ) {
2017-10-20 00:26:52 +03:00
double toffset ;
int i ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering offset_timers... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
toffset = MPI_Wtime ( ) - t [ tcount ] ;
for ( i = 0 ; i < tcount + 1 ; i + + ) {
t [ i ] + = toffset ;
}
2017-06-28 19:42:20 +03:00
}
void parse_dirpath ( char * dirpath_arg ) {
2017-10-20 00:26:52 +03:00
char * tmp , * token ;
char delimiter_string [ 3 ] = { ' @ ' , ' \n ' , ' \0 ' } ;
int i = 0 ;
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering parse_dirpath... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
tmp = dirpath_arg ;
if ( * tmp ! = ' \0 ' ) path_count + + ;
while ( * tmp ! = ' \0 ' ) {
if ( * tmp = = ' @ ' ) {
path_count + + ;
}
tmp + + ;
}
filenames = ( char * * ) malloc ( path_count * sizeof ( char * * ) ) ;
if ( filenames = = NULL ) {
FAIL ( " out of memory " ) ;
}
token = strtok ( dirpath_arg , delimiter_string ) ;
while ( token ! = NULL ) {
filenames [ i ] = token ;
token = strtok ( NULL , delimiter_string ) ;
i + + ;
}
2017-06-28 19:42:20 +03:00
}
2018-07-26 12:37:01 +03:00
static void prep_testdir ( int j , int dir_iter ) {
int pos = sprintf ( testdir , " %s " , testdirpath ) ;
if ( testdir [ strlen ( testdir ) - 1 ] ! = ' / ' ) {
pos + = sprintf ( & testdir [ pos ] , " / " ) ;
}
pos + = sprintf ( & testdir [ pos ] , " %s " , TEST_DIR ) ;
pos + = sprintf ( & testdir [ pos ] , " .%d-%d " , j , dir_iter ) ;
}
2017-06-28 19:42:20 +03:00
/*
* This function copies the unique directory name for a given option to
* the " to " parameter . Some memory must be allocated to the " to " parameter .
*/
void unique_dir_access ( int opt , char * to ) {
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering unique_dir_access... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
if ( opt = = MK_UNI_DIR ) {
MPI_Barrier ( testComm ) ;
2018-07-26 12:37:01 +03:00
sprintf ( to , " %s/%s " , testdir , unique_chdir_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = STAT_SUB_DIR ) {
2018-07-26 12:37:01 +03:00
sprintf ( to , " %s/%s " , testdir , unique_stat_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = READ_SUB_DIR ) {
2018-07-26 12:37:01 +03:00
sprintf ( to , " %s/%s " , testdir , unique_read_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = RM_SUB_DIR ) {
2018-07-26 12:37:01 +03:00
sprintf ( to , " %s/%s " , testdir , unique_rm_dir ) ;
2017-10-20 00:26:52 +03:00
} else if ( opt = = RM_UNI_DIR ) {
2018-07-26 12:37:01 +03:00
sprintf ( to , " %s/%s " , testdir , unique_rm_uni_dir ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
static void create_remove_dirs ( const char * path , bool create , uint64_t itemNum ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
const char * operation = create ? " create " : " remove " ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & &
( verbose > = 3 ) & &
( itemNum % ITEM_COUNT = = 0 & & ( itemNum ! = 0 ) ) ) {
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: %s dir: " LLU " \n " , operation , itemNum ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//create dirs
sprintf ( curr_item , " %s/dir.%s% " PRIu64 , path , create ? mk_name : rm_name , itemNum ) ;
if ( rank = = 0 & & verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: create_remove_items_helper (dirs %s): curr_item is \" %s \" \n " , operation , curr_item ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( create ) {
if ( backend - > mkdir ( curr_item , DIRMODE , & param ) = = - 1 ) {
FAIL ( " unable to create directory " ) ;
}
} else {
if ( backend - > rmdir ( curr_item , & param ) = = - 1 ) {
FAIL ( " unable to remove directory " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
static void remove_file ( const char * path , uint64_t itemNum ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & &
( verbose > = 3 ) & &
( itemNum % ITEM_COUNT = = 0 & & ( itemNum ! = 0 ) ) ) {
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: remove file: " LLU " \n " , itemNum ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//remove files
2018-07-07 12:29:27 +03:00
sprintf ( curr_item , " %s/file.%s " LLU " " , path , rm_name , itemNum ) ;
2017-10-20 00:26:52 +03:00
if ( rank = = 0 & & verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: create_remove_items_helper (non-dirs remove): curr_item is \" %s \" \n " , curr_item ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ! ( shared_file & & rank ! = 0 ) ) {
backend - > delete ( curr_item , & param ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
static void create_file ( const char * path , uint64_t itemNum ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
void * aiori_fh ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & &
( verbose > = 3 ) & &
( itemNum % ITEM_COUNT = = 0 & & ( itemNum ! = 0 ) ) ) {
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: create file: " LLU " \n " , itemNum ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//create files
2018-07-07 12:29:27 +03:00
sprintf ( curr_item , " %s/file.%s " LLU " " , path , mk_name , itemNum ) ;
2017-10-20 00:26:52 +03:00
if ( rank = = 0 & & verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: create_remove_items_helper (non-dirs create): curr_item is \" %s \" \n " , curr_item ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( collective_creates ) {
param . openFlags = IOR_WRONLY ;
2017-06-28 19:42:20 +03:00
if ( rank = = 0 & & verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: create_remove_items_helper (collective): open... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
aiori_fh = backend - > open ( curr_item , & param ) ;
if ( NULL = = aiori_fh ) {
2017-06-28 19:42:20 +03:00
FAIL ( " unable to open file " ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/*
* ! collective_creates
*/
} else {
param . openFlags = IOR_CREAT | IOR_WRONLY ;
param . filePerProc = ! shared_file ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( rank = = 0 & & verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: create_remove_items_helper (non-collective, shared): open... \n " ) ;
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
aiori_fh = backend - > create ( curr_item , & param ) ;
if ( NULL = = aiori_fh ) {
FAIL ( " unable to create file " ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( write_bytes > 0 ) {
if ( rank = = 0 & & verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: create_remove_items_helper: write... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/*
* According to Bill Loewe , writes are only done one time , so they are always at
* offset 0 ( zero ) .
*/
param . offset = 0 ;
param . fsyncPerWrite = sync_file ;
2018-07-07 12:29:27 +03:00
if ( write_bytes ! = ( size_t ) backend - > xfer ( WRITE , aiori_fh , ( IOR_size_t * ) write_buffer , write_bytes , & param ) ) {
2017-06-28 19:42:20 +03:00
FAIL ( " unable to write file " ) ;
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( rank = = 0 & & verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: create_remove_items_helper: close... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
backend - > close ( aiori_fh , & param ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* helper for creating/removing items */
void create_remove_items_helper ( const int dirs , const int create , const char * path ,
2018-07-07 12:29:27 +03:00
uint64_t itemNum , rank_progress_t * progress ) {
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering create_remove_items_helper... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2018-07-08 01:56:39 +03:00
for ( uint64_t i = progress - > items_start ; i < progress - > items_per_dir ; + + i ) {
2017-10-20 00:26:52 +03:00
if ( ! dirs ) {
if ( create ) {
create_file ( path , itemNum + i ) ;
} else {
remove_file ( path , itemNum + i ) ;
}
2017-06-28 19:42:20 +03:00
} else {
2017-10-20 00:26:52 +03:00
create_remove_dirs ( path , create , itemNum + i ) ;
2017-06-28 19:42:20 +03:00
}
2018-07-07 12:29:27 +03:00
if ( CHECK_STONE_WALL ( progress ) ) {
progress - > items_done = i + 1 ;
return ;
}
2017-10-20 00:26:52 +03:00
}
2018-07-08 01:56:39 +03:00
progress - > items_done = progress - > items_per_dir ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* helper function to do collective operations */
2018-07-07 12:29:27 +03:00
void collective_helper ( const int dirs , const int create , const char * path , uint64_t itemNum , rank_progress_t * progress ) {
2018-07-08 00:39:14 +03:00
char curr_item [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering collective_helper... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-08 01:56:39 +03:00
for ( uint64_t i = progress - > items_start ; i < progress - > items_per_dir ; + + i ) {
2017-10-20 00:26:52 +03:00
if ( dirs ) {
create_remove_dirs ( path , create , itemNum + i ) ;
continue ;
2017-06-28 19:42:20 +03:00
}
2018-07-07 12:29:27 +03:00
sprintf ( curr_item , " %s/file.%s " LLU " " , path , create ? mk_name : rm_name , itemNum + i ) ;
2017-06-28 19:42:20 +03:00
if ( rank = = 0 & & verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: create file: %s \n " , curr_item ) ;
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( create ) {
void * aiori_fh ;
//create files
param . openFlags = IOR_WRONLY | IOR_CREAT ;
aiori_fh = backend - > create ( curr_item , & param ) ;
if ( NULL = = aiori_fh ) {
FAIL ( " unable to create file " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
backend - > close ( aiori_fh , & param ) ;
} else if ( ! ( shared_file & & rank ! = 0 ) ) {
//remove files
backend - > delete ( curr_item , & param ) ;
2017-06-28 19:42:20 +03:00
}
2018-07-07 12:29:27 +03:00
if ( CHECK_STONE_WALL ( progress ) ) {
progress - > items_done = i + 1 ;
return ;
}
2017-06-28 19:42:20 +03:00
}
2018-07-08 01:56:39 +03:00
progress - > items_done = progress - > items_per_dir ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* recusive function to create and remove files/directories from the
directory tree */
2018-07-07 12:29:27 +03:00
void create_remove_items ( int currDepth , const int dirs , const int create , const int collective , const char * path , uint64_t dirNum , rank_progress_t * progress ) {
unsigned i ;
2018-07-08 00:39:14 +03:00
char dir [ MAX_PATHLEN ] ;
char temp_path [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
unsigned long long currDir = dirNum ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering create_remove_items, currDepth = %d... \n " , currDepth ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2018-07-08 00:39:14 +03:00
memset ( dir , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
strcpy ( temp_path , path ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( rank = = 0 & & verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: create_remove_items (start): temp_path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( currDepth = = 0 ) {
/* create items at this depth */
if ( ! leaf_only | | ( depth = = 0 & & leaf_only ) ) {
if ( collective ) {
2018-07-07 12:29:27 +03:00
collective_helper ( dirs , create , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-07 12:29:27 +03:00
create_remove_items_helper ( dirs , create , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( depth > 0 ) {
create_remove_items ( + + currDepth , dirs , create ,
2018-07-07 12:29:27 +03:00
collective , temp_path , + + dirNum , progress ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
} else if ( currDepth < = depth ) {
/* iterate through the branches */
for ( i = 0 ; i < branch_factor ; i + + ) {
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* determine the current branch and append it to the path */
sprintf ( dir , " %s.%llu/ " , base_tree_name , currDir ) ;
strcat ( temp_path , " / " ) ;
strcat ( temp_path , dir ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( rank = = 0 & & verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: create_remove_items (for loop): temp_path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create the items in this branch */
if ( ! leaf_only | | ( leaf_only & & currDepth = = depth ) ) {
if ( collective ) {
2018-07-07 12:29:27 +03:00
collective_helper ( dirs , create , temp_path , currDir * items_per_dir , progress ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-07 12:29:27 +03:00
create_remove_items_helper ( dirs , create , temp_path , currDir * items_per_dir , progress ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* make the recursive call for the next level below this branch */
create_remove_items (
+ + currDepth ,
dirs ,
create ,
collective ,
temp_path ,
2018-07-07 12:29:27 +03:00
( currDir * ( unsigned long long ) branch_factor ) + 1 ,
progress
) ;
2017-10-20 00:26:52 +03:00
currDepth - - ;
/* reset the path */
strcpy ( temp_path , path ) ;
currDir + + ;
}
}
2017-06-28 19:42:20 +03:00
}
/* stats all of the items created as specified by the input parameters */
2018-07-26 12:37:01 +03:00
void mdtest_stat ( const int random , const int dirs , const long dir_iter , const char * path , rank_progress_t * progress ) {
2017-10-20 00:26:52 +03:00
struct stat buf ;
uint64_t parent_dir , item_num = 0 ;
2018-07-08 00:39:14 +03:00
char item [ MAX_PATHLEN ] , temp [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering mdtest_stat... \n " ) ;
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2018-07-26 12:37:01 +03:00
uint64_t stop_items = items ;
if ( directory_loops ! = 1 ) {
stop_items = items_per_dir ;
2017-10-20 00:26:52 +03:00
}
/* iterate over all of the item IDs */
2018-07-26 12:37:01 +03:00
for ( uint64_t i = 0 ; i < stop_items ; + + i ) {
2017-10-20 00:26:52 +03:00
/*
* It doesn ' t make sense to pass the address of the array because that would
* be like passing char * * . Tested it on a Cray and it seems to work either
* way , but it seems that it is correct without the " & " .
*
2018-07-08 00:39:14 +03:00
memset ( & item , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
*/
2018-07-08 00:39:14 +03:00
memset ( item , 0 , MAX_PATHLEN ) ;
memset ( temp , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
/* determine the item number to stat */
if ( random ) {
item_num = rand_array [ i ] ;
} else {
item_num = i ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* make adjustments if in leaf only mode*/
if ( leaf_only ) {
item_num + = items_per_dir *
2018-07-14 18:46:24 +03:00
( num_dirs_in_tree - ( uint64_t ) pow ( branch_factor , depth ) ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create name of file/dir to stat */
if ( dirs ) {
if ( rank = = 0 & & verbose > = 3 & & ( i % ITEM_COUNT = = 0 ) & & ( i ! = 0 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: stat dir: " LLU " \n " , i ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-07 12:29:27 +03:00
sprintf ( item , " dir.%s " LLU " " , stat_name , item_num ) ;
2017-10-20 00:26:52 +03:00
} else {
if ( rank = = 0 & & verbose > = 3 & & ( i % ITEM_COUNT = = 0 ) & & ( i ! = 0 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: stat file: " LLU " \n " , i ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-07 12:29:27 +03:00
sprintf ( item , " file.%s " LLU " " , stat_name , item_num ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* determine the path to the file/dir to be stat'ed */
parent_dir = item_num / items_per_dir ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( parent_dir > 0 ) { //item is not in tree's root directory
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* prepend parent directory to item's path */
2018-07-07 12:29:27 +03:00
sprintf ( temp , " %s. " LLU " /%s " , base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
//still not at the tree's root dir
while ( parent_dir > branch_factor ) {
parent_dir = ( uint64_t ) ( ( parent_dir - 1 ) / branch_factor ) ;
2018-07-07 12:29:27 +03:00
sprintf ( temp , " %s. " LLU " /%s " , base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* Now get item to have the full path */
sprintf ( temp , " %s/%s " , path , item ) ;
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* below temp used to be hiername */
if ( rank = = 0 & & verbose > = 3 ) {
if ( dirs ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: mdtest_stat dir : %s \n " , item ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: mdtest_stat file: %s \n " , item ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-07 12:29:27 +03:00
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( - 1 = = backend - > stat ( item , & buf , & param ) ) {
if ( dirs ) {
if ( verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: Stat'ing directory \" %s \" \n " , item ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
FAIL ( " unable to stat directory " ) ;
} else {
if ( verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: Stat'ing file \" %s \" \n " , item ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
FAIL ( " unable to stat file " ) ;
}
2017-06-28 19:42:20 +03:00
}
}
}
/* reads all of the items created as specified by the input parameters */
2018-07-26 12:37:01 +03:00
void mdtest_read ( int random , int dirs , const long dir_iter , char * path ) {
2018-07-14 18:46:24 +03:00
uint64_t parent_dir , item_num = 0 ;
2018-07-08 00:39:14 +03:00
char item [ MAX_PATHLEN ] , temp [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
void * aiori_fh ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering mdtest_read... \n " ) ;
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* allocate read buffer */
if ( read_bytes > 0 ) {
read_buffer = ( char * ) malloc ( read_bytes ) ;
if ( read_buffer = = NULL ) {
FAIL ( " out of memory " ) ;
}
2017-06-28 19:42:20 +03:00
}
2018-07-26 12:37:01 +03:00
uint64_t stop_items = items ;
if ( directory_loops ! = 1 ) {
stop_items = items_per_dir ;
2017-10-20 00:26:52 +03:00
}
/* iterate over all of the item IDs */
2018-07-26 12:37:01 +03:00
for ( uint64_t i = 0 ; i < stop_items ; + + i ) {
2017-10-20 00:26:52 +03:00
/*
* It doesn ' t make sense to pass the address of the array because that would
* be like passing char * * . Tested it on a Cray and it seems to work either
* way , but it seems that it is correct without the " & " .
*
* NTH : Both are technically correct in C .
*
2018-07-08 00:39:14 +03:00
* memset ( & item , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
*/
2018-07-08 00:39:14 +03:00
memset ( item , 0 , MAX_PATHLEN ) ;
memset ( temp , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
/* determine the item number to read */
if ( random ) {
item_num = rand_array [ i ] ;
} else {
item_num = i ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* make adjustments if in leaf only mode*/
if ( leaf_only ) {
item_num + = items_per_dir *
( num_dirs_in_tree - ( uint64_t ) pow ( branch_factor , depth ) ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create name of file to read */
if ( ! dirs ) {
if ( rank = = 0 & & verbose > = 3 & & ( i % ITEM_COUNT = = 0 ) & & ( i ! = 0 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: read file: " LLU " \n " , i ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-07 12:29:27 +03:00
sprintf ( item , " file.%s " LLU " " , read_name , item_num ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* determine the path to the file/dir to be read'ed */
parent_dir = item_num / items_per_dir ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( parent_dir > 0 ) { //item is not in tree's root directory
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* prepend parent directory to item's path */
2018-07-07 12:29:27 +03:00
sprintf ( temp , " %s. " LLU " /%s " , base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* still not at the tree's root dir */
while ( parent_dir > branch_factor ) {
parent_dir = ( unsigned long long ) ( ( parent_dir - 1 ) / branch_factor ) ;
2018-07-07 12:29:27 +03:00
sprintf ( temp , " %s. " LLU " /%s " , base_tree_name , parent_dir , item ) ;
2017-10-20 00:26:52 +03:00
strcpy ( item , temp ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* Now get item to have the full path */
sprintf ( temp , " %s/%s " , path , item ) ;
strcpy ( item , temp ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* below temp used to be hiername */
if ( rank = = 0 & & verbose > = 3 ) {
if ( ! dirs ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: mdtest_read file: %s \n " , item ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-07 12:29:27 +03:00
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
/* open file for reading */
param . openFlags = O_RDONLY ;
aiori_fh = backend - > open ( item , & param ) ;
if ( NULL = = aiori_fh ) {
FAIL ( " unable to open file " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read file */
if ( read_bytes > 0 ) {
2018-07-07 12:29:27 +03:00
if ( read_bytes ! = ( size_t ) backend - > xfer ( READ , aiori_fh , ( IOR_size_t * ) read_buffer , read_bytes , & param ) ) {
2017-10-20 00:26:52 +03:00
FAIL ( " unable to read file " ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* close file */
backend - > close ( aiori_fh , & param ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* This method should be called by rank 0. It subsequently does all of
the creates and removes for the other ranks */
2018-07-07 12:29:27 +03:00
void collective_create_remove ( const int create , const int dirs , const int ntasks , const char * path , rank_progress_t * progress ) {
2018-07-08 00:39:14 +03:00
char temp [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering collective_create_remove... \n " ) ;
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* rank 0 does all of the creates and removes for all of the ranks */
for ( int i = 0 ; i < ntasks ; + + i ) {
2018-07-08 00:39:14 +03:00
memset ( temp , 0 , MAX_PATHLEN ) ;
2017-10-20 00:26:52 +03:00
strcpy ( temp , testdir ) ;
strcat ( temp , " / " ) ;
/* set the base tree name appropriately */
if ( unique_dir_per_task ) {
sprintf ( base_tree_name , " mdtest_tree.%d " , i ) ;
} else {
sprintf ( base_tree_name , " mdtest_tree " ) ;
}
/* Setup to do I/O to the appropriate test dir */
strcat ( temp , base_tree_name ) ;
strcat ( temp , " .0 " ) ;
/* set all item names appropriately */
if ( ! shared_file ) {
sprintf ( mk_name , " mdtest.%d. " , ( i + ( 0 * nstride ) ) % ntasks ) ;
sprintf ( stat_name , " mdtest.%d. " , ( i + ( 1 * nstride ) ) % ntasks ) ;
sprintf ( read_name , " mdtest.%d. " , ( i + ( 2 * nstride ) ) % ntasks ) ;
sprintf ( rm_name , " mdtest.%d. " , ( i + ( 3 * nstride ) ) % ntasks ) ;
}
if ( unique_dir_per_task ) {
sprintf ( unique_mk_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( i + ( 0 * nstride ) ) % ntasks ) ;
sprintf ( unique_chdir_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( i + ( 1 * nstride ) ) % ntasks ) ;
sprintf ( unique_stat_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( i + ( 2 * nstride ) ) % ntasks ) ;
sprintf ( unique_read_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( i + ( 3 * nstride ) ) % ntasks ) ;
sprintf ( unique_rm_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( i + ( 4 * nstride ) ) % ntasks ) ;
sprintf ( unique_rm_uni_dir , " %s " , testdir ) ;
}
/* Now that everything is set up as it should be, do the create or remove */
if ( rank = = 0 & & verbose > = 3 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: collective_create_remove (create_remove_items): temp is \" %s \" \n " , temp ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , dirs , create , 1 , temp , 0 , progress ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* reset all of the item names */
if ( unique_dir_per_task ) {
sprintf ( base_tree_name , " mdtest_tree.0 " ) ;
2017-06-28 19:42:20 +03:00
} else {
2017-10-20 00:26:52 +03:00
sprintf ( base_tree_name , " mdtest_tree " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( ! shared_file ) {
sprintf ( mk_name , " mdtest.%d. " , ( 0 + ( 0 * nstride ) ) % ntasks ) ;
sprintf ( stat_name , " mdtest.%d. " , ( 0 + ( 1 * nstride ) ) % ntasks ) ;
sprintf ( read_name , " mdtest.%d. " , ( 0 + ( 2 * nstride ) ) % ntasks ) ;
sprintf ( rm_name , " mdtest.%d. " , ( 0 + ( 3 * nstride ) ) % ntasks ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
sprintf ( unique_mk_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( 0 + ( 0 * nstride ) ) % ntasks ) ;
sprintf ( unique_chdir_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( 0 + ( 1 * nstride ) ) % ntasks ) ;
sprintf ( unique_stat_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( 0 + ( 2 * nstride ) ) % ntasks ) ;
sprintf ( unique_read_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( 0 + ( 3 * nstride ) ) % ntasks ) ;
sprintf ( unique_rm_dir , " %s/mdtest_tree.%d.0 " , testdir ,
( 0 + ( 4 * nstride ) ) % ntasks ) ;
sprintf ( unique_rm_uni_dir , " %s " , testdir ) ;
2017-06-28 19:42:20 +03:00
}
}
2018-07-07 12:29:27 +03:00
void directory_test ( const int iteration , const int ntasks , const char * path , rank_progress_t * progress ) {
2017-10-20 00:26:52 +03:00
int size ;
double t [ 5 ] = { 0 } ;
2018-07-08 00:39:14 +03:00
char temp_path [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
MPI_Comm_size ( testComm , & size ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering directory_test... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
MPI_Barrier ( testComm ) ;
t [ 0 ] = MPI_Wtime ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create phase */
if ( create_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( MK_UNI_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 0 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: directory_test: create path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* "touch" the files */
if ( collective_creates ) {
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
collective_create_remove ( 1 , 1 , ntasks , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
} else {
/* create directories */
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , 1 , 1 , 0 , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( barriers ) {
MPI_Barrier ( testComm ) ;
}
t [ 1 ] = MPI_Wtime ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat phase */
if ( stat_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( STAT_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 1 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: directory_test: stat path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat directories */
if ( random_seed > 0 ) {
2018-07-26 12:37:01 +03:00
mdtest_stat ( 1 , 1 , dir_iter , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-26 12:37:01 +03:00
mdtest_stat ( 0 , 1 , dir_iter , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( barriers ) {
MPI_Barrier ( testComm ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
t [ 2 ] = MPI_Wtime ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read phase */
if ( read_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( READ_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 2 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: directory_test: read path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read directories */
if ( random_seed > 0 ) {
; /* N/A */
} else {
; /* N/A */
}
2018-07-26 12:37:01 +03:00
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( barriers ) {
MPI_Barrier ( testComm ) ;
}
t [ 3 ] = MPI_Wtime ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( remove_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( RM_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 3 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: directory_test: remove directories path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* remove directories */
if ( collective_creates ) {
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
collective_create_remove ( 0 , 1 , ntasks , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
} else {
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , 1 , 0 , 0 , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( barriers ) {
MPI_Barrier ( testComm ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
t [ 4 ] = MPI_Wtime ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( remove_only ) {
if ( unique_dir_per_task ) {
unique_dir_access ( RM_UNI_DIR , temp_path ) ;
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: directory_test: remove unique directories path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task & & ! time_unique_dir_overhead ) {
offset_timers ( t , 4 ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* calculate times */
if ( create_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 0 ] = items * size / ( t [ 1 ] - t [ 0 ] ) ;
summary_table [ iteration ] . time [ 0 ] = t [ 1 ] - t [ 0 ] ;
summary_table [ iteration ] . items [ 0 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 0 ] = items ;
2017-10-20 00:26:52 +03:00
}
if ( stat_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 1 ] = items * size / ( t [ 2 ] - t [ 1 ] ) ;
summary_table [ iteration ] . time [ 1 ] = t [ 2 ] - t [ 1 ] ;
summary_table [ iteration ] . items [ 1 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 1 ] = items ;
2017-10-20 00:26:52 +03:00
}
if ( read_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 2 ] = items * size / ( t [ 3 ] - t [ 2 ] ) ;
summary_table [ iteration ] . time [ 2 ] = t [ 3 ] - t [ 2 ] ;
summary_table [ iteration ] . items [ 2 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 2 ] = items ;
2017-10-20 00:26:52 +03:00
}
if ( remove_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 3 ] = items * size / ( t [ 4 ] - t [ 3 ] ) ;
summary_table [ iteration ] . time [ 3 ] = t [ 4 ] - t [ 3 ] ;
summary_table [ iteration ] . items [ 3 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 3 ] = items ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( verbose > = 1 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Directory creation: %14.3f sec, %14.3f ops/sec \n " ,
t [ 1 ] - t [ 0 ] , summary_table [ iteration ] . rate [ 0 ] ) ;
fprintf ( out_logfile , " V-1: Directory stat : %14.3f sec, %14.3f ops/sec \n " ,
t [ 2 ] - t [ 1 ] , summary_table [ iteration ] . rate [ 1 ] ) ;
2017-10-20 00:26:52 +03:00
/* N/A
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Directory read : %14.3f sec, %14.3f ops/sec \n " ,
t [ 3 ] - t [ 2 ] , summary_table [ iteration ] . rate [ 2 ] ) ;
2017-10-20 00:26:52 +03:00
*/
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Directory removal : %14.3f sec, %14.3f ops/sec \n " ,
t [ 4 ] - t [ 3 ] , summary_table [ iteration ] . rate [ 3 ] ) ;
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2018-07-08 01:56:39 +03:00
/* Returns if the stonewall was hit */
int updateStoneWallIterations ( int iteration , rank_progress_t * progress , double tstart ) {
int hit = 0 ;
if ( verbose > = 1 ) {
2018-07-14 18:46:24 +03:00
fprintf ( out_logfile , " V-1: rank %d stonewall hit with %lld items \n " , rank , ( long long ) progress - > items_done ) ;
2018-07-08 01:56:39 +03:00
fflush ( out_logfile ) ;
}
2018-07-15 12:02:45 +03:00
uint64_t done = progress - > items_done ;
2018-07-08 01:56:39 +03:00
long long unsigned max_iter = 0 ;
2018-07-15 12:02:45 +03:00
MPI_Allreduce ( & progress - > items_done , & max_iter , 1 , MPI_LONG_LONG_INT , MPI_MAX , testComm ) ;
2018-07-08 01:56:39 +03:00
summary_table [ iteration ] . stonewall_time [ MDTEST_FILE_CREATE_NUM ] = MPI_Wtime ( ) - tstart ;
// continue to the maximum...
long long min_accessed = 0 ;
MPI_Reduce ( & progress - > items_done , & min_accessed , 1 , MPI_LONG_LONG_INT , MPI_MIN , 0 , testComm ) ;
long long sum_accessed = 0 ;
MPI_Reduce ( & progress - > items_done , & sum_accessed , 1 , MPI_LONG_LONG_INT , MPI_SUM , 0 , testComm ) ;
2018-07-15 12:02:45 +03:00
if ( items ! = ( sum_accessed / size ) & & rank = = 0 ) {
2018-07-08 01:56:39 +03:00
summary_table [ iteration ] . stonewall_item_sum [ MDTEST_FILE_CREATE_NUM ] = sum_accessed ;
summary_table [ iteration ] . stonewall_item_min [ MDTEST_FILE_CREATE_NUM ] = min_accessed * size ;
2018-07-15 12:02:45 +03:00
fprintf ( out_logfile , " Continue stonewall hit min: %lld max: %lld avg: %.1f \n " , min_accessed , max_iter , ( ( double ) sum_accessed ) / size ) ;
2018-07-08 01:56:39 +03:00
fflush ( out_logfile ) ;
2018-07-15 12:02:45 +03:00
}
if ( done ! = max_iter ) {
2018-07-08 01:56:39 +03:00
hit = 1 ;
}
2018-07-15 12:02:45 +03:00
progress - > items_start = done ;
2018-07-08 01:56:39 +03:00
progress - > items_per_dir = max_iter ;
return hit ;
}
2018-07-07 12:29:27 +03:00
void file_test ( const int iteration , const int ntasks , const char * path , rank_progress_t * progress ) {
2017-10-20 00:26:52 +03:00
int size ;
double t [ 5 ] = { 0 } ;
2018-07-08 00:39:14 +03:00
char temp_path [ MAX_PATHLEN ] ;
2018-07-07 12:29:27 +03:00
MPI_Comm_size ( testComm , & size ) ;
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering file_test... \n " ) ;
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
MPI_Barrier ( testComm ) ;
t [ 0 ] = MPI_Wtime ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create phase */
2018-07-08 00:19:42 +03:00
if ( create_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( MK_UNI_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 0 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: file_test: create path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* "touch" the files */
if ( collective_creates ) {
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
collective_create_remove ( 1 , 0 , ntasks , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
MPI_Barrier ( testComm ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* create files */
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , 0 , 1 , 0 , temp_path , 0 , progress ) ;
if ( stone_wall_timer_seconds ) {
2018-07-08 01:56:39 +03:00
int hit = updateStoneWallIterations ( iteration , progress , t [ 0 ] ) ;
if ( hit ) {
progress - > stone_wall_timer_seconds = 0 ;
2018-07-15 12:02:45 +03:00
printf ( " stonewall rank %d: %lld of %lld \n " , rank , ( long long ) progress - > items_start , ( long long ) progress - > items_per_dir ) ;
2018-07-08 01:56:39 +03:00
create_remove_items ( 0 , 0 , 1 , 0 , temp_path , 0 , progress ) ;
// now reset the values
progress - > stone_wall_timer_seconds = stone_wall_timer_seconds ;
2018-07-07 12:29:27 +03:00
}
2018-07-08 01:56:39 +03:00
if ( stoneWallingStatusFile ) {
2018-07-14 18:46:24 +03:00
StoreStoneWallingIterations ( stoneWallingStatusFile , progress - > items_done ) ;
2018-07-07 12:29:27 +03:00
}
}
2018-07-26 12:37:01 +03:00
}
2018-07-08 01:56:39 +03:00
} else {
if ( stoneWallingStatusFile ) {
2018-07-14 14:22:36 +03:00
int64_t expected_items ;
2018-07-08 01:56:39 +03:00
/* The number of items depends on the stonewalling file */
2018-07-14 14:22:36 +03:00
expected_items = ReadStoneWallingIterations ( stoneWallingStatusFile ) ;
if ( expected_items > = 0 ) {
items = expected_items ;
2018-07-14 18:46:24 +03:00
progress - > items_per_dir = items ;
2018-07-14 14:22:36 +03:00
}
if ( rank = = 0 ) {
if ( expected_items = = - 1 ) {
fprintf ( out_logfile , " WARNING: could not read stonewall status file \n " ) ;
} else if ( verbose > = 1 ) {
fprintf ( out_logfile , " Read stonewall status; items: " LLU " \n " , items ) ;
}
2018-07-08 01:56:39 +03:00
}
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( barriers ) {
2018-07-07 12:29:27 +03:00
MPI_Barrier ( testComm ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
t [ 1 ] = MPI_Wtime ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat phase */
2018-07-08 00:19:42 +03:00
if ( stat_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( STAT_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 1 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: file_test: stat path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* stat files */
2018-07-26 12:37:01 +03:00
mdtest_stat ( ( random_seed > 0 ? 1 : 0 ) , 0 , dir_iter , temp_path , progress ) ;
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( barriers ) {
MPI_Barrier ( testComm ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
t [ 2 ] = MPI_Wtime ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read phase */
2018-07-08 00:19:42 +03:00
if ( read_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( READ_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 2 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: file_test: read path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* read files */
if ( random_seed > 0 ) {
2018-07-26 12:37:01 +03:00
mdtest_read ( 1 , 0 , dir_iter , temp_path ) ;
2017-10-20 00:26:52 +03:00
} else {
2018-07-26 12:37:01 +03:00
mdtest_read ( 0 , 0 , dir_iter , temp_path ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( barriers ) {
MPI_Barrier ( testComm ) ;
}
t [ 3 ] = MPI_Wtime ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( remove_only ) {
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( iteration , dir_iter ) ;
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
unique_dir_access ( RM_SUB_DIR , temp_path ) ;
if ( ! time_unique_dir_overhead ) {
offset_timers ( t , 3 ) ;
}
} else {
2018-07-26 12:37:01 +03:00
sprintf ( temp_path , " %s/%s " , testdir , path ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: file_test: rm directories path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( collective_creates ) {
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
collective_create_remove ( 0 , 0 , ntasks , temp_path , progress ) ;
2017-10-20 00:26:52 +03:00
}
} else {
2018-07-07 12:29:27 +03:00
create_remove_items ( 0 , 0 , 0 , 0 , temp_path , 0 , progress ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( barriers ) {
MPI_Barrier ( testComm ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
t [ 4 ] = MPI_Wtime ( ) ;
if ( remove_only ) {
if ( unique_dir_per_task ) {
unique_dir_access ( RM_UNI_DIR , temp_path ) ;
} else {
strcpy ( temp_path , path ) ;
}
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: file_test: rm unique directories path is \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task & & ! time_unique_dir_overhead ) {
offset_timers ( t , 4 ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* calculate times */
if ( create_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 4 ] = items * size / ( t [ 1 ] - t [ 0 ] ) ;
summary_table [ iteration ] . time [ 4 ] = t [ 1 ] - t [ 0 ] ;
summary_table [ iteration ] . items [ 4 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 4 ] = items ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( stat_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 5 ] = items * size / ( t [ 2 ] - t [ 1 ] ) ;
summary_table [ iteration ] . time [ 5 ] = t [ 2 ] - t [ 1 ] ;
summary_table [ iteration ] . items [ 5 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 5 ] = items ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( read_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 6 ] = items * size / ( t [ 3 ] - t [ 2 ] ) ;
summary_table [ iteration ] . time [ 6 ] = t [ 3 ] - t [ 2 ] ;
summary_table [ iteration ] . items [ 6 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 6 ] = items ;
2017-10-20 00:26:52 +03:00
}
if ( remove_only ) {
2018-07-07 12:29:27 +03:00
summary_table [ iteration ] . rate [ 7 ] = items * size / ( t [ 4 ] - t [ 3 ] ) ;
summary_table [ iteration ] . time [ 7 ] = t [ 4 ] - t [ 3 ] ;
summary_table [ iteration ] . items [ 7 ] = items * size ;
summary_table [ iteration ] . stonewall_last_item [ 7 ] = items ;
2017-10-20 00:26:52 +03:00
}
if ( verbose > = 1 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: File creation : %14.3f sec, %14.3f ops/sec \n " ,
t [ 1 ] - t [ 0 ] , summary_table [ iteration ] . rate [ 4 ] ) ;
fprintf ( out_logfile , " V-1: File stat : %14.3f sec, %14.3f ops/sec \n " ,
t [ 2 ] - t [ 1 ] , summary_table [ iteration ] . rate [ 5 ] ) ;
fprintf ( out_logfile , " V-1: File read : %14.3f sec, %14.3f ops/sec \n " ,
t [ 3 ] - t [ 2 ] , summary_table [ iteration ] . rate [ 6 ] ) ;
fprintf ( out_logfile , " V-1: File removal : %14.3f sec, %14.3f ops/sec \n " ,
t [ 4 ] - t [ 3 ] , summary_table [ iteration ] . rate [ 7 ] ) ;
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
void print_help ( void ) {
int j ;
2018-08-15 19:07:17 +03:00
char APIs [ 1024 ] ;
aiori_supported_apis ( APIs ) ;
char apiStr [ 1024 ] ;
sprintf ( apiStr , " API for I/O [%s] " , APIs ) ;
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile ,
2017-10-20 00:26:52 +03:00
" Usage: mdtest [-b branching_factor] [-B] [-c] [-C] [-d testdir] [-D] [-e number_of_bytes_to_read] \n "
" [-E] [-f first] [-F] [-h] [-i iterations] [-I items_per_dir] [-l last] [-L] \n "
" [-n number_of_items] [-N stride_length] [-p seconds] [-r] \n "
" [-R[seed]] [-s stride] [-S] [-t] [-T] [-u] [-v] [-a API] \n "
2018-07-07 12:29:27 +03:00
" [-V verbosity_value] [-w number_of_bytes_to_write] [-W seconds] [-y] [-z depth] -Z \n "
2018-08-15 19:07:17 +03:00
" \t -a: %s \n "
2017-10-20 00:26:52 +03:00
" \t -b: branching factor of hierarchical directory structure \n "
" \t -B: no barriers between phases \n "
" \t -c: collective creates: task 0 does all creates \n "
" \t -C: only create files/dirs \n "
" \t -d: the directory in which the tests will run \n "
" \t -D: perform test on directories only (no files) \n "
" \t -e: bytes to read from each file \n "
" \t -E: only read files/dir \n "
" \t -f: first number of tasks on which the test will run \n "
" \t -F: perform test on files only (no directories) \n "
" \t -h: prints this help message \n "
" \t -i: number of iterations the test will run \n "
" \t -I: number of items per directory in tree \n "
" \t -l: last number of tasks on which the test will run \n "
" \t -L: files only at leaf level of tree \n "
" \t -n: every process will creat/stat/read/remove # directories and files \n "
" \t -N: stride # between neighbor tasks for file/dir operation (local=0) \n "
" \t -p: pre-iteration delay (in seconds) \n "
" \t -r: only remove files or directories left behind by previous runs \n "
" \t -R: randomly stat files (optional argument for random seed) \n "
" \t -s: stride between the number of tasks for each test \n "
" \t -S: shared file access (file only, no directories) \n "
" \t -t: time unique working directory overhead \n "
" \t -T: only stat files/dirs \n "
" \t -u: unique working directory for each task \n "
" \t -v: verbosity (each instance of option increments by one) \n "
" \t -V: verbosity value \n "
" \t -w: bytes to write to each file after it is created \n "
2018-07-08 01:56:39 +03:00
" \t -W: number in seconds; stonewall timer, write as many seconds and ensure all processes did the same number of operations (currently only stops during create phase) \n "
" \t -x: StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs \n "
2017-10-20 00:26:52 +03:00
" \t -y: sync file after writing \n "
" \t -z: depth of hierarchical directory structure \n "
2018-08-15 19:07:17 +03:00
" \t -Z: print time instead of rate \n " ,
apiStr
2017-10-20 00:26:52 +03:00
) ;
MPI_Initialized ( & j ) ;
if ( j ) {
MPI_Finalize ( ) ;
}
exit ( 0 ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
void summarize_results ( int iterations ) {
2018-07-08 00:39:14 +03:00
char access [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
int i , j , k ;
2018-07-07 12:29:27 +03:00
int start , stop , tableSize = MDTEST_LAST_NUM ;
2017-10-20 00:26:52 +03:00
double min , max , mean , sd , sum = 0 , var = 0 , curr = 0 ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
double all [ iterations * size * tableSize ] ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering summarize_results... \n " ) ;
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2018-07-07 12:29:27 +03:00
MPI_Barrier ( testComm ) ;
for ( int i = 0 ; i < iterations ; i + + ) {
if ( print_time ) {
MPI_Gather ( & summary_table [ i ] . time [ 0 ] , tableSize , MPI_DOUBLE , & all [ i * tableSize * size ] , tableSize , MPI_DOUBLE , 0 , testComm ) ;
} else {
MPI_Gather ( & summary_table [ i ] . rate [ 0 ] , tableSize , MPI_DOUBLE , & all [ i * tableSize * size ] , tableSize , MPI_DOUBLE , 0 , testComm ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " \n SUMMARY %s: (of %d iterations) \n " , print_time ? " time " : " rate " , iterations ) ;
fprintf ( out_logfile ,
2017-10-20 00:26:52 +03:00
" Operation Max Min Mean Std Dev \n " ) ;
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile ,
2017-10-20 00:26:52 +03:00
" --------- --- --- ---- ------- \n " ) ;
2018-07-07 12:29:27 +03:00
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* if files only access, skip entries 0-3 (the dir tests) */
if ( files_only & & ! dirs_only ) {
start = 4 ;
} else {
start = 0 ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* if directories only access, skip entries 4-7 (the file tests) */
if ( dirs_only & & ! files_only ) {
stop = 4 ;
} else {
stop = 8 ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* special case: if no directory or file tests, skip all */
if ( ! dirs_only & & ! files_only ) {
start = stop = 0 ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* calculate aggregates */
if ( barriers ) {
double maxes [ iterations ] ;
/* Because each proc times itself, in the case of barriers we
* have to backwards calculate the time to simulate the use
* of barriers .
*/
for ( i = start ; i < stop ; i + + ) {
for ( j = 0 ; j < iterations ; j + + ) {
maxes [ j ] = all [ j * tableSize + i ] ;
for ( k = 0 ; k < size ; k + + ) {
2018-07-07 12:29:27 +03:00
curr = all [ ( k * tableSize * iterations ) + ( j * tableSize ) + i ] ;
2017-10-20 00:26:52 +03:00
if ( maxes [ j ] < curr ) {
maxes [ j ] = curr ;
}
}
}
min = max = maxes [ 0 ] ;
for ( j = 0 ; j < iterations ; j + + ) {
if ( min > maxes [ j ] ) {
min = maxes [ j ] ;
}
if ( max < maxes [ j ] ) {
max = maxes [ j ] ;
}
sum + = maxes [ j ] ;
}
mean = sum / iterations ;
for ( j = 0 ; j < iterations ; j + + ) {
var + = pow ( ( mean - maxes [ j ] ) , 2 ) ;
}
var = var / iterations ;
sd = sqrt ( var ) ;
switch ( i ) {
case 0 : strcpy ( access , " Directory creation: " ) ; break ;
case 1 : strcpy ( access , " Directory stat : " ) ; break ;
/* case 2: strcpy(access, "Directory read :"); break; */
case 2 : ; break ; /* N/A */
case 3 : strcpy ( access , " Directory removal : " ) ; break ;
case 4 : strcpy ( access , " File creation : " ) ; break ;
case 5 : strcpy ( access , " File stat : " ) ; break ;
case 6 : strcpy ( access , " File read : " ) ; break ;
case 7 : strcpy ( access , " File removal : " ) ; break ;
default : strcpy ( access , " ERR " ) ; break ;
}
if ( i ! = 2 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " %s " , access ) ;
fprintf ( out_logfile , " %14.3f " , max ) ;
fprintf ( out_logfile , " %14.3f " , min ) ;
fprintf ( out_logfile , " %14.3f " , mean ) ;
fprintf ( out_logfile , " %14.3f \n " , sd ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
sum = var = 0 ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
} else {
for ( i = start ; i < stop ; i + + ) {
min = max = all [ i ] ;
for ( k = 0 ; k < size ; k + + ) {
for ( j = 0 ; j < iterations ; j + + ) {
curr = all [ ( k * tableSize * iterations )
+ ( j * tableSize ) + i ] ;
if ( min > curr ) {
min = curr ;
}
if ( max < curr ) {
max = curr ;
}
sum + = curr ;
}
}
mean = sum / ( iterations * size ) ;
for ( k = 0 ; k < size ; k + + ) {
for ( j = 0 ; j < iterations ; j + + ) {
var + = pow ( ( mean - all [ ( k * tableSize * iterations )
+ ( j * tableSize ) + i ] ) , 2 ) ;
}
}
var = var / ( iterations * size ) ;
sd = sqrt ( var ) ;
switch ( i ) {
case 0 : strcpy ( access , " Directory creation: " ) ; break ;
case 1 : strcpy ( access , " Directory stat : " ) ; break ;
/* case 2: strcpy(access, "Directory read :"); break; */
case 2 : ; break ; /* N/A */
case 3 : strcpy ( access , " Directory removal : " ) ; break ;
case 4 : strcpy ( access , " File creation : " ) ; break ;
case 5 : strcpy ( access , " File stat : " ) ; break ;
case 6 : strcpy ( access , " File read : " ) ; break ;
case 7 : strcpy ( access , " File removal : " ) ; break ;
default : strcpy ( access , " ERR " ) ; break ;
}
if ( i ! = 2 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " %s " , access ) ;
fprintf ( out_logfile , " %14.3f " , max ) ;
fprintf ( out_logfile , " %14.3f " , min ) ;
fprintf ( out_logfile , " %14.3f " , mean ) ;
fprintf ( out_logfile , " %14.3f \n " , sd ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
sum = var = 0 ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* calculate tree create/remove rates */
for ( i = 8 ; i < tableSize ; i + + ) {
min = max = all [ i ] ;
for ( j = 0 ; j < iterations ; j + + ) {
2018-07-07 12:29:27 +03:00
if ( print_time ) {
curr = summary_table [ j ] . time [ i ] ;
} else {
curr = summary_table [ j ] . rate [ i ] ;
}
2017-10-20 00:26:52 +03:00
if ( min > curr ) {
min = curr ;
}
if ( max < curr ) {
max = curr ;
}
sum + = curr ;
}
mean = sum / ( iterations ) ;
for ( j = 0 ; j < iterations ; j + + ) {
2018-07-07 12:29:27 +03:00
if ( print_time ) {
curr = summary_table [ j ] . time [ i ] ;
} else {
curr = summary_table [ j ] . rate [ i ] ;
}
var + = pow ( ( mean - curr ) , 2 ) ;
2017-10-20 00:26:52 +03:00
}
var = var / ( iterations ) ;
sd = sqrt ( var ) ;
switch ( i ) {
case 8 : strcpy ( access , " Tree creation : " ) ; break ;
case 9 : strcpy ( access , " Tree removal : " ) ; break ;
default : strcpy ( access , " ERR " ) ; break ;
}
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " %s " , access ) ;
fprintf ( out_logfile , " %14.3f " , max ) ;
fprintf ( out_logfile , " %14.3f " , min ) ;
fprintf ( out_logfile , " %14.3f " , mean ) ;
fprintf ( out_logfile , " %14.3f \n " , sd ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
sum = var = 0 ;
}
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* Checks to see if the test setup is valid. If it isn't, fail. */
void valid_tests ( ) {
2017-06-28 19:42:20 +03:00
2018-08-08 19:18:54 +03:00
if ( ( ( stone_wall_timer_seconds > 0 ) & & ( branch_factor > 1 ) ) | | ! barriers ) {
fprintf ( out_logfile , " Error, stone wall timer does only work with a branch factor <= 1 and with barriers \n " ) ;
MPI_Abort ( testComm , 1 ) ;
2018-07-12 20:09:13 +03:00
}
if ( ! create_only & & ! stat_only & & ! read_only & & ! remove_only ) {
create_only = stat_only = read_only = remove_only = 1 ;
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
fprintf ( out_logfile , " V-1: main: Setting create/stat/read/remove_only to True \n " ) ;
fflush ( out_logfile ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering valid_tests... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* if dirs_only and files_only were both left unset, set both now */
if ( ! dirs_only & & ! files_only ) {
dirs_only = files_only = 1 ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* if shared file 'S' access, no directory tests */
if ( shared_file ) {
dirs_only = 0 ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* check for no barriers with shifting processes for different phases.
that is , one may not specify both - B and - N as it will introduce
race conditions that may cause errors stat ' ing or deleting after
creates .
*/
if ( ( barriers = = 0 ) & & ( nstride ! = 0 ) & & ( rank = = 0 ) ) {
FAIL ( " Possible race conditions will occur: -B not compatible with -N " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* check for collective_creates incompatibilities */
if ( shared_file & & collective_creates & & rank = = 0 ) {
FAIL ( " -c not compatible with -S " ) ;
}
if ( path_count > 1 & & collective_creates & & rank = = 0 ) {
FAIL ( " -c not compatible with multiple test directories " ) ;
}
if ( collective_creates & & ! barriers ) {
FAIL ( " -c not compatible with -B " ) ;
2017-06-28 19:42:20 +03:00
}
2018-08-30 00:14:19 +03:00
if ( strcasecmp ( backend_name , " POSIX " ) ! = 0 & & strcasecmp ( backend_name , " DUMMY " ) ! = 0 & &
strcasecmp ( backend_name , " DFS " ) ! = 0 ) {
FAIL ( " -a only supported interface is POSIX, DFS and DUMMY right now! " ) ;
2018-07-07 14:14:34 +03:00
}
2017-10-20 00:26:52 +03:00
/* check for shared file incompatibilities */
if ( unique_dir_per_task & & shared_file & & rank = = 0 ) {
FAIL ( " -u not compatible with -S " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check multiple directory paths and strided option */
if ( path_count > 1 & & nstride > 0 ) {
FAIL ( " cannot have multiple directory paths with -N strides between neighbor tasks " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check for shared directory and multiple directories incompatibility */
if ( path_count > 1 & & unique_dir_per_task ! = 1 ) {
FAIL ( " shared directory mode is not compatible with multiple directory paths " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check if more directory paths than ranks */
if ( path_count > size ) {
FAIL ( " cannot have more directory paths than MPI tasks " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* check depth */
if ( depth < 0 ) {
FAIL ( " depth must be greater than or equal to zero " ) ;
}
/* check branch_factor */
if ( branch_factor < 1 & & depth > 0 ) {
FAIL ( " branch factor must be greater than or equal to zero " ) ;
}
/* check for valid number of items */
if ( ( items > 0 ) & & ( items_per_dir > 0 ) ) {
2018-07-26 12:37:01 +03:00
if ( unique_dir_per_task ) {
FAIL ( " only specify the number of items or the number of items per directory " ) ;
} else if ( items % items_per_dir ! = 0 ) {
FAIL ( " items must be a multiple of items per directory " ) ;
} else if ( stone_wall_timer_seconds ! = 0 ) {
FAIL ( " items + items_per_dir can only be set without stonewalling " ) ;
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
void show_file_system_size ( char * file_system ) {
2018-07-08 00:39:14 +03:00
char real_path [ MAX_PATHLEN ] ;
char file_system_unit_str [ MAX_PATHLEN ] = " GiB " ;
char inode_unit_str [ MAX_PATHLEN ] = " Mi " ;
2017-10-20 00:26:52 +03:00
int64_t file_system_unit_val = 1024 * 1024 * 1024 ;
int64_t inode_unit_val = 1024 * 1024 ;
int64_t total_file_system_size ,
free_file_system_size ,
total_inodes ,
free_inodes ;
double total_file_system_size_hr ,
used_file_system_percentage ,
used_inode_percentage ;
ior_aiori_statfs_t stat_buf ;
int ret ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering show_file_system_size... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
ret = backend - > statfs ( file_system , & stat_buf , & param ) ;
if ( 0 ! = ret ) {
FAIL ( " unable to stat file system " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
total_file_system_size = stat_buf . f_blocks * stat_buf . f_bsize ;
free_file_system_size = stat_buf . f_bfree * stat_buf . f_bsize ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
used_file_system_percentage = ( 1 - ( ( double ) free_file_system_size
/ ( double ) total_file_system_size ) ) * 100 ;
total_file_system_size_hr = ( double ) total_file_system_size
/ ( double ) file_system_unit_val ;
if ( total_file_system_size_hr > 1024 ) {
total_file_system_size_hr = total_file_system_size_hr / 1024 ;
strcpy ( file_system_unit_str , " TiB " ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* inodes */
total_inodes = stat_buf . f_files ;
free_inodes = stat_buf . f_ffree ;
used_inode_percentage = ( 1 - ( ( double ) free_inodes / ( double ) total_inodes ) )
* 100 ;
2017-06-28 19:42:20 +03:00
2018-01-24 21:26:53 +03:00
if ( realpath ( file_system , real_path ) = = NULL ) {
FAIL ( " unable to use realpath() " ) ;
}
2018-07-07 12:29:27 +03:00
2017-10-20 00:26:52 +03:00
/* show results */
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " Path: %s \n " , real_path ) ;
fprintf ( out_logfile , " FS: %.1f %s Used FS: %2.1f%% " ,
2017-10-20 00:26:52 +03:00
total_file_system_size_hr , file_system_unit_str ,
used_file_system_percentage ) ;
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " Inodes: %.1f %s Used Inodes: %2.1f%% \n " ,
2017-10-20 00:26:52 +03:00
( double ) total_inodes / ( double ) inode_unit_val ,
inode_unit_str , used_inode_percentage ) ;
2018-07-07 12:29:27 +03:00
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
return ;
2017-06-28 19:42:20 +03:00
}
void display_freespace ( char * testdirpath )
{
2018-07-08 00:39:14 +03:00
char dirpath [ MAX_PATHLEN ] = { 0 } ;
2017-10-20 00:26:52 +03:00
int i ;
int directoryFound = 0 ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering display_freespace... \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: testdirpath is \" %s \" \n " , testdirpath ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
strcpy ( dirpath , testdirpath ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* get directory for outfile */
i = strlen ( dirpath ) ;
while ( i - - > 0 ) {
if ( dirpath [ i ] = = ' / ' ) {
dirpath [ i ] = ' \0 ' ;
directoryFound = 1 ;
break ;
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* if no directory/, use '.' */
if ( directoryFound = = 0 ) {
strcpy ( dirpath , " . " ) ;
}
2017-06-28 19:42:20 +03:00
2018-08-30 00:14:19 +03:00
if ( strcasecmp ( backend_name , " DFS " ) = = 0 )
return ;
2017-10-20 00:26:52 +03:00
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: Before show_file_system_size, dirpath is \" %s \" \n " , dirpath ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
show_file_system_size ( dirpath ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: After show_file_system_size, dirpath is \" %s \" \n " , dirpath ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
return ;
2017-06-28 19:42:20 +03:00
}
void create_remove_directory_tree ( int create ,
2018-07-07 12:29:27 +03:00
int currDepth , char * path , int dirNum , rank_progress_t * progress ) {
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
unsigned i ;
2018-07-08 00:39:14 +03:00
char dir [ MAX_PATHLEN ] ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-1: Entering create_remove_directory_tree, currDepth = %d... \n " , currDepth ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( currDepth = = 0 ) {
sprintf ( dir , " %s/%s.%d/ " , path , base_tree_name , dirNum ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( create ) {
if ( rank = = 0 & & verbose > = 2 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-2: Making directory \" %s \" \n " , dir ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( - 1 = = backend - > mkdir ( dir , DIRMODE , & param ) ) {
2018-07-26 12:37:01 +03:00
fprintf ( out_logfile , " error could not create directory \" %s \" \n " , dir ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2018-07-07 12:29:27 +03:00
create_remove_directory_tree ( create , + + currDepth , dir , + + dirNum , progress ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ! create ) {
if ( rank = = 0 & & verbose > = 2 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-2: Remove directory \" %s \" \n " , dir ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( - 1 = = backend - > rmdir ( dir , & param ) ) {
FAIL ( " Unable to remove directory " ) ;
}
}
} else if ( currDepth < = depth ) {
2017-06-28 19:42:20 +03:00
2018-07-08 00:39:14 +03:00
char temp_path [ MAX_PATHLEN ] ;
2017-10-20 00:26:52 +03:00
strcpy ( temp_path , path ) ;
int currDir = dirNum ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
for ( i = 0 ; i < branch_factor ; i + + ) {
sprintf ( dir , " %s.%d/ " , base_tree_name , currDir ) ;
strcat ( temp_path , dir ) ;
if ( create ) {
if ( rank = = 0 & & verbose > = 2 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-2: Making directory \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
if ( - 1 = = backend - > mkdir ( temp_path , DIRMODE , & param ) ) {
FAIL ( " Unable to create directory " ) ;
}
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
create_remove_directory_tree ( create , + + currDepth ,
2018-07-07 12:29:27 +03:00
temp_path , ( branch_factor * currDir ) + 1 , progress ) ;
2017-10-20 00:26:52 +03:00
currDepth - - ;
if ( ! create ) {
if ( rank = = 0 & & verbose > = 2 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-2: Remove directory \" %s \" \n " , temp_path ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
if ( - 1 = = backend - > rmdir ( temp_path , & param ) ) {
FAIL ( " Unable to remove directory " ) ;
}
}
strcpy ( temp_path , path ) ;
currDir + + ;
}
2017-06-28 19:42:20 +03:00
}
}
2018-07-08 01:56:39 +03:00
static void mdtest_iteration ( int i , int j , MPI_Group testgroup , mdtest_results_t * summary_table ) {
rank_progress_t progress_o ;
memset ( & progress_o , 0 , sizeof ( progress_o ) ) ;
progress_o . start_time = GetTimeStamp ( ) ;
progress_o . stone_wall_timer_seconds = stone_wall_timer_seconds ;
progress_o . items_per_dir = items_per_dir ;
rank_progress_t * progress = & progress_o ;
2018-07-07 12:29:27 +03:00
/* start and end times of directory tree create/remove */
double startCreate , endCreate ;
2018-08-08 19:18:54 +03:00
int k ;
2018-07-07 12:29:27 +03:00
if ( rank = = 0 & & verbose > = 1 ) {
fprintf ( out_logfile , " V-1: main: * iteration %d * \n " , j + 1 ) ;
fflush ( out_logfile ) ;
}
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( j , dir_iter ) ;
2018-07-07 12:29:27 +03:00
2018-07-26 12:37:01 +03:00
if ( verbose > = 2 & & rank = = 0 ) {
fprintf ( out_logfile , " V-2: main (for j loop): making testdir, \" %s \" \n " , testdir ) ;
fflush ( out_logfile ) ;
}
if ( ( rank < path_count ) & & backend - > access ( testdir , F_OK , & param ) ! = 0 ) {
if ( backend - > mkdir ( testdir , DIRMODE , & param ) ! = 0 ) {
FAIL ( " Unable to create test directory " ) ;
2018-08-24 00:58:53 +03:00
}
2018-07-26 12:37:01 +03:00
}
2018-07-07 12:29:27 +03:00
}
2018-08-24 00:58:53 +03:00
2018-07-07 12:29:27 +03:00
if ( create_only ) {
2018-07-26 12:37:01 +03:00
/* create hierarchical directory structure */
MPI_Barrier ( testComm ) ;
2018-07-07 12:29:27 +03:00
startCreate = MPI_Wtime ( ) ;
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( j , dir_iter ) ;
if ( unique_dir_per_task ) {
if ( collective_creates & & ( rank = = 0 ) ) {
/*
* This is inside two loops , one of which already uses " i " and the other uses " j " .
* I don ' t know how this ever worked . I ' m changing this loop to use " k " .
*/
for ( k = 0 ; k < size ; k + + ) {
sprintf ( base_tree_name , " mdtest_tree.%d " , k ) ;
if ( verbose > = 3 & & rank = = 0 ) {
fprintf ( out_logfile ,
" V-3: main (create hierarchical directory loop-collective): Calling create_remove_directory_tree with \" %s \" \n " ,
testdir ) ;
fflush ( out_logfile ) ;
}
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 1 , 0 , testdir , 0 , progress ) ;
if ( CHECK_STONE_WALL ( progress ) ) {
size = k ;
break ;
}
}
} else if ( ! collective_creates ) {
if ( verbose > = 3 & & rank = = 0 ) {
fprintf ( out_logfile ,
" V-3: main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with \" %s \" \n " ,
testdir ) ;
fflush ( out_logfile ) ;
}
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 1 , 0 , testdir , 0 , progress ) ;
}
} else {
if ( rank = = 0 ) {
if ( verbose > = 3 & & rank = = 0 ) {
fprintf ( out_logfile ,
" V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with \" %s \" \n " ,
testdir ) ;
fflush ( out_logfile ) ;
}
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 1 , 0 , testdir , 0 , progress ) ;
}
}
2018-07-07 12:29:27 +03:00
}
MPI_Barrier ( testComm ) ;
endCreate = MPI_Wtime ( ) ;
summary_table - > rate [ 8 ] =
num_dirs_in_tree / ( endCreate - startCreate ) ;
summary_table - > time [ 8 ] = ( endCreate - startCreate ) ;
summary_table - > items [ 8 ] = num_dirs_in_tree ;
summary_table - > stonewall_last_item [ 8 ] = num_dirs_in_tree ;
if ( verbose > = 1 & & rank = = 0 ) {
fprintf ( out_logfile , " V-1: main: Tree creation : %14.3f sec, %14.3f ops/sec \n " ,
( endCreate - startCreate ) , summary_table - > rate [ 8 ] ) ;
fflush ( out_logfile ) ;
}
}
2018-07-26 12:37:01 +03:00
sprintf ( unique_mk_dir , " %s.0 " , base_tree_name ) ;
sprintf ( unique_chdir_dir , " %s.0 " , base_tree_name ) ;
sprintf ( unique_stat_dir , " %s.0 " , base_tree_name ) ;
sprintf ( unique_read_dir , " %s.0 " , base_tree_name ) ;
sprintf ( unique_rm_dir , " %s.0 " , base_tree_name ) ;
unique_rm_uni_dir [ 0 ] = 0 ;
2018-07-07 12:29:27 +03:00
if ( ! unique_dir_per_task ) {
if ( verbose > = 3 & & rank = = 0 ) {
fprintf ( out_logfile , " V-3: main: Using unique_mk_dir, \" %s \" \n " , unique_mk_dir ) ;
fflush ( out_logfile ) ;
}
}
if ( rank < i ) {
if ( ! shared_file ) {
sprintf ( mk_name , " mdtest.%d. " , ( rank + ( 0 * nstride ) ) % i ) ;
sprintf ( stat_name , " mdtest.%d. " , ( rank + ( 1 * nstride ) ) % i ) ;
sprintf ( read_name , " mdtest.%d. " , ( rank + ( 2 * nstride ) ) % i ) ;
sprintf ( rm_name , " mdtest.%d. " , ( rank + ( 3 * nstride ) ) % i ) ;
}
if ( unique_dir_per_task ) {
2018-07-26 12:37:01 +03:00
sprintf ( unique_mk_dir , " mdtest_tree.%d.0 " , ( rank + ( 0 * nstride ) ) % i ) ;
sprintf ( unique_chdir_dir , " mdtest_tree.%d.0 " , ( rank + ( 1 * nstride ) ) % i ) ;
sprintf ( unique_stat_dir , " mdtest_tree.%d.0 " , ( rank + ( 2 * nstride ) ) % i ) ;
sprintf ( unique_read_dir , " mdtest_tree.%d.0 " , ( rank + ( 3 * nstride ) ) % i ) ;
sprintf ( unique_rm_dir , " mdtest_tree.%d.0 " , ( rank + ( 4 * nstride ) ) % i ) ;
unique_rm_uni_dir [ 0 ] = 0 ;
2018-07-07 12:29:27 +03:00
}
if ( verbose > = 3 & & rank = = 0 ) {
fprintf ( out_logfile , " V-3: main: Copied unique_mk_dir, \" %s \" , to topdir \n " , unique_mk_dir ) ;
fflush ( out_logfile ) ;
}
if ( dirs_only & & ! shared_file ) {
if ( pre_delay ) {
2018-07-08 00:26:57 +03:00
DelaySecs ( pre_delay ) ;
2018-07-07 12:29:27 +03:00
}
directory_test ( j , i , unique_mk_dir , progress ) ;
}
if ( files_only ) {
if ( pre_delay ) {
2018-07-08 00:26:57 +03:00
DelaySecs ( pre_delay ) ;
2018-07-07 12:29:27 +03:00
}
file_test ( j , i , unique_mk_dir , progress ) ;
}
}
/* remove directory structure */
if ( ! unique_dir_per_task ) {
if ( verbose > = 3 & & rank = = 0 ) {
fprintf ( out_logfile , " V-3: main: Using testdir, \" %s \" \n " , testdir ) ;
fflush ( out_logfile ) ;
}
}
MPI_Barrier ( testComm ) ;
if ( remove_only ) {
startCreate = MPI_Wtime ( ) ;
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( j , dir_iter ) ;
if ( unique_dir_per_task ) {
if ( collective_creates & & ( rank = = 0 ) ) {
/*
* This is inside two loops , one of which already uses " i " and the other uses " j " .
* I don ' t know how this ever worked . I ' m changing this loop to use " k " .
*/
for ( k = 0 ; k < size ; k + + ) {
sprintf ( base_tree_name , " mdtest_tree.%d " , k ) ;
if ( verbose > = 3 & & rank = = 0 ) {
fprintf ( out_logfile ,
" V-3: main (remove hierarchical directory loop-collective): Calling create_remove_directory_tree with \" %s \" \n " ,
testdir ) ;
fflush ( out_logfile ) ;
}
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 0 , 0 , testdir , 0 , progress ) ;
if ( CHECK_STONE_WALL ( progress ) ) {
size = k ;
break ;
}
}
} else if ( ! collective_creates ) {
if ( verbose > = 3 & & rank = = 0 ) {
fprintf ( out_logfile ,
" V-3: main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with \" %s \" \n " ,
testdir ) ;
fflush ( out_logfile ) ;
}
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 0 , 0 , testdir , 0 , progress ) ;
}
} else {
if ( rank = = 0 ) {
if ( verbose > = 3 & & rank = = 0 ) {
fprintf ( out_logfile ,
" V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with \" %s \" \n " ,
testdir ) ;
fflush ( out_logfile ) ;
}
2018-08-24 00:58:53 +03:00
2018-07-26 12:37:01 +03:00
/*
* Let ' s pass in the path to the directory we most recently made so that we can use
* full paths in the other calls .
*/
create_remove_directory_tree ( 0 , 0 , testdir , 0 , progress ) ;
}
}
2018-07-07 12:29:27 +03:00
}
MPI_Barrier ( testComm ) ;
endCreate = MPI_Wtime ( ) ;
summary_table - > rate [ 9 ] = num_dirs_in_tree / ( endCreate - startCreate ) ;
summary_table - > time [ 9 ] = endCreate - startCreate ;
summary_table - > items [ 9 ] = num_dirs_in_tree ;
summary_table - > stonewall_last_item [ 8 ] = num_dirs_in_tree ;
if ( verbose > = 1 & & rank = = 0 ) {
fprintf ( out_logfile , " V-1: main Tree removal : %14.3f sec, %14.3f ops/sec \n " ,
( endCreate - startCreate ) , summary_table - > rate [ 9 ] ) ;
fflush ( out_logfile ) ;
}
if ( ( rank = = 0 ) & & ( verbose > = 2 ) ) {
fprintf ( out_logfile , " V-2: main (at end of for j loop): Removing testdir of \" %s \" \n " , testdir ) ;
fflush ( out_logfile ) ;
}
2018-07-26 12:37:01 +03:00
for ( int dir_iter = 0 ; dir_iter < directory_loops ; dir_iter + + ) {
prep_testdir ( j , dir_iter ) ;
if ( ( rank < path_count ) & & backend - > access ( testdir , F_OK , & param ) = = 0 ) {
//if (( rank == 0 ) && access(testdir, F_OK) == 0) {
if ( backend - > rmdir ( testdir , & param ) = = - 1 ) {
FAIL ( " unable to remove directory " ) ;
}
}
2018-07-07 12:29:27 +03:00
}
} else {
summary_table - > rate [ 9 ] = 0 ;
}
}
2018-08-24 00:58:53 +03:00
2018-07-07 12:29:27 +03:00
void mdtest_init_args ( ) {
barriers = 1 ;
branch_factor = 1 ;
throttle = 1 ;
2018-07-08 01:56:39 +03:00
stoneWallingStatusFile = NULL ;
2018-07-07 12:29:27 +03:00
create_only = 0 ;
stat_only = 0 ;
read_only = 0 ;
remove_only = 0 ;
leaf_only = 0 ;
depth = 0 ;
num_dirs_in_tree = 0 ;
items_per_dir = 0 ;
random_seed = 0 ;
print_time = 0 ;
shared_file = 0 ;
files_only = 0 ;
dirs_only = 0 ;
pre_delay = 0 ;
unique_dir_per_task = 0 ;
time_unique_dir_overhead = 0 ;
items = 0 ;
collective_creates = 0 ;
write_bytes = 0 ;
stone_wall_timer_seconds = 0 ;
read_bytes = 0 ;
sync_file = 0 ;
path_count = 0 ;
nstride = 0 ;
2018-08-24 00:58:53 +03:00
}
2018-07-12 20:09:13 +03:00
2018-07-07 12:29:27 +03:00
mdtest_results_t * mdtest_run ( int argc , char * * argv , MPI_Comm world_com , FILE * world_out ) {
testComm = world_com ;
out_logfile = world_out ;
mpi_comm_world = world_com ;
init_clock ( ) ;
mdtest_init_args ( ) ;
2018-08-08 19:18:54 +03:00
int i , j ;
2017-10-20 00:26:52 +03:00
int nodeCount ;
MPI_Group worldgroup , testgroup ;
struct {
int first ;
int last ;
int stride ;
} range = { 0 , 0 , 1 } ;
int first = 1 ;
int last = 0 ;
int stride = 1 ;
int iterations = 1 ;
2018-07-12 20:09:13 +03:00
verbose = 0 ;
int no_barriers = 0 ;
char * path = " ./out " ;
int randomize = 0 ;
2018-08-15 19:07:17 +03:00
char APIs [ 1024 ] ;
aiori_supported_apis ( APIs ) ;
char apiStr [ 1024 ] ;
sprintf ( apiStr , " API for I/O [%s] " , APIs ) ;
2018-07-12 20:09:13 +03:00
option_help options [ ] = {
2018-08-15 19:07:17 +03:00
{ ' a ' , NULL , apiStr , OPTION_OPTIONAL_ARGUMENT , ' s ' , & backend_name } ,
2018-07-12 20:09:13 +03:00
{ ' b ' , NULL , " branching factor of hierarchical directory structure " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & branch_factor } ,
{ ' d ' , NULL , " the directory in which the tests will run " , OPTION_OPTIONAL_ARGUMENT , ' s ' , & path } ,
{ ' B ' , NULL , " no barriers between phases " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & no_barriers } ,
{ ' C ' , NULL , " only create files/dirs " , OPTION_FLAG , ' d ' , & create_only } ,
{ ' T ' , NULL , " only stat files/dirs " , OPTION_FLAG , ' d ' , & stat_only } ,
{ ' E ' , NULL , " only read files/dir " , OPTION_FLAG , ' d ' , & read_only } ,
{ ' r ' , NULL , " only remove files or directories left behind by previous runs " , OPTION_FLAG , ' d ' , & remove_only } ,
{ ' D ' , NULL , " perform test on directories only (no files) " , OPTION_FLAG , ' d ' , & dirs_only } ,
{ ' e ' , NULL , " bytes to read from each file " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & read_bytes } ,
{ ' f ' , NULL , " first number of tasks on which the test will run " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & first } ,
{ ' F ' , NULL , " perform test on files only (no directories) " , OPTION_FLAG , ' d ' , & files_only } ,
{ ' i ' , NULL , " number of iterations the test will run " , OPTION_OPTIONAL_ARGUMENT , ' i ' , & iterations } ,
{ ' I ' , NULL , " number of items per directory in tree " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & items_per_dir } ,
{ ' l ' , NULL , " last number of tasks on which the test will run " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & last } ,
{ ' L ' , NULL , " files only at leaf level of tree " , OPTION_FLAG , ' d ' , & leaf_only } ,
{ ' n ' , NULL , " every process will creat/stat/read/remove # directories and files " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & items } ,
{ ' N ' , NULL , " stride # between neighbor tasks for file/dir operation (local=0) " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & nstride } ,
{ ' p ' , NULL , " pre-iteration delay (in seconds) " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & pre_delay } ,
2018-07-14 18:46:24 +03:00
{ ' R ' , NULL , " random access to files (only for stat) " , OPTION_FLAG , ' d ' , & randomize } ,
2018-07-12 20:09:13 +03:00
{ 0 , " random-seed " , " random seed for -R " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & random_seed } ,
{ ' s ' , NULL , " stride between the number of tasks for each test " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & stride } ,
{ ' S ' , NULL , " shared file access (file only, no directories) " , OPTION_FLAG , ' d ' , & shared_file } ,
{ ' c ' , NULL , " collective creates: task 0 does all creates " , OPTION_FLAG , ' d ' , & collective_creates } ,
{ ' t ' , NULL , " time unique working directory overhead " , OPTION_FLAG , ' d ' , & time_unique_dir_overhead } ,
{ ' u ' , NULL , " unique working directory for each task " , OPTION_FLAG , ' d ' , & unique_dir_per_task } ,
{ ' v ' , NULL , " verbosity (each instance of option increments by one) " , OPTION_FLAG , ' d ' , & verbose } ,
{ ' V ' , NULL , " verbosity value " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & verbose } ,
{ ' w ' , NULL , " bytes to write to each file after it is created " , OPTION_OPTIONAL_ARGUMENT , ' l ' , & write_bytes } ,
{ ' W ' , NULL , " number in seconds; stonewall timer, write as many seconds and ensure all processes did the same number of operations (currently only stops during create phase) " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & stone_wall_timer_seconds } ,
{ ' x ' , NULL , " StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs " , OPTION_OPTIONAL_ARGUMENT , ' s ' , & stoneWallingStatusFile } ,
{ ' y ' , NULL , " sync file after writing " , OPTION_FLAG , ' d ' , & sync_file } ,
{ ' z ' , NULL , " depth of hierarchical directory structure " , OPTION_OPTIONAL_ARGUMENT , ' d ' , & depth } ,
{ ' Z ' , NULL , " print time instead of rate " , OPTION_FLAG , ' d ' , & print_time } ,
LAST_OPTION
} ;
int printhelp = 0 ;
int parsed_options = option_parse ( argc , argv , options , & printhelp ) ;
2018-07-14 10:41:35 +03:00
backend = aiori_select ( backend_name ) ;
2018-07-12 20:09:13 +03:00
if ( NULL = = backend ) {
FAIL ( " Could not find suitable backend to use " ) ;
}
if ( backend - > get_options ! = NULL ) {
option_parse ( argc - parsed_options , argv + parsed_options , backend - > get_options ( ) , & printhelp ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2018-08-30 00:14:19 +03:00
if ( backend - > initialize )
backend - > initialize ( NULL ) ;
2018-07-12 20:09:13 +03:00
if ( printhelp ! = 0 ) {
2018-07-14 10:41:35 +03:00
printf ( " Usage: %s " , argv [ 0 ] ) ;
2018-07-12 20:09:13 +03:00
option_print_help ( options , 0 ) ;
if ( backend - > get_options ! = NULL ) {
printf ( " \n Plugin options for backend %s \n " , backend_name ) ;
option_print_help ( backend - > get_options ( ) , 1 ) ;
}
if ( printhelp = = 1 ) {
exit ( 0 ) ;
} else {
exit ( 1 ) ;
}
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-11-30 14:01:19 +03:00
2018-07-07 12:29:27 +03:00
MPI_Comm_rank ( testComm , & rank ) ;
MPI_Comm_size ( testComm , & size ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
pid = getpid ( ) ;
uid = getuid ( ) ;
2017-06-28 19:42:20 +03:00
2018-07-08 00:19:42 +03:00
nodeCount = size / CountTasksPerNode ( testComm ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
2018-07-07 16:01:11 +03:00
fprintf ( out_logfile , " -- started at %s -- \n \n " , PrintTimestamp ( ) ) ;
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " mdtest-%s was launched with %d total task(s) on %d node(s) \n " ,
2017-10-20 00:26:52 +03:00
RELEASE_VERS , size , nodeCount ) ;
2018-07-07 12:29:27 +03:00
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
if ( rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " Command line used: %s " , argv [ 0 ] ) ;
for ( i = 1 ; i < argc ; i + + ) {
fprintf ( out_logfile , " \" %s \" " , argv [ i ] ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " \n " ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2018-07-12 20:09:13 +03:00
/* adjust special variables */
barriers = ! no_barriers ;
if ( path ! = NULL ) {
parse_dirpath ( path ) ;
2017-06-28 19:42:20 +03:00
}
2018-07-12 20:09:13 +03:00
if ( randomize > 0 ) {
if ( random_seed = = 0 ) {
/* Ensure all procs have the same random number */
random_seed = time ( NULL ) ;
MPI_Barrier ( testComm ) ;
MPI_Bcast ( & random_seed , 1 , MPI_INT , 0 , testComm ) ;
}
random_seed + = rank ;
2017-10-20 00:26:52 +03:00
}
2018-07-26 12:37:01 +03:00
if ( ( items > 0 ) & & ( items_per_dir > 0 ) & & ( ! unique_dir_per_task ) ) {
directory_loops = items / items_per_dir ;
} else {
directory_loops = 1 ;
2017-10-20 00:26:52 +03:00
}
valid_tests ( ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( ( rank = = 0 ) & & ( verbose > = 1 ) ) {
2018-07-12 20:09:13 +03:00
// option_print_current(options);
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " api : %s \n " , backend_name ) ;
fprintf ( out_logfile , " barriers : %s \n " , ( barriers ? " True " : " False " ) ) ;
fprintf ( out_logfile , " collective_creates : %s \n " , ( collective_creates ? " True " : " False " ) ) ;
fprintf ( out_logfile , " create_only : %s \n " , ( create_only ? " True " : " False " ) ) ;
fprintf ( out_logfile , " dirpath(s): \n " ) ;
2017-10-20 00:26:52 +03:00
for ( i = 0 ; i < path_count ; i + + ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " \t %s \n " , filenames [ i ] ) ;
}
fprintf ( out_logfile , " dirs_only : %s \n " , ( dirs_only ? " True " : " False " ) ) ;
fprintf ( out_logfile , " read_bytes : " LLU " \n " , read_bytes ) ;
fprintf ( out_logfile , " read_only : %s \n " , ( read_only ? " True " : " False " ) ) ;
fprintf ( out_logfile , " first : %d \n " , first ) ;
fprintf ( out_logfile , " files_only : %s \n " , ( files_only ? " True " : " False " ) ) ;
fprintf ( out_logfile , " iterations : %d \n " , iterations ) ;
fprintf ( out_logfile , " items_per_dir : " LLU " \n " , items_per_dir ) ;
fprintf ( out_logfile , " last : %d \n " , last ) ;
fprintf ( out_logfile , " leaf_only : %s \n " , ( leaf_only ? " True " : " False " ) ) ;
fprintf ( out_logfile , " items : " LLU " \n " , items ) ;
fprintf ( out_logfile , " nstride : %d \n " , nstride ) ;
fprintf ( out_logfile , " pre_delay : %d \n " , pre_delay ) ;
fprintf ( out_logfile , " remove_only : %s \n " , ( leaf_only ? " True " : " False " ) ) ;
fprintf ( out_logfile , " random_seed : %d \n " , random_seed ) ;
fprintf ( out_logfile , " stride : %d \n " , stride ) ;
fprintf ( out_logfile , " shared_file : %s \n " , ( shared_file ? " True " : " False " ) ) ;
fprintf ( out_logfile , " time_unique_dir_overhead: %s \n " , ( time_unique_dir_overhead ? " True " : " False " ) ) ;
fprintf ( out_logfile , " stone_wall_timer_seconds: %d \n " , stone_wall_timer_seconds ) ;
fprintf ( out_logfile , " stat_only : %s \n " , ( stat_only ? " True " : " False " ) ) ;
fprintf ( out_logfile , " unique_dir_per_task : %s \n " , ( unique_dir_per_task ? " True " : " False " ) ) ;
fprintf ( out_logfile , " write_bytes : " LLU " \n " , write_bytes ) ;
fprintf ( out_logfile , " sync_file : %s \n " , ( sync_file ? " True " : " False " ) ) ;
fprintf ( out_logfile , " depth : %d \n " , depth ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
/* setup total number of items and number of items per dir */
if ( depth < = 0 ) {
num_dirs_in_tree = 1 ;
2017-06-28 19:42:20 +03:00
} else {
2017-10-20 00:26:52 +03:00
if ( branch_factor < 1 ) {
num_dirs_in_tree = 1 ;
} else if ( branch_factor = = 1 ) {
num_dirs_in_tree = depth + 1 ;
} else {
num_dirs_in_tree =
( 1 - pow ( branch_factor , depth + 1 ) ) / ( 1 - branch_factor ) ;
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( items_per_dir > 0 ) {
2018-07-26 12:37:01 +03:00
if ( unique_dir_per_task ) {
items = items_per_dir * num_dirs_in_tree ;
}
2017-06-28 19:42:20 +03:00
} else {
2017-10-20 00:26:52 +03:00
if ( leaf_only ) {
if ( branch_factor < = 1 ) {
items_per_dir = items ;
} else {
2018-07-14 18:46:24 +03:00
items_per_dir = ( uint64_t ) ( items / pow ( branch_factor , depth ) ) ;
items = items_per_dir * ( uint64_t ) pow ( branch_factor , depth ) ;
2017-10-20 00:26:52 +03:00
}
} else {
items_per_dir = items / num_dirs_in_tree ;
items = items_per_dir * num_dirs_in_tree ;
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* initialize rand_array */
if ( random_seed > 0 ) {
srand ( random_seed ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
uint64_t s ;
2017-06-28 19:42:20 +03:00
2018-07-14 18:46:24 +03:00
rand_array = ( uint64_t * ) malloc ( items * sizeof ( * rand_array ) ) ;
2017-06-28 19:42:20 +03:00
2018-07-14 18:46:24 +03:00
for ( s = 0 ; s < items ; s + + ) {
2017-10-20 00:26:52 +03:00
rand_array [ s ] = s ;
}
/* shuffle list randomly */
2018-07-14 18:46:24 +03:00
uint64_t n = items ;
2017-10-20 00:26:52 +03:00
while ( n > 1 ) {
n - - ;
/*
* Generate a random number in the range 0 . . n
*
* rand ( ) returns a number from 0 . . RAND_MAX . Divide that
* by RAND_MAX and you get a floating point number in the
* range 0 . . 1. Multiply that by n and you get a number in
* the range 0 . . n .
*/
2018-07-08 00:19:42 +03:00
uint64_t k = ( uint64_t ) ( ( ( double ) rand ( ) / ( double ) RAND_MAX ) * ( double ) n ) ;
2017-10-20 00:26:52 +03:00
/*
* Now move the nth element to the kth ( randomly chosen )
* element , and the kth element to the nth element .
*/
uint64_t tmp = rand_array [ k ] ;
rand_array [ k ] = rand_array [ n ] ;
rand_array [ n ] = tmp ;
}
}
/* allocate and initialize write buffer with # */
if ( write_bytes > 0 ) {
write_buffer = ( char * ) malloc ( write_bytes ) ;
if ( write_buffer = = NULL ) {
FAIL ( " out of memory " ) ;
}
memset ( write_buffer , 0x23 , write_bytes ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* setup directory path to work in */
if ( path_count = = 0 ) { /* special case where no directory path provided with '-d' option */
2018-08-08 19:18:54 +03:00
char * ret = getcwd ( testdirpath , MAX_PATHLEN ) ;
if ( ret = = NULL ) {
FAIL ( " Unable to get current working directory " ) ;
}
2017-10-20 00:26:52 +03:00
path_count = 1 ;
} else {
strcpy ( testdirpath , filenames [ rank % path_count ] ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* if directory does not exist, create it */
if ( ( rank < path_count ) & & backend - > access ( testdirpath , F_OK , & param ) ! = 0 ) {
if ( backend - > mkdir ( testdirpath , DIRMODE , & param ) ! = 0 ) {
FAIL ( " Unable to create test directory path " ) ;
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
/* display disk usage */
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: main (before display_freespace): testdirpath is \" %s \" \n " , testdirpath ) ;
fflush ( out_logfile ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) display_freespace ( testdirpath ) ;
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( verbose > = 3 & & rank = = 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " V-3: main (after display_freespace): testdirpath is \" %s \" \n " , testdirpath ) ;
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
if ( random_seed > 0 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " random seed: %d \n " , random_seed ) ;
2017-10-20 00:26:52 +03:00
}
}
2017-06-28 19:42:20 +03:00
2018-07-08 00:39:14 +03:00
if ( gethostname ( hostname , MAX_PATHLEN ) = = - 1 ) {
2017-10-20 00:26:52 +03:00
perror ( " gethostname " ) ;
2018-07-07 12:29:27 +03:00
MPI_Abort ( testComm , 2 ) ;
2017-10-20 00:26:52 +03:00
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( last = = 0 ) {
first = size ;
last = size ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
/* setup summary table for recording results */
2018-07-07 12:29:27 +03:00
summary_table = ( mdtest_results_t * ) malloc ( iterations * sizeof ( mdtest_results_t ) ) ;
memset ( summary_table , 0 , sizeof ( mdtest_results_t ) ) ;
for ( int i = 0 ; i < iterations ; i + + ) {
for ( int j = 0 ; j < MDTEST_LAST_NUM ; j + + ) {
summary_table [ i ] . rate [ j ] = 0.0 ;
summary_table [ i ] . time [ j ] = 0.0 ;
}
}
2017-10-20 00:26:52 +03:00
if ( summary_table = = NULL ) {
FAIL ( " out of memory " ) ;
}
2017-06-28 19:42:20 +03:00
2017-10-20 00:26:52 +03:00
if ( unique_dir_per_task ) {
sprintf ( base_tree_name , " mdtest_tree.%d " , rank ) ;
} else {
sprintf ( base_tree_name , " mdtest_tree " ) ;
}
/* default use shared directory */
strcpy ( mk_name , " mdtest.shared. " ) ;
strcpy ( stat_name , " mdtest.shared. " ) ;
strcpy ( read_name , " mdtest.shared. " ) ;
strcpy ( rm_name , " mdtest.shared. " ) ;
2018-07-07 12:29:27 +03:00
MPI_Comm_group ( testComm , & worldgroup ) ;
2017-10-20 00:26:52 +03:00
/* Run the tests */
for ( i = first ; i < = last & & i < = size ; i + = stride ) {
range . last = i - 1 ;
MPI_Group_range_incl ( worldgroup , 1 , ( void * ) & range , & testgroup ) ;
2018-07-07 12:29:27 +03:00
MPI_Comm_create ( testComm , testgroup , & testComm ) ;
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
if ( files_only & & dirs_only ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " \n %d tasks, " LLU " files/directories \n " , i , i * items ) ;
2017-10-20 00:26:52 +03:00
} else if ( files_only ) {
if ( ! shared_file ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " \n %d tasks, " LLU " files \n " , i , i * items ) ;
2017-10-20 00:26:52 +03:00
}
else {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " \n %d tasks, 1 file \n " , i ) ;
2017-10-20 00:26:52 +03:00
}
} else if ( dirs_only ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " \n %d tasks, " LLU " directories \n " , i , i * items ) ;
2017-10-20 00:26:52 +03:00
}
}
if ( rank = = 0 & & verbose > = 1 ) {
2018-07-07 12:29:27 +03:00
fprintf ( out_logfile , " \n " ) ;
fprintf ( out_logfile , " Operation Duration Rate \n " ) ;
fprintf ( out_logfile , " --------- -------- ---- \n " ) ;
2017-10-20 00:26:52 +03:00
}
2018-07-07 12:29:27 +03:00
for ( j = 0 ; j < iterations ; j + + ) {
2018-07-08 01:56:39 +03:00
// keep track of the current status for stonewalling
mdtest_iteration ( i , j , testgroup , & summary_table [ j ] ) ;
2017-10-20 00:26:52 +03:00
}
summarize_results ( iterations ) ;
if ( i = = 1 & & stride > 1 ) {
i = 0 ;
}
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( rank = = 0 ) {
2018-07-07 16:01:11 +03:00
fprintf ( out_logfile , " \n -- finished at %s -- \n " , PrintTimestamp ( ) ) ;
2018-07-07 12:29:27 +03:00
fflush ( out_logfile ) ;
2017-06-28 19:42:20 +03:00
}
2017-10-20 00:26:52 +03:00
if ( random_seed > 0 ) {
free ( rand_array ) ;
2017-06-28 19:42:20 +03:00
}
2018-08-30 00:14:19 +03:00
if ( backend - > finalize )
backend - > finalize ( NULL ) ;
2018-06-13 21:37:37 +03:00
2018-07-07 12:29:27 +03:00
return summary_table ;
2017-06-28 19:42:20 +03:00
}