vitalif
/
mdtest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #184 from hpc/feature-aiori 

Feature aiori
master
Julian Kunkel 2019-09-09 15:20:43 +01:00 committed by GitHub
parent ced9b0183f c83edfe39b
commit 0921556456
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 75 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/aiori-DUMMY.c
View File

@ -143,6 +143,10 @@ static int DUMMY_stat (const char *path, struct stat *buf, IOR_param_t * param){
return 0;
}
static int DUMMY_check_params(IOR_param_t * test){
return 1;
}
ior_aiori_t dummy_aiori = {
.name = "DUMMY",
.name_legacy = NULL,
@ -163,4 +167,5 @@ ior_aiori_t dummy_aiori = {
.finalize = NULL,
.get_options = DUMMY_options,
.enable_mdtest = true,
.check_params = DUMMY_check_params
};

12
src/aiori-POSIX.c
View File

@ -71,6 +71,7 @@
static IOR_offset_t POSIX_Xfer(int, void *, IOR_size_t *,
IOR_offset_t, IOR_param_t *);
static void POSIX_Fsync(void *, IOR_param_t *);
static void POSIX_Sync(IOR_param_t * );
/************************** O P T I O N S *****************************/
typedef struct{
@ -122,6 +123,7 @@ ior_aiori_t posix_aiori = {
.stat = aiori_posix_stat,
.get_options = POSIX_options,
.enable_mdtest = true,
.sync = POSIX_Sync
};
/***************************** F U N C T I O N S ******************************/
@ -588,6 +590,16 @@ static void POSIX_Fsync(void *fd, IOR_param_t * param)
EWARNF("fsync(%d) failed", *(int *)fd);
}
static void POSIX_Sync(IOR_param_t * param)
{
int ret = system("sync");
if (ret != 0){
FAIL("Error executing the sync command, ensure it exists.");
}
}
/*
* Close a file through the POSIX interface.
*/

21
src/aiori-S3.c
View File

@ -159,6 +159,8 @@ static void S3_Fsync(void*, IOR_param_t*);
static IOR_offset_t S3_GetFileSize(IOR_param_t*, MPI_Comm, char*);
static void S3_init();
static void S3_finalize();
static int S3_check_params(IOR_param_t *);
/************************** D E C L A R A T I O N S ***************************/
@ -177,7 +179,8 @@ ior_aiori_t s3_aiori = {
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
.initialize = S3_init,
.finalize = S3_finalize
.finalize = S3_finalize,
.check_params = S3_check_params
};
// "S3", plus EMC-extensions enabled
@ -228,6 +231,22 @@ static void S3_finalize(){
aws_cleanup();
}
static int S3_check_params(IOR_param_t * test){
/* N:1 and N:N */
IOR_offset_t NtoN = test->filePerProc;
IOR_offset_t Nto1 = ! NtoN;
IOR_offset_t s = test->segmentCount;
IOR_offset_t t = test->transferSize;
IOR_offset_t b = test->blockSize;
if (Nto1 && (s != 1) && (b != t)) {
ERR("N:1 (strided) requires xfer-size == block-size");
return 0;
}
return 1;
}
/* modelled on similar macros in iordef.h */
#define CURL_ERR(MSG, CURL_ERRNO, PARAM) \
do { \

2
src/aiori.h
View File

@ -86,6 +86,8 @@ typedef struct ior_aiori {
void (*finalize)(); /* called once per program after MPI is shutdown */
option_help * (*get_options)(void ** init_backend_options, void* init_values); /* initializes the backend options as well and returns the pointer to the option help structure */
bool enable_mdtest;
int (*check_params)(IOR_param_t *); /* check if the provided parameters for the given test and the module options are correct, if they aren't print a message and exit(1) or return 1*/
void (*sync)(IOR_param_t * ); /* synchronize every pending operation for this storage */
} ior_aiori_t;
enum bench_type {

15
src/ior.c
View File

@ -1678,15 +1678,12 @@ static void ValidateTests(IOR_param_t * test)
if (test->useExistingTestFile && test->lustre_set_striping)
ERR("Lustre stripe options are incompatible with useExistingTestFile");
/* N:1 and N:N */
IOR_offset_t NtoN = test->filePerProc;
IOR_offset_t Nto1 = ! NtoN;
IOR_offset_t s = test->segmentCount;
IOR_offset_t t = test->transferSize;
IOR_offset_t b = test->blockSize;
if (Nto1 && (s != 1) && (b != t)) {
ERR("N:1 (strided) requires xfer-size == block-size");
/* allow the backend to validate the options */
if(test->backend->check_params){
int check = test->backend->check_params(test);
if (check == 0){
ERR("The backend returned that the test parameters are invalid.");
}
}
}

50
src/mdtest.c
View File

@ -148,6 +148,7 @@ static size_t write_bytes;
static int stone_wall_timer_seconds;
static size_t read_bytes;
static int sync_file;
static int call_sync;
static int path_count;
static int nstride; /* neighbor stride */
static int make_node = 0;
@ -263,6 +264,19 @@ static void prep_testdir(int j, int dir_iter){
pos += sprintf(& testdir[pos], ".%d-%d", j, dir_iter);
}
static void phase_end(){
if (call_sync){
if(! backend->sync){
FAIL("Error, backend does not provide the sync method, but your requested to use sync.");
}
backend->sync(& param);
}
if (barriers) {
MPI_Barrier(testComm);
}
}
/*
* This function copies the unique directory name for a given option to
* the "to" parameter. Some memory must be allocated to the "to" parameter.
@ -836,9 +850,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
}
}
if (barriers) {
MPI_Barrier(testComm);
}
phase_end();
t[1] = GetTimeStamp();
/* stat phase */
@ -864,10 +876,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
}
}
}
if (barriers) {
MPI_Barrier(testComm);
}
phase_end();
t[2] = GetTimeStamp();
/* read phase */
@ -894,9 +903,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
}
}
if (barriers) {
MPI_Barrier(testComm);
}
phase_end();
t[3] = GetTimeStamp();
if (remove_only) {
@ -924,9 +931,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran
}
}
if (barriers) {
MPI_Barrier(testComm);
}
phase_end();
t[4] = GetTimeStamp();
if (remove_only) {
@ -1082,9 +1087,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
}
}
if (barriers) {
MPI_Barrier(testComm);
}
phase_end();
t[1] = GetTimeStamp();
/* stat phase */
@ -1107,9 +1110,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
}
}
if (barriers) {
MPI_Barrier(testComm);
}
phase_end();
t[2] = GetTimeStamp();
/* read phase */
@ -1136,9 +1137,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
}
}
if (barriers) {
MPI_Barrier(testComm);
}
phase_end();
t[3] = GetTimeStamp();
if (remove_only) {
@ -1168,9 +1167,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro
}
}
if (barriers) {
MPI_Barrier(testComm);
}
phase_end();
t[4] = GetTimeStamp();
if (remove_only) {
if (unique_dir_per_task) {
@ -1853,6 +1850,7 @@ void mdtest_init_args(){
stone_wall_timer_seconds = 0;
read_bytes = 0;
sync_file = 0;
call_sync = 0;
path_count = 0;
nstride = 0;
make_node = 0;
@ -1925,6 +1923,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
{'x', NULL, "StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs", OPTION_OPTIONAL_ARGUMENT, 's', & stoneWallingStatusFile},
{'X', "verify-read", "Verify the data read", OPTION_FLAG, 'd', & verify_read},
{'y', NULL, "sync file after writing", OPTION_FLAG, 'd', & sync_file},
{'Y', NULL, "call the sync command after each phase (included in the timing; note it causes all IO to be flushed from your node)", OPTION_FLAG, 'd', & call_sync},
{'z', NULL, "depth of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & depth},
{'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & print_time},
LAST_OPTION
@ -2008,6 +2007,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
VERBOSE(1,-1, "unique_dir_per_task : %s", ( unique_dir_per_task ? "True" : "False" ));
VERBOSE(1,-1, "write_bytes : "LLU"", write_bytes );
VERBOSE(1,-1, "sync_file : %s", ( sync_file ? "True" : "False" ));
VERBOSE(1,-1, "call_sync : %s", ( call_sync ? "True" : "False" ));
VERBOSE(1,-1, "depth : %d", depth );
VERBOSE(1,-1, "make_node : %d", make_node );

10
src/utilities.c
View File

@ -77,15 +77,15 @@ void* safeMalloc(uint64_t size){
}
void FailMessage(int rank, const char *location, char *format, ...) {
char msg[4096];
char msg[4096];
va_list args;
va_start(args, format);
vsnprintf(msg, 4096, format, args);
va_end(args);
fprintf(out_logfile, "%s: Process %d: FAILED in %s, %s: %s\n",
PrintTimestamp(), rank, location, msg, strerror(errno));
fflush(out_logfile);
MPI_Abort(testComm, 1);
fprintf(out_logfile, "%s: Process %d: FAILED in %s, %s: %s\n",
PrintTimestamp(), rank, location, msg, strerror(errno));
fflush(out_logfile);
MPI_Abort(testComm, 1);
}
size_t NodeMemoryStringToBytes(char *size_str)