From 20e960d0205acb4d8bda8ea50bc4f9f65b16494a Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 27 Mar 2019 20:04:48 +0000 Subject: [PATCH] Support the setting of module-specific options per test. --- src/aiori-DUMMY.c | 52 ++++---- src/aiori-POSIX.c | 36 +++++- src/aiori.c | 47 ++++--- src/aiori.h | 7 +- src/ior-output.c | 1 - src/ior.c | 21 +--- src/ior.h | 12 +- src/mdtest.c | 16 +-- src/option.c | 247 ++++++++++++++++++++----------------- src/option.h | 8 +- src/parse_options.c | 293 +++++++++++++++++++------------------------- src/utilities.c | 77 ++++++++++++ src/utilities.h | 3 + 13 files changed, 463 insertions(+), 357 deletions(-) diff --git a/src/aiori-DUMMY.c b/src/aiori-DUMMY.c index 675a91b..0494cb4 100755 --- a/src/aiori-DUMMY.c +++ b/src/aiori-DUMMY.c @@ -17,29 +17,33 @@ /************************** O P T I O N S *****************************/ -struct dummy_options{ +typedef struct { uint64_t delay_creates; uint64_t delay_xfer; int delay_rank_0_only; -}; - -static struct dummy_options o = { - .delay_creates = 0, - .delay_xfer = 0, - .delay_rank_0_only = 0, -}; - -static option_help options [] = { - {0, "dummy.delay-create", "Delay per create in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_creates}, - {0, "dummy.delay-xfer", "Delay per xfer in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_xfer}, - {0, "dummy.delay-only-rank0", "Delay only Rank0", OPTION_FLAG, 'd', & o.delay_rank_0_only}, - LAST_OPTION -}; +} dummy_options_t; static char * current = (char*) 1; -static option_help * DUMMY_options(){ - return options; +static option_help * DUMMY_options(void ** init_backend_options, void * init_values){ + dummy_options_t * o = malloc(sizeof(dummy_options_t)); + if (init_values != NULL){ + memcpy(o, init_values, sizeof(dummy_options_t)); + }else{ + memset(o, 0, sizeof(dummy_options_t)); + } + + *init_backend_options = o; + + option_help h [] = { + {0, "dummy.delay-create", "Delay per create in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o->delay_creates}, + {0, "dummy.delay-xfer", "Delay per xfer in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o->delay_xfer}, + {0, "dummy.delay-only-rank0", "Delay only Rank0", OPTION_FLAG, 'd', & o->delay_rank_0_only}, + LAST_OPTION + }; + option_help * help = malloc(sizeof(h)); + memcpy(help, h, sizeof(h)); + return help; } static void *DUMMY_Create(char *testFileName, IOR_param_t * param) @@ -47,9 +51,10 @@ static void *DUMMY_Create(char *testFileName, IOR_param_t * param) if(verbose > 4){ fprintf(out_logfile, "DUMMY create: %s = %p\n", testFileName, current); } - if (o.delay_creates){ - if (! o.delay_rank_0_only || (o.delay_rank_0_only && rank == 0)){ - struct timespec wait = { o.delay_creates / 1000 / 1000, 1000l * (o.delay_creates % 1000000)}; + dummy_options_t * o = (dummy_options_t*) param->backend_options; + if (o->delay_creates){ + if (! o->delay_rank_0_only || (o->delay_rank_0_only && rank == 0)){ + struct timespec wait = { o->delay_creates / 1000 / 1000, 1000l * (o->delay_creates % 1000000)}; nanosleep( & wait, NULL); } } @@ -102,9 +107,10 @@ static IOR_offset_t DUMMY_Xfer(int access, void *file, IOR_size_t * buffer, IOR_ if(verbose > 4){ fprintf(out_logfile, "DUMMY xfer: %p\n", file); } - if (o.delay_xfer){ - if (! o.delay_rank_0_only || (o.delay_rank_0_only && rank == 0)){ - struct timespec wait = {o.delay_xfer / 1000 / 1000, 1000l * (o.delay_xfer % 1000000)}; + dummy_options_t * o = (dummy_options_t*) param->backend_options; + if (o->delay_xfer){ + if (! o->delay_rank_0_only || (o->delay_rank_0_only && rank == 0)){ + struct timespec wait = {o->delay_xfer / 1000 / 1000, 1000l * (o->delay_xfer % 1000000)}; nanosleep( & wait, NULL); } } diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index ca7c4a2..1d41c64 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -72,8 +72,36 @@ static IOR_offset_t POSIX_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *); static void POSIX_Fsync(void *, IOR_param_t *); +/************************** O P T I O N S *****************************/ +typedef struct{ + int direct_io; +} posix_options_t; + + +static option_help * posix_options(void ** init_backend_options, void * init_values){ + posix_options_t * o = malloc(sizeof(posix_options_t)); + + if (init_values != NULL){ + memcpy(o, init_values, sizeof(posix_options_t)); + }else{ + o->direct_io = 0; + } + + *init_backend_options = o; + + option_help h [] = { + {0, "posix.odirect", "Direct I/O Mode", OPTION_FLAG, 'd', & o->direct_io}, + LAST_OPTION + }; + option_help * help = malloc(sizeof(h)); + memcpy(help, h, sizeof(h)); + return help; +} + + /************************** D E C L A R A T I O N S ***************************/ + ior_aiori_t posix_aiori = { .name = "POSIX", .name_legacy = NULL, @@ -90,6 +118,7 @@ ior_aiori_t posix_aiori = { .rmdir = aiori_posix_rmdir, .access = aiori_posix_access, .stat = aiori_posix_stat, + .get_options = posix_options, .enable_mdtest = true, }; @@ -274,8 +303,8 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) fd = (int *)malloc(sizeof(int)); if (fd == NULL) ERR("Unable to malloc file descriptor"); - - if (param->useO_DIRECT == TRUE) + posix_options_t * o = (posix_options_t*) param->backend_options; + if (o->direct_io == TRUE) set_o_direct_flag(&fd_oflag); if(param->dryRun) @@ -389,7 +418,8 @@ void *POSIX_Open(char *testFileName, IOR_param_t * param) if (fd == NULL) ERR("Unable to malloc file descriptor"); - if (param->useO_DIRECT == TRUE) + posix_options_t * o = (posix_options_t*) param->backend_options; + if (o->direct_io == TRUE) set_o_direct_flag(&fd_oflag); fd_oflag |= O_RDWR; diff --git a/src/aiori.c b/src/aiori.c index 11f11b4..52001c8 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -72,24 +72,37 @@ ior_aiori_t *available_aiori[] = { NULL }; -void airoi_parse_options(int argc, char ** argv, option_help * global_options){ - int airoi_c = aiori_count(); - options_all opt; - opt.module_count = airoi_c + 1; - opt.modules = malloc(sizeof(option_module) * (airoi_c + 1)); - opt.modules[0].prefix = NULL; - opt.modules[0].options = global_options; - ior_aiori_t **tmp = available_aiori; - for (int i=1; *tmp != NULL; ++tmp, i++) { - opt.modules[i].prefix = (*tmp)->name; - if((*tmp)->get_options != NULL){ - opt.modules[i].options = (*tmp)->get_options(); - }else{ - opt.modules[i].options = NULL; - } +void * airoi_update_module_options(const ior_aiori_t * backend, options_all_t * opt){ + if (backend->get_options == NULL) + return NULL; + char * name = backend->name; + ior_aiori_t **tmp = available_aiori; + for (int i=1; *tmp != NULL; ++tmp, i++) { + if (strcmp(opt->modules[i].prefix, name) == 0){ + opt->modules[i].options = (*tmp)->get_options(& opt->modules[i].defaults, opt->modules[i].defaults); + return opt->modules[i].defaults; } - option_parse(argc, argv, &opt); - free(opt.modules); + } + return NULL; +} + +options_all_t * airoi_create_all_module_options(option_help * global_options){ + int airoi_c = aiori_count(); + options_all_t * opt = malloc(sizeof(options_all_t)); + opt->module_count = airoi_c + 1; + opt->modules = malloc(sizeof(option_module) * (airoi_c + 1)); + opt->modules[0].prefix = NULL; + opt->modules[0].options = global_options; + ior_aiori_t **tmp = available_aiori; + for (int i=1; *tmp != NULL; ++tmp, i++) { + opt->modules[i].prefix = (*tmp)->name; + if((*tmp)->get_options != NULL){ + opt->modules[i].options = (*tmp)->get_options(& opt->modules[i].defaults, NULL); + }else{ + opt->modules[i].options = NULL; + } + } + return opt; } void aiori_supported_apis(char * APIs, char * APIs_legacy, enum bench_type type) diff --git a/src/aiori.h b/src/aiori.h index 727af58..782d1ac 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -83,7 +83,7 @@ typedef struct ior_aiori { int (*stat) (const char *path, struct stat *buf, IOR_param_t * param); void (*initialize)(); /* called once per program before MPI is started */ void (*finalize)(); /* called once per program after MPI is shutdown */ - option_help * (*get_options)(); + option_help * (*get_options)(void ** init_backend_options, void* init_values); /* initializes the backend options as well and returns the pointer to the option help structure */ bool enable_mdtest; } ior_aiori_t; @@ -110,7 +110,10 @@ void aiori_finalize(IOR_test_t * tests); const ior_aiori_t *aiori_select (const char *api); int aiori_count (void); void aiori_supported_apis(char * APIs, char * APIs_legacy, enum bench_type type); -void airoi_parse_options(int argc, char ** argv, option_help * global_options); +options_all_t * airoi_create_all_module_options(option_help * global_options); + +void * airoi_update_module_options(const ior_aiori_t * backend, options_all_t * module_defaults); + const char *aiori_default (void); /* some generic POSIX-based backend calls */ diff --git a/src/ior-output.c b/src/ior-output.c index 32a6054..40764ff 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -367,7 +367,6 @@ void ShowTestStart(IOR_param_t *test) PrintKeyValInt("setAlignment", test->setAlignment); PrintKeyValInt("storeFileOffset", test->storeFileOffset); PrintKeyValInt("useSharedFilePointer", test->useSharedFilePointer); - PrintKeyValInt("useO_DIRECT", test->useO_DIRECT); PrintKeyValInt("useStridedDatatype", test->useStridedDatatype); PrintKeyValInt("keepFile", test->keepFile); PrintKeyValInt("keepFileWithError", test->keepFileWithError); diff --git a/src/ior.c b/src/ior.c index 12b9390..a1dc321 100755 --- a/src/ior.c +++ b/src/ior.c @@ -493,20 +493,13 @@ static void aligned_buffer_free(void *buf) free(*(void **)((char *)buf - sizeof(char *))); } -static void* safeMalloc(uint64_t size){ - void * d = malloc(size); - if (d == NULL){ - ERR("Could not malloc an array"); - } - memset(d, 0, size); - return d; -} - void AllocResults(IOR_test_t *test) { int reps; if (test->results != NULL) - return; + return; + + IOR_param_t * params = & test->params; reps = test->params.repetitions; test->results = (IOR_results_t *) safeMalloc(sizeof(IOR_results_t) * reps); @@ -1235,13 +1228,7 @@ static void TestIoSys(IOR_test_t *test) fflush(out_logfile); } params->tasksPerNode = CountTasksPerNode(testComm); - - /* bind I/O calls to specific API */ - backend = aiori_select(params->api); - if (backend == NULL) - ERR_SIMPLE("unrecognized I/O API"); - - + backend = params->backend; /* show test setup */ if (rank == 0 && verbose >= VERBOSE_0) ShowSetup(params); diff --git a/src/ior.h b/src/ior.h index ad543ff..acf3a7d 100755 --- a/src/ior.h +++ b/src/ior.h @@ -36,6 +36,8 @@ typedef void *rados_ioctx_t; #endif +#include "option.h" + #include "iordef.h" /******************** DATA Packet Type ***************************************/ /* Holds the types of data packets: generic, offset, timestamp, incompressible */ @@ -93,6 +95,7 @@ typedef struct char * testFileName_fppReadCheck;/* filename for fpp read check */ char * hintsFileName; /* full name for hints file */ char * options; /* options string */ + // intermediate options int dryRun; /* do not perform any I/Os just run evtl. inputs print dummy output */ int numTasks; /* number of tasks for test */ int nodes; /* number of nodes for test */ @@ -126,7 +129,6 @@ typedef struct int useFileView; /* use MPI_File_set_view */ int useSharedFilePointer; /* use shared file pointer */ int useStridedDatatype; /* put strided access into datatype */ - int useO_DIRECT; /* use O_DIRECT, bypassing I/O buffers */ int showHints; /* show hints */ int summary_every_test; /* flag to print summary every test, not just at end */ int uniqueDir; /* use unique directory for each fpp */ @@ -148,8 +150,12 @@ typedef struct int randomOffset; /* access is to random offsets */ size_t memoryPerTask; /* additional memory used per task */ size_t memoryPerNode; /* additional memory used per node */ - enum PACKET_TYPE dataPacketType; /* The type of data packet. */ + char * memoryPerNodeStr; /* for parsing */ + char * testscripts; /* for parsing */ + char * buffer_type; /* for parsing */ + enum PACKET_TYPE dataPacketType; /* The type of data packet. */ + void * backend_options; /* Backend-specific options */ /* POSIX variables */ int singleXferAttempt; /* do not retry transfer if incomplete */ @@ -180,7 +186,7 @@ typedef struct size_t part_number; /* multi-part upload increment (PER-RANK!) */ char* UploadId; /* key for multi-part-uploads */ int collective_md; /* use collective metatata optimization */ - + /* RADOS variables */ rados_t rados_cluster; /* RADOS cluster handle */ diff --git a/src/mdtest.c b/src/mdtest.c index 8a81979..387bd86 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -151,7 +151,6 @@ static pid_t pid; static uid_t uid; /* Use the POSIX backend by default */ -static const char *backend_name = "POSIX"; static const ior_aiori_t *backend; static IOR_param_t param; @@ -2099,7 +2098,6 @@ void mdtest_init_args(){ nstride = 0; } - mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out) { testComm = world_com; out_logfile = world_out; @@ -2132,7 +2130,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * sprintf(apiStr, "API for I/O [%s]", APIs); option_help options [] = { - {'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & backend_name}, + {'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & param.api}, {'b', NULL, "branching factor of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & branch_factor}, {'d', NULL, "the directory in which the tests will run", OPTION_OPTIONAL_ARGUMENT, 's', & path}, {'B', NULL, "no barriers between phases", OPTION_OPTIONAL_ARGUMENT, 'd', & no_barriers}, @@ -2168,12 +2166,10 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & print_time}, LAST_OPTION }; - airoi_parse_options(argc, argv, options); - - backend = aiori_select(backend_name); - if (NULL == backend) { - FAIL("Could not find suitable backend to use"); - } + options_all_t * global_options = airoi_create_all_module_options(options); + option_parse(argc, argv, global_options); + updateParsedOptions(& param, global_options); + backend = param.backend; MPI_Comm_rank(testComm, &rank); MPI_Comm_size(testComm, &size); @@ -2222,7 +2218,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * if (( rank == 0 ) && ( verbose >= 1 )) { // option_print_current(options); - fprintf (out_logfile, "api : %s\n", backend_name); + fprintf (out_logfile, "api : %s\n", param.api); fprintf( out_logfile, "barriers : %s\n", ( barriers ? "True" : "False" )); fprintf( out_logfile, "collective_creates : %s\n", ( collective_creates ? "True" : "False" )); fprintf( out_logfile, "create_only : %s\n", ( create_only ? "True" : "False" )); diff --git a/src/option.c b/src/option.c index 4e88055..08833da 100644 --- a/src/option.c +++ b/src/option.c @@ -220,7 +220,136 @@ void option_print_current(option_help * args){ print_current_option_section(args, OPTION_FLAG); } -int option_parse(int argc, char ** argv, options_all * opt_all){ +static void option_parse_token(char ** argv, int * flag_parsed_next, int * requiredArgsSeen, options_all_t * opt_all, int * error, int * print_help){ + int foundOption = 0; + char * txt = argv[0]; + char * arg = strstr(txt, "="); + int replaced_equal = 0; + int i = 0; + if(arg != NULL){ + arg[0] = 0; + arg++; + replaced_equal = 1; + } + *flag_parsed_next = 0; + + for(int m = 0; m < opt_all->module_count; m++ ){ + option_help * args = opt_all->modules[m].options; + if(args == NULL) continue; + // try to find matching option help + for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 || o->help != NULL ; o++ ){ + if( o->shortVar == 0 && o->longVar == 0 ){ + // section + continue; + } + if ( (txt[0] == '-' && o->shortVar == txt[1]) || (strlen(txt) > 2 && txt[0] == '-' && txt[1] == '-' && o->longVar != NULL && strcmp(txt + 2, o->longVar) == 0)){ + foundOption = 1; + + // now process the option. + switch(o->arg){ + case (OPTION_FLAG):{ + assert(o->type == 'd'); + (*(int*) o->variable)++; + break; + } + case (OPTION_OPTIONAL_ARGUMENT): + case (OPTION_REQUIRED_ARGUMENT):{ + // check if next is an argument + if(arg == NULL){ + if(o->shortVar == txt[1] && txt[2] != 0){ + arg = & txt[2]; + }else{ + // simply take the next value as argument + i++; + arg = argv[1]; + *flag_parsed_next = 1; + } + } + + if(arg == NULL){ + const char str[] = {o->shortVar, 0}; + printf("Error, argument missing for option %s\n", (o->longVar != NULL) ? o->longVar : str); + exit(1); + } + + switch(o->type){ + case('p'):{ + // call the function in the variable + void(*fp)() = o->variable; + fp(arg); + break; + } + case('F'):{ + *(double*) o->variable = atof(arg); + break; + } + case('f'):{ + *(float*) o->variable = atof(arg); + break; + } + case('d'):{ + int64_t val = string_to_bytes(arg); + if (val > INT_MAX || val < INT_MIN){ + printf("WARNING: parsing the number %s to integer, this produced an overflow!\n", arg); + } + *(int*) o->variable = val; + break; + } + case('H'): + case('s'):{ + (*(char **) o->variable) = strdup(arg); + break; + } + case('c'):{ + (*(char *)o->variable) = arg[0]; + if(strlen(arg) > 1){ + printf("Error, ignoring remainder of string for option %c (%s).\n", o->shortVar, o->longVar); + } + break; + } + case('l'):{ + *(long long*) o->variable = string_to_bytes(arg); + break; + } + default: + printf("ERROR: Unknown option type %c\n", o->type); + } + } + } + if(replaced_equal){ + arg[-1] = '='; + } + + if(o->arg == OPTION_REQUIRED_ARGUMENT){ + (*requiredArgsSeen)++; + } + + break; + } + } + } + + + if (! foundOption){ + if(strcmp(txt, "-h") == 0 || strcmp(txt, "--help") == 0){ + *print_help = 1; + }else{ + *error = 1; + } + } +} + +int option_parse_key_value(char * key, char * value, options_all_t * opt_all){ + int flag_parsed_next; + char * argv[2] = {key, value}; + int error = 0; + int requiredArgsSeen = 0; + int print_help = 0; + option_parse_token(argv, & flag_parsed_next, & requiredArgsSeen, opt_all, & error, & print_help); + return error; +} + +int option_parse(int argc, char ** argv, options_all_t * opt_all){ int error = 0; int requiredArgsSeen = 0; int requiredArgsNeeded = 0; @@ -238,117 +367,13 @@ int option_parse(int argc, char ** argv, options_all * opt_all){ } for(i=1; i < argc; i++){ - char * txt = argv[i]; - int foundOption = 0; - char * arg = strstr(txt, "="); - int replaced_equal = 0; - if(arg != NULL){ - arg[0] = 0; - arg++; - replaced_equal = 1; + int flag_parsed_next; + option_parse_token(& argv[i], & flag_parsed_next, & requiredArgsSeen, opt_all, & error, & printhelp); + if (flag_parsed_next){ + i++; } - - for(int m = 0; m < opt_all->module_count; m++ ){ - option_help * args = opt_all->modules[m].options; - if(args == NULL) continue; - // try to find matching option help - for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 || o->help != NULL ; o++ ){ - if( o->shortVar == 0 && o->longVar == 0 ){ - // section - continue; - } - if ( (txt[0] == '-' && o->shortVar == txt[1]) || (strlen(txt) > 2 && txt[0] == '-' && txt[1] == '-' && o->longVar != NULL && strcmp(txt + 2, o->longVar) == 0)){ - foundOption = 1; - - // now process the option. - switch(o->arg){ - case (OPTION_FLAG):{ - assert(o->type == 'd'); - (*(int*) o->variable)++; - break; - } - case (OPTION_OPTIONAL_ARGUMENT): - case (OPTION_REQUIRED_ARGUMENT):{ - // check if next is an argument - if(arg == NULL){ - if(o->shortVar == txt[1] && txt[2] != 0){ - arg = & txt[2]; - }else{ - // simply take the next value as argument - i++; - arg = argv[i]; - } - } - - if(arg == NULL){ - const char str[] = {o->shortVar, 0}; - printf("Error, argument missing for option %s\n", (o->longVar != NULL) ? o->longVar : str); - exit(1); - } - - switch(o->type){ - case('p'):{ - // call the function in the variable - void(*fp)() = o->variable; - fp(arg); - break; - } - case('F'):{ - *(double*) o->variable = atof(arg); - break; - } - case('f'):{ - *(float*) o->variable = atof(arg); - break; - } - case('d'):{ - int64_t val = string_to_bytes(arg); - if (val > INT_MAX || val < INT_MIN){ - printf("WARNING: parsing the number %s to integer, this produced an overflow!\n", arg); - } - *(int*) o->variable = val; - break; - } - case('H'): - case('s'):{ - (*(char **) o->variable) = strdup(arg); - break; - } - case('c'):{ - (*(char *)o->variable) = arg[0]; - if(strlen(arg) > 1){ - printf("Error, ignoring remainder of string for option %c (%s).\n", o->shortVar, o->longVar); - } - break; - } - case('l'):{ - *(long long*) o->variable = string_to_bytes(arg); - break; - } - default: - printf("ERROR: Unknown option type %c\n", o->type); - } - } - } - if(replaced_equal){ - arg[-1] = '='; - } - - if(o->arg == OPTION_REQUIRED_ARGUMENT){ - requiredArgsSeen++; - } - - break; - } - } - } - if (! foundOption){ - if(strcmp(txt, "-h") == 0 || strcmp(txt, "--help") == 0){ - printhelp = 1; - }else{ - printf("Error invalid argument: %s\n", txt); - error = 1; - } + if(error){ + printf("Error invalid argument: %s\n", argv[i]); } } diff --git a/src/option.h b/src/option.h index 53a3732..7225921 100644 --- a/src/option.h +++ b/src/option.h @@ -26,12 +26,13 @@ typedef struct{ typedef struct{ char * prefix; // may be NULL to include it in the standard name option_help * options; + void * defaults; // these default values are taken from the command line } option_module; typedef struct{ int module_count; option_module * modules; -} options_all; +} options_all_t; #define LAST_OPTION {0, 0, 0, (option_value_type) 0, 0, NULL} @@ -39,6 +40,9 @@ int64_t string_to_bytes(char *size_str); void option_print_current(option_help * args); //@return the number of parsed arguments -int option_parse(int argc, char ** argv, options_all * args); +int option_parse(int argc, char ** argv, options_all_t * args); + +/* Parse a single line */ +int option_parse_key_value(char * key, char * value, options_all_t * opt_all); #endif diff --git a/src/parse_options.c b/src/parse_options.c index ea6dd63..f92e239 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -36,38 +36,11 @@ IOR_param_t initialTestParams; +option_help * createGlobalOptions(IOR_param_t * params); -static size_t NodeMemoryStringToBytes(char *size_str) -{ - int percent; - int rc; - long page_size; - long num_pages; - long long mem; - rc = sscanf(size_str, " %d %% ", &percent); - if (rc == 0) - return (size_t) string_to_bytes(size_str); - if (percent > 100 || percent < 0) - ERR("percentage must be between 0 and 100"); - -#ifdef HAVE_SYSCONF - page_size = sysconf(_SC_PAGESIZE); -#else - page_size = getpagesize(); -#endif - -#ifdef _SC_PHYS_PAGES - num_pages = sysconf(_SC_PHYS_PAGES); - if (num_pages == -1) - ERR("sysconf(_SC_PHYS_PAGES) is not supported"); -#else - ERR("sysconf(_SC_PHYS_PAGES) is not supported"); -#endif - mem = page_size * num_pages; - - return mem / 100 * percent; -} +static IOR_param_t * parameters; +static options_all_t * global_options; /* @@ -113,7 +86,7 @@ static void CheckRunSettings(IOR_test_t *tests) /* * Set flags from commandline string/value pairs. */ -void DecodeDirective(char *line, IOR_param_t *params) +void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_options) { char option[MAX_STR]; char value[MAX_STR]; @@ -132,6 +105,14 @@ void DecodeDirective(char *line, IOR_param_t *params) } if (strcasecmp(option, "api") == 0) { params->api = strdup(value); + + params->backend = aiori_select(params->api); + if (params->backend == NULL){ + fprintf(out_logfile, "Could not load backend API %s\n", params->api); + exit(-1); + } + /* copy the actual module options into the test */ + params->backend_options = airoi_update_module_options(params->backend, global_options); } else if (strcasecmp(option, "summaryFile") == 0) { if (rank == 0){ out_resultfile = fopen(value, "w"); @@ -240,8 +221,6 @@ void DecodeDirective(char *line, IOR_param_t *params) params->useFileView = atoi(value); } else if (strcasecmp(option, "usesharedfilepointer") == 0) { params->useSharedFilePointer = atoi(value); - } else if (strcasecmp(option, "useo_direct") == 0) { - params->useO_DIRECT = atoi(value); } else if (strcasecmp(option, "usestrideddatatype") == 0) { params->useStridedDatatype = atoi(value); } else if (strcasecmp(option, "showhints") == 0) { @@ -314,21 +293,28 @@ void DecodeDirective(char *line, IOR_param_t *params) } else if (strcasecmp(option, "summaryalways") == 0) { params->summary_every_test = atoi(value); } else { - if (rank == 0) - fprintf(out_logfile, "Unrecognized parameter \"%s\"\n", - option); - MPI_CHECK(MPI_Initialized(&initialized), "MPI_Initialized() error"); - if (initialized) - MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); - else - exit(-1); + int parsing_error = option_parse_key_value(option, value, module_options); + if(parsing_error){ + if (rank == 0) + fprintf(out_logfile, "Unrecognized parameter \"%s\"\n", + option); + MPI_CHECK(MPI_Initialized(&initialized), "MPI_Initialized() error"); + if (initialized) + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + else + exit(-1); + } } } +static void decodeDirectiveWrapper(char *line){ + DecodeDirective(line, parameters, global_options); +} + /* * Parse a single line, which may contain multiple comma-seperated directives */ -void ParseLine(char *line, IOR_param_t * test) +void ParseLine(char *line, IOR_param_t * test, options_all_t * module_options) { char *start, *end; @@ -337,7 +323,7 @@ void ParseLine(char *line, IOR_param_t * test) end = strchr(start, ','); if (end != NULL) *end = '\0'; - DecodeDirective(start, test); + DecodeDirective(start, test, module_options); start = end + 1; } while (end != NULL); } @@ -383,9 +369,12 @@ IOR_test_t *ReadConfigScript(char *scriptName) IOR_test_t *head = NULL; IOR_test_t *tail = NULL; + option_help ** option_p = & global_options->modules[0].options; + /* Initialize the first test */ - head = CreateTest(&initialTestParams, test_num++); + head = CreateTest(& initialTestParams, test_num++); tail = head; + *option_p = createGlobalOptions(& ((IOR_test_t*) head)->params); /* The current options */ /* open the script */ file = fopen(scriptName, "r"); @@ -423,6 +412,7 @@ IOR_test_t *ReadConfigScript(char *scriptName) tail->next = CreateTest(&tail->params, test_num++); AllocResults(tail); tail = tail->next; + *option_p = createGlobalOptions(& ((IOR_test_t*) tail->next)->params); } runflag = 1; } else if (runflag) { @@ -430,11 +420,12 @@ IOR_test_t *ReadConfigScript(char *scriptName) create and initialize a new test structure */ runflag = 0; tail->next = CreateTest(&tail->params, test_num++); + *option_p = createGlobalOptions(& ((IOR_test_t*) tail->next)->params); AllocResults(tail); tail = tail->next; - ParseLine(ptr, &tail->params); + ParseLine(ptr, &tail->params, global_options); } else { - ParseLine(ptr, &tail->params); + ParseLine(ptr, &tail->params, global_options); } } @@ -447,139 +438,105 @@ IOR_test_t *ReadConfigScript(char *scriptName) } -static IOR_param_t * parameters; +option_help * createGlobalOptions(IOR_param_t * params){ + char APIs[1024]; + char APIs_legacy[1024]; + aiori_supported_apis(APIs, APIs_legacy, IOR); + char apiStr[1024]; + sprintf(apiStr, "API for I/O [%s]", APIs); -static void decodeDirectiveWrapper(char *line){ - DecodeDirective(line, parameters); + option_help o [] = { + {'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & params->api}, + {'A', NULL, "refNum -- user supplied reference number to include in the summary", OPTION_OPTIONAL_ARGUMENT, 'd', & params->referenceNumber}, + {'b', NULL, "blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & params->blockSize}, + {'c', NULL, "collective -- collective I/O", OPTION_FLAG, 'd', & params->collective}, + {'C', NULL, "reorderTasks -- changes task ordering to n+1 ordering for readback", OPTION_FLAG, 'd', & params->reorderTasks}, + {'d', NULL, "interTestDelay -- delay between reps in seconds", OPTION_OPTIONAL_ARGUMENT, 'd', & params->interTestDelay}, + {'D', NULL, "deadlineForStonewalling -- seconds before stopping write or read phase", OPTION_OPTIONAL_ARGUMENT, 'd', & params->deadlineForStonewalling}, + {.help=" -O stoneWallingWearOut=1 -- once the stonewalling timout is over, all process finish to access the amount of data", .arg = OPTION_OPTIONAL_ARGUMENT}, + {.help=" -O stoneWallingWearOutIterations=N -- stop after processing this number of iterations, needed for reading data back written with stoneWallingWearOut", .arg = OPTION_OPTIONAL_ARGUMENT}, + {.help=" -O stoneWallingStatusFile=FILE -- this file keeps the number of iterations from stonewalling during write and allows to use them for read", .arg = OPTION_OPTIONAL_ARGUMENT}, + {'e', NULL, "fsync -- perform sync operation after each block write", OPTION_FLAG, 'd', & params->fsync}, + {'E', NULL, "useExistingTestFile -- do not remove test file before write access", OPTION_FLAG, 'd', & params->useExistingTestFile}, + {'f', NULL, "scriptFile -- test script name", OPTION_OPTIONAL_ARGUMENT, 's', & params->testscripts}, + {'F', NULL, "filePerProc -- file-per-process", OPTION_FLAG, 'd', & params->filePerProc}, + {'g', NULL, "intraTestBarriers -- use barriers between open, write/read, and close", OPTION_FLAG, 'd', & params->intraTestBarriers}, + /* This option toggles between Incompressible Seed and Time stamp sig based on -l, + * so we'll toss the value in both for now, and sort it out in initialization + * after all the arguments are in and we know which it keep. + */ + {'G', NULL, "setTimeStampSignature -- set value for time stamp signature/random seed", OPTION_OPTIONAL_ARGUMENT, 'd', & params->setTimeStampSignature}, + {'H', NULL, "showHints -- show hints", OPTION_FLAG, 'd', & params->showHints}, + {'i', NULL, "repetitions -- number of repetitions of test", OPTION_OPTIONAL_ARGUMENT, 'd', & params->repetitions}, + {'I', NULL, "individualDataSets -- datasets not shared by all procs [not working]", OPTION_FLAG, 'd', & params->individualDataSets}, + {'j', NULL, "outlierThreshold -- warn on outlier N seconds from mean", OPTION_OPTIONAL_ARGUMENT, 'd', & params->outlierThreshold}, + {'J', NULL, "setAlignment -- HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'd', & params->setAlignment}, + {'k', NULL, "keepFile -- don't remove the test file(s) on program exit", OPTION_FLAG, 'd', & params->keepFile}, + {'K', NULL, "keepFileWithError -- keep error-filled file(s) after data-checking", OPTION_FLAG, 'd', & params->keepFileWithError}, + {'l', NULL, "datapacket type-- type of packet that will be created [offset|incompressible|timestamp|o|i|t]", OPTION_OPTIONAL_ARGUMENT, 's', & params->buffer_type}, + {'m', NULL, "multiFile -- use number of reps (-i) for multiple file count", OPTION_FLAG, 'd', & params->multiFile}, + {'M', NULL, "memoryPerNode -- hog memory on the node (e.g.: 2g, 75%)", OPTION_OPTIONAL_ARGUMENT, 's', & params->memoryPerNodeStr}, + {'n', NULL, "noFill -- no fill in HDF5 file creation", OPTION_FLAG, 'd', & params->noFill}, + {'N', NULL, "numTasks -- number of tasks that should participate in the test", OPTION_OPTIONAL_ARGUMENT, 'd', & params->numTasks}, + {'o', NULL, "testFile -- full name for test", OPTION_OPTIONAL_ARGUMENT, 's', & params->testFileName}, + {'O', NULL, "string of IOR directives (e.g. -O checkRead=1,lustreStripeCount=32)", OPTION_OPTIONAL_ARGUMENT, 'p', & decodeDirectiveWrapper}, + {'p', NULL, "preallocate -- preallocate file size", OPTION_FLAG, 'd', & params->preallocate}, + {'P', NULL, "useSharedFilePointer -- use shared file pointer [not working]", OPTION_FLAG, 'd', & params->useSharedFilePointer}, + {'q', NULL, "quitOnError -- during file error-checking, abort on error", OPTION_FLAG, 'd', & params->quitOnError}, + {'Q', NULL, "taskPerNodeOffset for read tests use with -C & -Z options (-C constant N, -Z at least N)", OPTION_OPTIONAL_ARGUMENT, 'd', & params->taskPerNodeOffset}, + {'r', NULL, "readFile -- read existing file", OPTION_FLAG, 'd', & params->readFile}, + {'R', NULL, "checkRead -- verify that the output of read matches the expected signature (used with -G)", OPTION_FLAG, 'd', & params->checkRead}, + {'s', NULL, "segmentCount -- number of segments", OPTION_OPTIONAL_ARGUMENT, 'd', & params->segmentCount}, + {'S', NULL, "useStridedDatatype -- put strided access into datatype [not working]", OPTION_FLAG, 'd', & params->useStridedDatatype}, + {'t', NULL, "transferSize -- size of transfer in bytes (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & params->transferSize}, + {'T', NULL, "maxTimeDuration -- max time in minutes executing repeated test; it aborts only between iterations and not within a test!", OPTION_OPTIONAL_ARGUMENT, 'd', & params->maxTimeDuration}, + {'u', NULL, "uniqueDir -- use unique directory name for each file-per-process", OPTION_FLAG, 'd', & params->uniqueDir}, + {'U', NULL, "hintsFileName -- full name for hints file", OPTION_OPTIONAL_ARGUMENT, 's', & params->hintsFileName}, + {'v', NULL, "verbose -- output information (repeating flag increases level)", OPTION_FLAG, 'd', & params->verbose}, + {'V', NULL, "useFileView -- use MPI_File_set_view", OPTION_FLAG, 'd', & params->useFileView}, + {'w', NULL, "writeFile -- write file", OPTION_FLAG, 'd', & params->writeFile}, + {'W', NULL, "checkWrite -- check read after write", OPTION_FLAG, 'd', & params->checkWrite}, + {'x', NULL, "singleXferAttempt -- do not retry transfer if incomplete", OPTION_FLAG, 'd', & params->singleXferAttempt}, + {'X', NULL, "reorderTasksRandomSeed -- random seed for -Z option", OPTION_OPTIONAL_ARGUMENT, 'd', & params->reorderTasksRandomSeed}, + {'Y', NULL, "fsyncPerWrite -- perform sync operation after every write operation", OPTION_FLAG, 'd', & params->fsyncPerWrite}, + {'z', NULL, "randomOffset -- access is to random, not sequential, offsets within a file", OPTION_FLAG, 'd', & params->randomOffset}, + {'Z', NULL, "reorderTasksRandom -- changes task ordering to random ordering for readback", OPTION_FLAG, 'd', & params->reorderTasksRandom}, + {.help=" -O summaryFile=FILE -- store result data into this file", .arg = OPTION_OPTIONAL_ARGUMENT}, + {.help=" -O summaryFormat=[default,JSON,CSV] -- use the format for outputing the summary", .arg = OPTION_OPTIONAL_ARGUMENT}, + {0, "dryRun", "do not perform any I/Os just run evtl. inputs print dummy output", OPTION_FLAG, 'd', & params->dryRun}, + LAST_OPTION, + }; + option_help * options = malloc(sizeof(o)); + memcpy(options, & o, sizeof(o)); + return options; } + /* * Parse Commandline. */ IOR_test_t *ParseCommandLine(int argc, char **argv) { - char * testscripts = NULL; - int toggleG = FALSE; - char * buffer_type = ""; - char * memoryPerNode = NULL; init_IOR_Param_t(& initialTestParams); + + IOR_test_t *tests = NULL; + + GetPlatformName(initialTestParams.platform); + + option_help * options = createGlobalOptions( & initialTestParams); parameters = & initialTestParams; + global_options = airoi_create_all_module_options(options); + option_parse(argc, argv, global_options); + updateParsedOptions(& initialTestParams, global_options); - char APIs[1024]; - char APIs_legacy[1024]; - aiori_supported_apis(APIs, APIs_legacy, IOR); - char apiStr[1024]; - sprintf(apiStr, "API for I/O [%s]", APIs); + if (initialTestParams.testscripts){ + tests = ReadConfigScript(initialTestParams.testscripts); + }else{ + tests = CreateTest(&initialTestParams, 0); + AllocResults(tests); + } - option_help options [] = { - {'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & initialTestParams.api}, - {'A', NULL, "refNum -- user supplied reference number to include in the summary", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.referenceNumber}, - {'b', NULL, "blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & initialTestParams.blockSize}, - {'B', NULL, "useO_DIRECT -- uses O_DIRECT for POSIX, bypassing I/O buffers", OPTION_FLAG, 'd', & initialTestParams.useO_DIRECT}, - {'c', NULL, "collective -- collective I/O", OPTION_FLAG, 'd', & initialTestParams.collective}, - {'C', NULL, "reorderTasks -- changes task ordering to n+1 ordering for readback", OPTION_FLAG, 'd', & initialTestParams.reorderTasks}, - {'d', NULL, "interTestDelay -- delay between reps in seconds", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.interTestDelay}, - {'D', NULL, "deadlineForStonewalling -- seconds before stopping write or read phase", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.deadlineForStonewalling}, - {.help=" -O stoneWallingWearOut=1 -- once the stonewalling timout is over, all process finish to access the amount of data", .arg = OPTION_OPTIONAL_ARGUMENT}, - {.help=" -O stoneWallingWearOutIterations=N -- stop after processing this number of iterations, needed for reading data back written with stoneWallingWearOut", .arg = OPTION_OPTIONAL_ARGUMENT}, - {.help=" -O stoneWallingStatusFile=FILE -- this file keeps the number of iterations from stonewalling during write and allows to use them for read", .arg = OPTION_OPTIONAL_ARGUMENT}, - {'e', NULL, "fsync -- perform sync operation after each block write", OPTION_FLAG, 'd', & initialTestParams.fsync}, - {'E', NULL, "useExistingTestFile -- do not remove test file before write access", OPTION_FLAG, 'd', & initialTestParams.useExistingTestFile}, - {'f', NULL, "scriptFile -- test script name", OPTION_OPTIONAL_ARGUMENT, 's', & testscripts}, - {'F', NULL, "filePerProc -- file-per-process", OPTION_FLAG, 'd', & initialTestParams.filePerProc}, - {'g', NULL, "intraTestBarriers -- use barriers between open, write/read, and close", OPTION_FLAG, 'd', & initialTestParams.intraTestBarriers}, - /* This option toggles between Incompressible Seed and Time stamp sig based on -l, - * so we'll toss the value in both for now, and sort it out in initialization - * after all the arguments are in and we know which it keep. - */ - {'G', NULL, "setTimeStampSignature -- set value for time stamp signature/random seed", OPTION_OPTIONAL_ARGUMENT, 'd', & toggleG}, - {'H', NULL, "showHints -- show hints", OPTION_FLAG, 'd', & initialTestParams.showHints}, - {'i', NULL, "repetitions -- number of repetitions of test", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.repetitions}, - {'I', NULL, "individualDataSets -- datasets not shared by all procs [not working]", OPTION_FLAG, 'd', & initialTestParams.individualDataSets}, - {'j', NULL, "outlierThreshold -- warn on outlier N seconds from mean", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.outlierThreshold}, - {'J', NULL, "setAlignment -- HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.setAlignment}, - {'k', NULL, "keepFile -- don't remove the test file(s) on program exit", OPTION_FLAG, 'd', & initialTestParams.keepFile}, - {'K', NULL, "keepFileWithError -- keep error-filled file(s) after data-checking", OPTION_FLAG, 'd', & initialTestParams.keepFileWithError}, - {'l', NULL, "datapacket type-- type of packet that will be created [offset|incompressible|timestamp|o|i|t]", OPTION_OPTIONAL_ARGUMENT, 's', & buffer_type}, - {'m', NULL, "multiFile -- use number of reps (-i) for multiple file count", OPTION_FLAG, 'd', & initialTestParams.multiFile}, - {'M', NULL, "memoryPerNode -- hog memory on the node (e.g.: 2g, 75%)", OPTION_OPTIONAL_ARGUMENT, 's', & memoryPerNode}, - {'n', NULL, "noFill -- no fill in HDF5 file creation", OPTION_FLAG, 'd', & initialTestParams.noFill}, - {'N', NULL, "numTasks -- number of tasks that should participate in the test", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.numTasks}, - {'o', NULL, "testFile -- full name for test", OPTION_OPTIONAL_ARGUMENT, 's', & initialTestParams.testFileName}, - {'O', NULL, "string of IOR directives (e.g. -O checkRead=1,lustreStripeCount=32)", OPTION_OPTIONAL_ARGUMENT, 'p', & decodeDirectiveWrapper}, - {'p', NULL, "preallocate -- preallocate file size", OPTION_FLAG, 'd', & initialTestParams.preallocate}, - {'P', NULL, "useSharedFilePointer -- use shared file pointer [not working]", OPTION_FLAG, 'd', & initialTestParams.useSharedFilePointer}, - {'q', NULL, "quitOnError -- during file error-checking, abort on error", OPTION_FLAG, 'd', & initialTestParams.quitOnError}, - {'Q', NULL, "taskPerNodeOffset for read tests use with -C & -Z options (-C constant N, -Z at least N)", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.taskPerNodeOffset}, - {'r', NULL, "readFile -- read existing file", OPTION_FLAG, 'd', & initialTestParams.readFile}, - {'R', NULL, "checkRead -- verify that the output of read matches the expected signature (used with -G)", OPTION_FLAG, 'd', & initialTestParams.checkRead}, - {'s', NULL, "segmentCount -- number of segments", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.segmentCount}, - {'S', NULL, "useStridedDatatype -- put strided access into datatype [not working]", OPTION_FLAG, 'd', & initialTestParams.useStridedDatatype}, - {'t', NULL, "transferSize -- size of transfer in bytes (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & initialTestParams.transferSize}, - {'T', NULL, "maxTimeDuration -- max time in minutes executing repeated test; it aborts only between iterations and not within a test!", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.maxTimeDuration}, - {'u', NULL, "uniqueDir -- use unique directory name for each file-per-process", OPTION_FLAG, 'd', & initialTestParams.uniqueDir}, - {'U', NULL, "hintsFileName -- full name for hints file", OPTION_OPTIONAL_ARGUMENT, 's', & initialTestParams.hintsFileName}, - {'v', NULL, "verbose -- output information (repeating flag increases level)", OPTION_FLAG, 'd', & initialTestParams.verbose}, - {'V', NULL, "useFileView -- use MPI_File_set_view", OPTION_FLAG, 'd', & initialTestParams.useFileView}, - {'w', NULL, "writeFile -- write file", OPTION_FLAG, 'd', & initialTestParams.writeFile}, - {'W', NULL, "checkWrite -- check read after write", OPTION_FLAG, 'd', & initialTestParams.checkWrite}, - {'x', NULL, "singleXferAttempt -- do not retry transfer if incomplete", OPTION_FLAG, 'd', & initialTestParams.singleXferAttempt}, - {'X', NULL, "reorderTasksRandomSeed -- random seed for -Z option", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.reorderTasksRandomSeed}, - {'Y', NULL, "fsyncPerWrite -- perform sync operation after every write operation", OPTION_FLAG, 'd', & initialTestParams.fsyncPerWrite}, - {'z', NULL, "randomOffset -- access is to random, not sequential, offsets within a file", OPTION_FLAG, 'd', & initialTestParams.randomOffset}, - {'Z', NULL, "reorderTasksRandom -- changes task ordering to random ordering for readback", OPTION_FLAG, 'd', & initialTestParams.reorderTasksRandom}, - {.help=" -O summaryFile=FILE -- store result data into this file", .arg = OPTION_OPTIONAL_ARGUMENT}, - {.help=" -O summaryFormat=[default,JSON,CSV] -- use the format for outputing the summary", .arg = OPTION_OPTIONAL_ARGUMENT}, - {0, "dryRun", "do not perform any I/Os just run evtl. inputs print dummy output", OPTION_FLAG, 'd', & initialTestParams.dryRun}, - LAST_OPTION, - }; + CheckRunSettings(tests); - IOR_test_t *tests = NULL; - - GetPlatformName(initialTestParams.platform); - airoi_parse_options(argc, argv, options); - - if (toggleG){ - initialTestParams.setTimeStampSignature = toggleG; - initialTestParams.incompressibleSeed = toggleG; - } - - if (buffer_type[0] != 0){ - switch(buffer_type[0]) { - case 'i': /* Incompressible */ - initialTestParams.dataPacketType = incompressible; - break; - case 't': /* timestamp */ - initialTestParams.dataPacketType = timestamp; - break; - case 'o': /* offset packet */ - initialTestParams.storeFileOffset = TRUE; - initialTestParams.dataPacketType = offset; - break; - default: - fprintf(out_logfile, - "Unknown arguement for -l %s; generic assumed\n", buffer_type); - break; - } - } - if (memoryPerNode){ - initialTestParams.memoryPerNode = NodeMemoryStringToBytes(memoryPerNode); - } - const ior_aiori_t * backend = aiori_select(initialTestParams.api); - if (backend == NULL) - ERR_SIMPLE("unrecognized I/O API"); - - initialTestParams.backend = backend; - initialTestParams.apiVersion = backend->get_version(); - - if (testscripts){ - tests = ReadConfigScript(testscripts); - }else{ - tests = CreateTest(&initialTestParams, 0); - AllocResults(tests); - } - - CheckRunSettings(tests); - - return (tests); + return (tests); } diff --git a/src/utilities.c b/src/utilities.c index b71ce41..cc0918c 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -66,6 +66,83 @@ enum OutputFormat_t outputFormat; /***************************** F U N C T I O N S ******************************/ +void* safeMalloc(uint64_t size){ + void * d = malloc(size); + if (d == NULL){ + ERR("Could not malloc an array"); + } + memset(d, 0, size); + return d; +} + + +size_t NodeMemoryStringToBytes(char *size_str) +{ + int percent; + int rc; + long page_size; + long num_pages; + long long mem; + + rc = sscanf(size_str, " %d %% ", &percent); + if (rc == 0) + return (size_t) string_to_bytes(size_str); + if (percent > 100 || percent < 0) + ERR("percentage must be between 0 and 100"); + +#ifdef HAVE_SYSCONF + page_size = sysconf(_SC_PAGESIZE); +#else + page_size = getpagesize(); +#endif + +#ifdef _SC_PHYS_PAGES + num_pages = sysconf(_SC_PHYS_PAGES); + if (num_pages == -1) + ERR("sysconf(_SC_PHYS_PAGES) is not supported"); +#else + ERR("sysconf(_SC_PHYS_PAGES) is not supported"); +#endif + mem = page_size * num_pages; + + return mem / 100 * percent; +} + +void updateParsedOptions(IOR_param_t * options, options_all_t * global_options){ + if (options->setTimeStampSignature){ + options->incompressibleSeed = options->setTimeStampSignature; + } + + if (options->buffer_type && options->buffer_type[0] != 0){ + switch(options->buffer_type[0]) { + case 'i': /* Incompressible */ + options->dataPacketType = incompressible; + break; + case 't': /* timestamp */ + options->dataPacketType = timestamp; + break; + case 'o': /* offset packet */ + options->storeFileOffset = TRUE; + options->dataPacketType = offset; + break; + default: + fprintf(out_logfile, + "Unknown argument for -l %s; generic assumed\n", options->buffer_type); + break; + } + } + if (options->memoryPerNodeStr){ + options->memoryPerNode = NodeMemoryStringToBytes(options->memoryPerNodeStr); + } + const ior_aiori_t * backend = aiori_select(options->api); + if (backend == NULL) + ERR_SIMPLE("unrecognized I/O API"); + + options->backend = backend; + /* copy the actual module options into the test */ + options->backend_options = airoi_update_module_options(backend, global_options); + options->apiVersion = backend->get_version(); +} /* Used in aiori-POSIX.c and aiori-PLFS.c */ diff --git a/src/utilities.h b/src/utilities.h index 7933cbc..b646174 100755 --- a/src/utilities.h +++ b/src/utilities.h @@ -53,6 +53,7 @@ extern enum OutputFormat_t outputFormat; /* format of the output */ } while(0) #endif +void* safeMalloc(uint64_t size); void set_o_direct_flag(int *fd); char *CurrentTimeString(void); @@ -65,6 +66,8 @@ void ShowHints (MPI_Info *); char *HumanReadable(IOR_offset_t value, int base); int CountTasksPerNode(MPI_Comm comm); void DelaySecs(int delay); +void updateParsedOptions(IOR_param_t * options, options_all_t * global_options); +size_t NodeMemoryStringToBytes(char *size_str); /* Returns -1, if cannot be read */ int64_t ReadStoneWallingIterations(char * const filename);