vitalif
/
mdtest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Changed the parser. 

The parser now supports concurrent parsing of all plugin options.
Moved HDF5 collective_md option into the backend as an example.
Example: ./src/ior -a dummy --dummy.delay-xfer=50000
master
Julian M. Kunkel 2018-10-02 16:46:29 +01:00
parent acee84bedc
commit 6f7576aa8a
11 changed files with 175 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
doc/sphinx/userDoc/options.rst
View File

@ -289,8 +289,9 @@ HDF5-ONLY
* setAlignment - HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g) [1]
* collectiveMetadata - enable HDF5 collective metadata (available since
HDF5-1.10.0)
HDF5-backend options
^^^^^^^^^^^^^^^^^^^^
* -c | --collectiveMetadata - enable HDF5 collective metadata (available since HDF5-1.10.0)
MPIIO-, HDF5-, AND NCMPI-ONLY
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

6
src/aiori-DUMMY.c
View File

@ -29,9 +29,9 @@ static struct dummy_options o = {
};
static option_help options [] = {
{'c', "delay-create", "Delay per create in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_creates},
{'x', "delay-xfer", "Delay per xfer in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_xfer},
{'z', "delay-only-rank0", "Delay only Rank0", OPTION_FLAG, 'd', & o.delay_rank_0_only},
{0, "dummy.delay-create", "Delay per create in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_creates},
{0, "dummy.delay-xfer", "Delay per xfer in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_xfer},
{0, "dummy.delay-only-rank0", "Delay only Rank0", OPTION_FLAG, 'd', & o.delay_rank_0_only},
LAST_OPTION
};

23
src/aiori-HDF5.c
View File

@ -93,6 +93,7 @@ static char* HDF5_GetVersion();
static void HDF5_Fsync(void *, IOR_param_t *);
static IOR_offset_t HDF5_GetFileSize(IOR_param_t *, MPI_Comm, char *);
static int HDF5_Access(const char *, int, IOR_param_t *);
static option_help * HDF5_get_options();
/************************** D E C L A R A T I O N S ***************************/
@ -112,6 +113,7 @@ ior_aiori_t hdf5_aiori = {
.rmdir = aiori_posix_rmdir,
.access = HDF5_Access,
.stat = aiori_posix_stat,
.get_options = HDF5_get_options
};
static hid_t xferPropList; /* xfer property list */
@ -121,8 +123,27 @@ hid_t fileDataSpace; /* file data space id */
hid_t memDataSpace; /* memory data space id */
int newlyOpenedFile; /* newly opened file */
/************************** O P T I O N S *****************************/
struct HDF5_options{
int collective_md;
};
/***************************** F U N C T I O N S ******************************/
static struct HDF5_options o = {
.collective_md = 0
};
static option_help options [] = {
{0, "hdf5.collectiveMetadata", "Use collectiveMetadata (available since HDF5-1.10.0)", OPTION_FLAG, 'd', & o.collective_md},
LAST_OPTION
};
static option_help * HDF5_get_options(){
return options;
}
/*
* Create and open a file through the HDF5 interface.
*/
@ -230,7 +251,7 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
"cannot set alignment");
#ifdef HAVE_H5PSET_ALL_COLL_METADATA_OPS
if (param->collective_md) {
if (o.collective_md) {
/* more scalable metadata */
HDF5_CHECK(H5Pset_all_coll_metadata_ops(accessPropList, 1),

6
src/aiori-RADOS.c
View File

@ -41,9 +41,9 @@ static struct rados_options o = {
};
static option_help options [] = {
{'u', "user", "Username for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.user},
{'c', "conf", "Config file for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.conf},
{'p', "pool", "RADOS pool to use for I/O", OPTION_REQUIRED_ARGUMENT, 's', & o.pool},
{0, "rados.user", "Username for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.user},
{0, "rados.conf", "Config file for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.conf},
{0, "rados.pool", "RADOS pool to use for I/O", OPTION_REQUIRED_ARGUMENT, 's', & o.pool},
LAST_OPTION
};

20
src/aiori.c
View File

@ -63,6 +63,26 @@ ior_aiori_t *available_aiori[] = {
NULL
};
void airoi_parse_options(int argc, char ** argv, option_help * global_options){
int airoi_c = aiori_count();
options_all opt;
opt.module_count = airoi_c + 1;
opt.modules = malloc(sizeof(option_module) * (airoi_c + 1));
opt.modules[0].prefix = NULL;
opt.modules[0].options = global_options;
ior_aiori_t **tmp = available_aiori;
for (int i=1; *tmp != NULL; ++tmp, i++) {
opt.modules[i].prefix = (*tmp)->name;
if((*tmp)->get_options != NULL){
opt.modules[i].options = (*tmp)->get_options();
}else{
opt.modules[i].options = NULL;
}
}
option_parse(argc, argv, &opt);
free(opt.modules);
}
void aiori_supported_apis(char * APIs, char * APIs_legacy){
ior_aiori_t **tmp = available_aiori;
if(*tmp != NULL){

1
src/aiori.h
View File

@ -103,6 +103,7 @@ void aiori_finalize(IOR_test_t * tests);
const ior_aiori_t *aiori_select (const char *api);
int aiori_count (void);
void aiori_supported_apis(char * APIs, char * APIs_legacy);
void airoi_parse_options(int argc, char ** argv, option_help * global_options);
const char *aiori_default (void);
/* some generic POSIX-based backend calls */

1
src/ior.h
View File

@ -178,7 +178,6 @@ typedef struct
char* URI; /* "path" to target object */
size_t part_number; /* multi-part upload increment (PER-RANK!) */
char* UploadId; /* key for multi-part-uploads */
int collective_md; /* use collective metatata optimization */
/* RADOS variables */
rados_t rados_cluster; /* RADOS cluster handle */

24
src/mdtest.c
View File

@ -2213,35 +2213,13 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
{'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & print_time},
LAST_OPTION
};
int printhelp = 0;
int parsed_options = option_parse(argc, argv, options, & printhelp);
airoi_parse_options(argc, argv, options);
backend = aiori_select(backend_name);
if (NULL == backend) {
FAIL("Could not find suitable backend to use");
}
if(backend->get_options != NULL){
option_parse(argc - parsed_options, argv + parsed_options, backend->get_options(), & printhelp);
}
if(printhelp != 0){
printf("Usage: %s ", argv[0]);
option_print_help(options, 0);
if(backend->get_options != NULL){
printf("\nPlugin options for backend %s\n", backend_name);
option_print_help(backend->get_options(), 1);
}
if(printhelp == 1){
exit(0);
}else{
exit(1);
}
}
MPI_Comm_rank(testComm, &rank);
MPI_Comm_size(testComm, &size);

232
src/option.c
View File

@ -135,40 +135,7 @@ static void print_help_section(option_help * args, option_value_type type, char
}
}
void option_print_help(option_help * args, int is_plugin){
option_help * o;
int optionalArgs = 0;
for(o = args; o->shortVar != 0 || o->longVar != 0 ; o++){
if(o->arg != OPTION_REQUIRED_ARGUMENT){
optionalArgs = 1;
}
switch(o->arg){
case (OPTION_OPTIONAL_ARGUMENT):
case (OPTION_FLAG):{
if(o->shortVar != 0){
printf("[-%c] ", o->shortVar);
}else if(o->longVar != 0){
printf("[--%s] ", o->longVar);
}
break;
}case (OPTION_REQUIRED_ARGUMENT):{
if(o->shortVar != 0){
printf("-%c ", o->shortVar);
}else if(o->longVar != 0){
printf("--%s ", o->longVar);
}
break;
}
}
}
if (optionalArgs){
//printf(" [Optional Args]");
}
if (! is_plugin){
printf(" -- <Plugin options, see below>\n");
}
void option_print_help(option_help * args){
print_help_section(args, OPTION_REQUIRED_ARGUMENT, "Required arguments");
print_help_section(args, OPTION_FLAG, "Flags");
print_help_section(args, OPTION_OPTIONAL_ARGUMENT, "Optional arguments");
@ -253,17 +220,23 @@ void option_print_current(option_help * args){
print_current_option_section(args, OPTION_FLAG);
}
int option_parse(int argc, char ** argv, option_help * args, int * printhelp){
int option_parse(int argc, char ** argv, options_all * opt_all){
int error = 0;
int requiredArgsSeen = 0;
int requiredArgsNeeded = 0;
int i;
int printhelp = 0;
for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 ; o++ ){
if(o->arg == OPTION_REQUIRED_ARGUMENT){
requiredArgsNeeded++;
for(int m = 0; m < opt_all->module_count; m++ ){
option_help * args = opt_all->modules[m].options;
if(args == NULL) continue;
for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 ; o++ ){
if(o->arg == OPTION_REQUIRED_ARGUMENT){
requiredArgsNeeded++;
}
}
}
for(i=1; i < argc; i++){
char * txt = argv[i];
int foundOption = 0;
@ -274,105 +247,104 @@ int option_parse(int argc, char ** argv, option_help * args, int * printhelp){
arg++;
replaced_equal = 1;
}
if(strcmp(txt, "--") == 0){
// we found plugin options
break;
}
// try to find matching option help
for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 || o->help != NULL ; o++ ){
if( o->shortVar == 0 && o->longVar == 0 ){
// section
continue;
}
for(int m = 0; m < opt_all->module_count; m++ ){
option_help * args = opt_all->modules[m].options;
if(args == NULL) continue;
// try to find matching option help
for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 || o->help != NULL ; o++ ){
if( o->shortVar == 0 && o->longVar == 0 ){
// section
continue;
}
if ( (txt[0] == '-' && o->shortVar == txt[1]) || (strlen(txt) > 2 && txt[0] == '-' && txt[1] == '-' && o->longVar != NULL && strcmp(txt + 2, o->longVar) == 0)){
foundOption = 1;
if ( (txt[0] == '-' && o->shortVar == txt[1]) || (strlen(txt) > 2 && txt[0] == '-' && txt[1] == '-' && o->longVar != NULL && strcmp(txt + 2, o->longVar) == 0)){
foundOption = 1;
// now process the option.
switch(o->arg){
case (OPTION_FLAG):{
assert(o->type == 'd');
(*(int*) o->variable)++;
break;
}
case (OPTION_OPTIONAL_ARGUMENT):
case (OPTION_REQUIRED_ARGUMENT):{
// check if next is an argument
if(arg == NULL){
if(o->shortVar == txt[1] && txt[2] != 0){
arg = & txt[2];
}else{
// simply take the next value as argument
i++;
arg = argv[i];
}
// now process the option.
switch(o->arg){
case (OPTION_FLAG):{
assert(o->type == 'd');
(*(int*) o->variable)++;
break;
}
if(arg == NULL){
const char str[] = {o->shortVar, 0};
printf("Error, argument missing for option %s\n", (o->longVar != NULL) ? o->longVar : str);
exit(1);
}
switch(o->type){
case('p'):{
// call the function in the variable
void(*fp)() = o->variable;
fp(arg);
break;
}
case('F'):{
*(double*) o->variable = atof(arg);
break;
}
case('f'):{
*(float*) o->variable = atof(arg);
break;
}
case('d'):{
int64_t val = string_to_bytes(arg);
if (val > INT_MAX || val < INT_MIN){
printf("WARNING: parsing the number %s to integer, this produced an overflow!\n", arg);
case (OPTION_OPTIONAL_ARGUMENT):
case (OPTION_REQUIRED_ARGUMENT):{
// check if next is an argument
if(arg == NULL){
if(o->shortVar == txt[1] && txt[2] != 0){
arg = & txt[2];
}else{
// simply take the next value as argument
i++;
arg = argv[i];
}
*(int*) o->variable = val;
break;
}
case('H'):
case('s'):{
(*(char **) o->variable) = strdup(arg);
break;
if(arg == NULL){
const char str[] = {o->shortVar, 0};
printf("Error, argument missing for option %s\n", (o->longVar != NULL) ? o->longVar : str);
exit(1);
}
case('c'):{
(*(char *)o->variable) = arg[0];
if(strlen(arg) > 1){
printf("Error, ignoring remainder of string for option %c (%s).\n", o->shortVar, o->longVar);
switch(o->type){
case('p'):{
// call the function in the variable
void(*fp)() = o->variable;
fp(arg);
break;
}
break;
case('F'):{
*(double*) o->variable = atof(arg);
break;
}
case('f'):{
*(float*) o->variable = atof(arg);
break;
}
case('d'):{
int64_t val = string_to_bytes(arg);
if (val > INT_MAX || val < INT_MIN){
printf("WARNING: parsing the number %s to integer, this produced an overflow!\n", arg);
}
*(int*) o->variable = val;
break;
}
case('H'):
case('s'):{
(*(char **) o->variable) = strdup(arg);
break;
}
case('c'):{
(*(char *)o->variable) = arg[0];
if(strlen(arg) > 1){
printf("Error, ignoring remainder of string for option %c (%s).\n", o->shortVar, o->longVar);
}
break;
}
case('l'):{
*(long long*) o->variable = string_to_bytes(arg);
break;
}
default:
printf("ERROR: Unknown option type %c\n", o->type);
}
case('l'):{
*(long long*) o->variable = string_to_bytes(arg);
break;
}
default:
printf("ERROR: Unknown option type %c\n", o->type);
}
}
}
if(replaced_equal){
arg[-1] = '=';
}
if(replaced_equal){
arg[-1] = '=';
}
if(o->arg == OPTION_REQUIRED_ARGUMENT){
requiredArgsSeen++;
}
if(o->arg == OPTION_REQUIRED_ARGUMENT){
requiredArgsSeen++;
}
break;
break;
}
}
}
if (! foundOption){
if(strcmp(txt, "-h") == 0 || strcmp(txt, "--help") == 0){
*printhelp=1;
printhelp = 1;
}else{
printf("Error invalid argument: %s\n", txt);
error = 1;
@ -382,12 +354,26 @@ int option_parse(int argc, char ** argv, option_help * args, int * printhelp){
if( requiredArgsSeen != requiredArgsNeeded ){
printf("Error: Missing some required arguments\n\n");
*printhelp = -1;
printhelp = -1;
}
if(error != 0){
printf("Invalid options\n");
*printhelp = -1;
printhelp = -1;
}
if(printhelp == 1){
printf("Synopsis %s\n", argv[0]);
for(int m = 0; m < opt_all->module_count; m++ ){
option_help * args = opt_all->modules[m].options;
if(args == NULL) continue;
char * prefix = opt_all->modules[m].prefix;
if(prefix != NULL){
printf("\n\nModule %s\n", prefix);
}
option_print_help(args);
}
exit(0);
}
return i;

15
src/option.h
View File

@ -4,7 +4,7 @@
#include <stdint.h>
/*
* Initial revision by JK
* Initial version by JK
*/
typedef enum{
@ -23,13 +23,22 @@ typedef struct{
void * variable;
} option_help;
typedef struct{
char * prefix; // may be NULL to include it in the standard name
option_help * options;
} option_module;
typedef struct{
int module_count;
option_module * modules;
} options_all;
#define LAST_OPTION {0, 0, 0, (option_value_type) 0, 0, NULL}
int64_t string_to_bytes(char *size_str);
void option_print_help(option_help * args, int is_plugin);
void option_print_current(option_help * args);
//@return the number of parsed arguments
int option_parse(int argc, char ** argv, option_help * args, int * print_help);
int option_parse(int argc, char ** argv, options_all * args);
#endif

26
src/parse_options.c
View File

@ -302,8 +302,6 @@ void DecodeDirective(char *line, IOR_param_t *params)
params->numTasks = atoi(value);
} else if (strcasecmp(option, "summaryalways") == 0) {
params->summary_every_test = atoi(value);
} else if (strcasecmp(option, "collectiveMetadata") == 0) {
params->collective_md = atoi(value);
} else {
if (rank == 0)
fprintf(out_logfile, "Unrecognized parameter \"%s\"\n",
@ -515,8 +513,7 @@ IOR_test_t *ParseCommandLine(int argc, char **argv)
IOR_test_t *tests = NULL;
GetPlatformName(initialTestParams.platform);
int printhelp = 0;
int parsed_options = option_parse(argc, argv, options, & printhelp);
airoi_parse_options(argc, argv, options);
if (toggleG){
initialTestParams.setTimeStampSignature = toggleG;
@ -544,7 +541,6 @@ IOR_test_t *ParseCommandLine(int argc, char **argv)
if (memoryPerNode){
initialTestParams.memoryPerNode = NodeMemoryStringToBytes(optarg);
}
const ior_aiori_t * backend = aiori_select(initialTestParams.api);
if (backend == NULL)
ERR_SIMPLE("unrecognized I/O API");
@ -552,26 +548,6 @@ IOR_test_t *ParseCommandLine(int argc, char **argv)
initialTestParams.backend = backend;
initialTestParams.apiVersion = backend->get_version();
if(backend->get_options != NULL){
option_parse(argc - parsed_options, argv + parsed_options, backend->get_options(), & printhelp);
}
if(printhelp != 0){
printf("Usage: %s ", argv[0]);
option_print_help(options, 0);
if(backend->get_options != NULL){
printf("\nPlugin options for backend %s (%s)\n", initialTestParams.api, backend->get_version());
option_print_help(backend->get_options(), 1);
}
if(printhelp == 1){
exit(0);
}else{
exit(1);
}
}
if (testscripts){
tests = ReadConfigScript(testscripts);
}else{