mdtest/src/parse_options.c

477 lines
23 KiB
C
Executable File

/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
* vim:expandtab:shiftwidth=8:tabstop=8:
*/
/******************************************************************************\
* *
* Copyright (c) 2003, The Regents of the University of California *
* See the file COPYRIGHT for a complete copyright notice and license. *
* *
********************************************************************************
*
* Parse commandline functions.
*
\******************************************************************************/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>
#if defined(HAVE_STRINGS_H)
#include <strings.h>
#endif
#include "utilities.h"
#include "ior.h"
#include "aiori.h"
#include "parse_options.h"
#include "option.h"
#include "aiori.h"
IOR_param_t initialTestParams;
option_help * createGlobalOptions(IOR_param_t * params);
static IOR_param_t * parameters;
static options_all_t * global_options;
/*
* Check and correct all settings of each test in queue for correctness.
*/
static void CheckRunSettings(IOR_test_t *tests)
{
IOR_test_t *ptr;
IOR_param_t *params;
for (ptr = tests; ptr != NULL; ptr = ptr->next) {
params = &ptr->params;
/* If no write/read/check action requested, set both write and read */
if (params->writeFile == FALSE
&& params->readFile == FALSE
&& params->checkWrite == FALSE
&& params->checkRead == FALSE) {
params->readFile = TRUE;
params->writeFile = TRUE;
}
if(params->dualMount && !params->filePerProc) {
MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "Dual Mount can only be used with File Per Process");
}
}
}
/*
* Set flags from commandline string/value pairs.
*/
void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_options)
{
char option[MAX_STR];
char value[MAX_STR];
int rc;
int initialized;
rc = sscanf(line, " %[^=# \t\r\n] = %[^# \t\r\n] ", option, value);
if (rc != 2 && rank == 0) {
fprintf(out_logfile, "Syntax error in configuration options: %s\n",
line);
MPI_CHECK(MPI_Initialized(&initialized), "MPI_Initialized() error");
if (initialized)
MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error");
else
exit(-1);
}
if (strcasecmp(option, "api") == 0) {
params->api = strdup(value);
params->backend = aiori_select(params->api);
if (params->backend == NULL){
fprintf(out_logfile, "Could not load backend API %s\n", params->api);
exit(-1);
}
} else if (strcasecmp(option, "summaryFile") == 0) {
if (rank == 0){
out_resultfile = fopen(value, "w");
if (out_resultfile == NULL){
FAIL("Cannot open output file for writes!");
}
printf("Writing output to %s\n", value);
}
} else if (strcasecmp(option, "summaryFormat") == 0) {
if(strcasecmp(value, "default") == 0){
outputFormat = OUTPUT_DEFAULT;
}else if(strcasecmp(value, "JSON") == 0){
outputFormat = OUTPUT_JSON;
}else if(strcasecmp(value, "CSV") == 0){
outputFormat = OUTPUT_CSV;
}else{
FAIL("Unknown summaryFormat");
}
} else if (strcasecmp(option, "refnum") == 0) {
params->referenceNumber = atoi(value);
} else if (strcasecmp(option, "debug") == 0) {
params->debug = strdup(value);
} else if (strcasecmp(option, "platform") == 0) {
params->platform = strdup(value);
} else if (strcasecmp(option, "testfile") == 0) {
params->testFileName = strdup(value);
} else if (strcasecmp(option, "dualmount") == 0){
params->dualMount = atoi(value);
} else if (strcasecmp(option, "deadlineforstonewalling") == 0) {
params->deadlineForStonewalling = atoi(value);
} else if (strcasecmp(option, "stoneWallingWearOut") == 0) {
params->stoneWallingWearOut = atoi(value);
} else if (strcasecmp(option, "stoneWallingWearOutIterations") == 0) {
params->stoneWallingWearOutIterations = atoll(value);
} else if (strcasecmp(option, "stoneWallingStatusFile") == 0) {
params->stoneWallingStatusFile = strdup(value);
} else if (strcasecmp(option, "maxtimeduration") == 0) {
params->maxTimeDuration = atoi(value);
} else if (strcasecmp(option, "outlierthreshold") == 0) {
params->outlierThreshold = atoi(value);
} else if (strcasecmp(option, "numnodes") == 0) {
params->numNodes = atoi(value);
} else if (strcasecmp(option, "numtasks") == 0) {
params->numTasks = atoi(value);
} else if (strcasecmp(option, "numtasksonnode0") == 0) {
params->numTasksOnNode0 = atoi(value);
} else if (strcasecmp(option, "repetitions") == 0) {
params->repetitions = atoi(value);
} else if (strcasecmp(option, "intertestdelay") == 0) {
params->interTestDelay = atoi(value);
} else if (strcasecmp(option, "interiodelay") == 0) {
params->interIODelay = atoi(value);
} else if (strcasecmp(option, "readfile") == 0) {
params->readFile = atoi(value);
} else if (strcasecmp(option, "writefile") == 0) {
params->writeFile = atoi(value);
} else if (strcasecmp(option, "fileperproc") == 0) {
params->filePerProc = atoi(value);
} else if (strcasecmp(option, "taskpernodeoffset") == 0) {
params->taskPerNodeOffset = atoi(value);
} else if (strcasecmp(option, "reordertasksconstant") == 0) {
params->reorderTasks = atoi(value);
} else if (strcasecmp(option, "reordertasksrandom") == 0) {
params->reorderTasksRandom = atoi(value);
} else if (strcasecmp(option, "reordertasksrandomSeed") == 0) {
params->reorderTasksRandomSeed = atoi(value);
} else if (strcasecmp(option, "reordertasks") == 0) {
/* Backwards compatibility for the "reorderTasks" option.
MUST follow the other longer reordertasks checks. */
params->reorderTasks = atoi(value);
} else if (strcasecmp(option, "checkwrite") == 0) {
params->checkWrite = atoi(value);
} else if (strcasecmp(option, "checkread") == 0) {
params->checkRead = atoi(value);
} else if (strcasecmp(option, "keepfile") == 0) {
params->keepFile = atoi(value);
} else if (strcasecmp(option, "keepfilewitherror") == 0) {
params->keepFileWithError = atoi(value);
} else if (strcasecmp(option, "multiFile") == 0) {
params->multiFile = atoi(value);
} else if (strcasecmp(option, "warningAsErrors") == 0) {
params->warningAsErrors = atoi(value);
} else if (strcasecmp(option, "segmentcount") == 0) {
params->segmentCount = string_to_bytes(value);
} else if (strcasecmp(option, "blocksize") == 0) {
params->blockSize = string_to_bytes(value);
} else if (strcasecmp(option, "transfersize") == 0) {
params->transferSize = string_to_bytes(value);
} else if (strcasecmp(option, "singlexferattempt") == 0) {
params->singleXferAttempt = atoi(value);
} else if (strcasecmp(option, "intraTestBarriers") == 0) {
params->intraTestBarriers = atoi(value);
} else if (strcasecmp(option, "verbose") == 0) {
params->verbose = atoi(value);
} else if (strcasecmp(option, "settimestampsignature") == 0) {
params->setTimeStampSignature = atoi(value);
} else if (strcasecmp(option, "storefileoffset") == 0) {
params->storeFileOffset = atoi(value);
} else if (strcasecmp(option, "uniqueDir") == 0) {
params->uniqueDir = atoi(value);
} else if (strcasecmp(option, "useexistingtestfile") == 0) {
params->useExistingTestFile = atoi(value);
} else if (strcasecmp(option, "fsyncperwrite") == 0) {
params->fsyncPerWrite = atoi(value);
} else if (strcasecmp(option, "fsync") == 0) {
params->fsync = atoi(value);
} else if (strcasecmp(option, "randomoffset") == 0) {
params->randomOffset = atoi(value);
} else if (strcasecmp(option, "memoryPerTask") == 0) {
params->memoryPerTask = string_to_bytes(value);
params->memoryPerNode = 0;
} else if (strcasecmp(option, "memoryPerNode") == 0) {
params->memoryPerNode = NodeMemoryStringToBytes(value);
params->memoryPerTask = 0;
} else if (strcasecmp(option, "summaryalways") == 0) {
params->summary_every_test = atoi(value);
} else {
// backward compatibility for now
if (strcasecmp(option, "useo_direct") == 0) {
strcpy(option, "--posix.odirect");
}
int parsing_error = option_parse_key_value(option, value, module_options);
if(parsing_error){
if (rank == 0)
fprintf(out_logfile, "Unrecognized parameter \"%s\"\n",
option);
MPI_CHECK(MPI_Initialized(&initialized), "MPI_Initialized() error");
if (initialized)
MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error");
else
exit(-1);
}
}
}
/*
* Parse a single line, which may contain multiple comma-seperated directives
*/
void ParseLine(char *line, IOR_param_t * test, options_all_t * module_options)
{
char *start, *end;
char * newline = strdup(line);
start = newline;
do {
end = strchr(start, '#');
if (end != NULL){
*end = '\0';
end = NULL; // stop parsing after comment
}
end = strchr(start, ',');
if (end != NULL){
*end = '\0';
}
if(strlen(start) < 3){
fprintf(out_logfile, "Invalid option substring string: \"%s\" in \"%s\"\n", start, line);
exit(1);
}
DecodeDirective(start, test, module_options);
start = end + 1;
} while (end != NULL);
free(newline);
}
static void decodeDirectiveWrapper(char *line){
ParseLine(line, parameters, global_options);
}
/*
* Determines if the string "haystack" contains only the string "needle", and
* possibly whitespace before and after needle. Function is case insensitive.
*/
int contains_only(char *haystack, char *needle)
{
char *ptr, *end;
end = haystack + strlen(haystack);
/* skip over leading shitspace */
for (ptr = haystack; ptr < end; ptr++) {
if (!isspace(*ptr))
break;
}
/* check for "needle" */
if (strncasecmp(ptr, needle, strlen(needle)) != 0)
return 0;
/* make sure the rest of the line is only whitespace as well */
for (ptr += strlen(needle); ptr < end; ptr++) {
if (!isspace(*ptr))
return 0;
}
return 1;
}
/*
* Read the configuration script, allocating and filling in the structure of
* global parameters.
*/
IOR_test_t *ReadConfigScript(char *scriptName)
{
int test_num = 0;
int runflag = 0;
char linebuf[MAX_STR];
char empty[MAX_STR];
char *ptr;
FILE *file;
IOR_test_t *head = NULL;
IOR_test_t *tail = NULL;
option_help ** option_p = & global_options->modules[0].options;
/* Initialize the first test */
head = CreateTest(& initialTestParams, test_num++);
tail = head;
*option_p = createGlobalOptions(& ((IOR_test_t*) head)->params); /* The current options */
/* open the script */
file = fopen(scriptName, "r");
if (file == NULL)
ERR("fopen() failed");
/* search for the "IOR START" line */
while (fgets(linebuf, MAX_STR, file) != NULL) {
if (contains_only(linebuf, "ior start")) {
break;
}
}
/* Iterate over a block of IOR commands */
while (fgets(linebuf, MAX_STR, file) != NULL) {
/* skip over leading whitespace */
ptr = linebuf;
while (isspace(*ptr))
ptr++;
/* skip empty lines */
if (sscanf(ptr, "%s", empty) == -1)
continue;
/* skip lines containing only comments */
if (sscanf(ptr, " #%s", empty) == 1)
continue;
if (contains_only(ptr, "ior stop")) {
break;
} else if (contains_only(ptr, "run")) {
if (runflag) {
/* previous line was a "run" as well
create duplicate test */
tail->next = CreateTest(&tail->params, test_num++);
AllocResults(tail);
((IOR_test_t*) tail)->params.backend_options = airoi_update_module_options(((IOR_test_t*) tail)->params.backend, global_options);
tail = tail->next;
*option_p = createGlobalOptions(& ((IOR_test_t*) tail->next)->params);
}
runflag = 1;
} else if (runflag) {
/* If this directive was preceded by a "run" line, then
create and initialize a new test structure */
runflag = 0;
tail->next = CreateTest(&tail->params, test_num++);
*option_p = createGlobalOptions(& ((IOR_test_t*) tail->next)->params);
AllocResults(tail);
((IOR_test_t*) tail)->params.backend_options = airoi_update_module_options(((IOR_test_t*) tail)->params.backend, global_options);
tail = tail->next;
ParseLine(ptr, &tail->params, global_options);
} else {
ParseLine(ptr, &tail->params, global_options);
}
}
/* close the script */
if (fclose(file) != 0)
ERR("fclose() of script file failed");
AllocResults(tail); /* copy the actual module options into the test */
((IOR_test_t*) tail)->params.backend_options = airoi_update_module_options(((IOR_test_t*) tail)->params.backend, global_options);
return head;
}
option_help * createGlobalOptions(IOR_param_t * params){
char APIs[1024];
char APIs_legacy[1024];
aiori_supported_apis(APIs, APIs_legacy, IOR);
char apiStr[1024];
sprintf(apiStr, "API for I/O [%s]", APIs);
option_help o [] = {
{'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & params->api},
{'A', NULL, "refNum -- user supplied reference number to include in the summary", OPTION_OPTIONAL_ARGUMENT, 'd', & params->referenceNumber},
{'b', NULL, "blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & params->blockSize},
{'c', "collective", "Use collective I/O", OPTION_FLAG, 'd', & params->collective},
{'C', NULL, "reorderTasks -- changes task ordering for readback (useful to avoid client cache)", OPTION_FLAG, 'd', & params->reorderTasks},
{'d', NULL, "interTestDelay -- delay between reps in seconds", OPTION_OPTIONAL_ARGUMENT, 'd', & params->interTestDelay},
{'D', NULL, "deadlineForStonewalling -- seconds before stopping write or read phase", OPTION_OPTIONAL_ARGUMENT, 'd', & params->deadlineForStonewalling},
{.help=" -O stoneWallingWearOut=1 -- once the stonewalling timeout is over, all process finish to access the amount of data", .arg = OPTION_OPTIONAL_ARGUMENT},
{.help=" -O stoneWallingWearOutIterations=N -- stop after processing this number of iterations, needed for reading data back written with stoneWallingWearOut", .arg = OPTION_OPTIONAL_ARGUMENT},
{.help=" -O stoneWallingStatusFile=FILE -- this file keeps the number of iterations from stonewalling during write and allows to use them for read", .arg = OPTION_OPTIONAL_ARGUMENT},
{'e', NULL, "fsync -- perform a fsync() operation at the end of each read/write phase", OPTION_FLAG, 'd', & params->fsync},
{'E', NULL, "useExistingTestFile -- do not remove test file before write access", OPTION_FLAG, 'd', & params->useExistingTestFile},
{'f', NULL, "scriptFile -- test script name", OPTION_OPTIONAL_ARGUMENT, 's', & params->testscripts},
{'F', NULL, "filePerProc -- file-per-process", OPTION_FLAG, 'd', & params->filePerProc},
{'g', NULL, "intraTestBarriers -- use barriers between open, write/read, and close", OPTION_FLAG, 'd', & params->intraTestBarriers},
/* This option toggles between Incompressible Seed and Time stamp sig based on -l,
* so we'll toss the value in both for now, and sort it out in initialization
* after all the arguments are in and we know which it keep.
*/
{'G', NULL, "setTimeStampSignature -- set value for time stamp signature/random seed", OPTION_OPTIONAL_ARGUMENT, 'd', & params->setTimeStampSignature},
{'i', NULL, "repetitions -- number of repetitions of test", OPTION_OPTIONAL_ARGUMENT, 'd', & params->repetitions},
{'j', NULL, "outlierThreshold -- warn on outlier N seconds from mean", OPTION_OPTIONAL_ARGUMENT, 'd', & params->outlierThreshold},
{'k', NULL, "keepFile -- don't remove the test file(s) on program exit", OPTION_FLAG, 'd', & params->keepFile},
{'K', NULL, "keepFileWithError -- keep error-filled file(s) after data-checking", OPTION_FLAG, 'd', & params->keepFileWithError},
{'l', NULL, "datapacket type-- type of packet that will be created [offset|incompressible|timestamp|o|i|t]", OPTION_OPTIONAL_ARGUMENT, 's', & params->buffer_type},
{'m', NULL, "multiFile -- use number of reps (-i) for multiple file count", OPTION_FLAG, 'd', & params->multiFile},
{'M', NULL, "memoryPerNode -- hog memory on the node (e.g.: 2g, 75%)", OPTION_OPTIONAL_ARGUMENT, 's', & params->memoryPerNodeStr},
{'N', NULL, "numTasks -- number of tasks that are participating in the test (overrides MPI)", OPTION_OPTIONAL_ARGUMENT, 'd', & params->numTasks},
{'o', NULL, "testFile -- full name for test", OPTION_OPTIONAL_ARGUMENT, 's', & params->testFileName},
{'O', NULL, "string of IOR directives (e.g. -O checkRead=1,lustreStripeCount=32)", OPTION_OPTIONAL_ARGUMENT, 'p', & decodeDirectiveWrapper},
{'Q', NULL, "taskPerNodeOffset for read tests use with -C & -Z options (-C constant N, -Z at least N)", OPTION_OPTIONAL_ARGUMENT, 'd', & params->taskPerNodeOffset},
{'r', NULL, "readFile -- read existing file", OPTION_FLAG, 'd', & params->readFile},
{'R', NULL, "checkRead -- verify that the output of read matches the expected signature (used with -G)", OPTION_FLAG, 'd', & params->checkRead},
{'s', NULL, "segmentCount -- number of segments", OPTION_OPTIONAL_ARGUMENT, 'd', & params->segmentCount},
{'t', NULL, "transferSize -- size of transfer in bytes (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & params->transferSize},
{'T', NULL, "maxTimeDuration -- max time in minutes executing repeated test; it aborts only between iterations and not within a test!", OPTION_OPTIONAL_ARGUMENT, 'd', & params->maxTimeDuration},
{'u', NULL, "uniqueDir -- use unique directory name for each file-per-process", OPTION_FLAG, 'd', & params->uniqueDir},
{'v', NULL, "verbose -- output information (repeating flag increases level)", OPTION_FLAG, 'd', & params->verbose},
{'w', NULL, "writeFile -- write file", OPTION_FLAG, 'd', & params->writeFile},
{'W', NULL, "checkWrite -- check read after write", OPTION_FLAG, 'd', & params->checkWrite},
{'x', NULL, "singleXferAttempt -- do not retry transfer if incomplete", OPTION_FLAG, 'd', & params->singleXferAttempt},
{'X', NULL, "reorderTasksRandomSeed -- random seed for -Z option", OPTION_OPTIONAL_ARGUMENT, 'd', & params->reorderTasksRandomSeed},
{'y', NULL, "dualMount -- use dual mount points for a filesystem", OPTION_FLAG, 'd', & params->dualMount},
{'Y', NULL, "fsyncPerWrite -- perform sync operation after every write operation", OPTION_FLAG, 'd', & params->fsyncPerWrite},
{'z', NULL, "randomOffset -- access is to random, not sequential, offsets within a file", OPTION_FLAG, 'd', & params->randomOffset},
{'Z', NULL, "reorderTasksRandom -- changes task ordering to random ordering for readback", OPTION_FLAG, 'd', & params->reorderTasksRandom},
{0, "warningAsErrors", "Any warning should lead to an error.", OPTION_FLAG, 'd', & params->warningAsErrors},
{.help=" -O summaryFile=FILE -- store result data into this file", .arg = OPTION_OPTIONAL_ARGUMENT},
{.help=" -O summaryFormat=[default,JSON,CSV] -- use the format for outputting the summary", .arg = OPTION_OPTIONAL_ARGUMENT},
{0, "dryRun", "do not perform any I/Os just run evtl. inputs print dummy output", OPTION_FLAG, 'd', & params->dryRun},
LAST_OPTION,
};
option_help * options = malloc(sizeof(o));
memcpy(options, & o, sizeof(o));
return options;
}
/*
* Parse Commandline.
*/
IOR_test_t *ParseCommandLine(int argc, char **argv)
{
init_IOR_Param_t(& initialTestParams);
IOR_test_t *tests = NULL;
initialTestParams.platform = GetPlatformName();
option_help * options = createGlobalOptions( & initialTestParams);
parameters = & initialTestParams;
global_options = airoi_create_all_module_options(options);
option_parse(argc, argv, global_options);
updateParsedOptions(& initialTestParams, global_options);
if (initialTestParams.testscripts){
tests = ReadConfigScript(initialTestParams.testscripts);
}else{
tests = CreateTest(&initialTestParams, 0);
AllocResults(tests);
}
CheckRunSettings(tests);
return (tests);
}