vitalif
/
mdtest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added an uncompressible data packet to IOR. 

Changed how the -l option works.  Now you choose the type of datapacket
-l i  			incompressible data packets
-l incompressible  	incompressible data packets
-l timestamp 		timestamped data packets
-l t			timestamped data packets
-l offset		offset data packets
-l o			offset data packets

-G option now is either the seed for the incompressible random packets
   or the timestamp, depending on the input to the -l option.

-G will no long timestamp packets on its own without the additon of -l timestamp or -l t

I kept shorter versions of the options for the sake of typing sanity.
master
Blair Crossman 2015-05-21 12:05:56 -06:00
parent 37738dab26
commit 08d14e986d
4 changed files with 90 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/USER_GUIDE
View File

@ -74,7 +74,7 @@ These options are to be used on the command line. E.g., 'IOR -a POSIX -b 4K'.
-J N setAlignment -- HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g)
-k keepFile -- don't remove the test file(s) on program exit
-K keepFileWithError -- keep error-filled file(s) after data-checking
-l storeFileOffset -- use file offset as stored signature
-l data packet type-- type of packet that will be created [offset|incompressible|timestamp|o|i|t]
-m multiFile -- use number of reps (-i) for multiple file count
-M N memoryPerNode -- hog memory on the node (e.g.: 2g, 75%)
-n noFill -- no fill in HDF5 file creation

75
src/ior.c
View File

@ -224,6 +224,7 @@ void init_IOR_Param_t(IOR_param_t * p)
p->blockSize = 1048576;
p->transferSize = 262144;
p->randomSeed = -1;
p->incompressibleSeed = 573;
p->testComm = MPI_COMM_WORLD;
p->setAlignment = 1;
p->lustre_start_ost = -1;
@ -762,7 +763,7 @@ static void DisplayUsage(char **argv)
" -J N setAlignment -- HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g)",
" -k keepFile -- don't remove the test file(s) on program exit",
" -K keepFileWithError -- keep error-filled file(s) after data-checking",
" -l storeFileOffset -- use file offset as stored signature",
" -l datapacket type-- type of packet that will be created [offset|incompressible|timestamp|o|i|t]",
" -m multiFile -- use number of reps (-i) for multiple file count",
" -M N memoryPerNode -- hog memory on the node (e.g.: 2g, 75%)",
" -n noFill -- no fill in HDF5 file creation",
@ -852,6 +853,25 @@ void DistributeHints(void)
* ints, store transfer offset. If storeFileOffset option is used, the file
* (not transfer) offset is stored instead.
*/
static void
FillIncompressibleBuffer(void* buffer, IOR_param_t * test)
{
size_t i;
unsigned long long hi, lo;
unsigned long long *buf = (unsigned long long *)buffer;
for (i = 0; i < test->transferSize / sizeof(unsigned long long); i++) {
hi = ((unsigned long long) rand_r(&test->incompressibleSeed) << 32);
lo = (unsigned long long) rand_r(&test->incompressibleSeed);
buf[i] = hi | lo;
}
}
unsigned int reseed_incompressible_prng = TRUE;
static void
FillBuffer(void *buffer,
IOR_param_t * test, unsigned long long offset, int fillrank)
@ -859,27 +879,28 @@ FillBuffer(void *buffer,
size_t i;
unsigned long long hi, lo;
unsigned long long *buf = (unsigned long long *)buffer;
if(test->dataPacketType == incompressible ) { /* Make for some non compressable buffers with randomish data */
/*
* Consider adding a parameter to use incompressible data or what is here now.
* The way to get incompressible data would be to use some random transfer
* buffer content. In Linux we can read from /dev/urandom. In C we can use
* the rand() function in stdlib.h.
*
* # include <stdlib.h>
*
* hi = (( unsigned long long )rand() ) << 32;
* lo = (( unsigned long long )rand() );
*/
hi = ((unsigned long long)fillrank) << 32;
lo = (unsigned long long)test->timeStampSignatureValue;
for (i = 0; i < test->transferSize / sizeof(unsigned long long); i++) {
if ((i % 2) == 0) {
/* evens contain MPI rank and time in seconds */
buf[i] = hi | lo;
} else {
/* odds contain offset */
buf[i] = offset + (i * sizeof(unsigned long long));
/* In order for write checks to work, we have to restart the psuedo random sequence */
if(reseed_incompressible_prng == TRUE) {
test->incompressibleSeed = test->setTimeStampSignature ; /* We copied seed into timestampSignature at initialization */
reseed_incompressible_prng = FALSE;
}
FillIncompressibleBuffer(buffer, test);
}
else {
hi = ((unsigned long long)fillrank) << 32;
lo = (unsigned long long)test->timeStampSignatureValue;
for (i = 0; i < test->transferSize / sizeof(unsigned long long); i++) {
if ((i % 2) == 0) {
/* evens contain MPI rank and time in seconds */
buf[i] = hi | lo;
} else {
/* odds contain offset */
buf[i] = offset + (i * sizeof(unsigned long long));
}
}
}
}
@ -1622,6 +1643,8 @@ static void ShowSetup(IOR_param_t *params)
*/
static void ShowTest(IOR_param_t * test)
{
const char* data_packets[] = {"g", "t","o","i"};
fprintf(stdout, "TEST:\t%s=%d\n", "id", test->id);
fprintf(stdout, "\t%s=%d\n", "refnum", test->referenceNumber);
fprintf(stdout, "\t%s=%s\n", "api", test->api);
@ -1677,8 +1700,9 @@ static void ShowTest(IOR_param_t * test)
test->keepFileWithError);
fprintf(stdout, "\t%s=%d\n", "quitOnError", test->quitOnError);
fprintf(stdout, "\t%s=%d\n", "verbose", verbose);
fprintf(stdout, "\t%s=%d\n", "setTimeStampSignature",
test->setTimeStampSignature);
fprintf(stdout, "\t%s=%s\n", "data packet type", data_packets[test->dataPacketType]);
fprintf(stdout, "\t%s=%d\n", "setTimeStampSignature/incompressibleSeed",
test->setTimeStampSignature); /* Seed value was copied into setTimeStampSignature as well */
fprintf(stdout, "\t%s=%d\n", "collective", test->collective);
fprintf(stdout, "\t%s=%lld", "segmentCount", test->segmentCount);
#ifdef HAVE_GPFS_FCNTL_H
@ -2059,7 +2083,7 @@ static void TestIoSys(IOR_test_t *test)
/* Get iteration start time in seconds in task 0 and broadcast to
all tasks */
if (rank == 0) {
if (params->setTimeStampSignature) {
if (params->dataPacketType == timestamp && params->setTimeStampSignature) {
params->timeStampSignatureValue =
(unsigned int)params->setTimeStampSignature;
} else {
@ -2165,6 +2189,9 @@ static void TestIoSys(IOR_test_t *test)
rankOffset =
(2 * params->tasksPerNode) % params->numTasks;
}
reseed_incompressible_prng = TRUE; /* Re-Seed the PRNG to get same sequence back, if random */
GetTestFileName(testFileName, params);
params->open = WRITECHECK;
fd = backend->open(testFileName, params);

16
src/ior.h
View File

@ -48,6 +48,19 @@ extern int tasksPerNode;
extern int verbose;
extern MPI_Comm testComm;
/******************** DATA Packet Type ***************************************/
/* Holds the types of data packets: generic, offset, timestamp, incompressible */
enum PACKET_TYPE
{
generic = 0, /* No packet type specified */
timestamp=1, /* Timestamp packet set with -l */
offset=2, /* Offset packet set with -l */
incompressible=3 /* Incompressible packet set with -l */
};
/******************************************************************************/
/*
* The parameter struct holds all of the "global" data to be passed,
@ -121,9 +134,12 @@ typedef struct
unsigned int timeStampSignatureValue; /* value for time stamp signature */
void * fd_fppReadCheck; /* additional fd for fpp read check */
int randomSeed; /* random seed for write/read check */
int incompressibleSeed; /* random seed for incompressible file creation */
int randomOffset; /* access is to random offsets */
size_t memoryPerTask; /* additional memory used per task */
size_t memoryPerNode; /* additional memory used per node */
enum PACKET_TYPE dataPacketType; /* The type of data packet. */
/* POSIX variables */
int singleXferAttempt; /* do not retry transfer if incomplete */

24
src/parse_options.c
View File

@ -428,7 +428,7 @@ IOR_test_t *ReadConfigScript(char *scriptName)
IOR_test_t *ParseCommandLine(int argc, char **argv)
{
static const char *opts =
"a:A:b:BcCd:D:eEf:FgG:hHi:Ij:J:kKlmM:nN:o:O:pPqQ:rRs:St:T:uU:vVwWxX:YzZ";
"a:A:b:BcCd:D:eEf:FgG:hHi:Ij:J:kKl:mM:nN:o:O:pPqQ:rRs:St:T:uU:vVwWxX:YzZ";
int c, i;
static IOR_test_t *tests = NULL;
@ -484,7 +484,12 @@ IOR_test_t *ParseCommandLine(int argc, char **argv)
initialTestParams.intraTestBarriers = TRUE;
break;
case 'G':
/* This option toggles between Incompressible Seed and Time stamp sig based on -l,
* so we'll toss the value in both for now, and sort it out in initialization
* after all the arguments are in and we know which it keep.
*/
initialTestParams.setTimeStampSignature = atoi(optarg);
initialTestParams.incompressibleSeed = atoi(optarg);
break;
case 'h':
initialTestParams.showHelp = TRUE;
@ -511,7 +516,22 @@ IOR_test_t *ParseCommandLine(int argc, char **argv)
initialTestParams.keepFileWithError = TRUE;
break;
case 'l':
initialTestParams.storeFileOffset = TRUE;
switch(*optarg) {
case 'i': /* Incompressible */
initialTestParams.dataPacketType = incompressible;
break;
case 't': /* timestamp */
initialTestParams.dataPacketType = timestamp;
break;
case 'o': /* offset packet */
initialTestParams.storeFileOffset = TRUE;
initialTestParams.dataPacketType = offset;
break;
default:
fprintf(stdout,
"Unknown arguement for -l %s generic assumed\n", optarg);
break;
}
break;
case 'm':
initialTestParams.multiFile = TRUE;