Support random data generation for memory pattern in utilities. (#348)
* Support random data generation in utilities. Update first 8 byte element in each 4k block on updates to defy dedup. * Incorporate different packet types into mdtest/md-workbench. * Integrated utilities memory pattern tools into IOR. Now all tools use the same patterns. * Added IOR long option for compatibility between IOR and other tools. * Added new tests for random buffers.master
parent
3be3cfb274
commit
a436395570
|
@ -370,7 +370,7 @@ void ShowTestStart(IOR_param_t *test)
|
||||||
PrintKeyValInt("randomOffset", test->randomOffset);
|
PrintKeyValInt("randomOffset", test->randomOffset);
|
||||||
PrintKeyValInt("checkWrite", test->checkWrite);
|
PrintKeyValInt("checkWrite", test->checkWrite);
|
||||||
PrintKeyValInt("checkRead", test->checkRead);
|
PrintKeyValInt("checkRead", test->checkRead);
|
||||||
PrintKeyValInt("storeFileOffset", test->storeFileOffset);
|
PrintKeyValInt("dataPacketType", test->dataPacketType);
|
||||||
PrintKeyValInt("keepFile", test->keepFile);
|
PrintKeyValInt("keepFile", test->keepFile);
|
||||||
PrintKeyValInt("keepFileWithError", test->keepFileWithError);
|
PrintKeyValInt("keepFileWithError", test->keepFileWithError);
|
||||||
PrintKeyValInt("warningAsErrors", test->warningAsErrors);
|
PrintKeyValInt("warningAsErrors", test->warningAsErrors);
|
||||||
|
|
146
src/ior.c
146
src/ior.c
|
@ -409,81 +409,7 @@ static size_t
|
||||||
CompareData(void *expectedBuffer, size_t size, IOR_offset_t transferCount, IOR_param_t *test, IOR_offset_t offset, int fillrank, int access)
|
CompareData(void *expectedBuffer, size_t size, IOR_offset_t transferCount, IOR_param_t *test, IOR_offset_t offset, int fillrank, int access)
|
||||||
{
|
{
|
||||||
assert(access == WRITECHECK || access == READCHECK);
|
assert(access == WRITECHECK || access == READCHECK);
|
||||||
|
return verify_memory_pattern(offset, expectedBuffer, transferCount, test->setTimeStampSignature, fillrank, test->dataPacketType);
|
||||||
char testFileName[MAX_PATHLEN];
|
|
||||||
char * bufferLabel1 = "Expected: ";
|
|
||||||
char * bufferLabel2 = "Actual: ";
|
|
||||||
size_t i, j, length;
|
|
||||||
size_t errorCount = 0;
|
|
||||||
|
|
||||||
IOR_offset_t offsetSignature = 0;
|
|
||||||
unsigned long long hi, lo, val; // for data verification
|
|
||||||
hi = ((unsigned long long)fillrank) << 32;
|
|
||||||
lo = (unsigned long long)test->timeStampSignatureValue;
|
|
||||||
if (test->storeFileOffset){
|
|
||||||
offsetSignature = offset;
|
|
||||||
}
|
|
||||||
|
|
||||||
unsigned long long *testbuf = (unsigned long long *)expectedBuffer;
|
|
||||||
|
|
||||||
length = size / sizeof(IOR_size_t);
|
|
||||||
if (verbose >= VERBOSE_3) {
|
|
||||||
fprintf(out_logfile,
|
|
||||||
"[%d] At file byte offset %lld, comparing %llu-byte transfer\n",
|
|
||||||
rank, (long long) offset, (long long)size);
|
|
||||||
}
|
|
||||||
|
|
||||||
int incompressibleSeed = test->setTimeStampSignature + fillrank;
|
|
||||||
for (i = 0; i < length; i++) {
|
|
||||||
if(test->dataPacketType == incompressible ) {
|
|
||||||
/* same logic as in FillIncompressibleBuffer() */
|
|
||||||
/* WARNING: make sure that both functions are changed at the same time */
|
|
||||||
hi = ((unsigned long long) rand_r(& incompressibleSeed) << 32);
|
|
||||||
lo = (unsigned long long) rand_r(& incompressibleSeed);
|
|
||||||
val = hi | lo;
|
|
||||||
}else{
|
|
||||||
if ((i % 2) == 0) {
|
|
||||||
/* evens contain MPI rank and time in seconds */
|
|
||||||
val = hi | lo;
|
|
||||||
} else {
|
|
||||||
/* odds contain offset */
|
|
||||||
val = offsetSignature + (i * sizeof(unsigned long long));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (testbuf[i] != val) {
|
|
||||||
errorCount++;
|
|
||||||
if (verbose >= VERBOSE_2) {
|
|
||||||
fprintf(out_logfile,
|
|
||||||
"[%d] At transfer buffer #%lld, index #%lld (file byte offset %lld):\n",
|
|
||||||
rank, transferCount - 1, (long long)i,
|
|
||||||
(long long) offset +
|
|
||||||
(IOR_size_t) (i * sizeof(IOR_size_t)));
|
|
||||||
fprintf(out_logfile, "[%d] %s0x", rank, bufferLabel1);
|
|
||||||
fprintf(out_logfile, "%016llx\n", val);
|
|
||||||
fprintf(out_logfile, "[%d] %s0x", rank, bufferLabel2);
|
|
||||||
fprintf(out_logfile, "%016llx\n", testbuf[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
} else if (verbose >= VERBOSE_5) {
|
|
||||||
fprintf(out_logfile,
|
|
||||||
"[%d] PASSED offset = %llu bytes, transfer %lld\n",
|
|
||||||
rank, ((i * sizeof(unsigned long long)) + offset), transferCount);
|
|
||||||
fprintf(out_logfile, "[%d] GOOD %s0x", rank, bufferLabel1);
|
|
||||||
fprintf(out_logfile, "%016llx ", val);
|
|
||||||
fprintf(out_logfile, "\n[%d] GOOD %s0x", rank, bufferLabel2);
|
|
||||||
fprintf(out_logfile, "%016llx ", testbuf[i]);
|
|
||||||
fprintf(out_logfile, "\n");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (errorCount > 0 && verbose >= VERBOSE_1) {
|
|
||||||
GetTestFileName(testFileName, test);
|
|
||||||
EWARNF("[%d] FAILED comparison of buffer in file %s during transfer %lld offset %lld containing %d-byte ints (%zd errors)",
|
|
||||||
rank, testFileName, transferCount, offset, (int)sizeof(unsigned long long int),errorCount);
|
|
||||||
}else if(verbose >= VERBOSE_2){
|
|
||||||
fprintf(out_logfile, "[%d] comparison successful during transfer %lld offset %lld\n", rank, transferCount, offset);
|
|
||||||
}
|
|
||||||
|
|
||||||
return (errorCount);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -610,61 +536,6 @@ static void DistributeHints(MPI_Comm com)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Fill buffer, which is transfer size bytes long, with known 8-byte long long
|
|
||||||
* int values. In even-numbered 8-byte long long ints, store MPI task in high
|
|
||||||
* bits and timestamp signature in low bits. In odd-numbered 8-byte long long
|
|
||||||
* ints, store transfer offset. If storeFileOffset option is used, the file
|
|
||||||
* (not transfer) offset is stored instead.
|
|
||||||
*/
|
|
||||||
static unsigned int reseed_incompressible_prng = TRUE;
|
|
||||||
|
|
||||||
static void
|
|
||||||
FillIncompressibleBuffer(void* buffer, IOR_param_t * test)
|
|
||||||
{
|
|
||||||
size_t i;
|
|
||||||
unsigned long long hi, lo;
|
|
||||||
unsigned long long *buf = (unsigned long long *)buffer;
|
|
||||||
|
|
||||||
/* In order for write checks to work, we have to restart the pseudo random sequence */
|
|
||||||
/* This function has the same logic as CompareData() */
|
|
||||||
/* WARNING: make sure that both functions are changed at the same time */
|
|
||||||
if(reseed_incompressible_prng == TRUE) {
|
|
||||||
test->incompressibleSeed = test->setTimeStampSignature + rank; /* We copied seed into timestampSignature at initialization, also add the rank to add randomness between processes */
|
|
||||||
reseed_incompressible_prng = FALSE;
|
|
||||||
}
|
|
||||||
for (i = 0; i < test->transferSize / sizeof(unsigned long long); i++) {
|
|
||||||
hi = ((unsigned long long) rand_r(&test->incompressibleSeed) << 32);
|
|
||||||
lo = (unsigned long long) rand_r(&test->incompressibleSeed);
|
|
||||||
buf[i] = hi | lo;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void
|
|
||||||
FillBuffer(void *buffer,
|
|
||||||
IOR_param_t * test, unsigned long long offset, int fillrank)
|
|
||||||
{
|
|
||||||
size_t i;
|
|
||||||
unsigned long long hi, lo;
|
|
||||||
unsigned long long *buf = (unsigned long long *)buffer;
|
|
||||||
|
|
||||||
if(test->dataPacketType == incompressible ) { /* Make for some non compressible buffers with randomish data */
|
|
||||||
FillIncompressibleBuffer(buffer, test);
|
|
||||||
} else {
|
|
||||||
hi = ((unsigned long long)fillrank) << 32;
|
|
||||||
lo = (unsigned long long)test->timeStampSignatureValue;
|
|
||||||
for (i = 0; i < test->transferSize / sizeof(unsigned long long); i++) {
|
|
||||||
if ((i % 2) == 0) {
|
|
||||||
/* evens contain MPI rank and time in seconds */
|
|
||||||
buf[i] = hi | lo;
|
|
||||||
} else {
|
|
||||||
/* odds contain offset */
|
|
||||||
buf[i] = offset + (i * sizeof(unsigned long long));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Return string describing machine name and type.
|
* Return string describing machine name and type.
|
||||||
*/
|
*/
|
||||||
|
@ -1305,8 +1176,7 @@ static void TestIoSys(IOR_test_t *test)
|
||||||
params->timeStampSignatureValue = (unsigned int) params->setTimeStampSignature;
|
params->timeStampSignatureValue = (unsigned int) params->setTimeStampSignature;
|
||||||
}
|
}
|
||||||
XferBuffersSetup(&ioBuffers, params, pretendRank);
|
XferBuffersSetup(&ioBuffers, params, pretendRank);
|
||||||
reseed_incompressible_prng = TRUE; // reset pseudo random generator, necessary to guarantee the next call to FillBuffer produces the same value as it is right now
|
|
||||||
|
|
||||||
/* Initial time stamp */
|
/* Initial time stamp */
|
||||||
startTime = GetTimeStamp();
|
startTime = GetTimeStamp();
|
||||||
|
|
||||||
|
@ -1349,7 +1219,8 @@ static void TestIoSys(IOR_test_t *test)
|
||||||
(¶ms->timeStampSignatureValue, 1, MPI_UNSIGNED, 0,
|
(¶ms->timeStampSignatureValue, 1, MPI_UNSIGNED, 0,
|
||||||
testComm), "cannot broadcast start time value");
|
testComm), "cannot broadcast start time value");
|
||||||
|
|
||||||
FillBuffer(ioBuffers.buffer, params, 0, pretendRank);
|
generate_memory_pattern((char*) ioBuffers.buffer, params->transferSize, params->setTimeStampSignature, pretendRank, params->dataPacketType);
|
||||||
|
|
||||||
/* use repetition count for number of multiple files */
|
/* use repetition count for number of multiple files */
|
||||||
if (params->multiFile)
|
if (params->multiFile)
|
||||||
params->repCounter = rep;
|
params->repCounter = rep;
|
||||||
|
@ -1432,8 +1303,7 @@ static void TestIoSys(IOR_test_t *test)
|
||||||
}
|
}
|
||||||
rankOffset = (2 * shift) % params->numTasks;
|
rankOffset = (2 * shift) % params->numTasks;
|
||||||
}
|
}
|
||||||
reseed_incompressible_prng = TRUE; /* Re-Seed the PRNG to get same sequence back, if random */
|
|
||||||
|
|
||||||
GetTestFileName(testFileName, params);
|
GetTestFileName(testFileName, params);
|
||||||
params->open = WRITECHECK;
|
params->open = WRITECHECK;
|
||||||
fd = backend->open(testFileName, IOR_RDONLY, params->backend_options);
|
fd = backend->open(testFileName, IOR_RDONLY, params->backend_options);
|
||||||
|
@ -1643,7 +1513,7 @@ static void ValidateTests(IOR_param_t * test, MPI_Comm com)
|
||||||
ERR("random offset and constant reorder tasks specified with single-shared-file. Choose one and resubmit");
|
ERR("random offset and constant reorder tasks specified with single-shared-file. Choose one and resubmit");
|
||||||
if (test->randomOffset && test->checkRead && test->randomSeed == -1)
|
if (test->randomOffset && test->checkRead && test->randomSeed == -1)
|
||||||
ERR("random offset with read check option requires to set the random seed");
|
ERR("random offset with read check option requires to set the random seed");
|
||||||
if (test->randomOffset && test->storeFileOffset)
|
if (test->randomOffset && test->dataPacketType == DATA_OFFSET)
|
||||||
ERR("random offset not available with store file offset option)");
|
ERR("random offset not available with store file offset option)");
|
||||||
if ((strcasecmp(test->api, "HDF5") == 0) && test->randomOffset)
|
if ((strcasecmp(test->api, "HDF5") == 0) && test->randomOffset)
|
||||||
ERR("random offset not available with HDF5");
|
ERR("random offset not available with HDF5");
|
||||||
|
@ -1759,9 +1629,7 @@ static IOR_offset_t WriteOrReadSingle(IOR_offset_t offset, int pretendRank, IOR_
|
||||||
if (access == WRITE) {
|
if (access == WRITE) {
|
||||||
/* fills each transfer with a unique pattern
|
/* fills each transfer with a unique pattern
|
||||||
* containing the offset into the file */
|
* containing the offset into the file */
|
||||||
if (test->storeFileOffset == TRUE) {
|
update_write_memory_pattern(offset, ioBuffers->buffer, transfer, test->setTimeStampSignature, pretendRank, test->dataPacketType);
|
||||||
FillBuffer(buffer, test, offset, pretendRank);
|
|
||||||
}
|
|
||||||
amtXferred = backend->xfer(access, fd, buffer, transfer, offset, test->backend_options);
|
amtXferred = backend->xfer(access, fd, buffer, transfer, offset, test->backend_options);
|
||||||
if (amtXferred != transfer)
|
if (amtXferred != transfer)
|
||||||
ERR("cannot write to file");
|
ERR("cannot write to file");
|
||||||
|
|
14
src/ior.h
14
src/ior.h
|
@ -46,17 +46,6 @@
|
||||||
#endif /* not MPI_FILE_NULL */
|
#endif /* not MPI_FILE_NULL */
|
||||||
|
|
||||||
#define ISPOWEROFTWO(x) ((x != 0) && !(x & (x - 1)))
|
#define ISPOWEROFTWO(x) ((x != 0) && !(x & (x - 1)))
|
||||||
/******************** DATA Packet Type ***************************************/
|
|
||||||
/* Holds the types of data packets: generic, offset, timestamp, incompressible */
|
|
||||||
|
|
||||||
enum PACKET_TYPE
|
|
||||||
{
|
|
||||||
generic = 0, /* No packet type specified */
|
|
||||||
timestamp=1, /* Timestamp packet set with -l */
|
|
||||||
offset=2, /* Offset packet set with -l */
|
|
||||||
incompressible=3 /* Incompressible packet set with -l */
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
typedef enum{
|
typedef enum{
|
||||||
IOR_MEMORY_TYPE_CPU = 0,
|
IOR_MEMORY_TYPE_CPU = 0,
|
||||||
|
@ -142,7 +131,6 @@ typedef struct
|
||||||
int summary_every_test; /* flag to print summary every test, not just at end */
|
int summary_every_test; /* flag to print summary every test, not just at end */
|
||||||
int uniqueDir; /* use unique directory for each fpp */
|
int uniqueDir; /* use unique directory for each fpp */
|
||||||
int useExistingTestFile; /* do not delete test file before access */
|
int useExistingTestFile; /* do not delete test file before access */
|
||||||
int storeFileOffset; /* use file offset as stored signature */
|
|
||||||
int deadlineForStonewalling; /* max time in seconds to run any test phase */
|
int deadlineForStonewalling; /* max time in seconds to run any test phase */
|
||||||
int stoneWallingWearOut; /* wear out the stonewalling, once the timeout is over, each process has to write the same amount */
|
int stoneWallingWearOut; /* wear out the stonewalling, once the timeout is over, each process has to write the same amount */
|
||||||
uint64_t stoneWallingWearOutIterations; /* the number of iterations for the stonewallingWearOut, needed for readBack */
|
uint64_t stoneWallingWearOutIterations; /* the number of iterations for the stonewallingWearOut, needed for readBack */
|
||||||
|
@ -161,7 +149,7 @@ typedef struct
|
||||||
char * memoryPerNodeStr; /* for parsing */
|
char * memoryPerNodeStr; /* for parsing */
|
||||||
char * testscripts; /* for parsing */
|
char * testscripts; /* for parsing */
|
||||||
char * buffer_type; /* for parsing */
|
char * buffer_type; /* for parsing */
|
||||||
enum PACKET_TYPE dataPacketType; /* The type of data packet. */
|
ior_dataPacketType_e dataPacketType; /* The type of data packet. */
|
||||||
|
|
||||||
void * backend_options; /* Backend-specific options */
|
void * backend_options; /* Backend-specific options */
|
||||||
|
|
||||||
|
|
|
@ -19,6 +19,12 @@
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
DATA_TIMESTAMP, /* Will not include any offset, hence each buffer will be the same */
|
||||||
|
DATA_OFFSET,
|
||||||
|
DATA_INCOMPRESSIBLE /* Will include the offset as well */
|
||||||
|
} ior_dataPacketType_e;
|
||||||
|
|
||||||
#ifdef _WIN32
|
#ifdef _WIN32
|
||||||
# define _CRT_SECURE_NO_WARNINGS
|
# define _CRT_SECURE_NO_WARNINGS
|
||||||
# define _CRT_RAND_S
|
# define _CRT_RAND_S
|
||||||
|
|
|
@ -85,6 +85,8 @@ struct benchmark_options{
|
||||||
|
|
||||||
mdworkbench_results_t * results; // the results
|
mdworkbench_results_t * results; // the results
|
||||||
|
|
||||||
|
ior_dataPacketType_e dataPacketType;
|
||||||
|
char * packetTypeStr;
|
||||||
int offset;
|
int offset;
|
||||||
int iterations;
|
int iterations;
|
||||||
int global_iteration;
|
int global_iteration;
|
||||||
|
@ -117,7 +119,7 @@ struct benchmark_options{
|
||||||
int rank;
|
int rank;
|
||||||
int size;
|
int size;
|
||||||
int verify_read;
|
int verify_read;
|
||||||
int random_buffer_offset;
|
int random_seed;
|
||||||
|
|
||||||
float relative_waiting_factor;
|
float relative_waiting_factor;
|
||||||
int adaptive_waiting_mode;
|
int adaptive_waiting_mode;
|
||||||
|
@ -140,12 +142,13 @@ void init_options(){
|
||||||
.interface = "POSIX",
|
.interface = "POSIX",
|
||||||
.prefix = "./out",
|
.prefix = "./out",
|
||||||
.num = 1000,
|
.num = 1000,
|
||||||
.random_buffer_offset = -1,
|
.random_seed = -1,
|
||||||
.precreate = 3000,
|
.precreate = 3000,
|
||||||
.dset_count = 10,
|
.dset_count = 10,
|
||||||
.offset = 1,
|
.offset = 1,
|
||||||
.iterations = 3,
|
.iterations = 3,
|
||||||
.file_size = 3901,
|
.file_size = 3901,
|
||||||
|
.packetTypeStr = "t",
|
||||||
.run_info_file = "md-workbench.status"};
|
.run_info_file = "md-workbench.status"};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -553,7 +556,7 @@ void run_precreate(phase_stat_t * s, int current_index){
|
||||||
}
|
}
|
||||||
|
|
||||||
char * buf = aligned_buffer_alloc(o.file_size, o.gpu_memory_flags);
|
char * buf = aligned_buffer_alloc(o.file_size, o.gpu_memory_flags);
|
||||||
generate_memory_pattern(buf, o.file_size, o.random_buffer_offset, o.rank);
|
generate_memory_pattern(buf, o.file_size, o.random_seed, o.rank, o.dataPacketType);
|
||||||
double op_timer; // timer for individual operations
|
double op_timer; // timer for individual operations
|
||||||
size_t pos = -1; // position inside the individual measurement array
|
size_t pos = -1; // position inside the individual measurement array
|
||||||
double op_time;
|
double op_time;
|
||||||
|
@ -569,7 +572,7 @@ void run_precreate(phase_stat_t * s, int current_index){
|
||||||
if (NULL == aiori_fh){
|
if (NULL == aiori_fh){
|
||||||
FAIL("Unable to open file %s", obj_name);
|
FAIL("Unable to open file %s", obj_name);
|
||||||
}
|
}
|
||||||
update_write_memory_pattern(f * o.dset_count + d, buf, o.file_size, o.random_buffer_offset, o.rank);
|
update_write_memory_pattern(f * o.dset_count + d, buf, o.file_size, o.random_seed, o.rank, o.dataPacketType);
|
||||||
if ( o.file_size == (int) o.backend->xfer(WRITE, aiori_fh, (IOR_size_t *) buf, o.file_size, 0, o.backend_options)) {
|
if ( o.file_size == (int) o.backend->xfer(WRITE, aiori_fh, (IOR_size_t *) buf, o.file_size, 0, o.backend_options)) {
|
||||||
s->obj_create.suc++;
|
s->obj_create.suc++;
|
||||||
}else{
|
}else{
|
||||||
|
@ -650,7 +653,7 @@ void run_benchmark(phase_stat_t * s, int * current_index_p){
|
||||||
}
|
}
|
||||||
if ( o.file_size == (int) o.backend->xfer(READ, aiori_fh, (IOR_size_t *) buf, o.file_size, 0, o.backend_options) ) {
|
if ( o.file_size == (int) o.backend->xfer(READ, aiori_fh, (IOR_size_t *) buf, o.file_size, 0, o.backend_options) ) {
|
||||||
if(o.verify_read){
|
if(o.verify_read){
|
||||||
if(verify_memory_pattern(prevFile * o.dset_count + d, buf, o.file_size, o.random_buffer_offset, readRank) == 0){
|
if(verify_memory_pattern(prevFile * o.dset_count + d, buf, o.file_size, o.random_seed, readRank, o.dataPacketType) == 0){
|
||||||
s->obj_read.suc++;
|
s->obj_read.suc++;
|
||||||
}else{
|
}else{
|
||||||
s->obj_read.err++;
|
s->obj_read.err++;
|
||||||
|
@ -691,9 +694,9 @@ void run_benchmark(phase_stat_t * s, int * current_index_p){
|
||||||
op_timer = GetTimeStamp();
|
op_timer = GetTimeStamp();
|
||||||
aiori_fh = o.backend->create(obj_name, IOR_WRONLY | IOR_CREAT, o.backend_options);
|
aiori_fh = o.backend->create(obj_name, IOR_WRONLY | IOR_CREAT, o.backend_options);
|
||||||
if (NULL != aiori_fh){
|
if (NULL != aiori_fh){
|
||||||
generate_memory_pattern(buf, o.file_size, o.random_buffer_offset, writeRank);
|
generate_memory_pattern(buf, o.file_size, o.random_seed, writeRank, o.dataPacketType);
|
||||||
update_write_memory_pattern(newFileIndex * o.dset_count + d, buf, o.file_size, o.random_buffer_offset, writeRank);
|
update_write_memory_pattern(newFileIndex * o.dset_count + d, buf, o.file_size, o.random_seed, writeRank, o.dataPacketType);
|
||||||
|
|
||||||
if ( o.file_size == (int) o.backend->xfer(WRITE, aiori_fh, (IOR_size_t *) buf, o.file_size, 0, o.backend_options)) {
|
if ( o.file_size == (int) o.backend->xfer(WRITE, aiori_fh, (IOR_size_t *) buf, o.file_size, 0, o.backend_options)) {
|
||||||
s->obj_create.suc++;
|
s->obj_create.suc++;
|
||||||
}else{
|
}else{
|
||||||
|
@ -808,7 +811,7 @@ static option_help options [] = {
|
||||||
{0, "latency-all", "Keep the latency files from all ranks.", OPTION_FLAG, 'd', & o.latency_keep_all},
|
{0, "latency-all", "Keep the latency files from all ranks.", OPTION_FLAG, 'd', & o.latency_keep_all},
|
||||||
{'P', "precreate-per-set", "Number of object to precreate per data set.", OPTION_OPTIONAL_ARGUMENT, 'd', & o.precreate},
|
{'P', "precreate-per-set", "Number of object to precreate per data set.", OPTION_OPTIONAL_ARGUMENT, 'd', & o.precreate},
|
||||||
{'D', "data-sets", "Number of data sets covered per process and iteration.", OPTION_OPTIONAL_ARGUMENT, 'd', & o.dset_count},
|
{'D', "data-sets", "Number of data sets covered per process and iteration.", OPTION_OPTIONAL_ARGUMENT, 'd', & o.dset_count},
|
||||||
{'G', NULL, "Offset for the data in the read/write buffer, if not set, a random value is used", OPTION_OPTIONAL_ARGUMENT, 'd', & o.random_buffer_offset},
|
{'G', NULL, "Timestamp/Random seed for access pattern, if not set, a random value is used", OPTION_OPTIONAL_ARGUMENT, 'd', & o.random_seed},
|
||||||
{'o', NULL, "Output directory", OPTION_OPTIONAL_ARGUMENT, 's', & o.prefix},
|
{'o', NULL, "Output directory", OPTION_OPTIONAL_ARGUMENT, 's', & o.prefix},
|
||||||
{'q', "quiet", "Avoid irrelevant printing.", OPTION_FLAG, 'd', & o.quiet_output},
|
{'q', "quiet", "Avoid irrelevant printing.", OPTION_FLAG, 'd', & o.quiet_output},
|
||||||
//{'m', "lim-free-mem", "Allocate memory until this limit (in MiB) is reached.", OPTION_OPTIONAL_ARGUMENT, 'd', & o.limit_memory},
|
//{'m', "lim-free-mem", "Allocate memory until this limit (in MiB) is reached.", OPTION_OPTIONAL_ARGUMENT, 'd', & o.limit_memory},
|
||||||
|
@ -823,6 +826,7 @@ static option_help options [] = {
|
||||||
{'w', "stonewall-timer", "Stop each benchmark iteration after the specified seconds (if not used with -W this leads to process-specific progress!)", OPTION_OPTIONAL_ARGUMENT, 'd', & o.stonewall_timer},
|
{'w', "stonewall-timer", "Stop each benchmark iteration after the specified seconds (if not used with -W this leads to process-specific progress!)", OPTION_OPTIONAL_ARGUMENT, 'd', & o.stonewall_timer},
|
||||||
{'W', "stonewall-wear-out", "Stop with stonewall after specified time and use a soft wear-out phase -- all processes perform the same number of iterations", OPTION_FLAG, 'd', & o.stonewall_timer_wear_out},
|
{'W', "stonewall-wear-out", "Stop with stonewall after specified time and use a soft wear-out phase -- all processes perform the same number of iterations", OPTION_FLAG, 'd', & o.stonewall_timer_wear_out},
|
||||||
{'X', "verify-read", "Verify the data on read", OPTION_FLAG, 'd', & o.verify_read},
|
{'X', "verify-read", "Verify the data on read", OPTION_FLAG, 'd', & o.verify_read},
|
||||||
|
{0, "dataPacketType", "type of packet that will be created [offset|incompressible|timestamp|o|i|t]", OPTION_OPTIONAL_ARGUMENT, 's', & o.packetTypeStr},
|
||||||
{0, "allocateBufferOnGPU", "Allocate the buffer on the GPU.", OPTION_FLAG, 'd', & o.gpu_memory_flags},
|
{0, "allocateBufferOnGPU", "Allocate the buffer on the GPU.", OPTION_FLAG, 'd', & o.gpu_memory_flags},
|
||||||
{0, "start-item", "The iteration number of the item to start with, allowing to offset the operations", OPTION_OPTIONAL_ARGUMENT, 'l', & o.start_item_number},
|
{0, "start-item", "The iteration number of the item to start with, allowing to offset the operations", OPTION_OPTIONAL_ARGUMENT, 'l', & o.start_item_number},
|
||||||
{0, "print-detailed-stats", "Print detailed machine parsable statistics.", OPTION_FLAG, 'd', & o.print_detailed_stats},
|
{0, "print-detailed-stats", "Print detailed machine parsable statistics.", OPTION_FLAG, 'd', & o.print_detailed_stats},
|
||||||
|
@ -905,6 +909,8 @@ mdworkbench_results_t* md_workbench_run(int argc, char ** argv, MPI_Comm world_c
|
||||||
ERR("Backend doesn't support MDWorbench");
|
ERR("Backend doesn't support MDWorbench");
|
||||||
}
|
}
|
||||||
o.backend_options = airoi_update_module_options(o.backend, global_options);
|
o.backend_options = airoi_update_module_options(o.backend, global_options);
|
||||||
|
|
||||||
|
o.dataPacketType = parsePacketType(o.packetTypeStr[0]);
|
||||||
|
|
||||||
if (!(o.phase_cleanup || o.phase_precreate || o.phase_benchmark)){
|
if (!(o.phase_cleanup || o.phase_precreate || o.phase_benchmark)){
|
||||||
// enable all phases
|
// enable all phases
|
||||||
|
@ -915,9 +921,9 @@ mdworkbench_results_t* md_workbench_run(int argc, char ** argv, MPI_Comm world_c
|
||||||
ERR("Invalid options, if running only the benchmark phase using -2 with stonewall option then use stonewall wear-out");
|
ERR("Invalid options, if running only the benchmark phase using -2 with stonewall option then use stonewall wear-out");
|
||||||
exit(1);
|
exit(1);
|
||||||
}
|
}
|
||||||
if( o.random_buffer_offset == -1 ){
|
if( o.random_seed == -1 ){
|
||||||
o.random_buffer_offset = time(NULL);
|
o.random_seed = time(NULL);
|
||||||
MPI_Bcast(& o.random_buffer_offset, 1, MPI_INT, 0, o.com);
|
MPI_Bcast(& o.random_seed, 1, MPI_INT, 0, o.com);
|
||||||
}
|
}
|
||||||
|
|
||||||
if(o.backend->xfer_hints){
|
if(o.backend->xfer_hints){
|
||||||
|
|
23
src/mdtest.c
23
src/mdtest.c
|
@ -145,6 +145,7 @@ typedef struct {
|
||||||
int print_time;
|
int print_time;
|
||||||
int print_rate_and_time;
|
int print_rate_and_time;
|
||||||
int print_all_proc;
|
int print_all_proc;
|
||||||
|
ior_dataPacketType_e dataPacketType;
|
||||||
int random_seed;
|
int random_seed;
|
||||||
int shared_file;
|
int shared_file;
|
||||||
int files_only;
|
int files_only;
|
||||||
|
@ -392,12 +393,8 @@ static void create_file (const char *path, uint64_t itemNum) {
|
||||||
if (o.write_bytes > 0) {
|
if (o.write_bytes > 0) {
|
||||||
VERBOSE(3,5,"create_remove_items_helper: write..." );
|
VERBOSE(3,5,"create_remove_items_helper: write..." );
|
||||||
|
|
||||||
/*
|
|
||||||
* According to Bill Loewe, writes are only done one time, so they are always at
|
|
||||||
* offset 0 (zero).
|
|
||||||
*/
|
|
||||||
o.hints.fsyncPerWrite = o.sync_file;
|
o.hints.fsyncPerWrite = o.sync_file;
|
||||||
update_write_memory_pattern(itemNum, o.write_buffer, o.write_bytes, o.random_buffer_offset, rank);
|
update_write_memory_pattern(itemNum, o.write_buffer, o.write_bytes, o.random_buffer_offset, rank, o.dataPacketType);
|
||||||
|
|
||||||
if ( o.write_bytes != (size_t) o.backend->xfer(WRITE, aiori_fh, (IOR_size_t *) o.write_buffer, o.write_bytes, 0, o.backend_options)) {
|
if ( o.write_bytes != (size_t) o.backend->xfer(WRITE, aiori_fh, (IOR_size_t *) o.write_buffer, o.write_bytes, 0, o.backend_options)) {
|
||||||
EWARNF("unable to write file %s", curr_item);
|
EWARNF("unable to write file %s", curr_item);
|
||||||
|
@ -408,7 +405,7 @@ static void create_file (const char *path, uint64_t itemNum) {
|
||||||
if (o.write_bytes != (size_t) o.backend->xfer(READ, aiori_fh, (IOR_size_t *) o.write_buffer, o.write_bytes, 0, o.backend_options)) {
|
if (o.write_bytes != (size_t) o.backend->xfer(READ, aiori_fh, (IOR_size_t *) o.write_buffer, o.write_bytes, 0, o.backend_options)) {
|
||||||
EWARNF("unable to verify write (read/back) file %s", curr_item);
|
EWARNF("unable to verify write (read/back) file %s", curr_item);
|
||||||
}
|
}
|
||||||
o.verification_error += verify_memory_pattern(itemNum, o.write_buffer, o.write_bytes, o.random_buffer_offset, rank);
|
o.verification_error += verify_memory_pattern(itemNum, o.write_buffer, o.write_bytes, o.random_buffer_offset, rank, o.dataPacketType);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -725,15 +722,12 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) {
|
||||||
EWARNF("unable to read file %s", item);
|
EWARNF("unable to read file %s", item);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
int pretend_rank = (2 * o.nstride + rank) % o.size;
|
||||||
if(o.verify_read){
|
if(o.verify_read){
|
||||||
int pretend_rank = (2 * o.nstride + rank) % o.size;
|
|
||||||
if (o.shared_file) {
|
if (o.shared_file) {
|
||||||
pretend_rank = rank;
|
pretend_rank = rank;
|
||||||
}
|
}
|
||||||
o.verification_error += verify_memory_pattern(item_num, read_buffer, o.read_bytes, o.random_buffer_offset, pretend_rank);
|
o.verification_error += verify_memory_pattern(item_num, read_buffer, o.read_bytes, o.random_buffer_offset, pretend_rank, o.dataPacketType);
|
||||||
}else if((o.read_bytes >= 8 && ((uint64_t*) read_buffer)[0] != item_num) || (o.read_bytes < 8 && read_buffer[0] != (char) item_num)){
|
|
||||||
// do a lightweight check, which cost is neglectable
|
|
||||||
o.verification_error++;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2188,6 +2182,8 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
|
||||||
char apiStr[1024];
|
char apiStr[1024];
|
||||||
sprintf(apiStr, "API for I/O [%s]", APIs);
|
sprintf(apiStr, "API for I/O [%s]", APIs);
|
||||||
memset(& o.hints, 0, sizeof(o.hints));
|
memset(& o.hints, 0, sizeof(o.hints));
|
||||||
|
|
||||||
|
char * packetType = "t";
|
||||||
|
|
||||||
option_help options [] = {
|
option_help options [] = {
|
||||||
{'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & o.api},
|
{'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & o.api},
|
||||||
|
@ -2234,6 +2230,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
|
||||||
{'Y', NULL, "call the sync command after each phase (included in the timing; note it causes all IO to be flushed from your node)", OPTION_FLAG, 'd', & o.call_sync},
|
{'Y', NULL, "call the sync command after each phase (included in the timing; note it causes all IO to be flushed from your node)", OPTION_FLAG, 'd', & o.call_sync},
|
||||||
{'z', NULL, "depth of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & o.depth},
|
{'z', NULL, "depth of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & o.depth},
|
||||||
{'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & o.print_time},
|
{'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & o.print_time},
|
||||||
|
{0, "dataPacketType", "type of packet that will be created [offset|incompressible|timestamp|o|i|t]", OPTION_OPTIONAL_ARGUMENT, 's', & packetType},
|
||||||
{0, "allocateBufferOnGPU", "Allocate the buffer on the GPU.", OPTION_FLAG, 'd', & o.gpu_memory_flags},
|
{0, "allocateBufferOnGPU", "Allocate the buffer on the GPU.", OPTION_FLAG, 'd', & o.gpu_memory_flags},
|
||||||
{0, "warningAsErrors", "Any warning should lead to an error.", OPTION_FLAG, 'd', & aiori_warning_as_errors},
|
{0, "warningAsErrors", "Any warning should lead to an error.", OPTION_FLAG, 'd', & aiori_warning_as_errors},
|
||||||
{0, "saveRankPerformanceDetails", "Save the individual rank information into this CSV file.", OPTION_OPTIONAL_ARGUMENT, 's', & o.saveRankDetailsCSV},
|
{0, "saveRankPerformanceDetails", "Save the individual rank information into this CSV file.", OPTION_OPTIONAL_ARGUMENT, 's', & o.saveRankDetailsCSV},
|
||||||
|
@ -2250,6 +2247,8 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
|
||||||
|
|
||||||
free(global_options->modules);
|
free(global_options->modules);
|
||||||
free(global_options);
|
free(global_options);
|
||||||
|
|
||||||
|
o.dataPacketType = parsePacketType(packetType[0]);
|
||||||
|
|
||||||
MPI_Comm_rank(testComm, &rank);
|
MPI_Comm_rank(testComm, &rank);
|
||||||
MPI_Comm_size(testComm, &o.size);
|
MPI_Comm_size(testComm, &o.size);
|
||||||
|
@ -2420,7 +2419,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *
|
||||||
/* allocate and initialize write buffer with # */
|
/* allocate and initialize write buffer with # */
|
||||||
if (o.write_bytes > 0) {
|
if (o.write_bytes > 0) {
|
||||||
o.write_buffer = aligned_buffer_alloc(o.write_bytes, o.gpu_memory_flags);
|
o.write_buffer = aligned_buffer_alloc(o.write_bytes, o.gpu_memory_flags);
|
||||||
generate_memory_pattern(o.write_buffer, o.write_bytes, o.random_buffer_offset, rank);
|
generate_memory_pattern(o.write_buffer, o.write_bytes, o.random_buffer_offset, rank, o.dataPacketType);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* setup directory path to work in */
|
/* setup directory path to work in */
|
||||||
|
|
|
@ -222,8 +222,8 @@ void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_opt
|
||||||
params->verbose = atoi(value);
|
params->verbose = atoi(value);
|
||||||
} else if (strcasecmp(option, "settimestampsignature") == 0) {
|
} else if (strcasecmp(option, "settimestampsignature") == 0) {
|
||||||
params->setTimeStampSignature = atoi(value);
|
params->setTimeStampSignature = atoi(value);
|
||||||
} else if (strcasecmp(option, "storefileoffset") == 0) {
|
} else if (strcasecmp(option, "dataPacketType") == 0) {
|
||||||
params->storeFileOffset = atoi(value);
|
params->dataPacketType = parsePacketType(value[0]);
|
||||||
} else if (strcasecmp(option, "uniqueDir") == 0) {
|
} else if (strcasecmp(option, "uniqueDir") == 0) {
|
||||||
params->uniqueDir = atoi(value);
|
params->uniqueDir = atoi(value);
|
||||||
} else if (strcasecmp(option, "useexistingtestfile") == 0) {
|
} else if (strcasecmp(option, "useexistingtestfile") == 0) {
|
||||||
|
@ -450,7 +450,7 @@ option_help * createGlobalOptions(IOR_param_t * params){
|
||||||
{'j', NULL, "outlierThreshold -- warn on outlier N seconds from mean", OPTION_OPTIONAL_ARGUMENT, 'd', & params->outlierThreshold},
|
{'j', NULL, "outlierThreshold -- warn on outlier N seconds from mean", OPTION_OPTIONAL_ARGUMENT, 'd', & params->outlierThreshold},
|
||||||
{'k', NULL, "keepFile -- don't remove the test file(s) on program exit", OPTION_FLAG, 'd', & params->keepFile},
|
{'k', NULL, "keepFile -- don't remove the test file(s) on program exit", OPTION_FLAG, 'd', & params->keepFile},
|
||||||
{'K', NULL, "keepFileWithError -- keep error-filled file(s) after data-checking", OPTION_FLAG, 'd', & params->keepFileWithError},
|
{'K', NULL, "keepFileWithError -- keep error-filled file(s) after data-checking", OPTION_FLAG, 'd', & params->keepFileWithError},
|
||||||
{'l', NULL, "datapacket type-- type of packet that will be created [offset|incompressible|timestamp|o|i|t]", OPTION_OPTIONAL_ARGUMENT, 's', & params->buffer_type},
|
{'l', "dataPacketType", "datapacket type-- type of packet that will be created [offset|incompressible|timestamp|o|i|t]", OPTION_OPTIONAL_ARGUMENT, 's', & params->buffer_type},
|
||||||
{'m', NULL, "multiFile -- use number of reps (-i) for multiple file count", OPTION_FLAG, 'd', & params->multiFile},
|
{'m', NULL, "multiFile -- use number of reps (-i) for multiple file count", OPTION_FLAG, 'd', & params->multiFile},
|
||||||
{'M', NULL, "memoryPerNode -- hog memory on the node (e.g.: 2g, 75%)", OPTION_OPTIONAL_ARGUMENT, 's', & params->memoryPerNodeStr},
|
{'M', NULL, "memoryPerNode -- hog memory on the node (e.g.: 2g, 75%)", OPTION_OPTIONAL_ARGUMENT, 's', & params->memoryPerNodeStr},
|
||||||
{'N', NULL, "numTasks -- number of tasks that are participating in the test (overrides MPI)", OPTION_OPTIONAL_ARGUMENT, 'd', & params->numTasks},
|
{'N', NULL, "numTasks -- number of tasks that are participating in the test (overrides MPI)", OPTION_OPTIONAL_ARGUMENT, 'd', & params->numTasks},
|
||||||
|
|
102
src/utilities.c
102
src/utilities.c
|
@ -75,46 +75,74 @@ enum OutputFormat_t outputFormat;
|
||||||
|
|
||||||
/***************************** F U N C T I O N S ******************************/
|
/***************************** F U N C T I O N S ******************************/
|
||||||
|
|
||||||
void update_write_memory_pattern(uint64_t item, char * buf, size_t bytes, int buff_offset, int rank){
|
void update_write_memory_pattern(uint64_t item, char * buf, size_t bytes, int rand_seed, int pretendRank, ior_dataPacketType_e dataPacketType){
|
||||||
if(bytes >= 8){ // set the item number as first element of the buffer to be as much unique as possible
|
if(dataPacketType == DATA_TIMESTAMP || bytes < 8) return;
|
||||||
((uint64_t*) buf)[0] = item;
|
int k=1;
|
||||||
}else{
|
uint64_t * buffi = (uint64_t*) buf;
|
||||||
buf[0] = (char) item;
|
for(size_t i=0; i < bytes/sizeof(uint64_t); i+=512, k++){
|
||||||
|
buffi[i] = ((uint32_t) item * k) | ((uint64_t) pretendRank) << 32;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void generate_memory_pattern(char * buf, size_t bytes, int buff_offset, int rank){
|
void generate_memory_pattern(char * buf, size_t bytes, int rand_seed, int pretendRank, ior_dataPacketType_e dataPacketType){
|
||||||
uint64_t * buffi = (uint64_t*) buf;
|
uint64_t * buffi = (uint64_t*) buf;
|
||||||
// first half of 64 bits use the rank
|
// first half of 64 bits use the rank
|
||||||
const uint64_t ranki = ((uint64_t)(rank + 1) << 32) + buff_offset;
|
|
||||||
const size_t size = bytes / 8;
|
const size_t size = bytes / 8;
|
||||||
// the first 8 bytes are set to item number
|
// the first 8 bytes of each 4k block are updated at runtime
|
||||||
for(size_t i=1; i < size; i++){
|
unsigned seed = rand_seed + pretendRank;
|
||||||
buffi[i] = (i + 1) + ranki;
|
for(size_t i=0; i < size; i++){
|
||||||
|
switch(dataPacketType){
|
||||||
|
case(DATA_INCOMPRESSIBLE):{
|
||||||
|
uint64_t hi = ((uint64_t) rand_r(& seed) << 32);
|
||||||
|
uint64_t lo = (uint64_t) rand_r(& seed);
|
||||||
|
buffi[i] = hi | lo;
|
||||||
|
break;
|
||||||
|
}case(DATA_OFFSET):{
|
||||||
|
}case(DATA_TIMESTAMP):{
|
||||||
|
buffi[i] = ((uint64_t) pretendRank) << 32 | rand_seed + i;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for(size_t i=(bytes/8)*8; i < bytes; i++){
|
|
||||||
|
for(size_t i=size*8; i < bytes; i++){
|
||||||
buf[i] = (char) i;
|
buf[i] = (char) i;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
int verify_memory_pattern(int item, char * buffer, size_t bytes, int buff_offset, int pretendRank){
|
int verify_memory_pattern(uint64_t item, char * buffer, size_t bytes, int rand_seed, int pretendRank, ior_dataPacketType_e dataPacketType){
|
||||||
int error = 0;
|
int error = 0;
|
||||||
// always read all data to ensure that performance numbers stay the same
|
// always read all data to ensure that performance numbers stay the same
|
||||||
if((bytes >= 8 && ((uint64_t*) buffer)[0] != item) || (bytes < 8 && buffer[0] != (char) item)){
|
|
||||||
error = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t * buffi = (uint64_t*) buffer;
|
uint64_t * buffi = (uint64_t*) buffer;
|
||||||
// first half of 64 bits use the rank, here need to apply rank shifting
|
|
||||||
uint64_t rank_mod = ((uint64_t)(pretendRank + 1) << 32) + buff_offset;
|
|
||||||
// the first 8 bytes are set to item number
|
// the first 8 bytes are set to item number
|
||||||
for(size_t i=1; i < bytes/8; i++){
|
int k=1;
|
||||||
uint64_t exp = (i + 1) + rank_mod;
|
unsigned seed = rand_seed + pretendRank;
|
||||||
|
const size_t size = bytes / 8;
|
||||||
|
for(size_t i=0; i < size; i++){
|
||||||
|
uint64_t exp;
|
||||||
|
|
||||||
|
switch(dataPacketType){
|
||||||
|
case(DATA_INCOMPRESSIBLE):{
|
||||||
|
uint64_t hi = ((uint64_t) rand_r(& seed) << 32);
|
||||||
|
uint64_t lo = (uint64_t) rand_r(& seed);
|
||||||
|
exp = hi | lo;
|
||||||
|
break;
|
||||||
|
}case(DATA_OFFSET):{
|
||||||
|
}case(DATA_TIMESTAMP):{
|
||||||
|
exp = ((uint64_t) pretendRank) << 32 | rand_seed + i;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(i % 512 == 0 && dataPacketType != DATA_TIMESTAMP){
|
||||||
|
exp = ((uint32_t) item * k) | ((uint64_t) pretendRank) << 32;
|
||||||
|
k++;
|
||||||
|
}
|
||||||
if(buffi[i] != exp){
|
if(buffi[i] != exp){
|
||||||
error = 1;
|
error = 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for(size_t i=(bytes/8)*8; i < bytes; i++){
|
for(size_t i=size*8; i < bytes; i++){
|
||||||
if(buffer[i] != (char) i){
|
if(buffer[i] != (char) i){
|
||||||
error = 1;
|
error = 1;
|
||||||
}
|
}
|
||||||
|
@ -175,28 +203,28 @@ size_t NodeMemoryStringToBytes(char *size_str)
|
||||||
return mem / 100 * percent;
|
return mem / 100 * percent;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ior_dataPacketType_e parsePacketType(char t){
|
||||||
|
switch(t) {
|
||||||
|
case '\0': return DATA_TIMESTAMP;
|
||||||
|
case 'i': /* Incompressible */
|
||||||
|
return DATA_INCOMPRESSIBLE;
|
||||||
|
case 't': /* timestamp */
|
||||||
|
return DATA_TIMESTAMP;
|
||||||
|
case 'o': /* offset packet */
|
||||||
|
return DATA_OFFSET;
|
||||||
|
default:
|
||||||
|
ERRF("Unknown packet type \"%c\"; generic assumed\n", t);
|
||||||
|
return DATA_OFFSET;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void updateParsedOptions(IOR_param_t * options, options_all_t * global_options){
|
void updateParsedOptions(IOR_param_t * options, options_all_t * global_options){
|
||||||
if (options->setTimeStampSignature){
|
if (options->setTimeStampSignature){
|
||||||
options->incompressibleSeed = options->setTimeStampSignature;
|
options->incompressibleSeed = options->setTimeStampSignature;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (options->buffer_type && options->buffer_type[0] != 0){
|
if (options->buffer_type && options->buffer_type[0] != 0){
|
||||||
switch(options->buffer_type[0]) {
|
options->dataPacketType = parsePacketType(options->buffer_type[0]);
|
||||||
case 'i': /* Incompressible */
|
|
||||||
options->dataPacketType = incompressible;
|
|
||||||
break;
|
|
||||||
case 't': /* timestamp */
|
|
||||||
options->dataPacketType = timestamp;
|
|
||||||
break;
|
|
||||||
case 'o': /* offset packet */
|
|
||||||
options->storeFileOffset = TRUE;
|
|
||||||
options->dataPacketType = offset;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
fprintf(out_logfile,
|
|
||||||
"Unknown argument for -l %s; generic assumed\n", options->buffer_type);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (options->memoryPerNodeStr){
|
if (options->memoryPerNodeStr){
|
||||||
options->memoryPerNode = NodeMemoryStringToBytes(options->memoryPerNodeStr);
|
options->memoryPerNode = NodeMemoryStringToBytes(options->memoryPerNodeStr);
|
||||||
|
|
|
@ -35,10 +35,11 @@ extern enum OutputFormat_t outputFormat; /* format of the output */
|
||||||
void* safeMalloc(uint64_t size);
|
void* safeMalloc(uint64_t size);
|
||||||
void set_o_direct_flag(int *fd);
|
void set_o_direct_flag(int *fd);
|
||||||
|
|
||||||
void update_write_memory_pattern(uint64_t item, char * buf, size_t bytes, int buff_offset, int rank);
|
ior_dataPacketType_e parsePacketType(char t);
|
||||||
void generate_memory_pattern(char * buf, size_t bytes, int buff_offset, int rank);
|
void update_write_memory_pattern(uint64_t item, char * buf, size_t bytes, int rand_seed, int rank, ior_dataPacketType_e dataPacketType);
|
||||||
|
void generate_memory_pattern(char * buf, size_t bytes, int rand_seed, int rank, ior_dataPacketType_e dataPacketType);
|
||||||
/* check a data buffer, @return 0 if all is correct, otherwise 1 */
|
/* check a data buffer, @return 0 if all is correct, otherwise 1 */
|
||||||
int verify_memory_pattern(int item, char * buffer, size_t bytes, int buff_offset, int pretendRank);
|
int verify_memory_pattern(uint64_t item, char * buffer, size_t bytes, int rand_seed, int pretendRank, ior_dataPacketType_e dataPacketType);
|
||||||
|
|
||||||
char *CurrentTimeString(void);
|
char *CurrentTimeString(void);
|
||||||
int Regex(char *, char *);
|
int Regex(char *, char *);
|
||||||
|
|
|
@ -44,4 +44,9 @@ MDWB 3 -a POSIX -O=1 -D=2 -G=10 -P=4 -I=3 -2 -W -w 1 --read-only --run-info-file
|
||||||
MDWB 3 -a POSIX -O=1 -D=2 -G=10 -P=4 -I=3 -2 -W -w 1 --read-only --run-info-file=mdw.tst --print-detailed-stats
|
MDWB 3 -a POSIX -O=1 -D=2 -G=10 -P=4 -I=3 -2 -W -w 1 --read-only --run-info-file=mdw.tst --print-detailed-stats
|
||||||
MDWB 3 -a POSIX -O=1 -D=2 -G=10 -P=4 -I=3 -3 -W -w 1 --run-info-file=mdw.tst --print-detailed-stats
|
MDWB 3 -a POSIX -O=1 -D=2 -G=10 -P=4 -I=3 -3 -W -w 1 --run-info-file=mdw.tst --print-detailed-stats
|
||||||
|
|
||||||
|
MDWB 2 -a POSIX -O=1 -D=1 -G=3 -P=2 -I=2 -R=2 -X -S 772 --dataPacketType=t
|
||||||
|
DELETE=0
|
||||||
|
MDWB 2 -a POSIX -D=1 -P=2 -I=2 -R=2 -X -G=2252 -S 772 --dataPacketType=i -1
|
||||||
|
MDWB 2 -a POSIX -D=1 -P=2 -I=2 -R=2 -X -G=2252 -S 772 --dataPacketType=i -2
|
||||||
|
MDWB 2 -a POSIX -D=1 -P=2 -I=2 -R=2 -X -G=2252 -S 772 --dataPacketType=i -3
|
||||||
END
|
END
|
||||||
|
|
|
@ -90,6 +90,9 @@ function MDTEST(){
|
||||||
function MDWB(){
|
function MDWB(){
|
||||||
RANKS=$1
|
RANKS=$1
|
||||||
shift
|
shift
|
||||||
|
if [[ "$DELETE" != "0" ]] ; then
|
||||||
|
rm -rf "${IOR_TMP}/md-workbench"
|
||||||
|
fi
|
||||||
WHAT="${IOR_MPIRUN} $RANKS ${IOR_BIN_DIR}/md-workbench ${@} -o ${IOR_TMP}/md-workbench ${MDWB_EXTRA}"
|
WHAT="${IOR_MPIRUN} $RANKS ${IOR_BIN_DIR}/md-workbench ${@} -o ${IOR_TMP}/md-workbench ${MDWB_EXTRA}"
|
||||||
LOG="${IOR_OUT}/test_out.$I"
|
LOG="${IOR_OUT}/test_out.$I"
|
||||||
$WHAT 1>"$LOG" 2>&1
|
$WHAT 1>"$LOG" 2>&1
|
||||||
|
|
Loading…
Reference in New Issue