vitalif
/
mdtest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Removed offset from IOR structure.

master
Julian M. Kunkel 2020-06-10 17:47:07 +01:00
parent 8fa8ef0c02
commit eb2cd27fac
9 changed files with 42 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/aiori-DUMMY.c
View File

@ -116,7 +116,7 @@ static IOR_offset_t DUMMY_GetFileSize(aiori_mod_opt_t * options, MPI_Comm testCo
return 0;
}
static IOR_offset_t DUMMY_Xfer(int access, aiori_fd_t *file, IOR_size_t * buffer, IOR_offset_t length, aiori_mod_opt_t * options){
static IOR_offset_t DUMMY_Xfer(int access, aiori_fd_t *file, IOR_size_t * buffer, IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t * options){
if(verbose > 4){
fprintf(out_logfile, "DUMMY xfer: %p\n", file);
}

8
src/aiori-HDF5.c
View File

@ -86,7 +86,7 @@ static void SetupDataSet(void *, int flags, aiori_mod_opt_t *);
static aiori_fd_t *HDF5_Create(char *, int flags, aiori_mod_opt_t *);
static aiori_fd_t *HDF5_Open(char *, int flags, aiori_mod_opt_t *);
static IOR_offset_t HDF5_Xfer(int, aiori_fd_t *, IOR_size_t *,
IOR_offset_t, aiori_mod_opt_t *);
IOR_offset_t, IOR_offset_t, aiori_mod_opt_t *);
static void HDF5_Close(aiori_fd_t *, aiori_mod_opt_t *);
static void HDF5_Delete(char *, aiori_mod_opt_t *);
static char* HDF5_GetVersion();
@ -417,7 +417,7 @@ static aiori_fd_t *HDF5_Open(char *testFileName, int flags, aiori_mod_opt_t * pa
* Write or read access to file using the HDF5 interface.
*/
static IOR_offset_t HDF5_Xfer(int access, aiori_fd_t *fd, IOR_size_t * buffer,
IOR_offset_t length, aiori_mod_opt_t * param)
IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t * param)
{
static int firstReadCheck = FALSE, startNewDataSet;
IOR_offset_t segmentPosition, segmentSize;
@ -445,7 +445,7 @@ static IOR_offset_t HDF5_Xfer(int access, aiori_fd_t *fd, IOR_size_t * buffer,
* hints->blockSize;
segmentSize = (IOR_offset_t) (hints->numTasks) * hints->blockSize;
}
if ((IOR_offset_t) ((hints->offset - segmentPosition) % segmentSize) ==
if ((IOR_offset_t) ((offset - segmentPosition) % segmentSize) ==
0) {
/*
* ordinarily start a new data set, unless this is the
@ -471,7 +471,7 @@ static IOR_offset_t HDF5_Xfer(int access, aiori_fd_t *fd, IOR_size_t * buffer,
SetupDataSet(fd, access == WRITE ? IOR_CREAT : IOR_RDWR, param);
}
SeekOffset(fd, hints->offset, param);
SeekOffset(fd, offset, param);
/* this is necessary to reset variables for reaccessing file */
startNewDataSet = FALSE;

12
src/aiori-MMAP.c
View File

@ -29,7 +29,7 @@
static aiori_fd_t *MMAP_Create(char *, int flags, aiori_mod_opt_t *);
static aiori_fd_t *MMAP_Open(char *, int flags, aiori_mod_opt_t *);
static IOR_offset_t MMAP_Xfer(int, aiori_fd_t *, IOR_size_t *,
IOR_offset_t, aiori_mod_opt_t *);
IOR_offset_t, IOR_offset_t, aiori_mod_opt_t *);
static void MMAP_Close(aiori_fd_t *, aiori_mod_opt_t *);
static void MMAP_Fsync(aiori_fd_t *, aiori_mod_opt_t *);
static option_help * MMAP_options(aiori_mod_opt_t ** init_backend_options, aiori_mod_opt_t * init_values);
@ -156,19 +156,19 @@ static aiori_fd_t *MMAP_Open(char *testFileName, int flags, aiori_mod_opt_t * pa
* Write or read access to file using mmap
*/
static IOR_offset_t MMAP_Xfer(int access, aiori_fd_t *file, IOR_size_t * buffer,
IOR_offset_t length, aiori_mod_opt_t * param)
IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t * param)
{
mmap_options_t *o = (mmap_options_t*) param;
if (access == WRITE) {
memcpy(o->mmap_ptr + hints->offset, buffer, length);
memcpy(o->mmap_ptr + offset, buffer, length);
} else {
memcpy(buffer, o->mmap_ptr + hints->offset, length);
memcpy(buffer, o->mmap_ptr + offset, length);
}
if (hints->fsyncPerWrite == TRUE) {
if (msync(o->mmap_ptr + hints->offset, length, MS_SYNC) != 0)
if (msync(o->mmap_ptr + offset, length, MS_SYNC) != 0)
ERR("msync() failed");
if (posix_madvise(o->mmap_ptr + hints->offset, length,
if (posix_madvise(o->mmap_ptr + offset, length,
POSIX_MADV_DONTNEED) != 0)
ERR("madvise() failed");
}

12
src/aiori-MPIIO.c
View File

@ -36,7 +36,7 @@ static IOR_offset_t SeekOffset(MPI_File, IOR_offset_t, aiori_mod_opt_t *);
static aiori_fd_t *MPIIO_Create(char *, int iorflags, aiori_mod_opt_t *);
static aiori_fd_t *MPIIO_Open(char *, int flags, aiori_mod_opt_t *);
static IOR_offset_t MPIIO_Xfer(int, aiori_fd_t *, IOR_size_t *,
IOR_offset_t, aiori_mod_opt_t *);
IOR_offset_t, IOR_offset_t, aiori_mod_opt_t *);
static void MPIIO_Close(aiori_fd_t *, aiori_mod_opt_t *);
static char* MPIIO_GetVersion();
static void MPIIO_Fsync(aiori_fd_t *, aiori_mod_opt_t *);
@ -327,7 +327,7 @@ static aiori_fd_t *MPIIO_Open(char *testFileName, int flags, aiori_mod_opt_t * m
* Write or read access to file using the MPIIO interface.
*/
static IOR_offset_t MPIIO_Xfer(int access, aiori_fd_t * fdp, IOR_size_t * buffer,
IOR_offset_t length, aiori_mod_opt_t * module_options)
IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t * module_options)
{
/* NOTE: The second arg is (void *) for reads, and (const void *)
for writes. Therefore, one of the two sets of assignments below
@ -386,7 +386,7 @@ static IOR_offset_t MPIIO_Xfer(int access, aiori_fd_t * fdp, IOR_size_t * buffer
*/
if (param->useFileView) {
/* find offset in file */
if (SeekOffset(mfd->fd, hints->offset, module_options) <
if (SeekOffset(mfd->fd, offset, module_options) <
0) {
/* if unsuccessful */
length = -1;
@ -425,7 +425,7 @@ static IOR_offset_t MPIIO_Xfer(int access, aiori_fd_t * fdp, IOR_size_t * buffer
if (param->useSharedFilePointer) {
/* find offset in file */
if (SeekOffset
(mfd->fd, hints->offset, module_options) < 0) {
(mfd->fd, offset, module_options) < 0) {
/* if unsuccessful */
length = -1;
} else {
@ -444,13 +444,13 @@ static IOR_offset_t MPIIO_Xfer(int access, aiori_fd_t * fdp, IOR_size_t * buffer
if (hints->collective) {
/* explicit, collective call */
MPI_CHECK(Access_at_all
(mfd->fd, hints->offset,
(mfd->fd, offset,
buffer, length, MPI_BYTE, &status),
"cannot access explicit, collective");
} else {
/* explicit, noncollective call */
MPI_CHECK(Access_at
(mfd->fd, hints->offset,
(mfd->fd, offset,
buffer, length, MPI_BYTE, &status),
"cannot access explicit, noncollective");
}

14
src/aiori-POSIX.c
View File

@ -69,7 +69,7 @@
/**************************** P R O T O T Y P E S *****************************/
static IOR_offset_t POSIX_Xfer(int, aiori_fd_t *, IOR_size_t *,
IOR_offset_t, aiori_mod_opt_t *);
IOR_offset_t, IOR_offset_t, aiori_mod_opt_t *);
static void POSIX_Fsync(aiori_fd_t *, aiori_mod_opt_t *);
static void POSIX_Sync(aiori_mod_opt_t * );
static int POSIX_check_params(aiori_mod_opt_t * options);
@ -547,7 +547,7 @@ aiori_fd_t *POSIX_Open(char *testFileName, int flags, aiori_mod_opt_t * param)
* Write or read access to file using the POSIX interface.
*/
static IOR_offset_t POSIX_Xfer(int access, aiori_fd_t *file, IOR_size_t * buffer,
IOR_offset_t length, aiori_mod_opt_t * param)
IOR_offset_t length, IOR_offset_t offset, aiori_mod_opt_t * param)
{
int xferRetries = 0;
long long remaining = (long long)length;
@ -569,8 +569,8 @@ static IOR_offset_t POSIX_Xfer(int access, aiori_fd_t *file, IOR_size_t * buffer
/* seek to offset */
if (lseek64(fd, hints->offset, SEEK_SET) == -1)
ERRF("lseek64(%d, %lld, SEEK_SET) failed", fd, hints->offset);
if (lseek64(fd, offset, SEEK_SET) == -1)
ERRF("lseek64(%d, %lld, SEEK_SET) failed", fd, offset);
while (remaining > 0) {
/* write/read file */
@ -579,7 +579,7 @@ static IOR_offset_t POSIX_Xfer(int access, aiori_fd_t *file, IOR_size_t * buffer
fprintf(stdout,
"task %d writing to offset %lld\n",
rank,
hints->offset + length - remaining);
offset + length - remaining);
}
rc = write(fd, ptr, remaining);
if (rc == -1)
@ -593,7 +593,7 @@ static IOR_offset_t POSIX_Xfer(int access, aiori_fd_t *file, IOR_size_t * buffer
fprintf(stdout,
"task %d reading from offset %lld\n",
rank,
hints->offset + length - remaining);
offset + length - remaining);
}
rc = read(fd, ptr, remaining);
if (rc == 0)
@ -609,7 +609,7 @@ static IOR_offset_t POSIX_Xfer(int access, aiori_fd_t *file, IOR_size_t * buffer
rank,
access == WRITE ? "write()" : "read()",
rc, remaining,
hints->offset + length - remaining);
offset + length - remaining);
if (hints->singleXferAttempt == TRUE)
MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1),
"barrier error");

5
src/aiori.h
View File

@ -76,7 +76,6 @@ typedef struct aiori_xfer_hint_t{
IOR_offset_t segmentCount; /* number of segments (or HDF5 datasets) */
IOR_offset_t blockSize; /* contiguous bytes to write per task */
IOR_offset_t transferSize; /* size of transfer in bytes */
IOR_offset_t offset; /* offset for read/write */
IOR_offset_t expectedAggFileSize; /* calculated aggregate file size */
int singleXferAttempt; /* do not retry transfer if incomplete */
} aiori_xfer_hint_t;
@ -100,8 +99,8 @@ typedef struct ior_aiori {
Allow to set generic transfer options that shall be applied to any subsequent IO call.
*/
void (*xfer_hints)(aiori_xfer_hint_t * params);
IOR_offset_t (*xfer)(int, aiori_fd_t *, IOR_size_t *,
IOR_offset_t, aiori_mod_opt_t *);
IOR_offset_t (*xfer)(int access, aiori_fd_t *, IOR_size_t *,
IOR_offset_t size, IOR_offset_t offset, aiori_mod_opt_t *);
void (*close)(aiori_fd_t *, aiori_mod_opt_t *);
void (*delete)(char *, aiori_mod_opt_t *);
char* (*get_version)(void);

29
src/ior.c
View File

@ -70,7 +70,6 @@ static void ior_set_xfer_hints(IOR_param_t * p){
hints->segmentCount = p->segmentCount;
hints->blockSize = p->blockSize;
hints->transferSize = p->transferSize;
hints->offset = p->offset;
hints->expectedAggFileSize = p->expectedAggFileSize;
hints->singleXferAttempt = p->singleXferAttempt;
@ -385,7 +384,7 @@ CompareBuffers(void *expectedBuffer,
if (verbose >= VERBOSE_3) {
fprintf(out_logfile,
"[%d] At file byte offset %lld, comparing %llu-byte transfer\n",
rank, test->offset, (long long)size);
rank, (long long) offset, (long long)size);
}
for (i = 0; i < length; i++) {
if (testbuf[i] != goodbuf[i]) {
@ -394,7 +393,7 @@ CompareBuffers(void *expectedBuffer,
fprintf(out_logfile,
"[%d] At transfer buffer #%lld, index #%lld (file byte offset %lld):\n",
rank, transferCount - 1, (long long)i,
test->offset +
(long long) offset +
(IOR_size_t) (i * sizeof(IOR_size_t)));
fprintf(out_logfile, "[%d] %s0x", rank, bufferLabel1);
fprintf(out_logfile, "%016llx\n", goodbuf[i]);
@ -410,10 +409,10 @@ CompareBuffers(void *expectedBuffer,
}
} else if (verbose >= VERBOSE_5 && i % 4 == 0) {
fprintf(out_logfile,
"[%d] PASSED offset = %lld bytes, transfer %lld\n",
"[%d] PASSED offset = %lu bytes, transfer %lld\n",
rank,
((i * sizeof(unsigned long long)) +
test->offset), transferCount);
offset), transferCount);
fprintf(out_logfile, "[%d] GOOD %s0x", rank, bufferLabel1);
for (j = 0; j < 4; j++)
fprintf(out_logfile, "%016llx ", goodbuf[i + j]);
@ -436,8 +435,8 @@ CompareBuffers(void *expectedBuffer,
"%lld errors between buffer indices %lld and %lld.\n",
(long long)errorCount, (long long)first,
(long long)last);
fprintf(out_logfile, "[%d] File byte offset = %lld:\n", rank,
((first * sizeof(unsigned long long)) + test->offset));
fprintf(out_logfile, "[%d] File byte offset = %lu:\n", rank,
((first * sizeof(unsigned long long)) + offset));
fprintf(out_logfile, "[%d] %s0x", rank, bufferLabel1);
for (j = first; j < length && j < first + 4; j++)
@ -1808,16 +1807,16 @@ static IOR_offset_t WriteOrReadSingle(IOR_offset_t pairCnt, IOR_offset_t *offset
void *checkBuffer = ioBuffers->checkBuffer;
void *readCheckBuffer = ioBuffers->readCheckBuffer;
test->hints.offset = offsetArray[pairCnt]; // this looks inappropriate
IOR_offset_t offset = offsetArray[pairCnt]; // this looks inappropriate
transfer = test->transferSize;
if (access == WRITE) {
/* fills each transfer with a unique pattern
* containing the offset into the file */
if (test->storeFileOffset == TRUE) {
FillBuffer(buffer, test, test->offset, pretendRank);
FillBuffer(buffer, test, offset, pretendRank);
}
amtXferred = backend->xfer(access, fd, buffer, transfer, test->backend_options);
amtXferred = backend->xfer(access, fd, buffer, transfer, offset, test->backend_options);
if (amtXferred != transfer)
ERR("cannot write to file");
if (test->fsyncPerWrite)
@ -1827,7 +1826,7 @@ static IOR_offset_t WriteOrReadSingle(IOR_offset_t pairCnt, IOR_offset_t *offset
nanosleep( & wait, NULL);
}
} else if (access == READ) {
amtXferred = backend->xfer(access, fd, buffer, transfer, test->backend_options);
amtXferred = backend->xfer(access, fd, buffer, transfer, offset, test->backend_options);
if (amtXferred != transfer)
ERR("cannot read from file");
if (test->interIODelay > 0){
@ -1838,10 +1837,10 @@ static IOR_offset_t WriteOrReadSingle(IOR_offset_t pairCnt, IOR_offset_t *offset
memset(checkBuffer, 'a', transfer);
if (test->storeFileOffset == TRUE) {
FillBuffer(readCheckBuffer, test, test->offset, pretendRank);
FillBuffer(readCheckBuffer, test, offset, pretendRank);
}
amtXferred = backend->xfer(access, fd, checkBuffer, transfer, test->backend_options);
amtXferred = backend->xfer(access, fd, checkBuffer, transfer, offset, test->backend_options);
if (amtXferred != transfer)
ERR("cannot read from file write check");
(*transferCount)++;
@ -1851,12 +1850,12 @@ static IOR_offset_t WriteOrReadSingle(IOR_offset_t pairCnt, IOR_offset_t *offset
} else if (access == READCHECK) {
memset(checkBuffer, 'a', transfer);
amtXferred = backend->xfer(access, fd, checkBuffer, transfer, test->backend_options);
amtXferred = backend->xfer(access, fd, checkBuffer, transfer, offset, test->backend_options);
if (amtXferred != transfer){
ERR("cannot read from file");
}
if (test->storeFileOffset == TRUE) {
FillBuffer(readCheckBuffer, test, test->offset, pretendRank);
FillBuffer(readCheckBuffer, test, offset, pretendRank);
}
*errors += CompareBuffers(readCheckBuffer, checkBuffer, transfer, *transferCount, test, READCHECK);
}

1
src/ior.h
View File

@ -120,7 +120,6 @@ typedef struct
IOR_offset_t segmentCount; /* number of segments (or HDF5 datasets) */
IOR_offset_t blockSize; /* contiguous bytes to write per task */
IOR_offset_t transferSize; /* size of transfer in bytes */
IOR_offset_t offset; /* offset for read/write */
IOR_offset_t expectedAggFileSize; /* calculated aggregate file size */
int summary_every_test; /* flag to print summary every test, not just at end */

5
src/mdtest.c
View File

@ -391,9 +391,8 @@ static void create_file (const char *path, uint64_t itemNum) {
* According to Bill Loewe, writes are only done one time, so they are always at
* offset 0 (zero).
*/
hints.offset = 0;
hints.fsyncPerWrite = sync_file;
if ( write_bytes != (size_t) backend->xfer (WRITE, aiori_fh, (IOR_size_t *) write_buffer, write_bytes, backend_options)) {
if ( write_bytes != (size_t) backend->xfer (WRITE, aiori_fh, (IOR_size_t *) write_buffer, write_bytes, 0, backend_options)) {
FAIL("unable to write file %s", curr_item);
}
}
@ -716,7 +715,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) {
/* read file */
if (read_bytes > 0) {
read_buffer[0] = 42; /* use a random value to ensure that the read_buffer is now different from the expected buffer and read isn't sometimes NOOP */
if (read_bytes != (size_t) backend->xfer (READ, aiori_fh, (IOR_size_t *) read_buffer, read_bytes, backend_options)) {
if (read_bytes != (size_t) backend->xfer (READ, aiori_fh, (IOR_size_t *) read_buffer, read_bytes, 0, backend_options)) {
FAIL("unable to read file %s", item);
}
if(verify_read){