S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
/* -*- mode: c; indent-tabs-mode: nil; -*-
|
|
|
|
* vim:expandtab:
|
|
|
|
*
|
|
|
|
* NOTE: Someone was setting indent-sizes in the mode-line. Don't do that.
|
|
|
|
* 8-chars of indenting is ridiculous. If you really want 8-spaces,
|
|
|
|
* then change the mode-line to use tabs, and configure your personal
|
|
|
|
* editor environment to use 8-space tab-stops.
|
2017-10-20 19:02:24 +03:00
|
|
|
*
|
2012-01-09 00:51:04 +04:00
|
|
|
*/
|
2011-06-17 23:20:43 +04:00
|
|
|
/******************************************************************************\
|
|
|
|
* *
|
|
|
|
* Copyright (c) 2003, The Regents of the University of California *
|
|
|
|
* See the file COPYRIGHT for a complete copyright notice and license. *
|
|
|
|
* *
|
|
|
|
\******************************************************************************/
|
|
|
|
|
|
|
|
#ifndef _IOR_H
|
|
|
|
#define _IOR_H
|
|
|
|
|
2011-10-28 03:50:05 +04:00
|
|
|
#ifdef HAVE_CONFIG_H
|
2014-08-14 02:53:24 +04:00
|
|
|
# include "config.h"
|
|
|
|
#endif
|
2014-09-18 21:20:37 +04:00
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
#ifdef USE_HDFS_AIORI
|
|
|
|
# include <hdfs.h> /* hdfsFS */
|
2014-09-18 21:20:37 +04:00
|
|
|
#else
|
|
|
|
# include <stdint.h>
|
|
|
|
typedef uint16_t tPort; /* unused, but needs a type */
|
|
|
|
typedef void* hdfsFS; /* unused, but needs a type */
|
2011-10-28 03:50:05 +04:00
|
|
|
#endif
|
|
|
|
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
#ifdef USE_S3_AIORI
|
|
|
|
# include <curl/curl.h>
|
|
|
|
# include "aws4c.h"
|
|
|
|
#else
|
|
|
|
typedef void CURL; /* unused, but needs a type */
|
|
|
|
typedef void IOBuf; /* unused, but needs a type */
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
#include "iordef.h"
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
extern int numTasksWorld;
|
|
|
|
extern int rank;
|
|
|
|
extern int rankOffset;
|
|
|
|
extern int tasksPerNode;
|
|
|
|
extern int verbose;
|
|
|
|
extern MPI_Comm testComm;
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2015-05-21 21:05:56 +03:00
|
|
|
|
|
|
|
/******************** DATA Packet Type ***************************************/
|
|
|
|
/* Holds the types of data packets: generic, offset, timestamp, incompressible */
|
|
|
|
|
|
|
|
enum PACKET_TYPE
|
|
|
|
{
|
|
|
|
generic = 0, /* No packet type specified */
|
|
|
|
timestamp=1, /* Timestamp packet set with -l */
|
|
|
|
offset=2, /* Offset packet set with -l */
|
|
|
|
incompressible=3 /* Incompressible packet set with -l */
|
|
|
|
|
|
|
|
};
|
|
|
|
|
2015-05-27 19:24:52 +03:00
|
|
|
|
|
|
|
|
|
|
|
/***************** IOR_BUFFERS *************************************************/
|
|
|
|
/* A struct to hold the buffers so we can pass 1 pointer around instead of 3
|
|
|
|
*/
|
|
|
|
|
2017-10-20 19:02:24 +03:00
|
|
|
typedef struct IO_BUFFERS
|
2015-05-27 19:24:52 +03:00
|
|
|
{
|
|
|
|
void* buffer;
|
|
|
|
void* checkBuffer;
|
|
|
|
void* readCheckBuffer;
|
|
|
|
|
|
|
|
} IOR_io_buffers;
|
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
/******************************************************************************/
|
|
|
|
/*
|
|
|
|
* The parameter struct holds all of the "global" data to be passed,
|
|
|
|
* as well as results to be parsed.
|
|
|
|
*
|
|
|
|
* NOTE: If IOR_Param_t is changed, also change:
|
|
|
|
* init_IOR_Param_t() [ior.c]
|
|
|
|
* DisplayUsage() [ior.c]
|
|
|
|
* ShowTest() [ior.c]
|
|
|
|
* DecodeDirective() [parse_options.c]
|
|
|
|
* ParseCommandLine() [parse_options.c]
|
|
|
|
* USER_GUIDE
|
|
|
|
*/
|
|
|
|
|
|
|
|
typedef struct
|
|
|
|
{
|
|
|
|
char debug[MAX_STR]; /* debug info string */
|
|
|
|
unsigned int mode; /* file permissions */
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
unsigned int openFlags; /* open flags (see also <open>) */
|
2012-01-14 02:05:13 +04:00
|
|
|
int referenceNumber; /* user supplied reference number */
|
2011-11-12 04:40:45 +04:00
|
|
|
char api[MAX_STR]; /* API for I/O */
|
|
|
|
char apiVersion[MAX_STR]; /* API version */
|
|
|
|
char platform[MAX_STR]; /* platform type */
|
|
|
|
char testFileName[MAXPATHLEN]; /* full name for test */
|
|
|
|
char testFileName_fppReadCheck[MAXPATHLEN];/* filename for fpp read check */
|
|
|
|
char hintsFileName[MAXPATHLEN]; /* full name for hints file */
|
|
|
|
char options[MAXPATHLEN]; /* options string */
|
|
|
|
int numTasks; /* number of tasks for test */
|
|
|
|
int nodes; /* number of nodes for test */
|
|
|
|
int tasksPerNode; /* number of tasks per node */
|
|
|
|
int repetitions; /* number of repetitions of test */
|
|
|
|
int repCounter; /* rep counter */
|
|
|
|
int multiFile; /* multiple files */
|
|
|
|
int interTestDelay; /* delay between reps in seconds */
|
|
|
|
int open; /* flag for writing or reading */
|
|
|
|
int readFile; /* read of existing file */
|
|
|
|
int writeFile; /* write of file */
|
|
|
|
int filePerProc; /* single file or file-per-process */
|
|
|
|
int reorderTasks; /* reorder tasks for read back and check */
|
|
|
|
int taskPerNodeOffset; /* task node offset for reading files */
|
|
|
|
int reorderTasksRandom; /* reorder tasks for random file read back */
|
|
|
|
int reorderTasksRandomSeed; /* reorder tasks for random file read seed */
|
|
|
|
int checkWrite; /* check read after write */
|
|
|
|
int checkRead; /* check read after read */
|
|
|
|
int keepFile; /* don't delete the testfile on exit */
|
|
|
|
int keepFileWithError; /* don't delete the testfile with errors */
|
|
|
|
int errorFound; /* error found in data check */
|
|
|
|
int quitOnError; /* quit code when error in check */
|
|
|
|
int collective; /* collective I/O */
|
|
|
|
IOR_offset_t segmentCount; /* number of segments (or HDF5 datasets) */
|
|
|
|
IOR_offset_t blockSize; /* contiguous bytes to write per task */
|
|
|
|
IOR_offset_t transferSize; /* size of transfer in bytes */
|
|
|
|
IOR_offset_t offset; /* offset for read/write */
|
2011-12-13 09:00:18 +04:00
|
|
|
IOR_offset_t expectedAggFileSize; /* calculated aggregate file size */
|
2011-11-12 04:40:45 +04:00
|
|
|
int preallocate; /* preallocate file size */
|
|
|
|
int useFileView; /* use MPI_File_set_view */
|
|
|
|
int useSharedFilePointer; /* use shared file pointer */
|
|
|
|
int useStridedDatatype; /* put strided access into datatype */
|
|
|
|
int useO_DIRECT; /* use O_DIRECT, bypassing I/O buffers */
|
|
|
|
int showHints; /* show hints */
|
|
|
|
int showHelp; /* show options and help */
|
2012-01-14 01:27:55 +04:00
|
|
|
int summary_every_test; /* flag to print summary every test, not just at end */
|
2011-11-12 04:40:45 +04:00
|
|
|
int uniqueDir; /* use unique directory for each fpp */
|
|
|
|
int useExistingTestFile; /* do not delete test file before access */
|
|
|
|
int storeFileOffset; /* use file offset as stored signature */
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
int deadlineForStonewalling; /* max time in seconds to run any test phase */
|
2017-10-20 19:02:24 +03:00
|
|
|
int stoneWallingWearOut; /* wear out the stonewalling, once the timout is over, each process has to write the same amount */
|
2017-10-20 19:13:29 +03:00
|
|
|
int stoneWallingWearOutIterations; /* the number of iterations for the stonewallingWearOut, needed for readBack */
|
2012-01-09 00:30:05 +04:00
|
|
|
int maxTimeDuration; /* max time in minutes to run each test */
|
2011-11-12 04:40:45 +04:00
|
|
|
int outlierThreshold; /* warn on outlier N seconds from mean */
|
|
|
|
int verbose; /* verbosity */
|
|
|
|
int setTimeStampSignature; /* set time stamp signature */
|
|
|
|
unsigned int timeStampSignatureValue; /* value for time stamp signature */
|
|
|
|
void * fd_fppReadCheck; /* additional fd for fpp read check */
|
|
|
|
int randomSeed; /* random seed for write/read check */
|
2015-05-21 21:05:56 +03:00
|
|
|
int incompressibleSeed; /* random seed for incompressible file creation */
|
2011-11-12 04:40:45 +04:00
|
|
|
int randomOffset; /* access is to random offsets */
|
2012-01-09 06:41:30 +04:00
|
|
|
size_t memoryPerTask; /* additional memory used per task */
|
|
|
|
size_t memoryPerNode; /* additional memory used per node */
|
2015-05-21 21:05:56 +03:00
|
|
|
enum PACKET_TYPE dataPacketType; /* The type of data packet. */
|
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
|
|
|
|
/* POSIX variables */
|
|
|
|
int singleXferAttempt; /* do not retry transfer if incomplete */
|
|
|
|
int fsyncPerWrite; /* fsync() after each write */
|
|
|
|
int fsync; /* fsync() after write */
|
|
|
|
|
|
|
|
/* MPI variables */
|
2014-08-14 02:53:24 +04:00
|
|
|
MPI_Comm testComm; /* MPI communicator */
|
2011-11-12 04:40:45 +04:00
|
|
|
MPI_Datatype transferType; /* datatype for transfer */
|
|
|
|
MPI_Datatype fileType; /* filetype for file view */
|
|
|
|
|
|
|
|
/* HDF5 variables */
|
|
|
|
int individualDataSets; /* datasets not shared by all procs */
|
|
|
|
int noFill; /* no fill in file creation */
|
|
|
|
IOR_offset_t setAlignment; /* alignment in bytes */
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
/* HDFS variables */
|
2014-08-29 01:39:44 +04:00
|
|
|
char hdfs_user[MAX_STR]; /* copied from ENV, for now */
|
2014-08-14 02:53:24 +04:00
|
|
|
const char* hdfs_name_node;
|
|
|
|
tPort hdfs_name_node_port; /* (uint16_t) */
|
|
|
|
hdfsFS hdfs_fs; /* file-system handle */
|
|
|
|
int hdfs_replicas; /* n block replicas. (0 gets default) */
|
|
|
|
int hdfs_block_size; /* internal blk-size. (0 gets default) */
|
|
|
|
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
/* REST/S3 variables */
|
|
|
|
// CURL* curl; /* for libcurl "easy" fns (now managed by aws4c) */
|
Algorithms 'S3', 'S3_plus', and 'S3_EMC' all available.
These are variants on S3. S3 uses the "pure" S3 interface, e.g. using
Multi-Part-Upload. The "plus" variant enables EMC-extensions in the aws4c
library. This allows the N:N case to use "append", in the case where
"transfer_size" != "block_size" for IOR. In pure S3, the N:N case will
fail, because the EMC-extensions won't be enabled, and appending (which
attempts to use the EMC byte-range tricks to do this) will throw an error.
In the S3_EMC alg, N:1 uses EMCs other byte-range tricks to write different
parts of an N:1 file, and also uses append to write the parts of an N:N
file. Preliminary tests show these EMC extensions look to improve BW by
~20%.
I put all three algs in aiori-S3.c, because it seemed some code was getting
reused. Not sure if that's still going to make sense after the TBD, below.
TBD: Recently realized that the "pure' S3 shouldn't be trying to use
appends for anything. In the N:N case, it should just use MPU, within each
file. Then, there's no need for S3_plus. We just have S3, which does MPU
for all writes where transfer_size != block_size, and uses (standard)
byte-range reads for reading. Then S3_EMC uses "appends for N:N writes,
and byte-range writes for N:1 writes. This separates the code for the two
algs a little more, but we might still want them in the same file.
2014-10-30 01:04:30 +03:00
|
|
|
# define IOR_CURL_INIT 0x01 /* curl top-level inits were perfomed once? */
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
# define IOR_CURL_NOCONTINUE 0x02
|
Algorithms 'S3', 'S3_plus', and 'S3_EMC' all available.
These are variants on S3. S3 uses the "pure" S3 interface, e.g. using
Multi-Part-Upload. The "plus" variant enables EMC-extensions in the aws4c
library. This allows the N:N case to use "append", in the case where
"transfer_size" != "block_size" for IOR. In pure S3, the N:N case will
fail, because the EMC-extensions won't be enabled, and appending (which
attempts to use the EMC byte-range tricks to do this) will throw an error.
In the S3_EMC alg, N:1 uses EMCs other byte-range tricks to write different
parts of an N:1 file, and also uses append to write the parts of an N:N
file. Preliminary tests show these EMC extensions look to improve BW by
~20%.
I put all three algs in aiori-S3.c, because it seemed some code was getting
reused. Not sure if that's still going to make sense after the TBD, below.
TBD: Recently realized that the "pure' S3 shouldn't be trying to use
appends for anything. In the N:N case, it should just use MPU, within each
file. Then, there's no need for S3_plus. We just have S3, which does MPU
for all writes where transfer_size != block_size, and uses (standard)
byte-range reads for reading. Then S3_EMC uses "appends for N:N writes,
and byte-range writes for N:1 writes. This separates the code for the two
algs a little more, but we might still want them in the same file.
2014-10-30 01:04:30 +03:00
|
|
|
# define IOR_CURL_S3_EMC_EXT 0x04 /* allow EMC extensions to S3? */
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
char curl_flags;
|
|
|
|
char* URI; /* "path" to target object */
|
|
|
|
IOBuf* io_buf; /* aws4c places parsed header values here */
|
|
|
|
IOBuf* etags; /* accumulate ETags for N:1 parts */
|
|
|
|
size_t part_number; /* multi-part upload increment (PER-RANK!) */
|
|
|
|
# define MAX_UPLOAD_ID_SIZE 256 /* seems to be 32, actually */
|
|
|
|
char UploadId[MAX_UPLOAD_ID_SIZE +1]; /* key for multi-part-uploads */
|
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
/* NCMPI variables */
|
|
|
|
int var_id; /* variable id handle for data set */
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
/* Lustre variables */
|
|
|
|
int lustre_stripe_count;
|
|
|
|
int lustre_stripe_size;
|
|
|
|
int lustre_start_ost;
|
|
|
|
int lustre_set_striping; /* flag that we need to set lustre striping */
|
|
|
|
int lustre_ignore_locks;
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2013-09-26 17:48:50 +04:00
|
|
|
/* gpfs variables */
|
|
|
|
int gpfs_hint_access; /* use gpfs "access range" hint */
|
|
|
|
int gpfs_release_token; /* immediately release GPFS tokens after
|
|
|
|
creating or opening a file */
|
2017-09-21 18:12:31 +03:00
|
|
|
/* beegfs variables */
|
|
|
|
int beegfs_numTargets; /* number storage targets to use */
|
|
|
|
int beegfs_chunkSize; /* srtipe pattern for new files */
|
2013-09-26 17:48:50 +04:00
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
int id; /* test's unique ID */
|
|
|
|
int intraTestBarriers; /* barriers between open/op and op/close */
|
|
|
|
} IOR_param_t;
|
|
|
|
|
2011-12-13 09:00:18 +04:00
|
|
|
/* each pointer is to an array, each of length equal to the number of
|
|
|
|
repetitions in the test */
|
|
|
|
typedef struct {
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
double *writeTime;
|
|
|
|
double *readTime;
|
2017-10-20 19:02:24 +03:00
|
|
|
size_t pairs_accessed; // number of I/Os done, useful for deadlineForStonewalling
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
IOR_offset_t *aggFileSizeFromStat;
|
|
|
|
IOR_offset_t *aggFileSizeFromXfer;
|
|
|
|
IOR_offset_t *aggFileSizeForBW;
|
2011-12-13 09:00:18 +04:00
|
|
|
} IOR_results_t;
|
2011-11-12 04:40:45 +04:00
|
|
|
|
2011-12-13 09:00:18 +04:00
|
|
|
/* define the queuing structure for the test parameters */
|
|
|
|
typedef struct IOR_test_t {
|
S3 with Multi-Part Upload for N:1 is working.
Testing on our EMC ViPR installation. Therefore, we also have available
some EMC extensions. For example, EMC supports a special "byte-range"
header-option ("Range: bytes=-1-") which allows appending to an object.
This is not needed for N:1 (where every write creates an independent part),
but is vital for N:N (where every write is considered an append, unless
"transfer-size" is the same as "block-size").
We also use a LANL-extended implementation of aws4c 0.5, which provides
some special features, and allows greater efficiency. That is included in
this commit as a tarball. Untar it somewhere else and build it, to produce
a library, which is linked with IOR. (configure with --with-S3).
TBD: EMC also supports a simpler alternative to Multi-Part Upload, which
appears to have several advantages. We'll add that in next, but wanted to
capture this as is, before I break it.
2014-10-27 22:16:20 +03:00
|
|
|
IOR_param_t params;
|
|
|
|
IOR_results_t *results;
|
|
|
|
struct IOR_test_t *next;
|
2011-12-13 09:00:18 +04:00
|
|
|
} IOR_test_t;
|
|
|
|
|
2014-08-14 02:53:24 +04:00
|
|
|
|
2011-12-13 09:00:18 +04:00
|
|
|
IOR_test_t *CreateTest(IOR_param_t *init_params, int test_num);
|
|
|
|
void AllocResults(IOR_test_t *test);
|
2011-11-12 04:40:45 +04:00
|
|
|
void GetPlatformName(char *);
|
|
|
|
void init_IOR_Param_t(IOR_param_t *p);
|
2011-06-17 23:20:43 +04:00
|
|
|
|
2011-11-12 04:40:45 +04:00
|
|
|
#endif /* !_IOR_H */
|