diff --git a/README.md b/README.md index dd9ee1e..81a1fa3 100755 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ See also NOTES.txt # Testing + Run "make check" to invoke the unit test framework of Automake. + * To run basic functionality tests that we use for continuous integration, see ./testing/ * There are docker scripts provided to test various distributions at once. * See ./testing/docker/ diff --git a/configure.ac b/configure.ac index a80908e..3a34720 100755 --- a/configure.ac +++ b/configure.ac @@ -26,6 +26,7 @@ AX_PROG_CC_MPI(,,[ AC_MSG_FAILURE([MPI compiler requested, but couldn't use MPI.]) ]) +AC_PROG_RANLIB # No reason not to require modern C at this point AC_PROG_CC_C99 @@ -244,5 +245,6 @@ AM_CONDITIONAL([USE_CAPS], [test x$enable_caps = xyes]) AC_CONFIG_FILES([Makefile src/Makefile contrib/Makefile + src/test/Makefile doc/Makefile]) AC_OUTPUT diff --git a/src/Makefile.am b/src/Makefile.am index 9c27d89..0f1c913 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,3 +1,5 @@ +SUBDIRS = . test + bin_PROGRAMS = ior mdtest if USE_CAPS bin_PROGRAMS += IOR MDTEST @@ -5,6 +7,9 @@ endif noinst_HEADERS = ior.h utilities.h parse_options.h aiori.h iordef.h ior-internal.h option.h +lib_LIBRARIES = libaiori.a +libaiori_a_SOURCES = ior.c mdtest.c utilities.c parse_options.c ior-output.c option.c + extraSOURCES = aiori.c aiori-DUMMY.c extraLDADD = extraLDFLAGS = @@ -88,3 +93,5 @@ IOT_CPPFLAGS = $(ior_CPPFLAGS) MDTEST_SOURCES = $(mdtest_SOURCES) MDTEST_LDADD = $(mdtest_LDADD) MDTEST_CPPFLAGS = $(mdtest_CPPFLAGS) + +libaiori_a_SOURCES += $(extraSOURCES) diff --git a/src/aiori-S3.c b/src/aiori-S3.c index 955e0f0..4836dd1 100755 --- a/src/aiori-S3.c +++ b/src/aiori-S3.c @@ -110,6 +110,39 @@ #include "aws4c_extra.h" // utilities, e.g. for parsing XML in responses + + +/* buffer is used to generate URLs, err_msgs, etc */ +#define BUFF_SIZE 1024 +static char buff[BUFF_SIZE]; + +const int ETAG_SIZE = 32; + +CURLcode rc; + +/* Any objects we create or delete will be under this bucket */ +const char* bucket_name = "ior"; + +/* TODO: The following stuff goes into options! */ +/* REST/S3 variables */ +// CURL* curl; /* for libcurl "easy" fns (now managed by aws4c) */ +# define IOR_CURL_INIT 0x01 /* curl top-level inits were perfomed once? */ +# define IOR_CURL_NOCONTINUE 0x02 +# define IOR_CURL_S3_EMC_EXT 0x04 /* allow EMC extensions to S3? */ + +#ifdef USE_S3_AIORI +# include +# include "aws4c.h" +#else + typedef void CURL; /* unused, but needs a type */ + typedef void IOBuf; /* unused, but needs a type */ +#endif + + IOBuf* io_buf; /* aws4c places parsed header values here */ + IOBuf* etags; /* accumulate ETags for N:1 parts */ + +/////////////////////////////////////////////// + /**************************** P R O T O T Y P E S *****************************/ static void* S3_Create(char*, IOR_param_t*); static void* S3_Open(char*, IOR_param_t*); @@ -180,17 +213,23 @@ ior_aiori_t s3_emc_aiori = { .finalize = S3_finalize }; +static int is_initialized = FALSE; + static void S3_init(){ - /* This is supposed to be done before *any* threads are created. - * Could MPI_Init() create threads (or call multi-threaded - * libraries)? We'll assume so. */ - AWS4C_CHECK( aws_init() ); + if (is_initialized) return; + is_initialized = TRUE; + /* This is supposed to be done before *any* threads are created. + * Could MPI_Init() create threads (or call multi-threaded + * libraries)? We'll assume so. */ + AWS4C_CHECK( aws_init() ); } static void S3_finalize(){ - /* done once per program, after exiting all threads. - * NOTE: This fn doesn't return a value that can be checked for success. */ - aws_cleanup(); + if (! is_initialized) return; + is_initialized = FALSE; + /* done once per program, after exiting all threads. + * NOTE: This fn doesn't return a value that can be checked for success. */ + aws_cleanup(); } /* modelled on similar macros in iordef.h */ @@ -213,20 +252,6 @@ static void S3_finalize(){ } while (0) - -/* buffer is used to generate URLs, err_msgs, etc */ -#define BUFF_SIZE 1024 -static char buff[BUFF_SIZE]; - -const int ETAG_SIZE = 32; - -CURLcode rc; - -/* Any objects we create or delete will be under this bucket */ -const char* bucket_name = "ior"; -//const char* bucket_name = "brettk"; - - /***************************** F U N C T I O N S ******************************/ @@ -251,9 +276,8 @@ const char* bucket_name = "ior"; * --------------------------------------------------------------------------- */ -static -void -s3_connect( IOR_param_t* param ) { + +static void s3_connect( IOR_param_t* param ) { if (param->verbose >= VERBOSE_2) { printf("-> s3_connect\n"); /* DEBUGGING */ } diff --git a/src/ior.c b/src/ior.c index 4307558..e493d09 100755 --- a/src/ior.c +++ b/src/ior.c @@ -210,11 +210,7 @@ void init_IOR_Param_t(IOR_param_t * p) p->hdfs_replicas = 0; /* invokes the default */ p->hdfs_block_size = 0; - // p->curl = NULL; p->URI = NULL; - p->curl_flags = 0; - p->io_buf = NULL; - p->etags = NULL; p->part_number = 0; p->beegfs_numTargets = -1; @@ -223,26 +219,6 @@ void init_IOR_Param_t(IOR_param_t * p) p->mmap_ptr = NULL; } -/** - * Bind the global "backend" pointer to the requested backend AIORI's - * function table. - */ -static void AioriBind(char* api, IOR_param_t* param) -{ - backend = aiori_select (api); - if (NULL != backend) { - if (! strncmp(api, "S3", 2)) { - if (! strcasecmp(api, "S3_EMC")) { - param->curl_flags |= IOR_CURL_S3_EMC_EXT; - } else { - param->curl_flags &= ~(IOR_CURL_S3_EMC_EXT); - } - } - } else { - ERR("unrecognized IO API"); - } -} - static void DisplayOutliers(int numTasks, double timerVal, @@ -1235,7 +1211,7 @@ static void TestIoSys(IOR_test_t *test) } /* bind I/O calls to specific API */ - AioriBind(params->api, params); + backend = aiori_select(params->api); /* show test setup */ if (rank == 0 && verbose >= VERBOSE_0) diff --git a/src/ior.h b/src/ior.h index b6655b3..24a6d53 100755 --- a/src/ior.h +++ b/src/ior.h @@ -29,14 +29,6 @@ typedef void* hdfsFS; /* unused, but needs a type */ #endif -#ifdef USE_S3_AIORI -# include -# include "aws4c.h" -#else - typedef void CURL; /* unused, but needs a type */ - typedef void IOBuf; /* unused, but needs a type */ -#endif - #ifdef USE_RADOS_AIORI # include #else @@ -180,15 +172,7 @@ typedef struct int hdfs_replicas; /* n block replicas. (0 gets default) */ int hdfs_block_size; /* internal blk-size. (0 gets default) */ - /* REST/S3 variables */ - // CURL* curl; /* for libcurl "easy" fns (now managed by aws4c) */ -# define IOR_CURL_INIT 0x01 /* curl top-level inits were perfomed once? */ -# define IOR_CURL_NOCONTINUE 0x02 -# define IOR_CURL_S3_EMC_EXT 0x04 /* allow EMC extensions to S3? */ - char curl_flags; char* URI; /* "path" to target object */ - IOBuf* io_buf; /* aws4c places parsed header values here */ - IOBuf* etags; /* accumulate ETags for N:1 parts */ size_t part_number; /* multi-part upload increment (PER-RANK!) */ char* UploadId; /* key for multi-part-uploads */ diff --git a/src/option.c b/src/option.c index 13e1908..831afb2 100644 --- a/src/option.c +++ b/src/option.c @@ -267,9 +267,11 @@ int option_parse(int argc, char ** argv, option_help * args, int * printhelp){ char * txt = argv[i]; int foundOption = 0; char * arg = strstr(txt, "="); + int replaced_equal = 0; if(arg != NULL){ arg[0] = 0; arg++; + replaced_equal = 1; } if(strcmp(txt, "--") == 0){ // we found plugin options @@ -346,7 +348,7 @@ int option_parse(int argc, char ** argv, option_help * args, int * printhelp){ } } } - if(arg != NULL){ + if(replaced_equal){ arg[-1] = '='; } diff --git a/src/test/Makefile.am b/src/test/Makefile.am new file mode 100644 index 0000000..751423f --- /dev/null +++ b/src/test/Makefile.am @@ -0,0 +1,7 @@ +TESTS = testlib +bin_PROGRAMS = testlib + +testlib_SOURCES = lib.c +testlib_LDFLAGS = +testlib_LDADD = ../libaiori.a +testlib_CPPFLAGS = diff --git a/src/test/lib.c b/src/test/lib.c new file mode 100644 index 0000000..64ebf31 --- /dev/null +++ b/src/test/lib.c @@ -0,0 +1,20 @@ +#include "../ior.h" +#include "../mdtest.h" + +int main(int argc, char ** argv){ + int rank; + MPI_Init(& argc, & argv); + MPI_Comm_rank(MPI_COMM_WORLD, & rank); + + if (rank == 0){ + char * param[] = {"./ior", "-a", "DUMMY"}; + IOR_test_t * res = ior_run(3, param, MPI_COMM_SELF, stdout); + } + if (rank == 0){ + char * param[] = {"./mdtest", "-a", "DUMMY"}; + mdtest_results_t * res = mdtest_run(3, param, MPI_COMM_SELF, stdout); + } + MPI_Finalize(); + + return 0; +}