From 6cae44b512ce9eae77bccf2e1b5a24e45ce80637 Mon Sep 17 00:00:00 2001 From: Osamu Tatebe Date: Fri, 21 Jun 2019 21:05:41 +0900 Subject: [PATCH] abstract I/O interface for Gfarm file system --- configure.ac | 20 ++- src/Makefile.am | 5 + src/aiori-Gfarm.c | 316 ++++++++++++++++++++++++++++++++++++++++++++++ src/aiori.c | 3 + src/aiori.h | 1 + 5 files changed, 344 insertions(+), 1 deletion(-) create mode 100644 src/aiori-Gfarm.c diff --git a/configure.ac b/configure.ac index 80e8533..7d8e4a7 100755 --- a/configure.ac +++ b/configure.ac @@ -185,7 +185,25 @@ AM_COND_IF([USE_RADOS_AIORI],[ AC_DEFINE([USE_RADOS_AIORI], [], [Build RADOS backend AIORI]) ]) - +# Gfarm support +AC_MSG_CHECKING([for Gfarm file system]) +AC_ARG_WITH([gfarm], + [AS_HELP_STRING([--with-gfarm=GFARM_ROOT], + [support IO with libgfarm backend @<:@default=no@:>@])], + [], [with_gfarm=no]) +AC_MSG_RESULT([$with_gfarm]) +AM_CONDITIONAL([USE_GFARM_AIORI], [test x$with_gfarm != xno]) +if test x$with_gfarm != xno; then + AC_DEFINE([USE_GFARM_AIORI], [], [Build Gfarm backend AIORI]) + case x$with_gfarm in + xyes) ;; + *) + CPPFLAGS="$CPPFLAGS -I$with_gfarm/include" + LDFLAGS="$LDFLAGS -L$with_gfarm/lib" ;; + esac + AC_CHECK_LIB([gfarm], [gfarm_initialize],, [AC_MSG_ERROR([libgfarm not found])]) + AC_CHECK_MEMBERS([struct stat.st_mtim.tv_nsec]) +fi # aws4c is needed for the S3 backend (see --with-S3, below). # Version 0.5.2 of aws4c is available at https://github.com/jti-lanl/aws4c.git diff --git a/src/Makefile.am b/src/Makefile.am index 51fb873..74dcd31 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -70,6 +70,11 @@ extraSOURCES += aiori-RADOS.c extraLDADD += -lrados endif +if USE_GFARM_AIORI +extraSOURCES += aiori-Gfarm.c +extraLDADD += -lgfarm +endif + if USE_S3_AIORI extraSOURCES += aiori-S3.c if AWS4C_DIR diff --git a/src/aiori-Gfarm.c b/src/aiori-Gfarm.c new file mode 100644 index 0000000..cc835f1 --- /dev/null +++ b/src/aiori-Gfarm.c @@ -0,0 +1,316 @@ +#include +#include +#include +#include +#include +#undef PACKAGE_NAME +#undef PACKAGE_STRING +#undef PACKAGE_TARNAME +#undef PACKAGE_VERSION +#include "ior.h" +#include "aiori.h" + +struct gfarm_file { + GFS_File gf; +}; + +void +Gfarm_initialize() +{ + gfarm_initialize(NULL, NULL); +} + +void +Gfarm_finalize() +{ + gfarm_terminate(); +} + +void * +Gfarm_create(char *fn, IOR_param_t *param) +{ + GFS_File gf; + struct gfarm_file *fp; + gfarm_error_t e; + + if (param->dryRun) + return (NULL); + + e = gfs_pio_create(fn, GFARM_FILE_RDWR, 0664, &gf); + if (e != GFARM_ERR_NO_ERROR) + ERR("gfs_pio_create failed"); + GFARM_MALLOC(fp); + if (fp == NULL) + ERR("no memory"); + fp->gf = gf; + return (fp); +} + +void * +Gfarm_open(char *fn, IOR_param_t *param) +{ + GFS_File gf; + struct gfarm_file *fp; + gfarm_error_t e; + + if (param->dryRun) + return (NULL); + + e = gfs_pio_open(fn, GFARM_FILE_RDWR, &gf); + if (e != GFARM_ERR_NO_ERROR) + ERR("gfs_pio_open failed"); + GFARM_MALLOC(fp); + if (fp == NULL) + ERR("no memory"); + fp->gf = gf; + return (fp); +} + +IOR_offset_t +Gfarm_xfer(int access, void *fd, IOR_size_t *buffer, IOR_offset_t len, + IOR_param_t *param) +{ + struct gfarm_file *fp = fd; + IOR_offset_t rem = len; + gfarm_off_t off; + gfarm_error_t e; +#define MAX_SZ (1024 * 1024 * 1024) + int sz, n; + char *buf = (char *)buffer; + + if (param->dryRun) + return (len); + + if (len > MAX_SZ) + sz = MAX_SZ; + else + sz = len; + + e = gfs_pio_seek(fp->gf, param->offset, GFARM_SEEK_SET, &off); + if (e != GFARM_ERR_NO_ERROR) + ERR("gfs_pio_seek failed"); + while (rem > 0) { + if (access == WRITE) + e = gfs_pio_write(fp->gf, buf, sz, &n); + else + e = gfs_pio_read(fp->gf, buf, sz, &n); + if (e != GFARM_ERR_NO_ERROR) + ERR("xfer failed"); + if (n == 0) + ERR("EOF encountered"); + rem -= n; + buf += n; + } + return (len); +} + +void +Gfarm_close(void *fd, IOR_param_t *param) +{ + struct gfarm_file *fp = fd; + + if(param->dryRun) + return; + + if (gfs_pio_close(fp->gf) != GFARM_ERR_NO_ERROR) + ERR("gfs_pio_close failed"); + free(fp); +} + +void +Gfarm_delete(char *fn, IOR_param_t *param) +{ + gfarm_error_t e; + + if (param->dryRun) + return; + + e = gfs_unlink(fn); + if (e != GFARM_ERR_NO_ERROR) + errno = gfarm_error_to_errno(e); +} + +char * +Gfarm_version() +{ + return ((char *)gfarm_version()); +} + +void +Gfarm_fsync(void *fd, IOR_param_t *param) +{ + struct gfarm_file *fp = fd; + + if(param->dryRun) + return; + + if (gfs_pio_sync(fp->gf) != GFARM_ERR_NO_ERROR) + ERR("gfs_pio_sync failed"); +} + +IOR_offset_t +Gfarm_get_file_size(IOR_param_t *param, MPI_Comm comm, char *fn) +{ + struct gfs_stat st; + IOR_offset_t size, sum, min, max; + + if (param->dryRun) + return (0); + + if (gfs_stat(fn, &st) != GFARM_ERR_NO_ERROR) + ERR("gfs_stat failed"); + size = st.st_size; + gfs_stat_free(&st); + + if (param->filePerProc == TRUE) { + MPI_CHECK(MPI_Allreduce(&size, &sum, 1, MPI_LONG_LONG_INT, + MPI_SUM, comm), "cannot total data moved"); + size = sum; + } else { + MPI_CHECK(MPI_Allreduce(&size, &min, 1, MPI_LONG_LONG_INT, + MPI_MIN, comm), "cannot total data moved"); + MPI_CHECK(MPI_Allreduce(&size, &max, 1, MPI_LONG_LONG_INT, + MPI_MAX, comm), "cannot total data moved"); + if (min != max) { + if (rank == 0) + WARN("inconsistent file size by different " + "tasks"); + /* incorrect, but now consistent across tasks */ + size = min; + } + } + return (size); +} + +int +Gfarm_statfs(const char *fn, ior_aiori_statfs_t *st, IOR_param_t *param) +{ + gfarm_off_t used, avail, files; + gfarm_error_t e; + int bsize = 4096; + + if (param->dryRun) + return (0); + + e = gfs_statfs_by_path(fn, &used, &avail, &files); + if (e != GFARM_ERR_NO_ERROR) { + errno = gfarm_error_to_errno(e); + return (-1); + } + st->f_bsize = bsize; + st->f_blocks = (used + avail) / bsize; + st->f_bfree = avail / bsize; + st->f_files = 2 * files; /* XXX */ + st->f_ffree = files; /* XXX */ + return (0); +} + +int +Gfarm_mkdir(const char *fn, mode_t mode, IOR_param_t *param) +{ + gfarm_error_t e; + + if (param->dryRun) + return (0); + + e = gfs_mkdir(fn, mode); + if (e == GFARM_ERR_NO_ERROR) + return (0); + errno = gfarm_error_to_errno(e); + return (-1); +} + +int +Gfarm_rmdir(const char *fn, IOR_param_t *param) +{ + gfarm_error_t e; + + if (param->dryRun) + return (0); + + e = gfs_rmdir(fn); + if (e == GFARM_ERR_NO_ERROR) + return (0); + errno = gfarm_error_to_errno(e); + return (-1); +} + +int +Gfarm_access(const char *fn, int mode, IOR_param_t *param) +{ + struct gfs_stat st; + gfarm_error_t e; + + if (param->dryRun) + return (0); + + e = gfs_stat(fn, &st); + if (e != GFARM_ERR_NO_ERROR) { + errno = gfarm_error_to_errno(e); + return (-1); + } + gfs_stat_free(&st); + return (0); +} + +/* XXX FIXME */ +#define GFS_DEV ((dev_t)-1) +#define GFS_BLKSIZE 8192 +#define STAT_BLKSIZ 512 /* for st_blocks */ + +int +Gfarm_stat(const char *fn, struct stat *buf, IOR_param_t *param) +{ + struct gfs_stat st; + gfarm_error_t e; + + if (param->dryRun) + return (0); + + e = gfs_stat(fn, &st); + if (e != GFARM_ERR_NO_ERROR) { + errno = gfarm_error_to_errno(e); + return (-1); + } + buf->st_dev = GFS_DEV; + buf->st_ino = st.st_ino; + buf->st_mode = st.st_mode; + buf->st_nlink = st.st_nlink; + buf->st_uid = getuid(); /* XXX */ + buf->st_gid = getgid(); /* XXX */ + buf->st_size = st.st_size; + buf->st_blksize = GFS_BLKSIZE; + buf->st_blocks = (st.st_size + STAT_BLKSIZ - 1) / STAT_BLKSIZ; + buf->st_atime = st.st_atimespec.tv_sec; + buf->st_mtime = st.st_mtimespec.tv_sec; + buf->st_ctime = st.st_ctimespec.tv_sec; +#if defined(HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC) + buf->st_atim.tv_nsec = st.st_atimespec.tv_nsec; + buf->st_mtim.tv_nsec = st.st_mtimespec.tv_nsec; + buf->st_ctim.tv_nsec = st.st_ctimespec.tv_nsec; +#endif + gfs_stat_free(&st); + return (0); +} + +ior_aiori_t gfarm_aiori = { + .name = "Gfarm", + .name_legacy = NULL, + .create = Gfarm_create, + .open = Gfarm_open, + .xfer = Gfarm_xfer, + .close = Gfarm_close, + .delete = Gfarm_delete, + .get_version = Gfarm_version, + .fsync = Gfarm_fsync, + .get_file_size = Gfarm_get_file_size, + .statfs = Gfarm_statfs, + .mkdir = Gfarm_mkdir, + .rmdir = Gfarm_rmdir, + .access = Gfarm_access, + .stat = Gfarm_stat, + .initialize = Gfarm_initialize, + .finalize = Gfarm_finalize, + .get_options = NULL, + .enable_mdtest = true, +}; diff --git a/src/aiori.c b/src/aiori.c index 52001c8..41773d9 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -68,6 +68,9 @@ ior_aiori_t *available_aiori[] = { #endif #ifdef USE_RADOS_AIORI &rados_aiori, +#endif +#ifdef USE_GFARM_AIORI + &gfarm_aiori, #endif NULL }; diff --git a/src/aiori.h b/src/aiori.h index c2074c2..56717fe 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -105,6 +105,7 @@ extern ior_aiori_t s3_aiori; extern ior_aiori_t s3_plus_aiori; extern ior_aiori_t s3_emc_aiori; extern ior_aiori_t rados_aiori; +extern ior_aiori_t gfarm_aiori; void aiori_initialize(IOR_test_t * tests); void aiori_finalize(IOR_test_t * tests);