HDF5: option to enable collective metadata

In HDF5-1.10 the library has an option to read and write the metadata of a file collectively.  One has to opt-in to this optimization with a property list but particularly at large scale it can have tremendous performance implications.

Further information:
- https://support.hdfgroup.org/HDF5/docNewFeatures/CollMDataIO/RFC-CollectiveMetadataReads.pdf
- https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetAllCollMetadataOps

configure.ac

@@ -98,8 +98,12 @@ AC_ARG_WITH([hdf5],
 AM_CONDITIONAL([USE_HDF5_AIORI], [test x$with_hdf5 = xyes])
 AM_COND_IF([USE_HDF5_AIORI],[
         AC_DEFINE([USE_HDF5_AIORI], [], [Build HDF5 backend AIORI])
+	AC_SEARCH_LIBS([H5Pset_all_coll_metadata_ops], [hdf5])
+	AC_CHECK_FUNCS([H5Pset_all_coll_metadata_ops])
 ])
 
+
+
 # HDFS support
 AC_ARG_WITH([hdfs],
         [AS_HELP_STRING([--with-hdfs],

doc/sphinx/userDoc/options.rst

@@ -289,6 +289,9 @@ HDF5-ONLY
 
   * setAlignment         - HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g) [1]
 
+  * collectiveMetadata   - enable HDF5 collective metadata (available since
+                           HDF5-1.10.0)
+
 MPIIO-, HDF5-, AND NCMPI-ONLY
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   * collective           - uses collective operations for access [0=FALSE]

src/aiori-HDF5.c

@@ -228,6 +228,17 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param)
                                     param->setAlignment),
                    "cannot set alignment");
 
+#ifdef HAVE_H5PSET_ALL_COLL_METADATA_OPS
+        if (param->collective_md) {
+                /* more scalable metadata */
+
+                HDF5_CHECK(H5Pset_all_coll_metadata_ops(accessPropList, 1),
+                        "cannot set collective md read");
+                HDF5_CHECK(H5Pset_coll_metadata_write(accessPropList, 1),
+                        "cannot set collective md write");
+        }
+#endif
+
         /* open file */
         if (param->open == WRITE) {     /* WRITE */
                 *fd = H5Fcreate(testFileName, fd_mode,

src/ior.h

@@ -175,6 +175,7 @@ typedef struct
     char*       URI;                 /* "path" to target object */
     size_t      part_number;         /* multi-part upload increment (PER-RANK!) */
     char*       UploadId; /* key for multi-part-uploads */
+    int         collective_md;       /* use collective metatata optimization */
 
     /* RADOS variables */
     rados_t rados_cluster;           /* RADOS cluster handle */

src/parse_options.c

@@ -300,6 +300,8 @@ void DecodeDirective(char *line, IOR_param_t *params)
                 params->numTasks = atoi(value);
         } else if (strcasecmp(option, "summaryalways") == 0) {
                 params->summary_every_test = atoi(value);
+        } else if (strcasecmp(option, "collectiveMetadata") == 0) {
+                params->collective_md = atoi(value);
         } else {
                 if (rank == 0)
                         fprintf(out_logfile, "Unrecognized parameter \"%s\"\n",