commit a56d39e6f1cf64baed12c28bb446792ae0d39d20 Author: Christopher J. Morrone Date: Fri Jun 17 12:20:43 2011 -0700 Start with the code from IOR-2.10.3 diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000..70008a2 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,256 @@ +Copyright (c) 2003, The Regents of the University of California. +Produced at the Lawrence Livermore National Laboratory. +Written by William Loewe , Tyce McLarty , +and Christopher Morrone . +UCRL-CODE-2003-016. +All rights reserved. + +This file is part of IOR. + +Please also read Our Notice and GNU General Public License. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License (as published by the Free Software +Foundation) version 2, dated June 1991. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the terms and conditions of the GNU General Public +License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA + + +OUR NOTICE AND TERMS AND CONDITIONS OF THE GNU GENERAL PUBLIC LICENSE + +Our Preamble Notice + +A. This notice is required to be provided under our contract with the U.S. +Department of Energy (DOE). This work was produced at the University of +California, Lawrence Livermore National Laboratory under Contract No. +W-7405-ENG-48 with the DOE. + +B. Neither the United States Government nor the University of California nor +any of their employees, makes any warranty, express or implied, or assumes any +liability or responsibility for the accuracy, completeness, or usefulness of +any information, apparatus, product, or process disclosed, or represents that +its use would not infringe privately-owned rights. + +C. Also, reference herein to any specific commercial products, process, or +services by trade name, trademark, manufacturer or otherwise does not +necessarily constitute or imply its endorsement, recommendation, or favoring +by the United States Government or the University of California. The views and +opinions of authors expressed herein do not necessarily state or reflect those +of the United States Government or the University of California, and shall not +be used for advertising or product endorsement purposes. + +The precise terms and conditions for copying, distribution and modification +follows. + +GNU Terms and Conditions for Copying, Distribution, and Modification + +0. This License applies to any program or other work which contains a notice +placed by the copyright holder saying it may be distributed under the terms of +this General Public License. The "Program," below, refers to any such program +or work, and a "work based on the Program" means either the Program or any +derivative work under copyright law: that is to say, a work containing the +Program or a portion of it, either verbatim or with modifications and/or +translated into another language. (Hereinafter, translation is included +without limitation in the term "modification".) Each licensee is addressed as +"you." + +Activities other than copying, distribution and modification are not covered by +this License; they are outside its scope. The act of running the Program is +not restricted, and the output from the Program is covered only if its contents +constitute a work based on the Program (independent of having been made by +running the Program). Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source code as +you receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice and +disclaimer of warranty; keep intact all the notices that refer to this License +and to the absence of any warranty; and give any other recipients of the +Program a copy of this License along with the Program. + +You may charge a fee for the physical act of transferring a copy, and you may +at your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of it, +thus forming a work based on the Program, and copy and distribute such +modifications or work under the terms of Section 1 above, provided that you +also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices stating + that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in whole + or in part contains or is derived from the Program or any part thereof, + to be licensed as a whole at no charge to all third parties under the terms + of this License. + + c) If the modified program normally reads commands interactively when run, + you must cause it, when started running for such interactive use in the + most ordinary way, to print or display an announcement including an + appropriate copyright notice and a notice that there is no warranty (or + else, saying that you provide a warranty) and that users may redistribute + the program under these conditions, and telling the user how to view a copy + of this License. (Exception: if the Program itself is interactive but does + not normally print such an announcement, your work based on the Program is + not required to print an announcement.) + +These requirements apply to the modified work as a whole. If identifiable +sections of that work are not derived from the Program, and can be reasonably +considered independent and separate works in themselves, then this License, and +its terms, do not apply to those sections when you distribute them as separate +work. But when you distribute the same section as part of a whole which is a +work based on the Program, the distribution of the whole must be on the terms +of this License, whose permissions for other licensees extend to the entire +whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest your +rights to work written entirely by you; rather, the intent is to exercise the +right to control the distribution of derivative or collective works based on +the Program. + +In addition, mere aggregation of another work not based on the Program with the +Program (or with a work based on the Program) on a volume of a storage or +distribution medium does not bring the other work under the scope of this +License. + +3. You may copy and distribute the Program (or a work based on it, under +Section 2) in object code or executable form under the terms of Sections 1 and +2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable source + code, which must be distributed under the terms of Sections 1 and 2 above + on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three years, to + give any third party, for a charge no more than your cost of physically + performing source distribution, a complete machine-readable copy of the + corresponding source code, to be distributed under the terms of Sections 1 + and 2 above on a medium customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer to + distribute corresponding source code. (This alternative is allowed only + for noncommercial distribution and only if you received the program in + object code or executable form with such an offer, in accord with + Subsection b above.) + +The source code for a work means the preferred form the work for making +modifications to it. For an executable work, complete source code means all +the source code for all modules it contains, plus any associated interface +definition files, plus the scripts used to control compilation and installation +of the executable. However, as a special exception, the source code +distributed need not include anything that is normally distributed (in either +source or binary form) with the major components (compiler, kernel, and so on) +of the operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the source +code from the same place counts as distribution of the source code, even though +third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as +expressly provided under this License. Any attempt otherwise to copy, modify, +sublicense or distribute the Program is void, and will automatically terminate +your rights under this License. However, parties who have received copies, or +rights, from you under this License will not have their licenses terminated so +long as such parties remain in full compliance. + +5. You are not required to accept this License, since you have not signed it. +However, nothing else grants you permission to modify or distribute the Program +or its derivative works. These actions are prohibited by law if you do not +accept this License. Therefore, by modifying or distributing the Program (or +any work based on the Program), you indicate your acceptance of this License to +do so, and all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the Program), +the recipient automatically receives a license from the original licensor to +copy, distribute or modify the Program subject to these terms and conditions. +You may not impose any further restrictions on the recipients' exercise of the +rights granted herein. You are not responsible for enforcing compliance by +third parties to this License. + +7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), conditions +are imposed on you (whether by court order, agreement or otherwise) that +contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot distribute so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not distribute the Program at all. +For example, if a patent license would not permit royalty-free redistribution +of the Program by all those who receive copies directly or indirectly through +you, then the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply and +the section as a whole is intended to apply in other circumstances. + +It is not the purpose to this section to induce you to infringe any patents or +other property right claims or to contest validity of any such claims; this +section has the sole purpose of protecting the integrity of the free software +distribution system, which is implemented by public license practices. Many +people have made generous contributions to the wide range of software +distributed through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing to +distribute software through any other system and a licensee cannot impose that +choice. + +This section is intended to make thoroughly clear what is believed to be a +consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in certain +countries either by patents or by copyrighted interfaces, the original +copyright holder who places the Program under this License may add an explicit +geographical distribution limitation excluding those countries, so that +distribution is permitted only in or among countries not thus excluded. In +such case, this License incorporates the limitation as if written in the body +of this License. + +9. The Free Software Foundation may publish revised and/or new versions of the +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems +or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any later +version," you have the option of following the terms and conditions either of +that version of any later version published by the Free Software Foundation. +If the Program does not specify a version number of this License, you may +choose any version ever published by the Free Software Foundation. + +10. If you wish to incorporate parts of the Program into other free programs +whose distribution conditions are different, write to the author to ask for +permission. For software which is copyrighted by the Free Software Foundation, +write to the Free Software Foundation; we sometimes make exceptions for this. +Our decision to grant permission will be guided by the two goals of preserving +the free status of all derivatives of our free software and or promoting the +sharing and reuse of software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR +THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE +STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE +PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND +PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, +YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL +ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE +PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR +INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA +BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER +OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS diff --git a/IOR.vcproj b/IOR.vcproj new file mode 100644 index 0000000..c075e0f --- /dev/null +++ b/IOR.vcproj @@ -0,0 +1,414 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..29497a1 --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +#/*****************************************************************************\ +#* * +#* Copyright (c) 2003, The Regents of the University of California * +#* See the file COPYRIGHT for a complete copyright notice and license. * +#* * +#******************************************************************************* +#* +#* CVS info: +#* $RCSfile: Makefile,v $ +#* $Revision: 1.1.1.1 $ +#* $Date: 2007/10/15 23:36:54 $ +#* $Author: rklundt $ +#* +#* Purpose: +#* Make IOR executable. +#* +#* gmake posix -- IOR with only POSIX interface +#* gmake mpiio -- IOR with only POSIX and MPIIO interfaces +#* gmake hdf5 -- IOR with POSIX, MPIIO, and HDF5 interfaces +#* gmake ncmpi -- IOR with POSIX, MPIIO, and NCMPI interfaces +#* gmake all -- IOR with POSIX, MPIIO, HDF5, and NCMPI interfaces +#* gmake clean -- remove executable and object files +#* +#\*****************************************************************************/ + +SRC = ./src/C + +posix: + (cd $(SRC) && $(MAKE) posix) + +%: + (cd $(SRC) && $(MAKE) $@) diff --git a/README b/README new file mode 100644 index 0000000..61b256d --- /dev/null +++ b/README @@ -0,0 +1,11 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +\******************************************************************************/ + +This collection of files contains the source (in /src) and Makefile for +building IOR. In /scripts, there are scripts for simulating I/O behavior of +various parallel applications. For more information, please read the User +Guide. diff --git a/RELEASE_LOG b/RELEASE_LOG new file mode 100644 index 0000000..00fc06e --- /dev/null +++ b/RELEASE_LOG @@ -0,0 +1,147 @@ + +Changes in IOR-2.10.3 + * bug 2962326 "Segmentation Fault When Summarizing Results" fixed + * bug 2786285 "-Wrong number of parameters to function H5Dcreate" fixed + (NOTE: to compile for HDF5 1.6 libs use "-D H5_USE_16_API") + * bug 1992514 "delay (-d) doesn't work" fixed +Contributed by demyn@users.sourceforge.net + * Ported to Windows. Required changes related to 'long' types, which on Windows + are always 32-bits, even on 64-bit systems. Missing system headers and + functions acount for most of the remaining changes. + New files for Windows: + - IOR/ior.vcproj - Visual C project file + - IOR/src/C/win/getopt.{h,c} - GNU getopt() support + See updates in the USER_GUIDE for build instructions on Windows. + * Fixed bug in incrementing transferCount + * Fixed bugs in SummarizeResults with mismatched format specifiers + * Fixed inconsistencies between option names, -h output, and the USER_GUIDE. + +Changes in IOR-2.10.2: +Hodson, 8/18/2008: + +The modifications to IOR-2.10.1 extend existing random I/O capabilities and +enhance performance output statistics. + + cli script Description +----- ----------------- ---------------------------------------- +1) -A N testnum - test reference number for easier test + identification in log files +2) -Q N taskpernodeoffset - for read tests. Use with -C & -Z options + If -C (reordertasks) specified, + then node offset read by CONSTANT N. + If -Z (reordertasksrandom) specified, + then node offset read by RANDOM >= N. +3) -Z reordertasksrandom - random node task ordering for read tests + In this case, processes read + data written by other processes with + node offsets specified by the -Q option + and -X option. +4) -X N reordertasksrandomseed - random seed for -Z (reordertasksrandom) + If N>=0, use same seed for all iters + If N< 0, use different seed for ea. iter +5) -Y fsyncperwrite - perform fsync after every POSIX write + +Fixes in IOR-2.10.1: + * added '-J' setAlignment option for HDF5 alignment in bytes; default value + is 1, which does not set alignment + * changed how HDF5 and PnetCDF calculate performance -- formerly each used + the size of the stat()'ed file; changed it to be number of data bytes + transferred. these library-generated files can have large headers and + filler as well as sparse file content + * found potential overflow error in cast -- using IOR_offset_t, not int now + * replaced HAVE_HDF5_NO_FILL macro to instead directly check if H5_VERS_MAJOR + H5_VERS_MINOR are defined; if so, then must be HDF5-1.6.x or higher for + no-fill usage. + * corrected IOR_GetFileSize() function to point to HDF5 and NCMPI versions of + IOR_GetFileSize() calls + * changed the netcdf dataset from 1D array to 4D array, where the 4 dimensions + are: [segmentCount][numTasksWorld][numTransfers][transferSize] + This patch from Wei-keng Liao allows for file sizes > 4GB (provided no + single dimension is > 4GB). + * finalized random-capability release + * changed statvfs() to be for __sun (SunOS) only + * retired Python GUI + +Fixes in IOR-2.10.0.1: + * Cleaned up WriteOrRead(), reducing much to smaller functions. + * Added random capability for transfer offset. + * modified strtok(NULL, " \t\r\n") in ExtractHints() so no trailing characters + * added capability to set hints in NCMPI + +Fixes in IOR-2.9.6.1: + * for 'pvfs2:' filename prefix, now skips DisplayFreeSpace(); formerly this + caused a problem with statvfs() + * changed gethostname() to MPI_Get_processor_name() so for certain cases + when gethostname() would only return the frontend node name + * added SunOS compiler settings for makefile + * updated O_DIRECT usage for SunOS compliance + * changed statfs() to instead use statvfs() for SunOS compliance + * renamed compiler directive _USE_LUSTRE to _MANUALLY_SET_LUSTRE_STRIPING + +Fixes in IOR-2.9.5: + * Wall clock deviation time relabeled to be "Start time skew across all tasks". + * Added notification for "Using reorderTasks '-C' (expecting block, not cyclic, + task assignment)" + * Corrected bug with read performance with stonewalling (was using full size, + stat'ed file instead of bytes transfered). + +Fixes in IOR-2.9.4: + * Now using IOR_offset_t instead of int for tmpOffset in IOR.c:WriteOrRead(). + Formerly, this would cause error in file(s) > 2GB for ReadCheck. The + more-commonly-used WriteCheck option was not affected by this. + +Fixes in IOR-2.9.3: + * Changed FILE_DELIMITER from ':' to '@'. + * Any time skew between nodes is automatically adjusted so that all nodes + are calibrated to the root node's time. + * Wall clock deviation time is still reported, but have removed the warning + message that was generated for this. (Didn't add much new information, + just annoyed folks.) + * The '-j' outlierThreshold option now is set to 0 (off) as default. To use + this, set to a positive integer N, and any task who's time (for open, + access, etc.) is not within N seconds of the mean of all the tasks will show + up as an outlier. + +Fixes in IOR-2.9.2: + * Simple cleanup of error messages, format, etc. + * Changed error reporting so that with VERBOSITY=2 (-vv) any error encountered + is displayed. Previously, this was with VERBOSITY=3, along with full test + parameters, environment, and all access times of operations. + * Added deadlineForStonewalling option (-D). This option is used for measuring + the amount of data moved in a fixed time. After the barrier, each task + starts its own timer, begins moving data, and the stops moving data at a + prearranged time. Instead of measuring the amount of time to move a fixed + amount of data, this option measures the amount of data moved in a fixed + amount of time. The objective is to prevent tasks slow to complete from + skewing the performance. + +Fixes in IOR-2.9.1: + * Updated test script to run through file1:file2 cases. + * Program exit code is now total numbers of errors (both writecheck and + readcheck for all iterations), unless quitOnError (-q) is set. + * For various failure situations, replace abort with warning, including: + - failed uname() for platform name now gives warning + - failed unlink() of data file now gives warning + - failed fsync() of data file now gives warning + - failed open() of nonexistent script file now gives warning + * Changed labelling for error checking output to be (hopefully) more + clear in details on errors for diagnostics. + * Another fix for -o file1:file2 option. + * Corrected bug in GetTestFileName() -- now possible to handle -o file1:file2 + cases for file-per-proc. + +Fixes in IOR-2.9.0: + * Improved checkRead option to reread data from different node (to avoid + cache) and then compare both reads. + * Added outlierThreshold (-j) option to give warning if any task is more than + this number of seconds from the mean of all participating tasks. If so, the + task is identified, its time (start, elapsed create, elapsed transfer, + elapsed close, or end) is reported, as is the mean and standard deviation for + all tasks. The default for this is 5, i.e. any task not within 5 seconds of + the mean for those times is displayed. This value can be set with + outlierThreshold= or -j . + * Correct for clock skew between nodes - if skew greater than wallclock + deviation threshold (WC_DEV_THRESHOLD) in seconds, then broadcast root + node's timestamp and adjust by the difference. WC_DEV_THRESHOLD currently + set to 5. + * Added a Users Guide. diff --git a/UNDOCUMENTED_OPTIONS b/UNDOCUMENTED_OPTIONS new file mode 100644 index 0000000..f517567 --- /dev/null +++ b/UNDOCUMENTED_OPTIONS @@ -0,0 +1,36 @@ +To use these options, modify USE_UNDOC_OPT=TRUE in iordef.h and then recompile. +For passing arguments to the executable for these options, use this format: + + '-O option1=1,option2=0' + +Options: + + * fillTheFileSystem -- limits unnecessary output for showing performance, etc. + The command line '-w -k -m -i 123 -F -O fillTheFileSystem=5' has each task + creating (and keeping) 123 files, showing the time between every fifth file. + + * includeDeleteTime -- use with '-O includeDeleteTime=1' to have time to + delete file included in write performance time. Note that if this file + is deleted after the write, there's nothing available for the read back + or checking. + + * multiReRead -- use with '-i 5 -v -k -w -W -O multiReRead=1' to have 1 write + of file, followed by 4 reads w/data checking. + + * NFS_serverCount -- use with other NFS_* options, -F, and -o to write/read + files across multiple NFS servers. Set the server count to > 0 to enable. + The format is: 'NFS_rootPath/NFS_serverName[0..NFS_serverCount-1]/outFile'. + Each file is written round-robin to a different NFS server. + + * NFS_rootPath -- use with NFS_server* options for path to NFS servers. + + * NFS_serverName -- ise with NFS_* options for NFS server prefix. Specific + NFS server number is appended to this name, e.g. nfs123. + + * corruptFile -- corrupts file after writing for determining data-checking + reliability. + +Note: + + * cbif.c (Change Byte In File) is a simple utility to check the value of + a file offset; also allows changing a single byte at an offset. diff --git a/USER_GUIDE b/USER_GUIDE new file mode 100644 index 0000000..ce1399f --- /dev/null +++ b/USER_GUIDE @@ -0,0 +1,735 @@ +/*****************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +\*****************************************************************************/ +/********************* Modifications to IOR-2.10.1 **************************** +* Hodson, 8/18/2008: * +* Documentation updated for the following new option: * +* The modifications to IOR-2.10.1 extend existing random I/O capabilities and * +* enhance performance output statistics. * +* * +* cli script Description * +* ----- ----------------- ----------------------------------------* +* 1) -A N testnum - test reference number for easier test * +* identification in log files * +* 2) -Q N taskpernodeoffset - for read tests. Use with -C & -Z options* +* If -C (reordertasks) specified, * +* then node offset read by CONSTANT N. * +* If -Z (reordertasksrandom) specified, * +* then node offset read by RANDOM >= N. * +* 3) -Z reordertasksrandom - random node task ordering for read tests* +* In this case, processes read * +* data written by other processes with * +* node offsets specified by the -Q option * +* and -X option. * +* 4) -X N reordertasksrandomseed - random seed for -Z (reordertasksrandom) * +* If N>=0, use same seed for all iters * +* If N< 0, use different seed for ea. iter* +* 5) -Y fsyncperwrite - perform fsync after every POSIX write * +\*****************************************************************************/ + + IOR USER GUIDE + +Index: + * Basics + 1. Description + 2. Building IOR + 3. Running IOR + 4. Options + + * More Information + 5. Option details + 6. Verbosity levels + 7. Using Scripts + + * Troubleshooting + 8. Compatibility with older versions + + * Frequently Asked Questions + 9. How do I . . . ? + + * Output + 10. Enhanced output description + + +******************* +* 1. DESCRIPTION * +******************* +IOR can be used for testing performance of parallel file systems using various +interfaces and access patterns. IOR uses MPI for process synchronization. +IOR version 2 is a complete rewrite of the original IOR (Interleaved-Or-Random) +version 1 code. + + +******************* +* 2. BUILDING IOR * +******************* +Build Instructions: + + Type 'gmake [posix|mpiio|hdf5|ncmpi|all]' from the IOR/ directory. + On some platforms, e.g., Cray XT, specify "CC=cc" to build using "cc" instead of "mpicc". + In IOR/src/C, the file Makefile.config currently has settings for AIX, Linux, + OSF1 (TRU64), and IRIX64 to model on. Note that MPI must be present for + building/running IOR, and that MPI I/O must be available for MPI I/O, HDF5, + and Parallel netCDF builds. As well, HDF5 and Parallel netCDF libraries are + necessary for those builds. All IOR builds include the POSIX interface. + + You can build IOR as a native Windows application. One way to do this is to + use the "Microsoft Windows SDK", which is available as a free download and + includes development tools like the Visual C++ compiler. To build IOR with + MPI-IO support, also download and install the "Microsoft HPC Pack SDK". + Once these packages are installed on your Windows build system, follow these + steps: + 1. Open a "CMD Shell" under the Start menu Microsoft Windows SDK group. + 2. cd to the IOR directory containing ior.vcproj + 3. Run: vcbuild ior.vcproj "Release|x64" + + ior.exe will be created in the directory IOR/x64/Release. "Debug|x64", + "Release|Win32", and "Debug|Win32" configurations can also be built. + To build IOR without MPI-IO support, first edit ior.vcproj and replace + aiori-MPIIO.c with aiori-noMPIIO.c. + + +****************** +* 3. RUNNING IOR * +****************** +Two ways to run IOR: + + * Command line with arguments -- executable followed by command line options. + + E.g., to execute: IOR -w -r -o filename + This performs a write and a read to the file 'filename'. + + * Command line with scripts -- any arguments on the command line will + establish the default for the test run, but a script may be used in + conjunction with this for varying specific tests during an execution of the + code. + + E.g., to execute: IOR -W -f script + This defaults all tests in 'script' to use write data checking. + + +************** +* 4. OPTIONS * +************** +These options are to be used on the command line. E.g., 'IOR -a POSIX -b 4K'. + -A N testNum -- test number for reference in some output + -a S api -- API for I/O [POSIX|MPIIO|HDF5|NCMPI] + -b N blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g) + -B useO_DIRECT -- uses O_DIRECT for POSIX, bypassing I/O buffers + -c collective -- collective I/O + -C reorderTasks -- changes task ordering to n+1 ordering for readback + -Q N taskPerNodeOffset for read tests use with -C & -Z options (-C constant N, -Z at least N) [!HDF5] + -Z reorderTasksRandom -- changes task ordering to random ordering for readback + -X N reorderTasksRandomSeed -- random seed for -Z option + -d N interTestDelay -- delay between reps in seconds + -D N deadlineForStonewalling -- seconds before stopping write or read phase + -Y fsyncPerWrite -- perform fsync after each POSIX write + -e fsync -- perform fsync upon POSIX write close + -E useExistingTestFile -- do not remove test file before write access + -f S scriptFile -- test script name + -F filePerProc -- file-per-process + -g intraTestBarriers -- use barriers between open, write/read, and close + -G N setTimeStampSignature -- set value for time stamp signature + -h showHelp -- displays options and help + -H showHints -- show hints + -i N repetitions -- number of repetitions of test + -I individualDataSets -- datasets not shared by all procs [not working] + -j N outlierThreshold -- warn on outlier N seconds from mean + -J N setAlignment -- HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g) + -k keepFile -- don't remove the test file(s) on program exit + -K keepFileWithError -- keep error-filled file(s) after data-checking + -l storeFileOffset -- use file offset as stored signature + -m multiFile -- use number of reps (-i) for multiple file count + -n noFill -- no fill in HDF5 file creation + -N N numTasks -- number of tasks that should participate in the test + -o S testFile -- full name for test + -O S string of IOR directives (e.g. -O checkRead=1,lustreStripeCount=32) + -p preallocate -- preallocate file size + -P useSharedFilePointer -- use shared file pointer [not working] + -q quitOnError -- during file error-checking, abort on error + -r readFile -- read existing file + -R checkRead -- check read after read + -s N segmentCount -- number of segments + -S useStridedDatatype -- put strided access into datatype [not working] + -t N transferSize -- size of transfer in bytes (e.g.: 8, 4k, 2m, 1g) + -T N maxTimeDuration -- max time in minutes to run tests + -u uniqueDir -- use unique directory name for each file-per-process + -U S hintsFileName -- full name for hints file + -v verbose -- output information (repeating flag increases level) + -V useFileView -- use MPI_File_set_view + -w writeFile -- write file + -W checkWrite -- check read after write + -x singleXferAttempt -- do not retry transfer if incomplete + -z randomOffset -- access is to random, not sequential, offsets within a file + + +NOTES: * S is a string, N is an integer number. + * For transfer and block sizes, the case-insensitive K, M, and G + suffices are recognized. I.e., '4k' or '4K' is accepted as 4096. + + +********************* +* 5. OPTION DETAILS * +********************* +For each of the general settings, note the default is shown in brackets. +IMPORTANT NOTE: For all true/false options below [1]=true, [0]=false +IMPORTANT NOTE: Contrary to appearance, the script options below are NOT case sensitive + +GENERAL: +======== + * testNum - test reference number for some output [-1] + + * api - must be set to one of POSIX, MPIIO, HDF5, or NCMPI + depending on test [POSIX] + + * testFile - name of the output file [testFile] + NOTE: with filePerProc set, the tasks can round + robin across multiple file names '-o S@S@S' + + * hintsFileName - name of the hints file [] + + * repetitions - number of times to run each test [1] + + * multiFile - creates multiple files for single-shared-file or + file-per-process modes; i.e. each iteration creates + a new file [0=FALSE] + + * reorderTasksConstant - reorders tasks by a constant node offset for writing/reading neighbor's + data from different nodes [0=FALSE] + + * taskPerNodeOffset - for read tests. Use with -C & -Z options. [1] + With reorderTasks, constant N. With reordertasksrandom, >= N + + * reorderTasksRandom - reorders tasks to random ordering for readback [0=FALSE] + + * reorderTasksRandomSeed - random seed for reordertasksrandom option. [0] + >0, same seed for all iterations. <0, different seed for each iteration + + * quitOnError - upon error encountered on checkWrite or checkRead, + display current error and then stop execution; + if not set, count errors and continue [0=FALSE] + + * numTasks - number of tasks that should participate in the test + [0] + NOTE: 0 denotes all tasks + + * interTestDelay - this is the time in seconds to delay before + beginning a write or read in a series of tests [0] + NOTE: it does not delay before a check write or + check read + + * outlierThreshold - gives warning if any task is more than this number + of seconds from the mean of all participating tasks. + If so, the task is identified, its time (start, + elapsed create, elapsed transfer, elapsed close, or + end) is reported, as is the mean and standard + deviation for all tasks. The default for this is 0, + which turns it off. If set to a positive value, for + example 3, any task not within 3 seconds of the mean + displays its times. [0] + + * intraTestBarriers - use barrier between open, write/read, and close [0=FALSE] + + * uniqueDir - create and use unique directory for each + file-per-process [0=FALSE] + + * writeFile - writes file(s), first deleting any existing file [1=TRUE] + NOTE: the defaults for writeFile and readFile are + set such that if there is not at least one of + the following -w, -r, -W, or -R, it is assumed + that -w and -r are expected and are + consequently used -- this is only true with + the command line, and may be overridden in + a script + + * readFile - reads existing file(s) (from current or previous + run) [1=TRUE] + NOTE: see writeFile notes + + * filePerProc - accesses a single file for each processor; default + is a single file accessed by all processors [0=FALSE] + + * checkWrite - read data back and check for errors against known + pattern; can be used independently of writeFile [0=FALSE] + NOTES: * data checking is not timed and does not + affect other performance timings + * all errors tallied and returned as program + exit code, unless quitOnError set + + * checkRead - reread data and check for errors between reads; can + be used independently of readFile [0=FALSE] + NOTE: see checkWrite notes + + * keepFile - stops removal of test file(s) on program exit [0=FALSE] + + * keepFileWithError - ensures that with any error found in data-checking, + the error-filled file(s) will not be deleted [0=FALSE] + + * useExistingTestFile - do not remove test file before write access [0=FALSE] + + * segmentCount - number of segments in file [1] + NOTES: * a segment is a contiguous chunk of data + accessed by multiple clients each writing/ + reading their own contiguous data; + comprised of blocks accessed by multiple + clients + * with HDF5 this repeats the pattern of an + entire shared dataset + + * blockSize - size (in bytes) of a contiguous chunk of data + accessed by a single client; it is comprised of one + or more transfers [1048576] + + * transferSize - size (in bytes) of a single data buffer to be + transferred in a single I/O call [262144] + + * verbose - output information [0] + NOTE: this can be set to levels 0-5 on the command + line; repeating the -v flag will increase + verbosity level + + * setTimeStampSignature - set value for time stamp signature [0] + NOTE: used to rerun tests with the exact data + pattern by setting data signature to contain + positive integer value as timestamp to be + written in data file; if set to 0, is + disabled + + * showHelp - display options and help [0=FALSE] + + * storeFileOffset - use file offset as stored signature when writing + file [0=FALSE] + NOTE: this will affect performance measurements + + * maxTimeDuration - max time in minutes to run tests [0] + NOTES: * setting this to zero (0) unsets this option + * this option allows the current read/write + to complete without interruption + + * deadlineForStonewalling - seconds before stopping write or read phase [0] + NOTES: * used for measuring the amount of data moved + in a fixed time. After the barrier, each + task starts its own timer, begins moving + data, and the stops moving data at a pre- + arranged time. Instead of measuring the + amount of time to move a fixed amount of + data, this option measures the amount of + data moved in a fixed amount of time. The + objective is to prevent tasks slow to + complete from skewing the performance. + * setting this to zero (0) unsets this option + * this option is incompatible w/data checking + + * randomOffset - access is to random, not sequential, offsets within a file [0=FALSE] + NOTES: * this option is currently incompatible with: + -checkRead + -storeFileOffset + -MPIIO collective or useFileView + -HDF5 or NCMPI + + +POSIX-ONLY: +=========== + * useO_DIRECT - use O_DIRECT for POSIX, bypassing I/O buffers [0] + + * singleXferAttempt - will not continue to retry transfer entire buffer + until it is transferred [0=FALSE] + NOTE: when performing a write() or read() in POSIX, + there is no guarantee that the entire + requested size of the buffer will be + transferred; this flag keeps the retrying a + single transfer until it completes or returns + an error + + * fsyncPerWrite - perform fsync after each POSIX write [0=FALSE] + * fsync - perform fsync after POSIX write close [0=FALSE] + +MPIIO-ONLY: +=========== + * preallocate - preallocate the entire file before writing [0=FALSE] + + * useFileView - use an MPI datatype for setting the file view option + to use individual file pointer [0=FALSE] + NOTE: default IOR uses explicit file pointers + + * useSharedFilePointer - use a shared file pointer [0=FALSE] (not working) + NOTE: default IOR uses explicit file pointers + + * useStridedDatatype - create a datatype (max=2GB) for strided access; akin + to MULTIBLOCK_REGION_SIZE [0] (not working) + +HDF5-ONLY: +========== + * individualDataSets - within a single file each task will access its own + dataset [0=FALSE] (not working) + NOTE: default IOR creates a dataset the size of + numTasks * blockSize to be accessed by all + tasks + + * noFill - no pre-filling of data in HDF5 file creation [0=FALSE] + + * setAlignment - HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g) [1] + +MPIIO-, HDF5-, AND NCMPI-ONLY: +============================== + * collective - uses collective operations for access [0=FALSE] + + * showHints - show hint/value pairs attached to open file [0=FALSE] + NOTE: not available in NCMPI + +LUSTRE-SPECIFIC: +================ + * lustreStripeCount - set the lustre stripe count for the test file(s) [0] + + * lustreStripeSize - set the lustre stripe size for the test file(s) [0] + + * lustreStartOST - set the starting OST for the test file(s) [-1] + + * lustreIgnoreLocks - disable lustre range locking [0] + + +*********************** +* 6. VERBOSITY LEVELS * +*********************** +The verbosity of output for IOR can be set with -v. Increasing the number of +-v instances on a command line sets the verbosity higher. + +Here is an overview of the information shown for different verbosity levels: + 0 - default; only bare essentials shown + 1 - max clock deviation, participating tasks, free space, access pattern, + commence/verify access notification w/time + 2 - rank/hostname, machine name, timer used, individual repetition + performance results, timestamp used for data signature + 3 - full test details, transfer block/offset compared, individual data + checking errors, environment variables, task writing/reading file name, + all test operation times + 4 - task id and offset for each transfer + 5 - each 8-byte data signature comparison (WARNING: more data to STDOUT + than stored in file, use carefully) + + +******************** +* 7. USING SCRIPTS * +******************** + +IOR can use a script with the command line. Any options on the command line +will be considered the default settings for running the script. (I.e., +'IOR -W -f script' will have all tests in the script run with the -W option as +default.) The script itself can override these settings and may be set to run +run many different tests of IOR under a single execution. The command line is: + + IOR/bin/IOR -f script + +In IOR/scripts, there are scripts of testcases for simulating I/O behavior of +various application codes. Details are included in each script as necessary. + +An example of a script: +===============> start script <=============== +IOR START + api=[POSIX|MPIIO|HDF5|NCMPI] + testFile=testFile + hintsFileName=hintsFile + repetitions=8 + multiFile=0 + interTestDelay=5 + readFile=1 + writeFile=1 + filePerProc=0 + checkWrite=0 + checkRead=0 + keepFile=1 + quitOnError=0 + segmentCount=1 + blockSize=32k + outlierThreshold=0 + setAlignment=1 + transferSize=32 + singleXferAttempt=0 + individualDataSets=0 + verbose=0 + numTasks=32 + collective=1 + preallocate=0 + useFileView=0 + keepFileWithError=0 + setTimeStampSignature=0 + useSharedFilePointer=0 + useStridedDatatype=0 + uniqueDir=0 + fsync=0 + storeFileOffset=0 + maxTimeDuration=60 + deadlineForStonewalling=0 + useExistingTestFile=0 + useO_DIRECT=0 + showHints=0 + showHelp=0 +RUN + # additional tests are optional + +RUN + +RUN +IOR STOP +===============> stop script <=============== + +NOTES: * Not all test parameters need be set. The defaults can be viewed in + IOR/src/C/defaults.h. + * White space is ignored in script, as are comments starting with '#'. + + +**************************************** +* 8. COMPATIBILITY WITH OLDER VERSIONS * +**************************************** +1) IOR version 1 (c. 1996-2002) and IOR version 2 (c. 2003-present) are + incompatible. Input decks from one will not work on the other. As version + 1 is not included in this release, this shouldn't be case for concern. All + subsequent compatibility issues are for IOR version 2. + +2) IOR versions prior to release 2.8 provided data size and rates in powers + of two. E.g., 1 MB/sec referred to 1,048,576 bytes per second. With the + IOR release 2.8 and later versions, MB is now defined as 1,000,000 bytes + and MiB is 1,048,576 bytes. + +3) In IOR versions 2.5.3 to 2.8.7, IOR could be run without any command line + options. This assumed that if both write and read options (-w -r) were + omitted, the run with them both set as default. Later, it became clear + that in certain cases (data checking, e.g.) this caused difficulties. In + IOR versions 2.8.8 and later, if not one of the -w -r -W or -R options is + set, then -w and -r are set implicitly. + + +********************************* +* 9. FREQUENTLY ASKED QUESTIONS * +********************************* +HOW DO I PERFORM MULTIPLE DATA CHECKS ON AN EXISTING FILE? + + Use this command line: IOR -k -E -W -i 5 -o file + + -k keeps the file after the access rather than deleting it + -E uses the existing file rather than truncating it first + -W performs the writecheck + -i number of iterations of checking + -o filename + + On versions of IOR prior to 2.8.8, you need the -r flag also, otherwise + you'll first overwrite the existing file. (In earlier versions, omitting -w + and -r implied using both. This semantic has been subsequently altered to be + omitting -w, -r, -W, and -R implied using both -w and -r.) + + If you're running new tests to create a file and want repeat data checking on + this file multiple times, there is an undocumented option for this. It's -O + multiReRead=1, and you'd need to have an IOR version compiled with the + USE_UNDOC_OPT=1 (in iordef.h). The command line would look like this: + + IOR -k -E -w -W -i 5 -o file -O multiReRead=1 + + For the first iteration, the file would be written (w/o data checking). Then + for any additional iterations (four, in this example) the file would be + reread for whatever data checking option is used. + + +HOW DOES IOR CALCULATE PERFORMANCE? + + IOR performs get a time stamp START, then has all participating tasks open a + shared or independent file, transfer data, close the file(s), and then get a + STOP time. A stat() or MPI_File_get_size() is performed on the file(s) and + compared against the aggregate amount of data transferred. If this value + does not match, a warning is issued and the amount of data transferred as + calculated from write(), e.g., return codes is used. The calculated + bandwidth is the amount of data transferred divided by the elapsed + STOP-minus-START time. + + IOR also gets time stamps to report the open, transfer, and close times. + Each of these times is based on the earliest start time for any task and the + latest stop time for any task. Without using barriers between these + operations (-g), the sum of the open, transfer, and close times may not equal + the elapsed time from the first open to the last close. + + +HOW DO I ACCESS MULTIPLE FILE SYSTEMS IN IOR? + + It is possible when using the filePerProc option to have tasks round-robin + across multiple file names. Rather than use a single file name '-o file', + additional names '-o file1@file2@file3' may be used. In this case, a file + per process would have three different file names (which may be full path + names) to access. The '@' delimiter is arbitrary, and may be set in the + FILENAME_DELIMITER definition in iordef.h. + + Note that this option of multiple filenames only works with the filePerProc + -F option. This will not work for shared files. + + +HOW DO I BALANCE LOAD ACROSS MULTIPLE FILE SYSTEMS? + + As for the balancing of files per file system where different file systems + offer different performance, additional instances of the same destination + path can generally achieve good balance. + + For example, with FS1 getting 50% better performance than FS2, set the '-o' + flag such that there are additional instances of the FS1 directory. In this + case, '-o FS1/file@FS1/file@FS1/file@FS2/file@FS2/file' should adjust for + the performance difference and balance accordingly. + + +HOW DO I USE STONEWALLING? + + To use stonewalling (-D), it's generally best to separate write testing from + read testing. Start with writing a file with '-D 0' (stonewalling disabled) + to determine how long the file takes to be written. If it takes 10 seconds + for the data transfer, run again with a shorter duration, '-D 7' e.g., to + stop before the file would be completed without stonewalling. For reading, + it's best to create a full file (not an incompletely written file from a + stonewalling run) and then run with stonewalling set on this preexisting + file. If a write and read test are performed in the same run with + stonewalling, it's likely that the read will encounter an error upon hitting + the EOF. Separating the runs can correct for this. E.g., + + IOR -w -k -o file -D 10 # write and keep file, stonewall after 10 seconds + IOR -r -E -o file -D 7 # read existing file, stonewall after 7 seconds + + Also, when running multiple iterations of a read-only stonewall test, it may + be necessary to set the -D value high enough so that each iteration is not + reading from cache. Otherwise, in some cases, the first iteration may show + 100 MB/s, the next 200 MB/s, the third 300 MB/s. Each of these tests is + actually reading the same amount from disk in the allotted time, but they + are also reading the cached data from the previous test each time to get the + increased performance. Setting -D high enough so that the cache is + overfilled will prevent this. + + +HOW DO I BYPASS CACHING WHEN READING BACK A FILE I'VE JUST WRITTEN? + + One issue with testing file systems is handling cached data. When a file is + written, that data may be stored locally on the node writing the file. When + the same node attempts to read the data back from the file system either for + performance or data integrity checking, it may be reading from its own cache + rather from the file system. + + The reorderTasks '-C' option attempts to address this by having a different + node read back data than wrote it. For example, node N writes the data to + file, node N+1 reads back the data for read performance, node N+2 reads back + the data for write data checking, and node N+3 reads the data for read data + checking, comparing this with the reread data from node N+4. The objective + is to make sure on file access that the data is not being read from cached + data. + + Node 0: writes data + Node 1: reads data + Node 2: reads written data for write checking + Node 3: reads written data for read checking + Node 4: reads written data for read checking, comparing with Node 3 + + The algorithm for skipping from N to N+1, e.g., expects consecutive task + numbers on nodes (block assignment), not those assigned round robin (cyclic + assignment). For example, a test running 6 tasks on 3 nodes would expect + tasks 0,1 on node 0; tasks 2,3 on node 1; and tasks 4,5 on node 2. Were the + assignment for tasks-to-node in round robin fashion, there would be tasks 0,3 + on node 0; tasks 1,4 on node 1; and tasks 2,5 on node 2. In this case, there + would be no expectation that a task would not be reading from data cached on + a node. + + +HOW DO I USE HINTS? + + It is possible to pass hints to the I/O library or file system layers + following this form: + 'setenv IOR_HINT____ ' + For example: + 'setenv IOR_HINT__MPI__IBM_largeblock_io true' + 'setenv IOR_HINT__GPFS__important_hint true' + or, in a file in the form: + 'IOR_HINT____=' + Note that hints to MPI from the HDF5 or NCMPI layers are of the form: + 'setenv IOR_HINT__MPI__ ' + + +HOW DO I EXPLICITY SET THE FILE DATA SIGNATURE? + + The data signature for a transfer contains the MPI task number, transfer- + buffer offset, and also timestamp for the start of iteration. As IOR works + with 8-byte long long ints, the even-numbered long longs written contain a + 32-bit MPI task number and a 32-bit timestamp. The odd-numbered long longs + contain a 64-bit transferbuffer offset (or file offset if the '-l' + storeFileOffset option is used). To set the timestamp value, use '-G' or + setTimeStampSignature. + + +HOW DO I EASILY CHECK OR CHANGE A BYTE IN AN OUTPUT DATA FILE? + + There is a simple utility IOR/src/C/cbif/cbif.c that may be built. This is a + stand-alone, serial application called cbif (Change Byte In File). The + utility allows a file offset to be checked, returning the data at that + location in IOR's data check format. It also allows a byte at that location + to be changed. + + +HOW DO I CORRECT FOR CLOCK SKEW BETWEEN NODES IN A CLUSTER? + + To correct for clock skew between nodes, IOR compares times between nodes, + then broadcasts the root node's timestamp so all nodes can adjust by the + difference. To see an egregious outlier, use the '-j' option. Be sure + to set this value high enough to only show a node outside a certain time + from the mean. + + +WHAT HAPPENED TO THE GUI? + + In versions of IOR earlier than 2.9.x, there was a GUI available. Over time + it became clear that it wasn't find enough use to warrant maintenance. It + was retired in IOR-2.10.x. + +************************** +* 10. OUTPUT DESCRIPTION * +************************** +(FIXME -- this section needs updating and some rewrite.) + +Output Statistics: + +The quantity "aggregate operations/sec" was added to the existing "aggregate data rate" test log print file. +An "operation" is defined to be a write or read within an open/close [open/write|read/close]. +Multiple writes or reads within an open/close are also counted as multiple operations. +Also various other test relevant quantities are printed on a single "grepable" line using the pattern EXCEL. +This way, output from large parameter space runs can easily be imported to excel for analysis. Below is an example. + +grep EXCEL :IOR.o406550 + +Operation Max (MiB) Min (MiB) Mean (MiB) Std Dev Max (OPs) Min (OPs) Mean (OPs) Std Dev Mean(s) +--------- --------- --------- ---------- ------- --------- --------- ---------- ------- ------- +read 309.30 17.20 164.67 73.80 309.30 17.20 164.67 73.80 0.06581 +(line-continued) +#Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize + 8 2 100 1 0 1 0 0 1 1048576 1048576 8388608 5 EXCEL + +Where: +Max (MiB) - Maximum aggregate data rate of 100 iterations (reps) +Min (MiB) - Minumum aggregate data rate of 100 iterations (reps) +Mean(MiB) - Mean aggregate data rate of 100 iterations (reps) +Std Dev - Standard deviation aggregate data rate of 100 iterations (reps) +Max (OPs) - Maximum aggregate operations per second of 100 iterations (reps) +Min (OPs) - Minimum aggregate operations per second of 100 iterations (reps) +Mean (OPs)- Mean aggregate operations per second of 100 iterations (reps) +Std Dev - Standard deviation aggregate operations per second of 100 iteration (reps) +Mean(s) - Mean time per iteration (seconds) +#Tasks - number of I/O processes +tPN - number of I/O processes per node (per shared memory environment) +reps - number of times (iterations) each test is run. + The max,min,mean,ave,sdev above are calculated over "reps" tests +fPP - files per process +reord - constant node offset flag for reads +reordoff - node offset for reads +reordrand - random node offset flag for reads +seed - random seed for node random node offset +segcnt - number of seqments per file +blksiz - total MBytes written/read per process +xsize - total MBytes written/read per process per operation +aggsize - total Mbytes written/read by all processes per operations +5 - testnum +EXCEL - grep pattern for this summary print + +More detail information can be obtained with AT LEAST "-v -v" [verbose=0] level and grepping "XXCEL". +This includes a file "contention" histogram showing the number of files accessed 0,1,2,3,... times for a specified random pattern. + diff --git a/scripts/exampleScript b/scripts/exampleScript new file mode 100644 index 0000000..c17e284 --- /dev/null +++ b/scripts/exampleScript @@ -0,0 +1,24 @@ +IOR START + testFile = /tmp/work/swh13/testfile + filePerProc=1 + api=POSIX + repetitions=2 + verbose=1 + reordertasksrandom=1 + reordertasksrandomseed=-113 + RUN + verbose = 2 + repetitions=1#more foo + reordertasksconstant=1 + #foobar + blockSize=10m + transferSize=128k + randomoffset=1 + + RUN + verbose = 0 + + #blockSize= + transferSize=64k + RUN +IOR STOP diff --git a/scripts/run_script.cnl b/scripts/run_script.cnl new file mode 100644 index 0000000..a38edbd --- /dev/null +++ b/scripts/run_script.cnl @@ -0,0 +1,93 @@ +#!/bin/bash -x + +#PBS -N IOR +#PBS -j oe +#PBS -q batch +#PBS -A stf006 +#PBS -V +#PBS -l walltime=0:60:00,size=8 + +VERS=IOR-2.10.1.ornl.16 +WORK=/tmp/work/${USER} +echo $PBS_O_WORKDIR + +cd /ccs/proj/quadcore +tar -czvf ${WORK}/${VERS}.tar.gz ./${VERS} +cd ${WORK} +rm -fr ./${VERS} +tar -xzvf ${WORK}/${VERS}.tar.gz +cd ${WORK}/${VERS} +gmake clean +gmake mpiio +EXEC=${WORK}/${VERS}/src/C/IOR +IODIR=/tmp/work/swh13/test_files_x +cd ${WORK}/${VERS}/tests + +which mpirun + +rm -fr $IODIR +mkdir $IODIR + +let "w=128" +let "s=1024*1024" +let "i=3" + +MPIRUN="aprun -n" + +RESULTS="." + +let "tid=1" +XFERS="1048576 262144 32768 4096 1024" +XFERS="262144" +for xfer in `echo $XFERS` +do + let "n=8" +until [ "$n" -gt 8 ] +do + + let "m=$n/4" + #TESTS="POSIX MPIIO HDF5 NCMPI" + TESTS="POSIX MPIIO" + for test in `echo $TESTS` + do + runid="p$n.$xfer.${test}" + date + + V=" " + BLOCKS="1 10 1 10 1 10" + for blocks in `echo $BLOCKS` + do + + let "block=${xfer} * ${blocks}" + + #fileperproc tests + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -w -z ${V} -F -o $IODIR/testwrite.${runid} -Y -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -w -z ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -C ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -C -Q $m ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -Z -Q $m ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -Z -Q $m -X 13 ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -Z -Q $m -X -13 ${V} -F -o $IODIR/testwrite.${runid} -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + + #shared tests + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -w -z ${V} -o $IODIR/testwrite.${runid} -Y -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -w ${V} -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z ${V} -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + + #test mutually exclusive options + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -C ${V} -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -Z ${V} -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -Z -C ${V} -o $IODIR/testwrite.${runid} -i${i} -m -t ${xfer} -b ${block} -d 0.0 + let "tid=$tid + 17" + + V=$V" -v" + + done #blocks + + date + done #test + let "n = $n * 2" + done #n +done #xfer +exit diff --git a/scripts/run_script.linux b/scripts/run_script.linux new file mode 100644 index 0000000..42d444f --- /dev/null +++ b/scripts/run_script.linux @@ -0,0 +1,93 @@ +#!/bin/bash -x + +#PBS -N IOR +#PBS -j oe +#PBS -q batch +#PBS -A stf006 +#PBS -V +#PBS -l walltime=0:60:00,nodes=8:ppn=2 + +VERS=IOR-2.10.1 +WORK=/tmp/work/${USER} +echo $PBS_O_WORKDIR + +cd /ccs/proj/quadcore +tar -czvf ${WORK}/${VERS}.tar.gz ./${VERS} +cd ${WORK} +rm -fr ./${VERS} +tar -xzvf ${WORK}/${VERS}.tar.gz +cd ${WORK}/${VERS} +gmake clean +gmake mpiio +EXEC=${WORK}/${VERS}/src/C/IOR +IODIR=/tmp/work/swh13/test_files_x +cd ${WORK}/${VERS}/tests + +which mpirun + +rm -fr $IODIR +mkdir $IODIR + +let "w=128" +let "s=1024*1024" +let "i=3" + +MPIRUN="mpirun -np" + +RESULTS="." + +let "tid=1" +XFERS="1048576 262144 32768 4096 1024" +XFERS="262144" +for xfer in `echo $XFERS` +do + let "n=8" +until [ "$n" -gt 8 ] +do + + let "m=$n/4" + #TESTS="POSIX MPIIO HDF5 NCMPI" + TESTS="POSIX MPIIO" + for test in `echo $TESTS` + do + runid="p$n.$xfer.${test}" + date + + V=" " + BLOCKS="1 10 1 10 1 10" + for blocks in `echo $BLOCKS` + do + + let "block=${xfer} * ${blocks}" + + #fileperproc tests + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -w -z ${V} -F -o $IODIR/testwrite.${runid} -Y -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -w -z ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -C ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -C -Q $m ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -Z -Q $m ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -Z -Q $m -X 13 ${V} -F -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -Z -Q $m -X -13 ${V} -F -o $IODIR/testwrite.${runid} -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + + #shared tests + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -w -z ${V} -o $IODIR/testwrite.${runid} -Y -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -w ${V} -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z ${V} -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + + #test mutually exclusive options + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -C ${V} -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -z -Z ${V} -o $IODIR/testwrite.${runid} -k -e -i${i} -m -t ${xfer} -b ${block} -d 0.1 + ${MPIRUN} $n ${EXEC} -A ${tid} -a ${test} -r -Z -C ${V} -o $IODIR/testwrite.${runid} -i${i} -m -t ${xfer} -b ${block} -d 0.0 + let "tid=$tid + 17" + + V=$V" -v" + + done #blocks + + date + done #test + let "n = $n * 2" + done #n +done #xfer +exit diff --git a/src/C/IOR-aiori.h b/src/C/IOR-aiori.h new file mode 100644 index 0000000..5c931fe --- /dev/null +++ b/src/C/IOR-aiori.h @@ -0,0 +1,39 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: IOR-aiori.h,v $ +* $Revision: 1.1.1.1 $ +* $Date: 2007/10/15 23:36:54 $ +* $Author: rklundt $ +* +* Purpose: +* This is a header file that contains the abstract prototypes +* needed for IOR.c. +* +\******************************************************************************/ + +#ifndef _IOR_AIORI_H +#define _IOR_AIORI_H + +#include "IOR.h" + + +/**************************** P R O T O T Y P E S *****************************/ + +/* abstract IOR interfaces used in aiori-*.c */ +void * (*IOR_Create) (char *, IOR_param_t *); +void * (*IOR_Open) (char *, IOR_param_t *); +IOR_offset_t (*IOR_Xfer) (int, void *, IOR_size_t *, + IOR_offset_t, IOR_param_t *); +void (*IOR_Close) (void *, IOR_param_t *); +void (*IOR_Delete) (char *, IOR_param_t *); +void (*IOR_SetVersion) (IOR_param_t *); +void (*IOR_Fsync) (void *, IOR_param_t *); +IOR_offset_t (*IOR_GetFileSize) (IOR_param_t *, MPI_Comm, char *); + +#endif /* not _IOR_AIORI_H */ diff --git a/src/C/IOR.c b/src/C/IOR.c new file mode 100644 index 0000000..c71bbda --- /dev/null +++ b/src/C/IOR.c @@ -0,0 +1,2693 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: IOR.c,v $ +* $Revision: 1.4 $ +* $Date: 2008/12/03 16:16:02 $ +* $Author: rklundt $ +* +* Purpose: +* Test file system I/O. +* +* Settings and Usage: +* View DisplayUsage() for settings. +* Usage is with either by commandline or with an input script +* file of the form shown in DisplayUsage(). +* +* History (see CVS log for detailed history): +* 2001.11.21 wel +* Started initial implementation of code. +* 2002.02.07 wel +* Added abstract IOR interface for I/O. +* 2002.03.29 wel +* Added MPI synchronization. +* 2002.04.15 wel +* Added MPIIO. +* 2002.08.07 wel +* Added MPI file hints, collective calls, file views, etc. +* 2002.11.06 wel +* Added HDF5. +* 2003.10.03 wel +* Added NCMPI. +* +* Known problems and limitations: +* None known. +* +\******************************************************************************/ +/********************* Modifications to IOR-2.10.1 **************************** +* Hodson, 8/18/2008: * +* Documentation updated for the following new option: * +* The modifications to IOR-2.10.1 extend existing random I/O capabilities and * +* enhance performance output statistics, such as IOPs. * +* * +* cli script Description * +* ----- ----------------- ----------------------------------------* +* 1) -A N testnum - test reference number for easier test * +* identification in log files * +* 2) -Q N taskpernodeoffset - for read tests. Use with -C & -Z options* +* If -C (reordertasks) specified, * +* then node offset read by CONSTANT N. * +* If -Z (reordertasksrandom) specified, * +* then node offset read by RANDOM >= N. * +* 3) -Z reordertasksrandom - random node task ordering for read tests* +* In this case, processes read * +* data written by other processes with * +* node offsets specified by the -Q option * +* and -X option. * +* 4) -X N reordertasksrandomseed - random seed for -Z (reordertasksrandom) * +* If N>=0, use same seed for all iters * +* If N< 0, use different seed for ea. iter* +* 5) -Y fsyncperwrite - perform fsync after every POSIX write * +\*****************************************************************************/ + +#include "aiori.h" /* IOR I/O interfaces */ +#include "IOR.h" /* IOR definitions + and prototypes */ +#include "IOR-aiori.h" /* IOR abstract + interfaces */ +#include /* tolower() */ +#include /* sys_errlist */ +#include +#include +#include +#include +#include +#include /* struct stat */ +#include +#ifndef _WIN32 +# include /* gettimeofday() */ +# include /* uname() */ +#endif + +/************************** D E C L A R A T I O N S ***************************/ + +extern IOR_param_t defaultParameters, + initialTestParams; +extern int errno; /* error number */ +extern char ** environ; +int totalErrorCount = 0; +int numTasksWorld = 0; +int rank = 0; +int rankOffset = 0; +int tasksPerNode = 0; /* tasks per node */ +int verbose = VERBOSE_0; /* verbose output */ +double wall_clock_delta = 0; +double wall_clock_deviation; +MPI_Comm testComm; + +/********************************** M A I N ***********************************/ + +int +main(int argc, + char ** argv) +{ + int i; + IOR_queue_t * tests; + + /* + * check -h option from commandline without starting MPI; + * if the help option is requested in a script file (showHelp=TRUE), + * the help output will be displayed in the MPI job + */ + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0) { + DisplayUsage(argv); + return(0); + } + } + + /* start the MPI code */ + MPI_CHECK(MPI_Init(&argc, &argv), "cannot initialize MPI"); + MPI_CHECK(MPI_Comm_size(MPI_COMM_WORLD, &numTasksWorld), + "cannot get number of tasks"); + MPI_CHECK(MPI_Comm_rank(MPI_COMM_WORLD, &rank), "cannot get rank"); + /* set error-handling */ + /*MPI_CHECK(MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN), + "cannot set errhandler");*/ + + /* setup tests before verifying test validity */ + tests = SetupTests(argc, argv); + verbose = tests->testParameters.verbose; + tests->testParameters.testComm = MPI_COMM_WORLD; + + /* check for commandline usage */ + if (rank == 0 && tests->testParameters.showHelp == TRUE) { + DisplayUsage(argv); + } + + /* perform each test */ + while (tests != NULL) { + verbose = tests->testParameters.verbose; + if (rank == 0 && verbose >= VERBOSE_0) { + ShowInfo(argc, argv, &tests->testParameters); + } + if (rank == 0 && verbose >= VERBOSE_3) { + ShowTest(&tests->testParameters); + } + TestIoSys(&tests->testParameters); + tests = tests->nextTest; + } + + /* display finish time */ + if (rank == 0 && verbose >= VERBOSE_0) { + fprintf(stdout, "Run finished: %s", CurrentTimeString()); + } + + MPI_CHECK(MPI_Finalize(), "cannot finalize MPI"); + + return(totalErrorCount); + +} /* main() */ + + +/***************************** F U N C T I O N S ******************************/ + +/******************************************************************************/ +/* + * Bind abstract I/O function pointers to API-specific functions. + */ + +void +AioriBind(char * api) +{ + if (strcmp(api, "POSIX") == 0) { + IOR_Create = IOR_Create_POSIX; + IOR_Open = IOR_Open_POSIX; + IOR_Xfer = IOR_Xfer_POSIX; + IOR_Close = IOR_Close_POSIX; + IOR_Delete = IOR_Delete_POSIX; + IOR_SetVersion = IOR_SetVersion_POSIX; + IOR_Fsync = IOR_Fsync_POSIX; + IOR_GetFileSize = IOR_GetFileSize_POSIX; + } else if (strcmp(api, "MPIIO") == 0) { + IOR_Create = IOR_Create_MPIIO; + IOR_Open = IOR_Open_MPIIO; + IOR_Xfer = IOR_Xfer_MPIIO; + IOR_Close = IOR_Close_MPIIO; + IOR_Delete = IOR_Delete_MPIIO; + IOR_SetVersion = IOR_SetVersion_MPIIO; + IOR_Fsync = IOR_Fsync_MPIIO; + IOR_GetFileSize = IOR_GetFileSize_MPIIO; + } else if (strcmp(api, "HDF5") == 0) { + IOR_Create = IOR_Create_HDF5; + IOR_Open = IOR_Open_HDF5; + IOR_Xfer = IOR_Xfer_HDF5; + IOR_Close = IOR_Close_HDF5; + IOR_Delete = IOR_Delete_HDF5; + IOR_SetVersion = IOR_SetVersion_HDF5; + IOR_Fsync = IOR_Fsync_HDF5; + IOR_GetFileSize = IOR_GetFileSize_HDF5; + } else if (strcmp(api, "NCMPI") == 0) { + IOR_Create = IOR_Create_NCMPI; + IOR_Open = IOR_Open_NCMPI; + IOR_Xfer = IOR_Xfer_NCMPI; + IOR_Close = IOR_Close_NCMPI; + IOR_Delete = IOR_Delete_NCMPI; + IOR_SetVersion = IOR_SetVersion_NCMPI; + IOR_Fsync = IOR_Fsync_NCMPI; + IOR_GetFileSize = IOR_GetFileSize_NCMPI; + } else { + WARN("unrecognized IO API"); + } +} /* AioriBind() */ + + +/******************************************************************************/ +/* + * Display outliers found. + */ + +void +DisplayOutliers(int numTasks, + double timerVal, + char * timeString, + int access, + int outlierThreshold) +{ + char accessString[MAX_STR]; + double sum, mean, sqrDiff, var, sd; + + /* for local timerVal, don't compensate for wall clock delta */ + timerVal += wall_clock_delta; + + MPI_CHECK(MPI_Allreduce(&timerVal, &sum, 1, MPI_DOUBLE, MPI_SUM, testComm), + "MPI_Allreduce()"); + mean = sum / numTasks; + sqrDiff = pow((mean - timerVal), 2); + MPI_CHECK(MPI_Allreduce(&sqrDiff, &var, 1, MPI_DOUBLE, MPI_SUM, testComm), + "MPI_Allreduce()"); + var = var / numTasks; + sd = sqrt(var); + + if (access == WRITE) { + strcpy(accessString, "write"); + } else { /* READ */ + strcpy(accessString, "read"); + } + if (fabs(timerVal - mean) > (double)outlierThreshold){ + fprintf(stdout, "WARNING: for task %d, %s %s is %f\n", + rank, accessString, timeString, timerVal); + fprintf(stdout, " (mean=%f, stddev=%f)\n", mean, sd); + fflush(stdout); + } +} /* DisplayOutliers() */ + + +/******************************************************************************/ +/* + * Check for outliers in start/end times and elapsed create/xfer/close times. + */ + +void +CheckForOutliers(IOR_param_t * test, + double ** timer, + int rep, + int access) +{ + int shift; + + if (access == WRITE) { + shift = 0; + } else { /* READ */ + shift = 6; + } + + DisplayOutliers(test->numTasks, timer[shift+0][rep], + "start time", access, test->outlierThreshold); + DisplayOutliers(test->numTasks, timer[shift+1][rep]-timer[shift+0][rep], + "elapsed create time", access, test->outlierThreshold); + DisplayOutliers(test->numTasks, timer[shift+3][rep]-timer[shift+2][rep], + "elapsed transfer time", access, test->outlierThreshold); + DisplayOutliers(test->numTasks, timer[shift+5][rep]-timer[shift+4][rep], + "elapsed close time", access, test->outlierThreshold); + DisplayOutliers(test->numTasks, timer[shift+5][rep], + "end time", access, test->outlierThreshold); + +} /* CheckForOutliers() */ + + +/******************************************************************************/ +/* + * Check if actual file size equals expected size; if not use actual for + * calculating performance rate. + */ + +void +CheckFileSize(IOR_param_t *test, + IOR_offset_t dataMoved, + int rep) +{ + MPI_CHECK(MPI_Allreduce(&dataMoved, &test->aggFileSizeFromXfer[rep], + 1, MPI_LONG_LONG_INT, MPI_SUM, testComm), + "cannot total data moved"); + + if (strcmp(test->api, "HDF5") != 0 && strcmp(test->api, "NCMPI") != 0) { + if (verbose >= VERBOSE_0 && rank == 0) { + if ((test->aggFileSizeFromCalc[rep] + != test->aggFileSizeFromXfer[rep]) + || + (test->aggFileSizeFromStat[rep] + != test->aggFileSizeFromXfer[rep])) { + fprintf(stdout, + "WARNING: Expected aggregate file size = %lld.\n", + (long long)test->aggFileSizeFromCalc[rep]); + fprintf(stdout, + "WARNING: Stat() of aggregate file size = %lld.\n", + (long long)test->aggFileSizeFromStat[rep]); + fprintf(stdout, + "WARNING: Using actual aggregate bytes moved = %lld.\n", + (long long)test->aggFileSizeFromXfer[rep]); + } + } + } + test->aggFileSizeForBW[rep] = test->aggFileSizeFromXfer[rep]; + +} /* CheckFileSize() */ + + +/******************************************************************************/ +/* + * Check if string is true or false. + */ + +char * +CheckTorF(char * string) +{ + string = LowerCase(string); + if (strcmp(string, "false") == 0) { + strcpy(string, "0"); + } else if (strcmp(string, "true") == 0) { + strcpy(string, "1"); + } + return(string); +} /* CheckTorF() */ + + +/******************************************************************************/ +/* + * Compare buffers after reading/writing each transfer. Displays only first + * difference in buffers and returns total errors counted. + */ + +size_t +CompareBuffers(void * expectedBuffer, + void * unknownBuffer, + size_t size, + IOR_offset_t transferCount, + IOR_param_t * test, + int access) +{ + char testFileName[MAXPATHLEN], + bufferLabel1[MAX_STR], + bufferLabel2[MAX_STR]; + size_t i, j, length, first, last; + size_t errorCount = 0; + int inError = 0; + unsigned long long *goodbuf = (unsigned long long *)expectedBuffer; + unsigned long long *testbuf = (unsigned long long *)unknownBuffer; + + if (access == WRITECHECK) { + strcpy(bufferLabel1, "Expected: "); + strcpy(bufferLabel2, "Actual: "); + } else if (access == READCHECK) { + strcpy(bufferLabel1, "1st Read: "); + strcpy(bufferLabel2, "2nd Read: "); + } else { + ERR("incorrect argument for CompareBuffers()"); + } + + length = size / sizeof(IOR_size_t); + first = -1; + if (verbose >= VERBOSE_3) { + fprintf(stdout, + "[%d] At file byte offset %lld, comparing %llu-byte transfer\n", + rank, test->offset, (long long)size); + } + for (i = 0; i < length; i++) { + if (testbuf[i] != goodbuf[i]) { + errorCount++; + if (verbose >= VERBOSE_2) { + fprintf(stdout, + "[%d] At transfer buffer #%lld, index #%lld (file byte offset %lld):\n", + rank, transferCount-1, (long long)i, + test->offset + (IOR_size_t)(i * sizeof(IOR_size_t))); + fprintf(stdout, "[%d] %s0x", rank, bufferLabel1); + fprintf(stdout, "%016llx\n", goodbuf[i]); + fprintf(stdout, "[%d] %s0x", rank, bufferLabel2); + fprintf(stdout, "%016llx\n", testbuf[i]); + } + if (!inError) { + inError = 1; + first = i; + last = i; + } else { + last = i; + } + } else if (verbose >= VERBOSE_5 && i % 4 == 0) { + fprintf(stdout, "[%d] PASSED offset = %lld bytes, transfer %lld\n", + rank, ((i * sizeof(unsigned long long)) + test->offset), + transferCount); + fprintf(stdout, "[%d] GOOD %s0x", rank, bufferLabel1); + for (j = 0; j < 4; j++) + fprintf(stdout, "%016llx ", goodbuf[i + j]); + fprintf(stdout, "\n[%d] GOOD %s0x", rank, bufferLabel2); + for (j = 0; j < 4; j++) + fprintf(stdout, "%016llx ", testbuf[i + j]); + fprintf(stdout, "\n"); + } + } + if (inError) { + inError = 0; + GetTestFileName(testFileName, test); + fprintf(stdout, + "[%d] FAILED comparison of buffer containing %d-byte ints:\n", + rank, (int)sizeof(unsigned long long int)); + fprintf(stdout, "[%d] File name = %s\n", rank, testFileName); + fprintf(stdout, "[%d] In transfer %lld, ", + rank, transferCount); + fprintf(stdout, "%lld errors between buffer indices %lld and %lld.\n", + (long long)errorCount, (long long)first, (long long)last); + fprintf(stdout, "[%d] File byte offset = %lld:\n", + rank, ((first * sizeof(unsigned long long)) + test->offset)); + + fprintf(stdout, "[%d] %s0x", rank, bufferLabel1); + for (j = first; j < length && j < first + 4; j++) + fprintf(stdout, "%016llx ", goodbuf[j]); + if (j == length) fprintf(stdout, "[end of buffer]"); + fprintf(stdout, "\n[%d] %s0x", rank, bufferLabel2); + for (j = first; j < length && j < first + 4; j++) + fprintf(stdout, "%016llx ", testbuf[j]); + if (j == length) fprintf(stdout, "[end of buffer]"); + fprintf(stdout, "\n"); + if (test->quitOnError == TRUE) + ERR("data check error, aborting execution"); + } + return(errorCount); +} /* CompareBuffers() */ + + +/******************************************************************************//* + * Count all errors across all tasks; report errors found. + */ + +int +CountErrors(IOR_param_t *test, + int access, + int errors) +{ + int allErrors = 0; + + if (test->checkWrite || test->checkRead) { + MPI_CHECK(MPI_Reduce(&errors, &allErrors, 1, MPI_INT, MPI_SUM, + 0, testComm), "cannot reduce errors"); + MPI_CHECK(MPI_Bcast(&allErrors, 1, MPI_INT, 0, testComm), + "cannot broadcast allErrors value"); + if (allErrors != 0) { + totalErrorCount += allErrors; + test->errorFound = TRUE; + } + if (rank == 0 && allErrors != 0) { + if (allErrors < 0) { + WARN("overflow in errors counted"); + allErrors = -1; + } + if (access == WRITECHECK) { + fprintf(stdout, "WARNING: incorrect data on write.\n"); + fprintf(stdout, + " %d errors found on write check.\n", allErrors); + } else { + fprintf(stdout, "WARNING: incorrect data on read.\n"); + fprintf(stdout, + " %d errors found on read check.\n", allErrors); + } + fprintf(stdout, "Used Time Stamp %u (0x%x) for Data Signature\n", + test->timeStampSignatureValue, test->timeStampSignatureValue); + } + } + return(allErrors); +} /* CountErrors() */ + + +/******************************************************************************/ +/* + * Compares hostnames to determine the number of tasks per node + */ + +int +CountTasksPerNode(int numTasks, MPI_Comm comm) +{ + char localhost[MAX_STR], + hostname[MAX_STR], + taskOnNode[MAX_STR]; + int count = 1, + resultsLen = MAX_STR, + i; + static int firstPass = TRUE; + MPI_Status status; + + MPI_CHECK(MPI_Get_processor_name(localhost, &resultsLen), + "cannot get processor name"); + + if (verbose >= VERBOSE_2 && firstPass) { + sprintf(taskOnNode, "task %d on %s", rank, localhost); + OutputToRoot(numTasks, comm, taskOnNode); + firstPass = FALSE; + } + + if (numTasks > 1) { + if (rank == 0) { + /* MPI_receive all hostnames, and compare to local hostname */ + for (i = 0; i < numTasks-1; i++) { + MPI_CHECK(MPI_Recv(hostname, MAX_STR, MPI_CHAR, MPI_ANY_SOURCE, + MPI_ANY_TAG, comm, &status), + "cannot receive hostnames"); + if (strcmp(hostname, localhost) == 0) + count++; + } + } else { + /* MPI_send hostname to root node */ + MPI_CHECK(MPI_Send(localhost, MAX_STR, MPI_CHAR, 0, 0, + comm), "cannot send hostname"); + } + MPI_CHECK(MPI_Bcast(&count, 1, MPI_INT, 0, comm), + "cannot broadcast tasks-per-node value"); + } + + return(count); +} /* CountTasksPerNode() */ + + +/******************************************************************************/ +/* + * Allocate a page-aligned (required by O_DIRECT) buffer. + */ + +void * +CreateBuffer(size_t size) +{ + size_t pageSize; + size_t pageMask; + char *buf, *tmp; + char *aligned; + + pageSize = getpagesize(); + pageMask = pageSize-1; + buf = malloc(size + pageSize + sizeof(void *)); + if (buf == NULL) ERR("out of memory"); + /* find the alinged buffer */ + tmp = buf + sizeof(char *); + aligned = tmp + pageSize - ((size_t)tmp & pageMask); + /* write a pointer to the original malloc()ed buffer into the bytes + preceding "aligned", so that the aligned buffer can later be free()ed */ + tmp = aligned - sizeof(void *); + *(void **)tmp = buf; + + return((void *)aligned); +} /* CreateBuffer() */ + + +/******************************************************************************/ +/* + * Create new test for list of tests. + */ + +IOR_queue_t * +CreateNewTest(int test_num) +{ + IOR_queue_t * newTest = NULL; + + newTest = (IOR_queue_t *)malloc(sizeof(IOR_queue_t)); + if (newTest == NULL) ERR("out of memory"); + newTest->testParameters = initialTestParams; + GetPlatformName(newTest->testParameters.platform); + newTest->testParameters.nodes = initialTestParams.numTasks / tasksPerNode; + newTest->testParameters.tasksPerNode = tasksPerNode; + newTest->testParameters.id = test_num; + newTest->nextTest = NULL; + return(newTest); +} /* CreateNewTest() */ + + +/******************************************************************************/ +/* + * Sleep for 'delay' seconds. + */ + +void +DelaySecs(int delay) +{ + if (rank == 0 && delay > 0 ) { + if (verbose >= VERBOSE_1) + fprintf(stdout, "delaying %d seconds . . .\n", delay); + sleep(delay); + } +} /* DelaySecs() */ + + +/******************************************************************************/ +/* + * Display freespace (df). + */ +void +DisplayFreespace(IOR_param_t * test) +{ + char fileName[MAX_STR] = {0}; + int i; + int directoryFound = FALSE; + + /* get outfile name */ + GetTestFileName(fileName, test); + + /* get directory for outfile */ + i = strlen(fileName); + while (i-- > 0) { + if (fileName[i] == '/') { + fileName[i] = '\0'; + directoryFound = TRUE; + break; + } + } + + /* if no directory/, use '.' */ + if (directoryFound == FALSE) { + strcpy(fileName, "."); + } + +#if USE_UNDOC_OPT /* NFS */ + if (test->NFS_serverCount) { + strcpy(fileName, test->NFS_rootPath); + } +#endif /* USE_UNDOC_OPT - NFS */ + + ShowFileSystemSize(fileName); + + return; +} /* DisplayFreespace() */ + + +/******************************************************************************/ +/* + * Display usage of script file. + */ + +void +DisplayUsage(char ** argv) +{ + char * opts[] = { +"OPTIONS:", +" -A N testNum -- test number for reference in some output", +" -a S api -- API for I/O [POSIX|MPIIO|HDF5|NCMPI]", +" -b N blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g)", +" -B useO_DIRECT -- uses O_DIRECT for POSIX, bypassing I/O buffers", +" -c collective -- collective I/O", +" -C reorderTasks -- changes task ordering to n+1 ordering for readback", +" -Q N taskPerNodeOffset for read tests use with -C & -Z options (-C constant N, -Z at least N)", +" -Z reorderTasksRandom -- changes task ordering to random ordering for readback", +" -X N reorderTasksRandomSeed -- random seed for -Z option", +" -d N interTestDelay -- delay between reps in seconds", +" -D N deadlineForStonewalling -- seconds before stopping write or read phase", +" -Y fsyncPerWrite -- perform fsync after each POSIX write", +" -e fsync -- perform fsync upon POSIX write close", +" -E useExistingTestFile -- do not remove test file before write access", +" -f S scriptFile -- test script name", +" -F filePerProc -- file-per-process", +" -g intraTestBarriers -- use barriers between open, write/read, and close", +" -G N setTimeStampSignature -- set value for time stamp signature", +" -h showHelp -- displays options and help", +" -H showHints -- show hints", +" -i N repetitions -- number of repetitions of test", +" -I individualDataSets -- datasets not shared by all procs [not working]", +" -j N outlierThreshold -- warn on outlier N seconds from mean", +" -J N setAlignment -- HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g)", +" -k keepFile -- don't remove the test file(s) on program exit", +" -K keepFileWithError -- keep error-filled file(s) after data-checking", +" -l storeFileOffset -- use file offset as stored signature", +" -m multiFile -- use number of reps (-i) for multiple file count", +" -n noFill -- no fill in HDF5 file creation", +" -N N numTasks -- number of tasks that should participate in the test", +" -o S testFile -- full name for test", +" -O S string of IOR directives (e.g. -O checkRead=1,lustreStripeCount=32)", +" -p preallocate -- preallocate file size", +" -P useSharedFilePointer -- use shared file pointer [not working]", +" -q quitOnError -- during file error-checking, abort on error", +" -r readFile -- read existing file", +" -R checkRead -- check read after read", +" -s N segmentCount -- number of segments", +" -S useStridedDatatype -- put strided access into datatype [not working]", +" -t N transferSize -- size of transfer in bytes (e.g.: 8, 4k, 2m, 1g)", +" -T N maxTimeDuration -- max time in minutes to run tests", +" -u uniqueDir -- use unique directory name for each file-per-process", +" -U S hintsFileName -- full name for hints file", +" -v verbose -- output information (repeating flag increases level)", +" -V useFileView -- use MPI_File_set_view", +" -w writeFile -- write file", +" -W checkWrite -- check read after write", +" -x singleXferAttempt -- do not retry transfer if incomplete", +" -z randomOffset -- access is to random, not sequential, offsets within a file", +" ", +" NOTE: S is a string, N is an integer number.", +" ", +"" }; + int i = 0; + + fprintf(stdout, "Usage: %s [OPTIONS]\n\n", * argv); + for (i=0; strlen(opts[i]) > 0; i++) + fprintf(stdout, "%s\n", opts[i]); + + return; +} /* DisplayUsage() */ + + +/******************************************************************************/ +/* + * Distribute IOR_HINTs to all tasks' environments. + */ + +void +DistributeHints(void) +{ + char hint[MAX_HINTS][MAX_STR], + fullHint[MAX_STR], + hintVariable[MAX_STR]; + int hintCount = 0, + i; + + if (rank == 0) { + for (i = 0; environ[i] != NULL; i++) { + if (strncmp(environ[i], "IOR_HINT", strlen("IOR_HINT")) == 0) { + hintCount++; + if (hintCount == MAX_HINTS) { + WARN("exceeded max hints; reset MAX_HINTS and recompile"); + hintCount = MAX_HINTS; + break; + } + /* assume no IOR_HINT is greater than MAX_STR in length */ + strncpy(hint[hintCount-1], environ[i], MAX_STR-1); + } + } + } + + MPI_CHECK(MPI_Bcast(&hintCount, sizeof(hintCount), MPI_BYTE, + 0, MPI_COMM_WORLD), "cannot broadcast hints"); + for (i = 0; i < hintCount; i++) { + MPI_CHECK(MPI_Bcast(&hint[i], MAX_STR, MPI_BYTE, + 0, MPI_COMM_WORLD), "cannot broadcast hints"); + strcpy(fullHint, hint[i]); + strcpy(hintVariable, strtok(fullHint, "=")); + if (getenv(hintVariable) == NULL) { + /* doesn't exist in this task's environment; better set it */ + if (putenv(hint[i]) != 0) WARN("cannot set environment variable"); + } + } +} /* DistributeHints() */ + + +/******************************************************************************/ +/* + * Fill buffer, which is transfer size bytes long, with known 8-byte long long + * int values. In even-numbered 8-byte long long ints, store MPI task in high + * bits and timestamp signature in low bits. In odd-numbered 8-byte long long + * ints, store transfer offset. If storeFileOffset option is used, the file + * (not transfer) offset is stored instead. + */ + +void +FillBuffer(void * buffer, + IOR_param_t * test, + unsigned long long offset, + int fillrank) +{ + size_t i; + unsigned long long hi, lo; + unsigned long long *buf = (unsigned long long *)buffer; + + hi = ((unsigned long long)fillrank) << 32; + lo = (unsigned long long)test->timeStampSignatureValue; + for (i = 0; i < test->transferSize / sizeof(unsigned long long); i++) { + if ((i%2) == 0) { + /* evens contain MPI rank and time in seconds */ + buf[i] = hi | lo; + } else { + /* odds contain offset */ + buf[i] = offset + (i * sizeof(unsigned long long)); + } + } +} /* FillBuffer() */ + + +/******************************************************************************//* + * Free transfer buffers. + */ + +void +FreeBuffers(int access, + void *checkBuffer, + void *readCheckBuffer, + void *buffer, + IOR_offset_t *offsetArray) +{ + /* free() aligned buffers */ + if (access == WRITECHECK || access == READCHECK) { + free( *(void **)((char *)checkBuffer - sizeof(char *)) ); + } + if (access == READCHECK) { + free( *(void **)((char *)readCheckBuffer - sizeof(char *)) ); + } + free( *(void **)((char *)buffer - sizeof(char *)) ); + + /* nothing special needed to free() this unaligned buffer */ + free(offsetArray); + return; +} /* FreeBuffers() */ + + +/******************************************************************************/ +/* + * Return string describing machine name and type. + */ + +void +GetPlatformName(char * platformName) +{ + char nodeName[MAX_STR], + * p, + * start, + sysName[MAX_STR]; + struct utsname name; + + if (uname(&name) != 0) { + WARN("cannot get platform name"); + sprintf(sysName, "%s", "Unknown"); + sprintf(nodeName, "%s", "Unknown"); + } else { + sprintf(sysName, "%s", name.sysname); + sprintf(nodeName, "%s", name.nodename); + } + + start = nodeName; + if (strlen(nodeName) == 0) { + p = start; + } else { + /* point to one character back from '\0' */ + p = start + strlen(nodeName) - 1; + } + /* + * to cut off trailing node number, search backwards + * for the first non-numeric character + */ + while (p != start) { + if (* p < '0' || * p > '9') { + * (p+1) = '\0'; + break; + } else { + p--; + } + } + + sprintf(platformName, "%s(%s)", nodeName, sysName); +} /* GetPlatformName() */ + + +/******************************************************************************/ +/* + * Return test file name to access. + * for single shared file, fileNames[0] is returned in testFileName + */ + +void +GetTestFileName(char * testFileName, IOR_param_t * test) +{ + char ** fileNames, + initialTestFileName[MAXPATHLEN], + testFileNameRoot[MAX_STR], + tmpString[MAX_STR]; + int count; + + /* parse filename for multiple file systems */ + strcpy(initialTestFileName, test->testFileName); + fileNames = ParseFileName(initialTestFileName, &count); + if (count > 1 && test->uniqueDir == TRUE) + ERR("cannot use multiple file names with unique directories"); + if (test->filePerProc) { + strcpy(testFileNameRoot, + fileNames[((rank+rankOffset)%test->numTasks) % count]); + } else { + strcpy(testFileNameRoot, fileNames[0]); + } + + /* give unique name if using multiple files */ + if (test->filePerProc) { + /* + * prepend rank subdirectory before filename + * e.g., /dir/file => /dir//file + */ + if (test->uniqueDir == TRUE) { + strcpy(testFileNameRoot, PrependDir(test, testFileNameRoot)); + } +#if USE_UNDOC_OPT /* NFS */ + if (test->NFS_serverCount) { + sprintf(tmpString, "%s/%s%d/%s", test->NFS_rootPath, + test->NFS_serverName, rank%(test->NFS_serverCount), + testFileNameRoot); + strcpy(testFileNameRoot, tmpString); + } +#endif /* USE_UNDOC_OPT - NFS */ + sprintf(testFileName, "%s.%08d", testFileNameRoot, + (rank+rankOffset)%test->numTasks); + } else { + strcpy(testFileName, testFileNameRoot); + } + + /* add suffix for multiple files */ + if (test->repCounter > -1) { + sprintf(tmpString, ".%d", test->repCounter); + strcat(testFileName, tmpString); + } +} /* GetTestFileName() */ + + +/******************************************************************************/ +/* + * Get time stamp. Use MPI_Timer() unless _NO_MPI_TIMER is defined, + * in which case use gettimeofday(). + */ + +double +GetTimeStamp(void) +{ + double timeVal; +#ifdef _NO_MPI_TIMER + struct timeval timer; + + if (gettimeofday(&timer, (struct timezone *)NULL) != 0) + ERR("cannot use gettimeofday()"); + timeVal = (double)timer.tv_sec + ((double)timer.tv_usec/1000000); +#else /* not _NO_MPI_TIMER */ + timeVal = MPI_Wtime(); /* no MPI_CHECK(), just check return value */ + if (timeVal < 0) ERR("cannot use MPI_Wtime()"); +#endif /* _NO_MPI_TIMER */ + + /* wall_clock_delta is difference from root node's time */ + timeVal -= wall_clock_delta; + + return(timeVal); +} /* GetTimeStamp() */ + + +/******************************************************************************/ +/* + * Convert IOR_offset_t value to human readable string. + */ + +char * +HumanReadable(IOR_offset_t value, int base) +{ + char * valueStr; + int m = 0, g = 0; + char m_str[8], g_str[8]; + + valueStr = (char *)malloc(MAX_STR); + if (valueStr == NULL) ERR("out of memory"); + + if (base == BASE_TWO) { + m = MEBIBYTE; + g = GIBIBYTE; + strcpy(m_str, "MiB"); + strcpy(g_str, "GiB"); + } else if (base == BASE_TEN) { + m = MEGABYTE; + g = GIGABYTE; + strcpy(m_str, "MB"); + strcpy(g_str, "GB"); + } + + if (value >= g) { + if (value % (IOR_offset_t)g) { + sprintf(valueStr, "%.2f %s", (double)((double)value/g), g_str); + } else { + sprintf(valueStr, "%d %s", (int)(value/g), g_str); + } + } else if (value >= m) { + if (value % (IOR_offset_t)m) { + sprintf(valueStr, "%.2f %s", (double)((double)value/m), m_str); + } else { + sprintf(valueStr, "%d %s", (int)(value/m), m_str); + } + } else if (value >= 0) { + sprintf(valueStr, "%d bytes", (int)value); + } else { + sprintf(valueStr, "-"); + } + return valueStr; +} /* HumanReadable() */ + + +/******************************************************************************/ +/* + * Change string to lower case. + */ + +char * +LowerCase(char * string) +{ + char * nextChar = string; + + while (* nextChar != '\0') { + * nextChar = (char)tolower((int)* nextChar); + nextChar++; + } + return(string); +} /* LowerCase() */ + + +/******************************************************************************/ +/* + * Parse file name. + */ + +char ** +ParseFileName(char * name, int * count) +{ + char ** fileNames, + * tmp, + * token; + char delimiterString[3] = { FILENAME_DELIMITER, '\n', '\0' }; + int i = 0; + + * count = 0; + tmp = name; + + /* pass one */ + /* if something there, count the first item */ + if (* tmp != '\0') { + (* count)++; + } + /* count the rest of the filenames */ + while (* tmp != '\0') { + if (* tmp == FILENAME_DELIMITER) { + (* count)++; + } + tmp++; + } + + fileNames = (char **)malloc((* count) * sizeof(char **)); + if (fileNames == NULL) ERR("out of memory"); + + /* pass two */ + token = strtok(name, delimiterString); + while (token != NULL) { + fileNames[i] = token; + token = strtok(NULL, delimiterString); + i++; + } + return(fileNames); +} /* ParseFileName() */ + + +/******************************************************************************/ +/* + * Pretty Print a Double. The First parameter is a flag determining if left + * justification should be used. The third parameter a null-terminated string + * that should be appended to the number field. + */ + +void +PPDouble(int leftjustify, + double number, + char * append) +{ + if (number < 0) { + fprintf(stdout, " - %s", append); + } else { + if (leftjustify) { + if (number < 1) + fprintf(stdout, "%-10.6f%s", number, append); + else if (number < 3600) + fprintf(stdout, "%-10.2f%s", number, append); + else + fprintf(stdout, "%-10.0f%s", number, append); + } else { + if (number < 1) + fprintf(stdout, "%10.6f%s", number, append); + else if (number < 3600) + fprintf(stdout, "%10.2f%s", number, append); + else + fprintf(stdout, "%10.0f%s", number, append); + } + } +} /* PPDouble() */ + + +/******************************************************************************/ +/* + * From absolute directory, insert rank as subdirectory. Allows each task + * to write to its own directory. E.g., /dir/file => /dir//file. + * + */ + +char * +PrependDir(IOR_param_t * test, char * rootDir) +{ + char * dir; + char fname[MAX_STR+1]; + char * p; + int i; + + dir = (char *)malloc(MAX_STR+1); + if (dir == NULL) ERR("out of memory"); + + /* get dir name */ + strcpy(dir, rootDir); + i = strlen(dir)-1; + while (i > 0) { + if (dir[i] == '\0' || dir[i] == '/') { + dir[i] = '/'; + dir[i+1] = '\0'; + break; + } + i--; + } + + /* get file name */ + strcpy(fname, rootDir); + p = fname; + while (i > 0) { + if (fname[i] == '\0' || fname[i] == '/') { + p = fname + (i+1); + break; + } + i--; + } + + /* create directory with rank as subdirectory */ + sprintf(dir, "%s%d", dir, (rank+rankOffset)%test->numTasks); + + /* dir doesn't exist, so create */ + if (access(dir, F_OK) != 0) { + if (mkdir(dir, S_IRWXU) < 0 ) { + ERR("cannot create directory"); + } + + /* check if correct permissions */ + } else if (access(dir, R_OK) != 0 || access(dir, W_OK) != 0 || + access(dir, X_OK) != 0) { + ERR("invalid directory permissions"); + } + + /* concatenate dir and file names */ + strcat(dir, "/"); + strcat(dir, p); + + return dir; +} /* PrependDir() */ + + +/******************************************************************************//* + * Read and then reread buffer to confirm data read twice matches. + */ + +void +ReadCheck(void *fd, + void *buffer, + void *checkBuffer, + void *readCheckBuffer, + IOR_param_t *test, + IOR_offset_t transfer, + IOR_offset_t blockSize, + IOR_offset_t *amtXferred, + IOR_offset_t *transferCount, + int access, + int *errors) +{ + int readCheckToRank, readCheckFromRank; + MPI_Status status; + IOR_offset_t tmpOffset; + IOR_offset_t segmentSize, segmentNum; + + memset(buffer, 'a', transfer); + *amtXferred = IOR_Xfer(access, fd, buffer, transfer, test); + tmpOffset = test->offset; + if (test->filePerProc == FALSE) { + /* offset changes for shared file, not for file-per-proc */ + segmentSize = test->numTasks * blockSize; + segmentNum = test->offset / segmentSize; + + /* work in current segment */ + test->offset = (((test->offset % segmentSize) + /* offset to neighbor's data */ + + ((test->reorderTasks ? + test->tasksPerNode : 0) * blockSize)) + /* stay within current segment */ + % segmentSize) + /* return segment to actual file offset */ + + (segmentNum * segmentSize); + } + if (*amtXferred != transfer) + ERR("cannot read from file on read check"); + memset(checkBuffer, 'a', transfer); /* empty buffer */ +#if USE_UNDOC_OPT /* corruptFile */ + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + /* intentionally corrupt file to determine if check works */ + if (test->corruptFile) { + CorruptFile(test->testFileName, test, 0, READCHECK); + } +#endif /* USE_UNDOC_OPT - corruptFile */ + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + if (test->filePerProc) { + *amtXferred = IOR_Xfer(access, test->fd_fppReadCheck, + checkBuffer, transfer, test); + } else { + *amtXferred = IOR_Xfer(access, fd, checkBuffer, transfer, test); + } + test->offset = tmpOffset; + if (*amtXferred != transfer) + ERR("cannot reread from file read check"); + (*transferCount)++; + /* exchange buffers */ + memset(readCheckBuffer, 'a', transfer); + readCheckToRank = (rank + (test->reorderTasks ? test->tasksPerNode : 0)) + % test->numTasks; + readCheckFromRank = (rank + (test->numTasks + - (test->reorderTasks ? test->tasksPerNode : 0))) + % test->numTasks; + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + MPI_Sendrecv(checkBuffer, transfer, MPI_CHAR, readCheckToRank, 1, + readCheckBuffer, transfer, MPI_CHAR, readCheckFromRank, 1, + testComm, &status); + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + *errors += CompareBuffers(buffer, readCheckBuffer, transfer, + *transferCount, test, READCHECK); + return; +} /* ReadCheck() */ + + +/******************************************************************************/ +/* + * Reduce test results, and show if verbose set. + */ + +void +ReduceIterResults(IOR_param_t * test, + double ** timer, + int rep, + int access) +{ + double reduced[12] = { 0 }, + diff[6], + currentWrite, + currentRead, + totalWriteTime, + totalReadTime; + enum {RIGHT, LEFT}; + int i; + static int firstIteration = TRUE; + IOR_offset_t aggFileSizeForBW; + MPI_Op op; + + /* Find the minimum start time of the even numbered timers, and the + maximum finish time for the odd numbered timers */ + for (i = 0; i < 12; i++) { + op = i % 2 ? MPI_MAX : MPI_MIN; + MPI_CHECK(MPI_Reduce(&timer[i][rep], &reduced[i], 1, MPI_DOUBLE, + op, 0, testComm), + "MPI_Reduce()"); + } + + if (rank == 0) { + /* Calculate elapsed times and throughput numbers */ + for (i = 0; i < 6; i++) + diff[i] = reduced[2*i+1] - reduced[2*i]; + totalReadTime = reduced[11] - reduced[6]; + totalWriteTime = reduced[5] - reduced[0]; + aggFileSizeForBW = test->aggFileSizeForBW[rep]; + if (access == WRITE) { + currentWrite = (double)((double)aggFileSizeForBW / totalWriteTime); + test->writeVal[0][rep] = currentWrite; + test->writeVal[1][rep] = totalWriteTime; + } + if (access == READ) { + currentRead = (double)((double)aggFileSizeForBW / totalReadTime); + test->readVal[0][rep] = currentRead; + test->readVal[1][rep] = totalReadTime; + } + } + + +#if USE_UNDOC_OPT /* fillTheFileSystem */ + if (test->fillTheFileSystem && rank == 0 && firstIteration + && rep == 0 && verbose >= VERBOSE_1) { + fprintf(stdout, " . . . skipping iteration results output . . .\n"); + fflush(stdout); + } +#endif /* USE_UNDOC_OPT - fillTheFileSystem */ + + if (rank == 0 && verbose >= VERBOSE_2 +#if USE_UNDOC_OPT /* fillTheFileSystem */ + && test->fillTheFileSystem == FALSE +#endif /* USE_UNDOC_OPT - fillTheFileSystem */ + ) { + /* print out the results */ + if (firstIteration && rep == 0) { + fprintf(stdout, "access bw(MiB/s) block(KiB) xfer(KiB)"); + fprintf(stdout, " open(s) wr/rd(s) close(s) total(s) iter\n"); + fprintf(stdout, "------ --------- ---------- ---------"); + fprintf(stdout, " -------- -------- -------- -------- ----\n"); + } + if (access == WRITE) { + fprintf(stdout, "write "); + PPDouble(LEFT, (currentWrite/MEBIBYTE), " \0"); + PPDouble(LEFT, (double)test->blockSize/KIBIBYTE, " \0"); + PPDouble(LEFT, (double)test->transferSize/KIBIBYTE, " \0"); + if (test->writeFile) { + PPDouble(LEFT, diff[0], " \0"); + PPDouble(LEFT, diff[1], " \0"); + PPDouble(LEFT, diff[2], " \0"); + PPDouble(LEFT, totalWriteTime, " \0"); + } + fprintf(stdout, "%-4d XXCEL\n", rep); + } + if (access == READ) { + fprintf(stdout, "read "); + PPDouble(LEFT, (currentRead/MEBIBYTE), " \0"); + PPDouble(LEFT, (double)test->blockSize/KIBIBYTE, " \0"); + PPDouble(LEFT, (double)test->transferSize/KIBIBYTE, " \0"); + if (test->readFile) { + PPDouble(LEFT, diff[3], " \0"); + PPDouble(LEFT, diff[4], " \0"); + PPDouble(LEFT, diff[5], " \0"); + PPDouble(LEFT, totalReadTime, " \0"); + } + fprintf(stdout, "%-4d XXCEL\n", rep); + } + fflush(stdout); + } + firstIteration = FALSE; /* set to TRUE to repeat this header */ +} /* ReduceIterResults() */ + + +/******************************************************************************/ +/* + * Check for file(s), then remove all files if file-per-proc, else single file. + */ + +void +RemoveFile(char * testFileName, + int filePerProc, + IOR_param_t * test) +{ + int tmpRankOffset; + if (filePerProc) + { + /* in random tasks, delete own file */ + if (test->reorderTasksRandom == TRUE) + { + tmpRankOffset = rankOffset; + rankOffset = 0; + GetTestFileName(testFileName, test); + } + if (access(testFileName, F_OK) == 0) + { + IOR_Delete(testFileName, test); + } + if (test->reorderTasksRandom == TRUE) + { + rankOffset = tmpRankOffset; + GetTestFileName(testFileName, test); + } + } else { + if ((rank == 0) && (access(testFileName, F_OK) == 0)) { + IOR_Delete(testFileName, test); + } + } +} /* RemoveFile() */ + + +/******************************************************************************/ +/* + * Setup tests by parsing commandline and creating test script. + */ + +IOR_queue_t * +SetupTests(int argc, char ** argv) +{ + IOR_queue_t * tests, + * testsHead; + + /* count the tasks per node */ + tasksPerNode = CountTasksPerNode(numTasksWorld, MPI_COMM_WORLD); + + testsHead = tests = ParseCommandLine(argc, argv); + /* + * Since there is no guarantee that anyone other than + * task 0 has the environment settings for the hints, pass + * the hint=value pair to everyone else in MPI_COMM_WORLD + */ + DistributeHints(); + + /* check validity of tests and create test queue */ + while (tests != NULL) { + ValidTests(&tests->testParameters); + tests = tests->nextTest; + } + + /* check for skew between tasks' start times */ + wall_clock_deviation = TimeDeviation(); + + /* seed random number generator */ + SeedRandGen(MPI_COMM_WORLD); + + return(testsHead); +} /* SetupTests() */ + + +/******************************************************************************//* + * Setup transfer buffers, creating and filling as needed. + */ + +void +SetupXferBuffers(void **buffer, + void **checkBuffer, + void **readCheckBuffer, + IOR_param_t *test, + int pretendRank, + int access) +{ + /* create buffer of filled data */ + *buffer = CreateBuffer(test->transferSize); + FillBuffer(*buffer, test, 0, pretendRank); + if (access == WRITECHECK || access == READCHECK) { + /* this allocates buffer only */ + *checkBuffer = CreateBuffer(test->transferSize); + if (access == READCHECK) { + *readCheckBuffer = CreateBuffer(test->transferSize); + } + } + return; +} /* SetupXferBuffers() */ + + +/******************************************************************************/ +/* + * Print header information for test output. + */ + +void +ShowInfo(int argc, char **argv, IOR_param_t *test) +{ + int i; + struct utsname unamebuf; + + fprintf(stdout, "%s: MPI Coordinated Test of Parallel I/O\n\n", + IOR_RELEASE); + + fprintf(stdout, "Run began: %s", CurrentTimeString()); +#if USE_UNDOC_OPT /* NFS */ + if (test->NFS_serverCount) { + fprintf(stdout, "NFS path: %s%s[0..%d]\n", test->NFS_rootPath, + test->NFS_serverName, test->NFS_serverCount-1); + } +#endif /* USE_UNDOC_OPT - NFS */ + fprintf(stdout, "Command line used:"); + for (i = 0; i < argc; i++) { + fprintf(stdout, " %s", argv[i]); + } + fprintf(stdout, "\n"); + if (uname(&unamebuf) != 0) { + WARN("uname failed"); + fprintf(stdout, "Machine: Unknown"); + } else { + fprintf(stdout, "Machine: %s %s", unamebuf.sysname, unamebuf.nodename); + if (verbose >= VERBOSE_2) { + fprintf(stdout, " %s %s %s", unamebuf.release, unamebuf.version, + unamebuf.machine); + } + } + fprintf(stdout, "\n"); +#ifdef _NO_MPI_TIMER + if (verbose >= VERBOSE_2) + fprintf(stdout, "Using unsynchronized POSIX timer\n"); +#else /* not _NO_MPI_TIMER */ + if (MPI_WTIME_IS_GLOBAL) { + if (verbose >= VERBOSE_2) + fprintf(stdout, "Using synchronized MPI timer\n"); + } else { + if (verbose >= VERBOSE_2) + fprintf(stdout, "Using unsynchronized MPI timer\n"); + } +#endif /* _NO_MPI_TIMER */ + if (verbose >= VERBOSE_1) { + fprintf(stdout, "Start time skew across all tasks: %.02f sec\n", + wall_clock_deviation); + /* if pvfs2:, then skip */ + if (Regex(test->testFileName, "^[a-z][a-z].*:") == 0) { + DisplayFreespace(test); + } + } + if (verbose >= VERBOSE_3) { /* show env */ + fprintf(stdout, "STARTING ENVIRON LOOP\n"); + for (i = 0; environ[i] != NULL; i++) { + fprintf(stdout, "%s\n", environ[i]); + } + fprintf(stdout, "ENDING ENVIRON LOOP\n"); + } + fflush(stdout); +} /* ShowInfo() */ + + +/******************************************************************************/ +/* + * Show simple test output with max results for iterations. + */ + +void +ShowSetup(IOR_param_t * test) +{ + IOR_offset_t aggFileSizeForBW; + + /* use expected file size of iteration #0 for display */ + aggFileSizeForBW = test->aggFileSizeFromCalc[0]; + + if (strcmp(test->debug, "") != 0) { + fprintf(stdout, "\n*** DEBUG MODE ***\n"); + fprintf(stdout, "*** %s ***\n\n", test->debug); + } + fprintf(stdout, "\nSummary:\n"); + fprintf(stdout, "\tapi = %s\n", + test->apiVersion); + fprintf(stdout, "\ttest filename = %s\n", test->testFileName); + fprintf(stdout, "\taccess = "); + if (test->filePerProc) { + fprintf(stdout, "file-per-process"); + } else { + fprintf(stdout, "single-shared-file"); + } + if (verbose >= VERBOSE_1 && strcmp(test->api, "POSIX") != 0) { + if (test->collective == FALSE) { + fprintf(stdout, ", independent"); + } else { + fprintf(stdout, ", collective"); + } + } + fprintf(stdout, "\n"); + if (verbose >= VERBOSE_1) { + if (test->segmentCount > 1) { + fprintf(stdout, "\tpattern = strided (%d segments)\n", + (int)test->segmentCount); + } else { + fprintf(stdout, "\tpattern = segmented (1 segment)\n"); + } + } + fprintf(stdout, "\tordering in a file ="); + if (test->randomOffset == FALSE) { + fprintf(stdout, " sequential offsets\n"); + } else { + fprintf(stdout, " random offsets\n"); + } + fprintf(stdout, "\tordering inter file="); + if (test->reorderTasks == FALSE && test->reorderTasksRandom == FALSE){ + fprintf(stdout, " no tasks offsets\n"); + } + if (test->reorderTasks == TRUE) + { + fprintf(stdout, "constant task offsets = %d\n",test->taskPerNodeOffset); + } + if (test->reorderTasksRandom == TRUE) + { + fprintf(stdout, "random task offsets >= %d, seed=%d\n",test->taskPerNodeOffset,test->reorderTasksRandomSeed); + } + fprintf(stdout, "\tclients = %d (%d per node)\n", + test->numTasks, test->tasksPerNode); + fprintf(stdout, "\trepetitions = %d\n", + test->repetitions); + fprintf(stdout, "\txfersize = %s\n", + HumanReadable(test->transferSize, BASE_TWO)); + fprintf(stdout, "\tblocksize = %s\n", + HumanReadable(test->blockSize, BASE_TWO)); + fprintf(stdout, "\taggregate filesize = %s\n", + HumanReadable(aggFileSizeForBW, BASE_TWO)); +#ifdef _MANUALLY_SET_LUSTRE_STRIPING + fprintf(stdout, "\tLustre stripe size = %s\n", + ((test->lustre_stripe_size == 0) ? "Use default" : + HumanReadable(test->lustre_stripe_size, BASE_TWO))); + if (test->lustre_stripe_count == 0) { + fprintf(stdout, "\t stripe count = %s\n", "Use default"); + } else { + fprintf(stdout, "\t stripe count = %d\n", + test->lustre_stripe_count); + } +#endif /* _MANUALLY_SET_LUSTRE_STRIPING */ + if (test->deadlineForStonewalling > 0) { + fprintf(stdout, "\tUsing stonewalling = %d second(s)\n", + test->deadlineForStonewalling); + } + fprintf(stdout, "\n"); + /*fprintf(stdout, "\n ================================\n\n");*/ + fflush(stdout); +} /* ShowSetup() */ + + +/******************************************************************************/ +/* + * Show test description. + */ + +void +ShowTest(IOR_param_t * test) +{ + fprintf(stdout, "\n\n ================================\n\n"); + fprintf(stdout, "TEST:\t%s=%d\n", "id", test->id); + fprintf(stdout, "\t%s=%d\n", "testnum", test->TestNum); + fprintf(stdout, "\t%s=%s\n", "api", test->api); + fprintf(stdout, "\t%s=%s\n", "platform", test->platform); + fprintf(stdout, "\t%s=%s\n", "testFileName", test->testFileName); + fprintf(stdout, "\t%s=%s\n", "hintsFileName", test->hintsFileName); + fprintf(stdout, "\t%s=%d\n", "deadlineForStonewall", + test->deadlineForStonewalling); + fprintf(stdout, "\t%s=%d\n", "maxTimeDuration", test->maxTimeDuration); + fprintf(stdout, "\t%s=%d\n", "outlierThreshold", test->outlierThreshold); + fprintf(stdout, "\t%s=%s\n", "options", test->options); + fprintf(stdout, "\t%s=%d\n", "nodes", test->nodes); + fprintf(stdout, "\t%s=%d\n", "tasksPerNode", tasksPerNode); + fprintf(stdout, "\t%s=%d\n", "repetitions", test->repetitions); + fprintf(stdout, "\t%s=%d\n", "multiFile", test->multiFile); + fprintf(stdout, "\t%s=%d\n", "interTestDelay", test->interTestDelay); + fprintf(stdout, "\t%s=%d\n", "fsync", test->fsync); + fprintf(stdout, "\t%s=%d\n", "fsYncperwrite", test->fsyncPerWrite); + fprintf(stdout, "\t%s=%d\n", "useExistingTestFile", + test->useExistingTestFile); + fprintf(stdout, "\t%s=%d\n", "showHints", test->showHints); + fprintf(stdout, "\t%s=%d\n", "uniqueDir", test->uniqueDir); + fprintf(stdout, "\t%s=%d\n", "showHelp", test->showHelp); + fprintf(stdout, "\t%s=%d\n", "individualDataSets",test->individualDataSets); + fprintf(stdout, "\t%s=%d\n", "singleXferAttempt", test->singleXferAttempt); + fprintf(stdout, "\t%s=%d\n", "readFile", test->readFile); + fprintf(stdout, "\t%s=%d\n", "writeFile", test->writeFile); + fprintf(stdout, "\t%s=%d\n", "filePerProc", test->filePerProc); + fprintf(stdout, "\t%s=%d\n", "reorderTasks", test->reorderTasks); + fprintf(stdout, "\t%s=%d\n", "reorderTasksRandom", test->reorderTasksRandom); + fprintf(stdout, "\t%s=%d\n", "reorderTasksRandomSeed", test->reorderTasksRandomSeed); + fprintf(stdout, "\t%s=%d\n", "randomOffset", test->randomOffset); + fprintf(stdout, "\t%s=%d\n", "checkWrite", test->checkWrite); + fprintf(stdout, "\t%s=%d\n", "checkRead", test->checkRead); + fprintf(stdout, "\t%s=%d\n", "preallocate", test->preallocate); + fprintf(stdout, "\t%s=%d\n", "useFileView", test->useFileView); + fprintf(stdout, "\t%s=%lld\n", "setAlignment", test->setAlignment); + fprintf(stdout, "\t%s=%d\n", "storeFileOffset", test->storeFileOffset); + fprintf(stdout, "\t%s=%d\n", "useSharedFilePointer", + test->useSharedFilePointer); + fprintf(stdout, "\t%s=%d\n", "useO_DIRECT", test->useO_DIRECT); + fprintf(stdout, "\t%s=%d\n", "useStridedDatatype", + test->useStridedDatatype); + fprintf(stdout, "\t%s=%d\n", "keepFile", test->keepFile); + fprintf(stdout, "\t%s=%d\n", "keepFileWithError", test->keepFileWithError); + fprintf(stdout, "\t%s=%d\n", "quitOnError", test->quitOnError); + fprintf(stdout, "\t%s=%d\n", "verbose", verbose); + fprintf(stdout, "\t%s=%d\n", "setTimeStampSignature", + test->setTimeStampSignature); + fprintf(stdout, "\t%s=%d\n", "collective", test->collective); + fprintf(stdout, "\t%s=%lld", "segmentCount", test->segmentCount); + if (strcmp(test->api, "HDF5") == 0) { + fprintf(stdout, " (datasets)"); + } + fprintf(stdout, "\n"); + fprintf(stdout, "\t%s=%lld\n", "transferSize", test->transferSize); + fprintf(stdout, "\t%s=%lld\n", "blockSize", test->blockSize); +} /* ShowTest() */ + + +/******************************************************************************/ +/* + * Summarize results, showing max rates (and min, mean, stddev if verbose) + */ + +void +SummarizeResults(IOR_param_t * test) +{ + int rep, ival; + double maxWrite[2], minWrite[2], maxRead[2], minRead[2], + meanWrite[2], meanRead[2], + varWrite[2], varRead[2], + sdWrite[2], sdRead[2], + sumWrite[2], sumRead[2], + writeTimeSum, readTimeSum; + + for (ival=0; ival<2; ival++) + { + varWrite[ival] = 0; + varRead[ival] = 0; + sdWrite[ival] = 0; + sdRead[ival] = 0; + sumWrite[ival] = 0; + sumRead[ival] = 0; + writeTimeSum = 0; + readTimeSum = 0; + + if (ival == 1) + { + for (rep = 0; rep < test->repetitions; rep++) + { + writeTimeSum += test->writeVal[ival][rep]; + readTimeSum += test->readVal [ival][rep]; + test->writeVal[ival][rep] = (double)test->numTasks*((double)test->blockSize/(double)test->transferSize)/test->writeVal[ival][rep]; + test->readVal [ival][rep] = (double)test->numTasks*((double)test->blockSize/(double)test->transferSize)/test->readVal [ival][rep]; + } + } + + maxWrite[ival] = minWrite[ival] = test->writeVal[ival][0]; + maxRead[ival] = minRead[ival] = test->readVal[ival][0]; + + for (rep = 0; rep < test->repetitions; rep++) { + if (maxWrite[ival] < test->writeVal[ival][rep]) { + maxWrite[ival] = test->writeVal[ival][rep]; + } + if (maxRead[ival] < test->readVal[ival][rep]) { + maxRead[ival] = test->readVal[ival][rep]; + } + if (minWrite[ival] > test->writeVal[ival][rep]) { + minWrite[ival] = test->writeVal[ival][rep]; + } + if (minRead[ival] > test->readVal[ival][rep]) { + minRead[ival] = test->readVal[ival][rep]; + } + sumWrite[ival] += test->writeVal[ival][rep]; + sumRead[ival] += test->readVal[ival][rep]; + } + + meanWrite[ival] = sumWrite[ival] / test->repetitions; + meanRead[ival] = sumRead[ival] / test->repetitions; + + for (rep = 0; rep < test->repetitions; rep++) { + varWrite[ival] += pow((meanWrite[ival] - test->writeVal[ival][rep]), 2); + varRead[ival] += pow((meanRead[ival] - test->readVal[ival][rep]), 2); + } + varWrite[ival] = varWrite[ival] / test->repetitions; + varRead[ival] = varRead[ival] / test->repetitions; + sdWrite[ival] = sqrt(varWrite[ival]); + sdRead[ival] = sqrt(varRead[ival]); + } + + if (rank == 0 && verbose >= VERBOSE_0) + { + fprintf(stdout, "Operation Max (MiB) Min (MiB) Mean (MiB) Std Dev Max (OPs) Min (OPs) Mean (OPs) Std Dev Mean (s) "); + if (verbose >= VERBOSE_1) + fprintf(stdout, "Op grep #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize\n"); + fprintf(stdout, "\n"); + fprintf(stdout, "--------- --------- --------- ---------- ------- --------- --------- ---------- ------- --------\n"); + if (maxWrite[0] > 0.) + { + fprintf(stdout,"%s ", "write"); + fprintf(stdout,"%10.2f ", maxWrite[0]/MEBIBYTE); + fprintf(stdout,"%10.2f ", minWrite[0]/MEBIBYTE); + fprintf(stdout,"%10.2f", meanWrite[0]/MEBIBYTE); + fprintf(stdout,"%10.2f ", sdWrite[0]/MEBIBYTE); + fprintf(stdout,"%10.2f ", maxWrite[1]); + fprintf(stdout,"%10.2f ", minWrite[1]); + fprintf(stdout,"%10.2f", meanWrite[1]); + fprintf(stdout,"%10.2f", sdWrite[1]); + fprintf(stdout,"%10.5f ", writeTimeSum/(double)(test->repetitions)); + if (verbose >= VERBOSE_1) + { + fprintf(stdout,"%d ", test->numTasks); + fprintf(stdout,"%d ", test->tasksPerNode); + fprintf(stdout,"%d ", test->repetitions); + fprintf(stdout,"%d ", test->filePerProc); + fprintf(stdout,"%d ", test->reorderTasks); + fprintf(stdout,"%d ", test->taskPerNodeOffset); + fprintf(stdout,"%d ", test->reorderTasksRandom); + fprintf(stdout,"%d ", test->reorderTasksRandomSeed); + fprintf(stdout,"%lld ", test->segmentCount); + fprintf(stdout,"%lld ", test->blockSize); + fprintf(stdout,"%lld ", test->transferSize); + fprintf(stdout,"%lld ", test->aggFileSizeForBW[0]); + fprintf(stdout,"%d ", test->TestNum); + fprintf(stdout,"%s ", test->api); + } + fprintf(stdout,"EXCEL\n"); + } + if (maxRead[0] > 0.) + { + fprintf(stdout,"%s ", "read"); + fprintf(stdout,"%10.2f ", maxRead[0]/MEBIBYTE); + fprintf(stdout,"%10.2f ", minRead[0]/MEBIBYTE); + fprintf(stdout,"%10.2f", meanRead[0]/MEBIBYTE); + fprintf(stdout,"%10.2f ", sdRead[0]/MEBIBYTE); + fprintf(stdout,"%10.2f ", maxRead[1]); + fprintf(stdout,"%10.2f ", minRead[1]); + fprintf(stdout,"%10.2f", meanRead[1]); + fprintf(stdout,"%10.2f", sdRead[1]); + fprintf(stdout,"%10.5f ", readTimeSum/(double)(test->repetitions)); + if (verbose >= VERBOSE_1) + { + fprintf(stdout,"%d ", test->numTasks); + fprintf(stdout,"%d ", test->tasksPerNode); + fprintf(stdout,"%d ", test->repetitions); + fprintf(stdout,"%d ", test->filePerProc); + fprintf(stdout,"%d ", test->reorderTasks); + fprintf(stdout,"%d ", test->taskPerNodeOffset); + fprintf(stdout,"%d ", test->reorderTasksRandom); + fprintf(stdout,"%d ", test->reorderTasksRandomSeed); + fprintf(stdout,"%lld ", test->segmentCount); + fprintf(stdout,"%lld ", test->blockSize); + fprintf(stdout,"%lld ", test->transferSize); + fprintf(stdout,"%lld ", test->aggFileSizeForBW[0]); + fprintf(stdout,"%d ", test->TestNum); + fprintf(stdout,"%s ", test->api); + } + fprintf(stdout,"EXCEL\n"); + } + fflush(stdout); + } + + if (rank == 0 && verbose >= VERBOSE_0) { + fprintf(stdout, "\n"); + if (test->writeFile) { + fprintf(stdout, "Max Write: %.2f MiB/sec (%.2f MB/sec)\n", + maxWrite[0]/MEBIBYTE, maxWrite[0]/MEGABYTE); + } + if (test->readFile) { + fprintf(stdout, "Max Read: %.2f MiB/sec (%.2f MB/sec)\n", + maxRead[0]/MEBIBYTE, maxRead[0]/MEGABYTE); + } + fprintf(stdout, "\n"); + } +} /* SummarizeResults() */ + + +/******************************************************************************/ +/* + * Using the test parameters, run iteration(s) of single test. + */ + +void +TestIoSys(IOR_param_t *test) +{ + char testFileName[MAX_STR]; + double * timer[12]; + double startTime; + int i, + rep, + maxTimeDuration; + void * fd; + MPI_Group orig_group, new_group; + int range[3]; + IOR_offset_t dataMoved; /* for data rate calculation */ + + /* set up communicator for test */ + if (test->numTasks > numTasksWorld) { + if (rank == 0) { + fprintf(stdout, + "WARNING: More tasks requested (%d) than available (%d),", + test->numTasks, numTasksWorld); + fprintf(stdout," running on %d tasks.\n", numTasksWorld); + } + test->numTasks = numTasksWorld; + } + MPI_CHECK(MPI_Comm_group(MPI_COMM_WORLD, &orig_group), + "MPI_Comm_group() error"); + range[0] = 0; /* first rank */ + range[1] = test->numTasks - 1; /* last rank */ + range[2] = 1; /* stride */ + MPI_CHECK(MPI_Group_range_incl(orig_group, 1, &range, &new_group), + "MPI_Group_range_incl() error"); + MPI_CHECK(MPI_Comm_create(MPI_COMM_WORLD, new_group, &testComm), + "MPI_Comm_create() error"); + test->testComm = testComm; + if (testComm == MPI_COMM_NULL) { + /* tasks not in the group do not participate in this test */ + MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error"); + return; + } + if (rank == 0 && verbose >= VERBOSE_1) { + fprintf(stdout, "Participating tasks: %d\n", test->numTasks); + fflush(stdout); + } + if (rank == 0 && test->reorderTasks == TRUE && verbose >= VERBOSE_1) { + fprintf(stdout, + "Using reorderTasks '-C' (expecting block, not cyclic, task assignment)\n"); + fflush(stdout); + } + test->tasksPerNode = CountTasksPerNode(test->numTasks, testComm); + + /* setup timers */ + for (i = 0; i < 12; i++) { + timer[i] = (double *)malloc(test->repetitions * sizeof(double)); + if (timer[i] == NULL) ERR("out of memory"); + } + for (i = 0; i < 2; i++) + { + test->writeVal[i] = (double *)malloc(test->repetitions * sizeof(double)); + if (test->writeVal[i] == NULL) ERR("out of memory"); + test->readVal[i] = (double *)malloc(test->repetitions * sizeof(double)); + if (test->readVal[i] == NULL) ERR("out of memory"); + for (rep = 0; rep < test->repetitions; rep++) { + test->writeVal[i][rep] = test->readVal[i][rep] = 0.0; + } + } + test->aggFileSizeFromCalc = + (IOR_offset_t *)malloc(test->repetitions * sizeof(IOR_offset_t)); + if (test->aggFileSizeFromCalc == NULL) ERR("out of memory"); + test->aggFileSizeFromStat = + (IOR_offset_t *)malloc(test->repetitions * sizeof(IOR_offset_t)); + if (test->aggFileSizeFromStat == NULL) ERR("out of memory"); + test->aggFileSizeFromXfer = + (IOR_offset_t *)malloc(test->repetitions * sizeof(IOR_offset_t)); + if (test->aggFileSizeFromXfer == NULL) ERR("out of memory"); + test->aggFileSizeForBW = + (IOR_offset_t *)malloc(test->repetitions * sizeof(IOR_offset_t)); + if (test->aggFileSizeForBW == NULL) ERR("out of memory"); + + /* bind I/O calls to specific API */ + AioriBind(test->api); + + /* file size is first calculated on for comparison */ + for (rep = 0; rep < test->repetitions; rep++) { + test->aggFileSizeFromCalc[rep] = test->blockSize * test->segmentCount * + test->numTasks; + } + + /* show test setup */ + if (rank == 0 && verbose >= VERBOSE_0) ShowSetup(test); + + startTime = GetTimeStamp(); + maxTimeDuration = test->maxTimeDuration * 60; /* convert to seconds */ + +#if USE_UNDOC_OPT /* fillTheFileSystem */ + if (rank == 0 && test->fillTheFileSystem && verbose >= VERBOSE_0) { + fprintf(stdout, "Run started: %s", CurrentTimeString()); + } +#endif /* USE_UNDOC_OPT - fillTheFileSystem */ + + /* loop over test iterations */ + for (rep = 0; rep < test->repetitions; rep++) { + + /* Get iteration start time in seconds in task 0 and broadcast to + all tasks */ + if (rank == 0) { + if (test->setTimeStampSignature) { + test->timeStampSignatureValue = + (unsigned int)test->setTimeStampSignature; + } else { + time_t currentTime; + if ((currentTime = time(NULL)) == -1) { + ERR("cannot get current time"); + } + test->timeStampSignatureValue = (unsigned int)currentTime; + } + if (verbose >= VERBOSE_2) { + fprintf(stdout, + "Using Time Stamp %u (0x%x) for Data Signature\n", + test->timeStampSignatureValue, + test->timeStampSignatureValue); + } + } + MPI_CHECK(MPI_Bcast(&test->timeStampSignatureValue, 1, MPI_UNSIGNED, 0, + testComm), "cannot broadcast start time value"); +#if USE_UNDOC_OPT /* fillTheFileSystem */ + if (test->fillTheFileSystem && + rep > 0 && rep % (test->fillTheFileSystem / test->numTasks) == 0) { + if (rank == 0 && verbose >= VERBOSE_0) { + fprintf(stdout, "at file #%d, time: %s", rep*test->numTasks, + CurrentTimeString()); + fflush(stdout); + } + } +#endif /* USE_UNDOC_OPT - fillTheFileSystem */ + /* use repetition count for number of multiple files */ + if (test->multiFile) test->repCounter = rep; + + /* + * write the file(s), getting timing between I/O calls + */ + + if (test->writeFile +#if USE_UNDOC_OPT /* multiReRead */ + && (!test->multiReRead || !rep) +#endif /* USE_UNDOC_OPT - multiReRead */ + && (maxTimeDuration + ? (GetTimeStamp() - startTime < maxTimeDuration) : 1)) { + GetTestFileName(testFileName, test); + if (verbose >= VERBOSE_3) { + fprintf(stdout, "task %d writing %s\n", rank, testFileName); + } + DelaySecs(test->interTestDelay); + if (test->useExistingTestFile == FALSE) { + RemoveFile(testFileName, test->filePerProc, test); + } + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + test->open = WRITE; + timer[0][rep] = GetTimeStamp(); + fd = IOR_Create(testFileName, test); + timer[1][rep] = GetTimeStamp(); + if (test->intraTestBarriers) + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + if (rank == 0 && verbose >= VERBOSE_1) { + fprintf(stdout, "Commencing write performance test.\n"); + fprintf(stdout, "%s\n", CurrentTimeString()); + } + timer[2][rep] = GetTimeStamp(); + dataMoved = WriteOrRead(test, fd, WRITE); + timer[3][rep] = GetTimeStamp(); + if (test->intraTestBarriers) + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + timer[4][rep] = GetTimeStamp(); + IOR_Close(fd, test); + +#if USE_UNDOC_OPT /* includeDeleteTime */ + if (test->includeDeleteTime) { + if (rank == 0 && verbose >= VERBOSE_1) { + fprintf(stdout, "** including delete time **\n"); + } + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + RemoveFile(testFileName, test->filePerProc, test); + } +#endif /* USE_UNDOC_OPT - includeDeleteTime */ + + timer[5][rep] = GetTimeStamp(); + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + +#if USE_UNDOC_OPT /* includeDeleteTime */ + if (test->includeDeleteTime) { + /* not accurate, but no longer a test file to examine */ + test->aggFileSizeFromStat[rep] = test->aggFileSizeFromCalc[rep]; + test->aggFileSizeFromXfer[rep] = test->aggFileSizeFromCalc[rep]; + test->aggFileSizeForBW[rep] = test->aggFileSizeFromCalc[rep]; + } else { +#endif /* USE_UNDOC_OPT - includeDeleteTime */ + + /* get the size of the file just written */ + test->aggFileSizeFromStat[rep] + = IOR_GetFileSize(test, testComm, testFileName); + + /* check if stat() of file doesn't equal expected file size, + use actual amount of byte moved */ + CheckFileSize(test, dataMoved, rep); + +#if USE_UNDOC_OPT /* includeDeleteTime */ + } +#endif /* USE_UNDOC_OPT - includeDeleteTime */ + + if (verbose >= VERBOSE_3) WriteTimes(test, timer, rep, WRITE); + ReduceIterResults(test, timer, rep, WRITE); + if (test->outlierThreshold) { + CheckForOutliers(test, timer, rep, WRITE); + } + } + + /* + * perform a check of data, reading back data and comparing + * against what was expected to be written + */ + if (test->checkWrite +#if USE_UNDOC_OPT /* multiReRead */ + && (!test->multiReRead || rep) +#endif /* USE_UNDOC_OPT - multiReRead */ + && (maxTimeDuration + ? (GetTimeStamp() - startTime < maxTimeDuration) : 1)) { +#if USE_UNDOC_OPT /* corruptFile */ + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + /* intentionally corrupt file to determine if check works */ + if (test->corruptFile) { + CorruptFile(testFileName, test, rep, WRITECHECK); + } +#endif /* USE_UNDOC_OPT - corruptFile */ + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + if (rank == 0 && verbose >= VERBOSE_1) { + fprintf(stdout, + "Verifying contents of the file(s) just written.\n"); + fprintf(stdout, "%s\n", CurrentTimeString()); + } + if (test->reorderTasks) { + /* move two nodes away from writing node */ + rankOffset = (2*test->tasksPerNode) % test->numTasks; + } + GetTestFileName(testFileName, test); + test->open = WRITECHECK; + fd = IOR_Open(testFileName, test); + dataMoved = WriteOrRead(test, fd, WRITECHECK); + IOR_Close(fd, test); + rankOffset = 0; + } + /* + * read the file(s), getting timing between I/O calls + */ + if (test->readFile + && (maxTimeDuration ? (GetTimeStamp() - startTime < maxTimeDuration) : 1)) { + /* Get rankOffset [file offset] for this process to read, based on -C,-Z,-Q,-X options */ + /* Constant process offset reading */ + if (test->reorderTasks) { + /* move taskPerNodeOffset nodes[1==default] away from writing node */ + rankOffset = (test->taskPerNodeOffset*test->tasksPerNode) % test->numTasks; + } + /* random process offset reading */ + if (test->reorderTasksRandom) + { + /* this should not intefere with randomOffset within a file because GetOffsetArrayRandom */ + /* seeds every random() call */ + int *rankoffs, *filecont, *filehits, ifile, jfile, nodeoffset; + unsigned int iseed0; + nodeoffset = test->taskPerNodeOffset; + nodeoffset = (nodeoffset < test->nodes) ? nodeoffset : test->nodes-1; + iseed0 = (test->reorderTasksRandomSeed < 0) ? (-1*test->reorderTasksRandomSeed+rep):test->reorderTasksRandomSeed; + srand(rank+iseed0); + { rankOffset = rand() % test->numTasks; } + while (rankOffset < (nodeoffset*test->tasksPerNode)) { rankOffset = rand() % test->numTasks; } + /* Get more detailed stats if requested by verbose level */ + if (verbose >= VERBOSE_2) + { + if (rank == 0) + { + rankoffs = (int *)malloc(test->numTasks*sizeof(int)); + filecont = (int *)malloc(test->numTasks*sizeof(int)); + filehits = (int *)malloc(test->numTasks*sizeof(int)); + } + MPI_CHECK(MPI_Gather(&rankOffset,1,MPI_INT,rankoffs,1,MPI_INT,0,MPI_COMM_WORLD), "MPI_Gather error"); + /*file hits histogram*/ + if (rank == 0) + { + memset((void*)filecont,0,test->numTasks*sizeof(int)); + for (ifile=0; ifilenumTasks; ifile++) { filecont[(ifile+rankoffs[ifile])%test->numTasks]++; } + memset((void*)filehits,0,test->numTasks*sizeof(int)); + for (ifile=0; ifilenumTasks; ifile++) + for (jfile=0; jfilenumTasks; jfile++) { if (ifile == filecont[jfile]) filehits[ifile]++; } + /* fprintf(stdout, "File Contention Dist:"); + for (ifile=0; ifilenumTasks; ifile++) { fprintf(stdout," %d",filecont[ifile]); } + fprintf(stdout,"\n"); */ + fprintf(stdout, "#File Hits Dist:"); + jfile = 0; ifile = 0; + while (jfilenumTasks && + ifilenumTasks) { fprintf(stdout," %d",filehits[ifile]);jfile+=filehits[ifile],ifile++;} + fprintf(stdout," XXCEL\n"); + free(rankoffs); + free(filecont); + free(filehits); + } + } + } + /* Using globally passed rankOffset, following function generates testFileName to read */ + GetTestFileName(testFileName, test); + + if (verbose >= VERBOSE_3) { + fprintf(stdout, "task %d reading %s\n", rank, testFileName); + } + DelaySecs(test->interTestDelay); + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + test->open = READ; + timer[6][rep] = GetTimeStamp(); + fd = IOR_Open(testFileName, test); + if (rank == 0 && verbose >= VERBOSE_2) { + fprintf(stdout, "[RANK %03d] open for reading file %s XXCEL\n", rank,testFileName); + } + timer[7][rep] = GetTimeStamp(); + if (test->intraTestBarriers) + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + if (rank == 0 && verbose >= VERBOSE_1) { + fprintf(stdout, "Commencing read performance test.\n"); + fprintf(stdout, "%s\n", CurrentTimeString()); + } + timer[8][rep] = GetTimeStamp(); + dataMoved = WriteOrRead(test, fd, READ); + timer[9][rep] = GetTimeStamp(); + if (test->intraTestBarriers) + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + timer[10][rep] = GetTimeStamp(); + IOR_Close(fd, test); + timer[11][rep] = GetTimeStamp(); + + /* get the size of the file just read */ + test->aggFileSizeFromStat[rep] = IOR_GetFileSize(test, testComm, + testFileName); + + /* check if stat() of file doesn't equal expected file size, + use actual amount of byte moved */ + CheckFileSize(test, dataMoved, rep); + + if (verbose >= VERBOSE_3) WriteTimes(test, timer, rep, READ); + ReduceIterResults(test, timer, rep, READ); + if (test->outlierThreshold) { + CheckForOutliers(test, timer, rep, READ); + } + } /* end readFile test */ + + /* + * perform a check of data, reading back data twice and + * comparing against what was expected to be read + */ + if (test->checkRead + && (maxTimeDuration + ? (GetTimeStamp() - startTime < maxTimeDuration) : 1)) { + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + if (rank == 0 && verbose >= VERBOSE_1) { + fprintf(stdout, "Re-reading the file(s) twice to "); + fprintf(stdout, "verify that reads are consistent.\n"); + fprintf(stdout, "%s\n", CurrentTimeString()); + } + if (test->reorderTasks) { + /* move three nodes away from reading node */ + rankOffset = (3*test->tasksPerNode) % test->numTasks; + } + GetTestFileName(testFileName, test); + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + test->open = READCHECK; + fd = IOR_Open(testFileName, test); + if (test->filePerProc) { + int tmpRankOffset; + tmpRankOffset = rankOffset; + /* increment rankOffset to open comparison file on other node */ + if (test->reorderTasks) { + /* move four nodes away from reading node */ + rankOffset = (4*test->tasksPerNode) % test->numTasks; + } + GetTestFileName(test->testFileName_fppReadCheck, test); + rankOffset = tmpRankOffset; + test->fd_fppReadCheck = + IOR_Open(test->testFileName_fppReadCheck, test); + } + dataMoved = WriteOrRead(test, fd, READCHECK); + if (test->filePerProc) { + IOR_Close(test->fd_fppReadCheck, test); + test->fd_fppReadCheck = NULL; + } + IOR_Close(fd, test); + } + /* + * this final barrier may not be necessary as IOR_Close should + * be a collective call -- but to make sure that the file has + * has not be removed by a task before another finishes writing, + * the MPI_Barrier() call has been included. + */ + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + if (!test->keepFile && !(test->keepFileWithError && test->errorFound)) { + RemoveFile(testFileName, test->filePerProc, test); + } + test->errorFound = FALSE; + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + rankOffset = 0; + } + +#if USE_UNDOC_OPT /* fillTheFileSystem */ + if (rank == 0 && test->fillTheFileSystem && verbose >= VERBOSE_0) { + fprintf(stdout, "Run ended: %s", CurrentTimeString()); + } +#endif /* USE_UNDOC_OPT - fillTheFileSystem */ + + SummarizeResults(test); + + MPI_CHECK(MPI_Comm_free(&testComm), "MPI_Comm_free() error"); + for (i = 0; i < 2; i++) + { + free(test->writeVal[i]); + free(test->readVal[i]); + } + free(test->aggFileSizeFromCalc); + free(test->aggFileSizeFromStat); + free(test->aggFileSizeFromXfer); + free(test->aggFileSizeForBW); + for (i = 0; i < 12; i++) { + free(timer[i]); + } + /* Sync with the tasks that did not participate in this test */ + MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error"); + +} /* TestIoSys() */ + + +/******************************************************************************/ +/* + * Determine any spread (range) between node times. + */ + +double +TimeDeviation(void) +{ + double timestamp, min = 0, max = 0, roottimestamp; + + MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error"); + timestamp = GetTimeStamp(); + MPI_CHECK(MPI_Reduce(×tamp, &min, 1, MPI_DOUBLE, + MPI_MIN, 0, MPI_COMM_WORLD), + "cannot reduce tasks' times"); + MPI_CHECK(MPI_Reduce(×tamp, &max, 1, MPI_DOUBLE, + MPI_MAX, 0, MPI_COMM_WORLD), + "cannot reduce tasks' times"); + + /* delta between individual nodes' time and root node's time */ + roottimestamp = timestamp; + MPI_CHECK(MPI_Bcast(&roottimestamp, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD), + "cannot broadcast root's time"); + wall_clock_delta = timestamp - roottimestamp; + + return max-min; +} /* TimeDeviation() */ + + +/******************************************************************************/ +/* + * Determine if valid tests from parameters. + */ + +void +ValidTests(IOR_param_t * test) +{ + /* get the version of the tests */ + + AioriBind(test->api); + IOR_SetVersion(test); + + if (test->repetitions <= 0) + WARN_RESET("too few test repetitions", repetitions); + if (test->numTasks <= 0) + ERR("too few tasks for testing"); + if (test->interTestDelay < 0) + WARN_RESET("inter-test delay must be nonnegative value", + interTestDelay); + if (test->readFile != TRUE && test->writeFile != TRUE + && test->checkRead != TRUE && test->checkWrite != TRUE) + ERR("test must write, read, or check file"); + if ((test->deadlineForStonewalling > 0) + && (test->checkWrite == TRUE || test->checkRead == TRUE)) + ERR("can not perform write or read check with stonewalling"); + if (test->segmentCount < 0) + ERR("segment count must be positive value"); + if ((test->blockSize % sizeof(IOR_size_t)) != 0) + ERR("block size must be a multiple of access size"); + if (test->blockSize < 0) + ERR("block size must be non-negative integer"); + if ((test->transferSize % sizeof(IOR_size_t)) != 0) + ERR("transfer size must be a multiple of access size"); + if (test->setAlignment < 0) + ERR("alignment must be non-negative integer"); + if (test->transferSize < 0) + ERR("transfer size must be non-negative integer"); + if (test->transferSize == 0) { + ERR("test will not complete with zero transfer size"); + } else { + if ((test->blockSize % test->transferSize) != 0) + ERR("block size must be a multiple of transfer size"); + } + if (test->blockSize < test->transferSize) + ERR("block size must not be smaller than transfer size"); + if ((strcmp(test->api, "MPIIO") == 0) + && (test->blockSize < sizeof(IOR_size_t) + || test->transferSize < sizeof(IOR_size_t))) + ERR("block/transfer size may not be smaller than IOR_size_t for MPIIO"); + if ((strcmp(test->api, "HDF5") == 0) + && (test->blockSize < sizeof(IOR_size_t) + || test->transferSize < sizeof(IOR_size_t))) + ERR("block/transfer size may not be smaller than IOR_size_t for HDF5"); + if ((strcmp(test->api, "NCMPI") == 0) + && (test->blockSize < sizeof(IOR_size_t) + || test->transferSize < sizeof(IOR_size_t))) + ERR("block/transfer size may not be smaller than IOR_size_t for NCMPI"); + if ((strcmp(test->api, "NCMPI") == 0) + && ((test->numTasks * test->blockSize * test->segmentCount) + > (2*(IOR_offset_t)GIBIBYTE))) + ERR("file size must be < 2GiB"); + if((test->useFileView == TRUE) + && (sizeof(MPI_Aint) < 8) /* used for 64-bit datatypes */ + && ((test->numTasks * test->blockSize) > (2*(IOR_offset_t)GIBIBYTE))) + ERR("segment size must be < 2GiB"); + if ((strcmp(test->api, "POSIX") != 0) && test->singleXferAttempt) + WARN_RESET("retry only available in POSIX", singleXferAttempt); + if ((strcmp(test->api, "POSIX") != 0) && test->fsync) + WARN_RESET("fsync() only available in POSIX", fsync); + if ((strcmp(test->api, "MPIIO") != 0) && test->preallocate) + WARN_RESET("preallocation only available in MPIIO", preallocate); + if ((strcmp(test->api, "MPIIO") != 0) && test->useFileView) + WARN_RESET("file view only available in MPIIO", useFileView); + if ((strcmp(test->api, "MPIIO") != 0) && test->useSharedFilePointer) + WARN_RESET("shared file pointer only available in MPIIO", + useSharedFilePointer); + if ((strcmp(test->api, "MPIIO") == 0) && test->useSharedFilePointer) + WARN_RESET("shared file pointer not implemented", useSharedFilePointer); + if ((strcmp(test->api, "MPIIO") != 0) && test->useStridedDatatype) + WARN_RESET("strided datatype only available in MPIIO", + useStridedDatatype); + if ((strcmp(test->api, "MPIIO") == 0) && test->useStridedDatatype) + WARN_RESET("strided datatype not implemented", useStridedDatatype); + if ((strcmp(test->api, "MPIIO") == 0) + && test->useStridedDatatype + && (test->blockSize < sizeof(IOR_size_t) + || test->transferSize < sizeof(IOR_size_t))) + ERR("need larger file size for strided datatype in MPIIO"); + if ((strcmp(test->api, "POSIX") == 0) && test->showHints) + WARN_RESET("hints not available in POSIX", showHints); + if ((strcmp(test->api, "POSIX") == 0) && test->collective) + WARN_RESET("collective not available in POSIX", collective); + if (test->reorderTasks == TRUE && test->reorderTasksRandom == TRUE) + ERR("Both Constant and Random task re-ordering specified. Choose one and resubmit"); + if (test->randomOffset && test->reorderTasksRandom && test->filePerProc == FALSE) + ERR("random offset and random reorder tasks specified with single-shared-file. Choose one and resubmit"); + if (test->randomOffset && test->reorderTasks && test->filePerProc == FALSE) + ERR("random offset and constant reorder tasks specified with single-shared-file. Choose one and resubmit"); + if (test->randomOffset && test->checkRead) + ERR("random offset not available with read check option (use write check)"); + if (test->randomOffset && test->storeFileOffset) + ERR("random offset not available with store file offset option)"); + if ((strcmp(test->api, "MPIIO") == 0) && test->randomOffset && test->collective) + ERR("random offset not available with collective MPIIO"); + if ((strcmp(test->api, "MPIIO") == 0) && test->randomOffset && test->useFileView) + ERR("random offset not available with MPIIO fileviews"); + if ((strcmp(test->api, "HDF5") == 0) && test->randomOffset) + ERR("random offset not available with HDF5"); + if ((strcmp(test->api, "NCMPI") == 0) && test->randomOffset) + ERR("random offset not available with NCMPI"); + if ((strcmp(test->api, "HDF5") != 0) && test->individualDataSets) + WARN_RESET("individual datasets only available in HDF5", + individualDataSets); + if ((strcmp(test->api, "HDF5") == 0) && test->individualDataSets) + WARN_RESET("individual data sets not implemented", individualDataSets); + if ((strcmp(test->api, "NCMPI") == 0) && test->filePerProc) + ERR("file-per-proc not available in current NCMPI"); + if (test->noFill) { + if (strcmp(test->api, "HDF5") != 0) { + ERR("'no fill' option only available in HDF5"); + } else { + /* check if hdf5 available */ + #if defined (H5_VERS_MAJOR) && defined (H5_VERS_MINOR) + /* no-fill option not available until hdf5-1.6.x */ + #if (H5_VERS_MAJOR > 0 && H5_VERS_MINOR > 5) + ; + #else + char errorString[MAX_STR]; + sprintf(errorString, "'no fill' option not available in %s", + test->apiVersion); + ERR(errorString); + #endif + #else + WARN("unable to determine HDF5 version for 'no fill' usage"); + #endif + } + } + if (test->useExistingTestFile + && (test->lustre_stripe_count != 0 + || test->lustre_stripe_size != 0 + || test->lustre_start_ost != -1)) + ERR("Lustre stripe options are incompatible with useExistingTestFile"); +} /* ValidTests() */ + +IOR_offset_t * +GetOffsetArraySequential(IOR_param_t *test, int pretendRank) +{ + IOR_offset_t i, j, k = 0; + IOR_offset_t offsets; + IOR_offset_t *offsetArray; + + /* count needed offsets */ + offsets = (test->blockSize / test->transferSize) * test->segmentCount; + + /* setup empty array */ + offsetArray = (IOR_offset_t *)malloc((offsets+1) * sizeof(IOR_offset_t)); + if (offsetArray == NULL) ERR("out of memory"); + offsetArray[offsets] = -1; /* set last offset with -1 */ + + /* fill with offsets */ + for (i = 0; i < test->segmentCount; i++) { + for (j = 0; j < (test->blockSize / test->transferSize); j++) { + offsetArray[k] = j * test->transferSize; + if (test->filePerProc) { + offsetArray[k] += i * test->blockSize; + } else { + offsetArray[k] += (i * test->numTasks * test->blockSize) + + (pretendRank * test->blockSize); + } + k++; + } + } + + return(offsetArray); +} /* GetOffsetArraySequential() */ + + +IOR_offset_t * +GetOffsetArrayRandom(IOR_param_t *test, int pretendRank, int access) +{ + int seed; + IOR_offset_t i, value, tmp; + IOR_offset_t offsets = 0, offsetCnt = 0; + IOR_offset_t fileSize; + IOR_offset_t *offsetArray; + + /* set up seed for random() */ + if (access == WRITE || access == READ) { + test->randomSeed = seed = random(); + } else { + seed = test->randomSeed; + } + srandom(seed); + + fileSize = test->blockSize * test->segmentCount; + if (test->filePerProc == FALSE) { + fileSize *= test->numTasks; + } + + /* count needed offsets (pass 1) */ + for (i = 0; i < fileSize; i += test->transferSize) { + if (test->filePerProc == FALSE) { + if ((random() % test->numTasks) == pretendRank) { + offsets++; + } + } else { + offsets++; + } + } + + /* setup empty array */ + offsetArray = (IOR_offset_t *)malloc((offsets+1) * sizeof(IOR_offset_t)); + if (offsetArray == NULL) ERR("out of memory"); + offsetArray[offsets] = -1; /* set last offset with -1 */ + + if (test->filePerProc) { + /* fill array */ + for (i = 0; i < offsets; i++) { + offsetArray[i] = i * test->transferSize; + } + } else { + /* fill with offsets (pass 2) */ + srandom(seed); /* need same seed */ + for (i = 0; i < fileSize; i += test->transferSize) { + if ((random() % test->numTasks) == pretendRank) { + offsetArray[offsetCnt] = i; + offsetCnt++; + } + } + } + /* reorder array */ + for (i = 0; i < offsets; i++) { + value = random() % offsets; + tmp = offsetArray[value]; + offsetArray[value] = offsetArray[i]; + offsetArray[i] = tmp; + } + SeedRandGen(test->testComm); /* synchronize seeds across tasks */ + + return(offsetArray); +} /* GetOffsetArrayRandom() */ + + +/******************************************************************************/ +/* + * Write or Read data to file(s). This loops through the strides, writing + * out the data to each block in transfer sizes, until the remainder left is 0. + */ + +IOR_offset_t +WriteOrRead(IOR_param_t * test, + void * fd, + int access) +{ + int errors = 0; + IOR_offset_t amtXferred, + transfer, + transferCount = 0, + pairCnt = 0, + * offsetArray; + int pretendRank; + void * buffer = NULL; + void * checkBuffer = NULL; + void * readCheckBuffer = NULL; + IOR_offset_t dataMoved = 0; /* for data rate calculation */ + double startForStonewall; + int hitStonewall; + + + /* initialize values */ + pretendRank = (rank + rankOffset) % test->numTasks; + + if (test->randomOffset) { + offsetArray = GetOffsetArrayRandom(test, pretendRank, access); + } else { + offsetArray = GetOffsetArraySequential(test, pretendRank); + } + + SetupXferBuffers(&buffer, &checkBuffer, &readCheckBuffer, + test, pretendRank, access); + + /* check for stonewall */ + startForStonewall = GetTimeStamp(); + hitStonewall = ((test->deadlineForStonewalling != 0) + && ((GetTimeStamp() - startForStonewall) + > test->deadlineForStonewalling)); + + /* loop over offsets to access */ + while ((offsetArray[pairCnt] != -1) && !hitStonewall) { + test->offset = offsetArray[pairCnt]; + /* + * fills each transfer with a unique pattern + * containing the offset into the file + */ + if (test->storeFileOffset == TRUE) { + FillBuffer(buffer, test, test->offset, pretendRank); + } + transfer = test->transferSize; + if (access == WRITE) { + amtXferred = IOR_Xfer(access, fd, buffer, transfer, test); + if (amtXferred != transfer) ERR("cannot write to file"); + } else if (access == READ) { + amtXferred = IOR_Xfer(access, fd, buffer, transfer, test); + if (amtXferred != transfer) ERR("cannot read from file"); + } else if (access == WRITECHECK) { + memset(checkBuffer, 'a', transfer); + amtXferred = IOR_Xfer(access, fd, checkBuffer, transfer, test); + if (amtXferred != transfer) + ERR("cannot read from file write check"); + transferCount++; + errors += CompareBuffers(buffer, checkBuffer, transfer, + transferCount, test, WRITECHECK); + } else if (access == READCHECK){ + ReadCheck(fd, buffer, checkBuffer, readCheckBuffer, test, + transfer, test->blockSize, &amtXferred, + &transferCount, access, &errors); + } + dataMoved += amtXferred; + pairCnt++; + + hitStonewall = ((test->deadlineForStonewalling != 0) + && ((GetTimeStamp() - startForStonewall) + > test->deadlineForStonewalling)); + } + + totalErrorCount += CountErrors(test, access, errors); + + FreeBuffers(access, checkBuffer, readCheckBuffer, buffer, offsetArray); + + if (access == WRITE && test->fsync == TRUE) { + IOR_Fsync(fd, test); /*fsync after all accesses*/ + } + return(dataMoved); +} /* WriteOrRead() */ + + +/******************************************************************************/ +/* + * Write times taken during each iteration of the test. + */ + +void +WriteTimes(IOR_param_t * test, + double ** timer, + int iteration, + int writeOrRead) +{ + char accessType[MAX_STR], + timerName[MAX_STR]; + int i, + start, + stop; + + if (writeOrRead == WRITE) { + start = 0; + stop = 6; + strcpy(accessType, "WRITE"); + } else if (writeOrRead == READ) { + start = 6; + stop = 12; + strcpy(accessType, "READ"); + } else { + ERR("incorrect WRITE/READ option"); + } + + for (i = start; i < stop; i++) { + switch(i) { + case 0: strcpy(timerName, "write open start"); break; + case 1: strcpy(timerName, "write open stop"); break; + case 2: strcpy(timerName, "write start"); break; + case 3: strcpy(timerName, "write stop"); break; + case 4: strcpy(timerName, "write close start"); break; + case 5: strcpy(timerName, "write close stop"); break; + case 6: strcpy(timerName, "read open start"); break; + case 7: strcpy(timerName, "read open stop"); break; + case 8: strcpy(timerName, "read start"); break; + case 9: strcpy(timerName, "read stop"); break; + case 10: strcpy(timerName, "read close start"); break; + case 11: strcpy(timerName, "read close stop"); break; + default: strcpy(timerName, "invalid timer"); break; + } + fprintf(stdout, "Test %d: Iter=%d, Task=%d, Time=%f, %s\n", + test->id, iteration, (int)rank, timer[i][iteration], timerName); + } +} /* WriteTimes() */ diff --git a/src/C/IOR.h b/src/C/IOR.h new file mode 100644 index 0000000..414d964 --- /dev/null +++ b/src/C/IOR.h @@ -0,0 +1,105 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: IOR.h,v $ +* $Revision: 1.2 $ +* $Date: 2008/12/02 17:12:14 $ +* $Author: rklundt $ +* +* Purpose: +* This is a header file that contains the definitions and prototypes +* needed for IOR.c. +* +\******************************************************************************/ + +#ifndef _IOR_H +#define _IOR_H + +#include "aiori.h" /* abstract IOR interfaces */ +#include "iordef.h" /* IOR Definitions */ + + +/*************************** D E F I N I T I O N S ****************************/ + +/* define the queuing structure for the test parameters */ +typedef struct IOR_queue_t { + IOR_param_t testParameters; + struct IOR_queue_t * nextTest; +} IOR_queue_t; + + +/**************************** P R O T O T Y P E S *****************************/ + +/* functions used in IOR.c */ +void AioriBind (char *); +void CheckForOutliers (IOR_param_t *, double **, int, int); +void CheckFileSize (IOR_param_t *, IOR_offset_t, int); +char * CheckTorF (char *); +size_t CompareBuffers (void *, void *, size_t, + IOR_offset_t, IOR_param_t *, int); +int CountErrors (IOR_param_t *, int, int); +int CountTasksPerNode(int, MPI_Comm); +void * CreateBuffer (size_t); +IOR_queue_t * CreateNewTest (int); +void DelaySecs (int); +void DisplayFreespace (IOR_param_t *); +void DisplayOutliers (int, double, char *, int, int); +void DisplayUsage (char **); +void DistributeHints (void); +void FillBuffer (void *, IOR_param_t *, + unsigned long long, int); +void FreeBuffers (int, void *, void *, void *, IOR_offset_t *); +void GetPlatformName (char *); +void GetTestFileName (char *, IOR_param_t *); +double GetTimeStamp (void); +char * HumanReadable (IOR_offset_t, int); +IOR_offset_t IOR_GetFileSize_POSIX (IOR_param_t *, MPI_Comm, char *); +IOR_offset_t IOR_GetFileSize_MPIIO (IOR_param_t *, MPI_Comm, char *); +char * LowerCase (char *); +void OutputToRoot (int, MPI_Comm, char *); +void PPDouble (int, double, char *); +char * PrependDir (IOR_param_t *, char *); +IOR_queue_t * ParseCommandLine (int, char **); +char ** ParseFileName (char *, int *); +void ReadCheck (void *, void *, void *, void *, IOR_param_t *, + IOR_offset_t, IOR_offset_t, IOR_offset_t *, + IOR_offset_t *, int, int *); +void ReduceIterResults(IOR_param_t *, double **, int, int); +int Regex (char *, char *); +void RemoveFile (char *, int, IOR_param_t *); +int Regex (char *, char *); +void SetupXferBuffers (void **, void **, void **, + IOR_param_t *, int, int); +IOR_queue_t * SetupTests (int, char **); +void ShowFileSystemSize (char *); +void ShowInfo (int, char **, IOR_param_t *); +void ShowSetup (IOR_param_t *); +void ShowTest (IOR_param_t *); +void SummarizeResults (IOR_param_t *); +void TestIoSys (IOR_param_t *); +double TimeDeviation (void); +void ValidTests (IOR_param_t *); +IOR_offset_t WriteOrRead (IOR_param_t *, void *, int); +void WriteTimes (IOR_param_t *, double **, int, int); + + +/* functions used in utilities.c */ +char * CurrentTimeString(void); +void DumpBuffer (void *, size_t); +void ExtractHint (char *, char *, char *); +void SetHints (MPI_Info *, char *); +void ShowHints (MPI_Info *); +IOR_offset_t StringToBytes (char *); + +#if USE_UNDOC_OPT +void CorruptFile (char *, IOR_param_t *, int, int); +void ModifyByteInFile (char *, IOR_offset_t, int); +#endif /* USE_UNDOC_OPTS */ +void SeedRandGen (MPI_Comm); + +#endif /* not _IOR_H */ diff --git a/src/C/Makefile b/src/C/Makefile new file mode 100644 index 0000000..d3f1506 --- /dev/null +++ b/src/C/Makefile @@ -0,0 +1,69 @@ +#/*****************************************************************************\ +#* * +#* Copyright (c) 2003, The Regents of the University of California * +#* See the file COPYRIGHT for a complete copyright notice and license. * +#* * +#******************************************************************************* +#* +#* CVS info: +#* $RCSfile: Makefile,v $ +#* $Revision: 1.1.1.1 $ +#* $Date: 2007/10/15 23:36:54 $ +#* $Author: rklundt $ +#* +#* Purpose: +#* Make IOR executable. +#* +#* gmake posix -- IOR with only POSIX interfaces +#* gmake mpiio -- IOR with only POSIX and MPIIO interfaces +#* gmake hdf5 -- IOR with POSIX, MPIIO, and HDF5 interfaces +#* gmake ncmpi -- IOR with POSIX, MPIIO, and NCMPI interfaces +#* gmake all -- IOR with POSIX, MPIIO, HDF5, and NCMPI interfaces +#* gmake clean -- remove executable and object files +#* +#\*****************************************************************************/ + +include Makefile.config + +# Requires GNU Make +OS=$(shell uname) + +SRCS = IOR.c utilities.c parse_options.c +OBJS = $(SRCS:.c=.o) + +posix: $(OBJS) aiori-POSIX.o aiori-noMPIIO.o aiori-noHDF5.o aiori-noNCMPI.o + $(CC) -o IOR $(OBJS) \ + aiori-POSIX.o aiori-noMPIIO.o aiori-noHDF5.o aiori-noNCMPI.o \ + $(LDFLAGS) + +mpiio: $(OBJS) aiori-POSIX.o aiori-MPIIO.o aiori-noHDF5.o aiori-noNCMPI.o + $(CC) -o IOR $(OBJS) \ + aiori-POSIX.o aiori-MPIIO.o aiori-noHDF5.o aiori-noNCMPI.o \ + $(LDFLAGS) + +hdf5: $(OBJS) aiori-POSIX.o aiori-MPIIO.o aiori-HDF5.o aiori-noNCMPI.o + $(CC) $(LDFLAGS_HDF5) -o IOR $(OBJS) \ + aiori-POSIX.o aiori-MPIIO.o aiori-HDF5.o aiori-noNCMPI.o + +ncmpi: $(OBJS) aiori-POSIX.o aiori-MPIIO.o aiori-noHDF5.o aiori-NCMPI.o + $(CC) $(LDFLAGS_NCMPI) -o IOR $(OBJS) \ + aiori-POSIX.o aiori-MPIIO.o aiori-noHDF5.o aiori-NCMPI.o + +all: $(OBJS) aiori-POSIX.o aiori-MPIIO.o aiori-HDF5.o aiori-NCMPI.o + $(CC) $(LDFLAGS_HDF5) $(LDFLAGS_NCMPI) -o IOR $(OBJS) \ + aiori-POSIX.o aiori-MPIIO.o aiori-HDF5.o aiori-NCMPI.o + +clean: + -rm -f *.o IOR + +aiori-MPIIO.o: aiori-MPIIO.c + $(CC) -c aiori-MPIIO.c + +aiori-HDF5.o: aiori-HDF5.c + $(CC) $(CCFLAGS_HDF5) -c aiori-HDF5.c + +aiori-NCMPI.o: aiori-NCMPI.c + $(CC) $(CCFLAGS_NCMPI) -c aiori-NCMPI.c + +.c.o: + $(CC) $(CCFLAGS) -c $< diff --git a/src/C/Makefile.config b/src/C/Makefile.config new file mode 100644 index 0000000..f6b3acb --- /dev/null +++ b/src/C/Makefile.config @@ -0,0 +1,90 @@ +#/*****************************************************************************\ +#* * +#* Copyright (c) 2003, The Regents of the University of California * +#* See the file COPYRIGHT for a complete copyright notice and license. * +#* * +#******************************************************************************* +#* +#* CVS info: +#* $RCSfile: Makefile.config,v $ +#* $Revision: 1.3 $ +#* $Date: 2008/12/02 17:12:14 $ +#* $Author: rklundt $ +#* +#* Purpose: +#* Maintain compilation settings for various platforms. +#* +#\*****************************************************************************/ + +################ +# AIX SETTINGS # +################ +CC.AIX = mpcc -q64 +CCFLAGS.AIX = -g -D_LARGE_FILES +# -qwarn64 -qinfo=all -D_NO_MPI_TIMER +LDFLAGS.AIX = #-bmaxdata:0x80000000 +HDF5_DIR.AIX = /usr/local/tools/hdf5/hdf5-1.6.5/parallel +#NCMPI_DIR.AIX = /usr/local/netcdf/parallel-netcdf-0.9.4_64bit +NCMPI_DIR.AIX = /g/g0/$(shell whoami)/LIBS/INSTALLS/parallel-netcdf-1.0.2pre2/aix_5_64_fed + +################### +# IRIX64 SETTINGS # +################### +CC.IRIX64 = /usr/local/mpich-1.2.2/irix-n32/bin/mpicc +CCFLAGS.IRIX64 = -g +# -D_NO_MPI_TIMER +LDFLAGS.IRIX64 = +HDF5_DIR.IRIX64 = /usr/local/tools/hdf5/hdf5-1.6.0/parallel +NCMPI_DIR.IRIX64 = + + +################## +# LINUX SETTINGS # +################## +CC.Linux = mpicc +CCFLAGS.Linux = -g -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 #-D_MANUALLY_SET_LUSTRE_STRIPING -Wall -pedantic -D_NO_MPI_TIMER +#LDFLAGS.Linux = -L/usr/lib/mpi/mpi_gnu/lib -lmpio +#LDFLAGS.Linux = -L/usr/lib/mpi/mpi_intel/lib -lmpio +LDFLAGS.Linux = +#HDF5_DIR.Linux = /usr/local/tools/hdf5/hdf5-1.6.5/parallel +#NCMPI_DIR.Linux = /g/g0/$(shell whoami)/LIBS/INSTALLS/parallel-netcdf-1.0.2pre2/chaos_3_x86_elan3 + + +################## +# SunOS SETTINGS # +################## +CC.SunOS = mpicc +CCFLAGS.SunOS = -g -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 +LDFLAGS.SunOS = -Llib -lmpich +HDF5_DIR.SunOS = +NCMPI_DIR.SunOS = + + +################ +# BGL SETTINGS # +################ +CC.BGL = mpgcc +CCFLAGS.BGL = -g -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 +LDFLAGS.BGL = +HDF5_DIR.BGL = +NCMPI_DIR.BGL = + + +######################### +# OSF1 (TRU64) SETTINGS # +######################### +CC.OSF1 = /usr/local/new_mpi/bin/mpicc +CCFLAGS.OSF1 = -g -ieee +LDFLAGS.OSF1 = +HDF5_DIR.OSF1 = /usr/local/hdf5/hdf5-1.4.5/mpich-1.2.4shm/prod/bit64/shared +NCMPI_DIR.OSF1 = + +################################################################################ + +CC = $(CC.$(OS)) +CCFLAGS = $(CCFLAGS.$(OS)) +LDFLAGS = $(LDFLAGS.$(OS)) -lm +CCFLAGS_HDF5 = $(CCFLAGS.$(OS)) -I$(HDF5_DIR.$(OS))/include +LDFLAGS_HDF5 = $(LDFLAGS.$(OS)) -L$(HDF5_DIR.$(OS))/lib -lhdf5 -lm -lz +CCFLAGS_NCMPI = $(CCFLAGS.$(OS)) -I$(NCMPI_DIR.$(OS))/include +LDFLAGS_NCMPI = $(LDFLAGS.$(OS)) -L$(NCMPI_DIR.$(OS))/lib -lpnetcdf -lm diff --git a/src/C/aiori-HDF5.c b/src/C/aiori-HDF5.c new file mode 100644 index 0000000..7dea602 --- /dev/null +++ b/src/C/aiori-HDF5.c @@ -0,0 +1,581 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: aiori-HDF5.c,v $ +* $Revision: 1.2 $ +* $Date: 2008/12/02 17:12:14 $ +* $Author: rklundt $ +* +* Purpose: +* Implementation of abstract I/O interface for HDF5. +* +\******************************************************************************/ + +#include "aiori.h" /* abstract IOR interface */ +#include /* sys_errlist */ +#include /* only for fprintf() */ +#include +#include +#include + +#define NUM_DIMS 1 /* number of dimensions to data set */ + +/******************************************************************************/ +/* + * HDF5_CHECK will display a custom error message and then exit the program + */ + +/* + * should use MPI_Abort(), not exit(), in this macro; some versions of + * MPI, however, hang with HDF5 property lists et al. left unclosed + */ + +/* + * for versions later than hdf5-1.6, the H5Eget_[major|minor]() functions + * have been deprecated and replaced with H5Eget_msg() + */ +#if H5_VERS_MAJOR > 1 && H5_VERS_MINOR > 6 +#define HDF5_CHECK(HDF5_RETURN, MSG) do { \ + char resultString[1024]; \ + \ + if (HDF5_RETURN < 0) { \ + fprintf(stdout, "** error **\n"); \ + fprintf(stdout, "ERROR in %s (line %d): %s.\n", \ + __FILE__, __LINE__, MSG); \ + strcpy(resultString, H5Eget_major((H5E_major_t)HDF5_RETURN)); \ + if (strcmp(resultString, "Invalid major error number") != 0) \ + fprintf(stdout, "HDF5 %s\n", resultString); \ + strcpy(resultString, H5Eget_minor((H5E_minor_t)HDF5_RETURN)); \ + if (strcmp(resultString, "Invalid minor error number") != 0) \ + fprintf(stdout, "%s\n", resultString); \ + fprintf(stdout, "** exiting **\n"); \ + exit(-1); \ + } \ +} while(0) +#else /* ! (H5_VERS_MAJOR > 1 && H5_VERS_MINOR > 6) */ +#define HDF5_CHECK(HDF5_RETURN, MSG) do { \ + char resultString[1024]; \ + \ + if (HDF5_RETURN < 0) { \ + fprintf(stdout, "** error **\n"); \ + fprintf(stdout, "ERROR in %s (line %d): %s.\n", \ + __FILE__, __LINE__, MSG); \ + /* \ + * H5Eget_msg(hid_t mesg_id, H5E_type_t* mesg_type, \ + * char* mesg, size_t size) \ + */ \ + fprintf(stdout, "** exiting **\n"); \ + exit(-1); \ + } \ +} while(0) +#endif /* H5_VERS_MAJOR > 1 && H5_VERS_MINOR > 6 */ +/**************************** P R O T O T Y P E S *****************************/ + +IOR_offset_t SeekOffset_HDF5 (void *, IOR_offset_t, IOR_param_t *); +void SetHints (MPI_Info *, char *); +void SetupDataSet_HDF5(void *, IOR_param_t *); +void ShowHints (MPI_Info *); + + +/************************** D E C L A R A T I O N S ***************************/ + +extern int errno, /* error number */ + rank, + rankOffset, + verbose; /* verbose output */ +extern MPI_Comm testComm; + + +static hid_t xferPropList; /* xfer property list */ +hid_t dataSet; /* data set id */ +hid_t dataSpace; /* data space id */ +hid_t fileDataSpace; /* file data space id */ +hid_t memDataSpace; /* memory data space id */ +int newlyOpenedFile; /* newly opened file */ + +/***************************** F U N C T I O N S ******************************/ + + +/******************************************************************************/ +/* + * Create and open a file through the HDF5 interface. + */ + +void * +IOR_Create_HDF5(char * testFileName, + IOR_param_t * param) +{ + return IOR_Open_HDF5(testFileName, param); +} /* IOR_Create_HDF5() */ + + +/******************************************************************************/ +/* + * Open a file through the HDF5 interface. + */ + +void * +IOR_Open_HDF5(char * testFileName, + IOR_param_t * param) +{ + hid_t accessPropList, + createPropList; + hsize_t memStart[NUM_DIMS], + dataSetDims[NUM_DIMS], + memStride[NUM_DIMS], + memCount[NUM_DIMS], + memBlock[NUM_DIMS], + memDataSpaceDims[NUM_DIMS]; + int tasksPerDataSet; + unsigned fd_mode = (unsigned)0; + hid_t *fd; + MPI_Comm comm; + MPI_Info mpiHints = MPI_INFO_NULL; + + fd = (hid_t *)malloc(sizeof(hid_t)); + if (fd == NULL) ERR("Unable to malloc file descriptor"); + /* + * HDF5 uses different flags than those for POSIX/MPIIO + */ + if (param->open == WRITE) { /* WRITE flags */ + param->openFlags = IOR_TRUNC; + } else { /* READ or check WRITE/READ flags */ + param->openFlags = IOR_RDONLY; + } + + /* set IOR file flags to HDF5 flags */ + /* -- file open flags -- */ + if (param->openFlags & IOR_RDONLY) {fd_mode |= H5F_ACC_RDONLY;} + if (param->openFlags & IOR_WRONLY) { + fprintf(stdout, "File write only not implemented in HDF5\n"); + } + if (param->openFlags & IOR_RDWR) {fd_mode |= H5F_ACC_RDWR;} + if (param->openFlags & IOR_APPEND) { + fprintf(stdout, "File append not implemented in HDF5\n"); + } + if (param->openFlags & IOR_CREAT) {fd_mode |= H5F_ACC_CREAT;} + if (param->openFlags & IOR_EXCL) {fd_mode |= H5F_ACC_EXCL;} + if (param->openFlags & IOR_TRUNC) {fd_mode |= H5F_ACC_TRUNC;} + if (param->openFlags & IOR_DIRECT) { + fprintf(stdout, "O_DIRECT not implemented in HDF5\n"); + } + + /* set up file creation property list */ + createPropList = H5Pcreate(H5P_FILE_CREATE); + HDF5_CHECK(createPropList, "cannot create file creation property list"); + /* set size of offset and length used to address HDF5 objects */ + HDF5_CHECK(H5Pset_sizes(createPropList, sizeof(hsize_t), sizeof(hsize_t)), + "cannot set property list properly"); + + /* set up file access property list */ + accessPropList = H5Pcreate(H5P_FILE_ACCESS); + HDF5_CHECK(accessPropList, "cannot create file access property list"); + + /* + * someday HDF5 implementation will allow subsets of MPI_COMM_WORLD + */ + /* store MPI communicator info for the file access property list */ + if (param->filePerProc) { + comm = MPI_COMM_SELF; + } else { + comm = testComm; + } + + SetHints(&mpiHints, param->hintsFileName); + /* + * note that with MP_HINTS_FILTERED=no, all key/value pairs will + * be in the info object. The info object that is attached to + * the file during MPI_File_open() will only contain those pairs + * deemed valid by the implementation. + */ + /* show hints passed to file */ + if (rank == 0 && param->showHints) { + fprintf(stdout, "\nhints passed to access property list {\n"); + ShowHints(&mpiHints); + fprintf(stdout, "}\n"); + } + HDF5_CHECK(H5Pset_fapl_mpio(accessPropList, comm, mpiHints), + "cannot set file access property list"); + + /* set alignment */ + HDF5_CHECK(H5Pset_alignment(accessPropList, param->setAlignment, + param->setAlignment), "cannot set alignment"); + + /* open file */ + if (param->open == WRITE) { /* WRITE */ + *fd = H5Fcreate(testFileName, fd_mode, + createPropList, accessPropList); + HDF5_CHECK(*fd, "cannot create file"); + } else { /* READ or CHECK */ + *fd = H5Fopen(testFileName, fd_mode, accessPropList); + HDF5_CHECK(*fd, "cannot open file"); + } + + /* show hints actually attached to file handle */ + if (param->showHints || (1) /* WEL - this needs fixing */) { + if (rank == 0 && (param->showHints) /* WEL - this needs fixing */) { + WARN("showHints not working for HDF5"); + } + } else { + MPI_Info mpiHintsCheck = MPI_INFO_NULL; + hid_t apl; + apl = H5Fget_access_plist(*fd); + HDF5_CHECK(H5Pget_fapl_mpio(apl, &comm, &mpiHintsCheck), + "cannot get info object through HDF5"); + if (rank == 0) { + fprintf(stdout, + "\nhints returned from opened file (HDF5) {\n"); + ShowHints(&mpiHintsCheck); + fprintf(stdout, "}\n"); + if (1 == 1) { /* request the MPIIO file handle and its hints */ + MPI_File * fd_mpiio; + HDF5_CHECK(H5Fget_vfd_handle(*fd, apl, (void **)&fd_mpiio), + "cannot get MPIIO file handle"); + MPI_CHECK(MPI_File_get_info(*fd_mpiio, &mpiHintsCheck), + "cannot get info object through MPIIO"); + fprintf(stdout, + "\nhints returned from opened file (MPIIO) {\n"); + ShowHints(&mpiHintsCheck); + fprintf(stdout, "}\n"); + } + } + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + } + + /* this is necessary for resetting various parameters + needed for reopening and checking the file */ + newlyOpenedFile = TRUE; + + HDF5_CHECK(H5Pclose(createPropList), "cannot close creation property list"); + HDF5_CHECK(H5Pclose(accessPropList), "cannot close access property list"); + + /* create property list for serial/parallel access */ + xferPropList = H5Pcreate(H5P_DATASET_XFER); + HDF5_CHECK(xferPropList, "cannot create transfer property list"); + + /* set data transfer mode */ + if (param->collective) { + HDF5_CHECK(H5Pset_dxpl_mpio(xferPropList, H5FD_MPIO_COLLECTIVE), + "cannot set collective data transfer mode"); + } else { + HDF5_CHECK(H5Pset_dxpl_mpio(xferPropList, H5FD_MPIO_INDEPENDENT), + "cannot set independent data transfer mode"); + } + + /* set up memory data space for transfer */ + memStart[0] = (hsize_t)0; + memCount[0] = (hsize_t)1; + memStride[0] = (hsize_t)(param->transferSize / sizeof(IOR_size_t)); + memBlock[0] = (hsize_t)(param->transferSize / sizeof(IOR_size_t)); + memDataSpaceDims[0] = (hsize_t)param->transferSize; + memDataSpace = H5Screate_simple(NUM_DIMS, memDataSpaceDims, NULL); + HDF5_CHECK(memDataSpace, "cannot create simple memory data space"); + + /* define hyperslab for memory data space */ + HDF5_CHECK(H5Sselect_hyperslab(memDataSpace, H5S_SELECT_SET, + memStart, memStride, memCount, memBlock), + "cannot create hyperslab"); + + /* set up parameters for fpp or different dataset count */ + if (param->filePerProc) { + tasksPerDataSet = 1; + } else { + if (param->individualDataSets) { + /* each task in segment has single data set */ + tasksPerDataSet = 1; + } else { + /* share single data set across all tasks in segment */ + tasksPerDataSet = param->numTasks; + } + } + dataSetDims[0] = (hsize_t)((param->blockSize / sizeof(IOR_size_t)) + * tasksPerDataSet); + + /* create a simple data space containing information on size + and shape of data set, and open it for access */ + dataSpace = H5Screate_simple(NUM_DIMS, dataSetDims, NULL); + HDF5_CHECK(dataSpace, "cannot create simple data space"); + + return(fd); +} /* IOR_Open_HDF5() */ + + +/******************************************************************************/ +/* + * Write or read access to file using the HDF5 interface. + */ + +IOR_offset_t +IOR_Xfer_HDF5(int access, + void * fd, + IOR_size_t * buffer, + IOR_offset_t length, + IOR_param_t * param) +{ + static int firstReadCheck = FALSE, + startNewDataSet; + IOR_offset_t segmentPosition, + segmentSize; + + /* + * this toggle is for the read check operation, which passes through + * this function twice; note that this function will open a data set + * only on the first read check and close only on the second + */ + if (access == READCHECK) { + if (firstReadCheck == TRUE) { + firstReadCheck = FALSE; + } else { + firstReadCheck = TRUE; + } + } + + /* determine by offset if need to start new data set */ + if (param->filePerProc == TRUE) { + segmentPosition = (IOR_offset_t)0; + segmentSize = param->blockSize; + } else { + segmentPosition = (IOR_offset_t)((rank + rankOffset) % param->numTasks) + * param->blockSize; + segmentSize = (IOR_offset_t)(param->numTasks) * param->blockSize; + } + if ((IOR_offset_t)((param->offset - segmentPosition) % segmentSize) == 0) { + /* + * ordinarily start a new data set, unless this is the + * second pass through during a read check + */ + startNewDataSet = TRUE; + if (access == READCHECK && firstReadCheck != TRUE) { + startNewDataSet = FALSE; + } + } + + /* create new data set */ + if (startNewDataSet == TRUE) { + /* if just opened this file, no data set to close yet */ + if (newlyOpenedFile != TRUE) { + HDF5_CHECK(H5Dclose(dataSet), "cannot close data set"); + HDF5_CHECK(H5Sclose(fileDataSpace), + "cannot close file data space"); + } + SetupDataSet_HDF5(fd, param); + } + + SeekOffset_HDF5(fd, param->offset, param); + + /* this is necessary to reset variables for reaccessing file */ + startNewDataSet = FALSE; + newlyOpenedFile = FALSE; + + /* access the file */ + if (access == WRITE) { /* WRITE */ + HDF5_CHECK(H5Dwrite(dataSet, H5T_NATIVE_LLONG, + memDataSpace, fileDataSpace, + xferPropList, buffer), + "cannot write to data set"); + } else { /* READ or CHECK */ + HDF5_CHECK(H5Dread(dataSet, H5T_NATIVE_LLONG, + memDataSpace, fileDataSpace, + xferPropList, buffer), + "cannot read from data set"); + } + return(length); +} /* IOR_Xfer_HDF5() */ + + +/******************************************************************************/ +/* + * Perform fsync(). + */ + +void +IOR_Fsync_HDF5(void * fd, IOR_param_t * param) +{ + ; +} /* IOR_Fsync_HDF5() */ + + +/******************************************************************************/ +/* + * Close a file through the HDF5 interface. + */ + +void +IOR_Close_HDF5(void * fd, + IOR_param_t * param) +{ + if (param->fd_fppReadCheck == NULL) { + HDF5_CHECK(H5Dclose(dataSet), "cannot close data set"); + HDF5_CHECK(H5Sclose(dataSpace), "cannot close data space"); + HDF5_CHECK(H5Sclose(fileDataSpace), "cannot close file data space"); + HDF5_CHECK(H5Sclose(memDataSpace), "cannot close memory data space"); + HDF5_CHECK(H5Pclose(xferPropList), + " cannot close transfer property list"); + } + HDF5_CHECK(H5Fclose(*(hid_t *)fd), "cannot close file"); + free(fd); +} /* IOR_Close_HDF5() */ + + +/******************************************************************************/ +/* + * Delete a file through the HDF5 interface. + */ + +void +IOR_Delete_HDF5(char * testFileName, IOR_param_t * param) +{ + if (unlink(testFileName) != 0) WARN("cannot delete file"); +} /* IOR_Delete_HDF5() */ + + +/******************************************************************************/ +/* + * Determine api version. + */ + +void +IOR_SetVersion_HDF5(IOR_param_t *test) +{ + unsigned major, minor, release; + if (H5get_libversion(&major, &minor, &release) < 0) { + WARN("cannot get HDF5 library version"); + } else { + sprintf(test->apiVersion, "%s-%u.%u.%u", + test->api, major, minor, release); + } +#ifndef H5_HAVE_PARALLEL + strcat(test->apiVersion, " (Serial)"); +#else /* H5_HAVE_PARALLEL */ + strcat(test->apiVersion, " (Parallel)"); +#endif /* not H5_HAVE_PARALLEL */ +} /* IOR_SetVersion_HDF5() */ + + +/************************ L O C A L F U N C T I O N S ***********************/ + +/******************************************************************************/ +/* + * Seek to offset in file using the HDF5 interface and set up hyperslab. + */ + +IOR_offset_t +SeekOffset_HDF5(void *fd, + IOR_offset_t offset, + IOR_param_t * param) +{ + IOR_offset_t segmentSize; + hsize_t hsStride[NUM_DIMS], + hsCount[NUM_DIMS], + hsBlock[NUM_DIMS]; + hsize_t hsStart[NUM_DIMS]; + + if (param->filePerProc == TRUE) { + segmentSize = (IOR_offset_t)param->blockSize; + } else { + segmentSize = (IOR_offset_t)(param->numTasks) * param->blockSize; + } + + /* create a hyperslab representing the file data space */ + if (param->individualDataSets) { + /* start at zero offset if not */ + hsStart[0] = (hsize_t)((offset % param->blockSize) + / sizeof(IOR_size_t)); + } else { + /* start at a unique offset if shared */ + hsStart[0] = (hsize_t)((offset % segmentSize) / sizeof(IOR_size_t)); + } + hsCount[0] = (hsize_t)1; + hsStride[0] = (hsize_t)(param->transferSize / sizeof(IOR_size_t)); + hsBlock[0] = (hsize_t)(param->transferSize / sizeof(IOR_size_t)); + + /* retrieve data space from data set for hyperslab */ + fileDataSpace = H5Dget_space(dataSet); + HDF5_CHECK(fileDataSpace, "cannot get data space from data set"); + HDF5_CHECK(H5Sselect_hyperslab(fileDataSpace, H5S_SELECT_SET, + hsStart, hsStride, hsCount, hsBlock), + "cannot select hyperslab"); + return(offset); +} /* SeekOffset_HDF5() */ + + +/******************************************************************************/ +/* + * Create HDF5 data set. + */ + +void +SetupDataSet_HDF5(void * fd, + IOR_param_t * param) +{ + char dataSetName[MAX_STR]; + hid_t dataSetPropList; + int dataSetID; + static int dataSetSuffix = 0; + + /* may want to use an extendable dataset (H5S_UNLIMITED) someday */ + /* may want to use a chunked dataset (H5S_CHUNKED) someday */ + + /* need to reset suffix counter if newly-opened file */ + if (newlyOpenedFile) dataSetSuffix = 0; + + /* may want to use individual access to each data set someday */ + if (param->individualDataSets) { + dataSetID = (rank + rankOffset) % param->numTasks; + } else { + dataSetID = 0; + } + + sprintf(dataSetName, "%s-%04d.%04d", "Dataset", dataSetID, dataSetSuffix++); + + if (param->open == WRITE) { /* WRITE */ + /* create data set */ + dataSetPropList = H5Pcreate(H5P_DATASET_CREATE); + /* check if hdf5 available */ + #if defined (H5_VERS_MAJOR) && defined (H5_VERS_MINOR) + /* no-fill option not available until hdf5-1.6.x */ + #if (H5_VERS_MAJOR > 0 && H5_VERS_MINOR > 5) + if (param->noFill == TRUE) { + if (rank == 0 && verbose >= VERBOSE_1) { + fprintf(stdout, "\nusing 'no fill' option\n"); + } + HDF5_CHECK(H5Pset_fill_time(dataSetPropList, + H5D_FILL_TIME_NEVER), + "cannot set fill time for property list"); + } + #else + char errorString[MAX_STR]; + sprintf(errorString, "'no fill' option not available in %s", test->apiVersion); + ERR(errorString); + #endif + #else + WARN("unable to determine HDF5 version for 'no fill' usage"); + #endif + dataSet = H5Dcreate(*(hid_t *)fd, dataSetName, H5T_NATIVE_LLONG, + dataSpace, dataSetPropList); + HDF5_CHECK(dataSet, "cannot create data set"); + } else { /* READ or CHECK */ + dataSet = H5Dopen(*(hid_t *)fd, dataSetName); + HDF5_CHECK(dataSet, "cannot create data set"); + } + +} /* SetupDataSet_HDF5() */ + + +/******************************************************************************/ +/* + * Use MPIIO call to get file size. + */ + +IOR_offset_t +IOR_GetFileSize_HDF5(IOR_param_t * test, + MPI_Comm testComm, + char * testFileName) +{ + return(IOR_GetFileSize_MPIIO(test, testComm, testFileName)); +} /* IOR_GetFileSize_HDF5() */ diff --git a/src/C/aiori-MPIIO.c b/src/C/aiori-MPIIO.c new file mode 100644 index 0000000..e6dfef4 --- /dev/null +++ b/src/C/aiori-MPIIO.c @@ -0,0 +1,470 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: aiori-MPIIO.c,v $ +* $Revision: 1.2 $ +* $Date: 2008/12/02 17:12:14 $ +* $Author: rklundt $ +* +* Purpose: +* Implementation of abstract I/O interface for MPIIO. +* +\******************************************************************************/ + +#include "aiori.h" /* abstract IOR interface */ +#include /* sys_errlist */ +#include /* only for fprintf() */ +#include +#include +#include + +#ifndef MPIAPI +# define MPIAPI /* defined as __stdcall on Windows */ +#endif + + +/**************************** P R O T O T Y P E S *****************************/ + +static IOR_offset_t SeekOffset_MPIIO (MPI_File, IOR_offset_t, + IOR_param_t *); +void SetHints (MPI_Info *, char *); +void ShowHints (MPI_Info *); + +/************************** D E C L A R A T I O N S ***************************/ + +extern int errno; +extern int rank; +extern int rankOffset; +extern int verbose; +extern MPI_Comm testComm; + +/***************************** F U N C T I O N S ******************************/ + +/******************************************************************************/ +/* + * Create and open a file through the MPIIO interface. + */ + +void * +IOR_Create_MPIIO(char * testFileName, + IOR_param_t * param) +{ + return IOR_Open_MPIIO(testFileName, param); +} /* IOR_Create_MPIIO() */ + + +/******************************************************************************/ +/* + * Open a file through the MPIIO interface. Setup file view. + */ + +void * +IOR_Open_MPIIO(char * testFileName, + IOR_param_t * param) +{ + int fd_mode = (int)0, + offsetFactor, + tasksPerFile, + transfersPerBlock = param->blockSize + / param->transferSize; + struct fileTypeStruct { + int globalSizes[2], + localSizes[2], + startIndices[2]; + } fileTypeStruct; + MPI_File * fd; + MPI_Comm comm; + MPI_Info mpiHints = MPI_INFO_NULL; + + fd = (MPI_File *)malloc(sizeof(MPI_File)); + if (fd == NULL) ERR("Unable to malloc MPI_File"); + + *fd = 0; + + /* set IOR file flags to MPIIO flags */ + /* -- file open flags -- */ + if (param->openFlags & IOR_RDONLY) {fd_mode |= MPI_MODE_RDONLY;} + if (param->openFlags & IOR_WRONLY) {fd_mode |= MPI_MODE_WRONLY;} + if (param->openFlags & IOR_RDWR) {fd_mode |= MPI_MODE_RDWR;} + if (param->openFlags & IOR_APPEND) {fd_mode |= MPI_MODE_APPEND;} + if (param->openFlags & IOR_CREAT) {fd_mode |= MPI_MODE_CREATE;} + if (param->openFlags & IOR_EXCL) {fd_mode |= MPI_MODE_EXCL;} + if (param->openFlags & IOR_TRUNC) { + fprintf(stdout, "File truncation not implemented in MPIIO\n"); + } + if (param->openFlags & IOR_DIRECT) { + fprintf(stdout, "O_DIRECT not implemented in MPIIO\n"); + } + + /* + * MPI_MODE_UNIQUE_OPEN mode optimization eliminates the overhead of file + * locking. Only open a file in this mode when the file will not be con- + * currently opened elsewhere, either inside or outside the MPI environment. + */ + fd_mode |= MPI_MODE_UNIQUE_OPEN; + + if (param->filePerProc) { + comm = MPI_COMM_SELF; + } else { + comm = testComm; + } + + SetHints(&mpiHints, param->hintsFileName); + /* + * note that with MP_HINTS_FILTERED=no, all key/value pairs will + * be in the info object. The info object that is attached to + * the file during MPI_File_open() will only contain those pairs + * deemed valid by the implementation. + */ + /* show hints passed to file */ + if (rank == 0 && param->showHints) { + fprintf(stdout, "\nhints passed to MPI_File_open() {\n"); + ShowHints(&mpiHints); + fprintf(stdout, "}\n"); + } + MPI_CHECK(MPI_File_open(comm, testFileName, fd_mode, mpiHints, fd), + "cannot open file"); + + /* show hints actually attached to file handle */ + if (rank == 0 && param->showHints) { + MPI_CHECK(MPI_File_get_info(*fd, &mpiHints), + "cannot get file info"); + fprintf(stdout, "\nhints returned from opened file {\n"); + ShowHints(&mpiHints); + fprintf(stdout, "}\n"); + } + + /* preallocate space for file */ + if (param->preallocate && param->open == WRITE) { + MPI_CHECK(MPI_File_preallocate(*fd, + (MPI_Offset)(param->segmentCount*param->blockSize*param->numTasks)), + "cannot preallocate file"); + } + /* create file view */ + if (param->useFileView) { + /* create contiguous transfer datatype */ + MPI_CHECK(MPI_Type_contiguous(param->transferSize / sizeof(IOR_size_t), + MPI_LONG_LONG_INT, ¶m->transferType), + "cannot create contiguous datatype"); + MPI_CHECK(MPI_Type_commit(¶m->transferType), + "cannot commit datatype"); + if (param->filePerProc) { + offsetFactor = 0; + tasksPerFile = 1; + } else { + offsetFactor = (rank + rankOffset) % param->numTasks; + tasksPerFile = param->numTasks; + } + + /* + * create file type using subarray + */ + fileTypeStruct.globalSizes[0] = 1; + fileTypeStruct.globalSizes[1] = transfersPerBlock * tasksPerFile; + fileTypeStruct.localSizes[0] = 1; + fileTypeStruct.localSizes[1] = transfersPerBlock; + fileTypeStruct.startIndices[0] = 0; + fileTypeStruct.startIndices[1] = transfersPerBlock * offsetFactor; + + MPI_CHECK(MPI_Type_create_subarray(2, fileTypeStruct.globalSizes, + fileTypeStruct.localSizes, + fileTypeStruct.startIndices, + MPI_ORDER_C, param->transferType, + ¶m->fileType), + "cannot create subarray"); + MPI_CHECK(MPI_Type_commit(¶m->fileType), "cannot commit datatype"); + + MPI_CHECK(MPI_File_set_view(*fd, (MPI_Offset)0, + param->transferType, param->fileType, + "native", (MPI_Info)MPI_INFO_NULL), + "cannot set file view"); + } + return((void *)fd); +} /* IOR_Open_MPIIO() */ + + +/******************************************************************************/ +/* + * Write or read access to file using the MPIIO interface. + */ + +IOR_offset_t +IOR_Xfer_MPIIO(int access, + void * fd, + IOR_size_t * buffer, + IOR_offset_t length, + IOR_param_t * param) +{ + int (MPIAPI *Access) (MPI_File, void *, int, + MPI_Datatype, MPI_Status *); + int (MPIAPI *Access_at) (MPI_File, MPI_Offset, void *, int, + MPI_Datatype,MPI_Status *); + int (MPIAPI *Access_all) (MPI_File, void *, int, + MPI_Datatype, MPI_Status *); + int (MPIAPI *Access_at_all) (MPI_File, MPI_Offset, void *, int, + MPI_Datatype, MPI_Status *); + /* + * this needs to be properly implemented: + * + * int (*Access_ordered)(MPI_File, void *, int, + * MPI_Datatype, MPI_Status *); + */ + MPI_Status status; + + /* point functions to appropriate MPIIO calls */ + if (access == WRITE) { /* WRITE */ + Access = MPI_File_write; + Access_at = MPI_File_write_at; + Access_all = MPI_File_write_all; + Access_at_all = MPI_File_write_at_all; + /* + * this needs to be properly implemented: + * + * Access_ordered = MPI_File_write_ordered; + */ + } else { /* READ or CHECK */ + Access = MPI_File_read; + Access_at = MPI_File_read_at; + Access_all = MPI_File_read_all; + Access_at_all = MPI_File_read_at_all; + /* + * this needs to be properly implemented: + * + * Access_ordered = MPI_File_read_ordered; + */ + } + + /* + * 'useFileView' uses derived datatypes and individual file pointers + */ + if (param->useFileView) { + /* find offset in file */ + if (SeekOffset_MPIIO(*(MPI_File *)fd, param->offset, param) < 0) { + /* if unsuccessful */ + length = -1; + } else { + /* + * 'useStridedDatatype' fits multi-strided pattern into a datatype; + * must use 'length' to determine repetitions (fix this for + * multi-segments someday, WEL): + * e.g., 'IOR -s 2 -b 32K -t 32K -a MPIIO -S' + */ + if (param->useStridedDatatype) { + length = param->segmentCount; + } else { + length = 1; + } + if (param->collective) { + /* individual, collective call */ + MPI_CHECK(Access_all(*(MPI_File *)fd, buffer, length, + param->transferType, &status), + "cannot access collective"); + } else { + /* individual, noncollective call */ + MPI_CHECK(Access(*(MPI_File *)fd, buffer, length, + param->transferType, &status), + "cannot access noncollective"); + } + length *= param->transferSize; /* for return value in bytes */ + } + } else { + /* + * !useFileView does not use derived datatypes, but it uses either + * shared or explicit file pointers + */ + if (param->useSharedFilePointer) { + /* find offset in file */ + if (SeekOffset_MPIIO(*(MPI_File *)fd, param->offset, param) < 0) { + /* if unsuccessful */ + length = -1; + } else { + /* shared, collective call */ + /* + * this needs to be properly implemented: + * + * MPI_CHECK(Access_ordered(fd.MPIIO, buffer, length, + * MPI_BYTE, &status), + * "cannot access shared, collective"); + */ + fprintf(stdout, "useSharedFilePointer not implemented\n"); + } + } else { + if (param->collective) { + /* explicit, collective call */ + MPI_CHECK(Access_at_all(*(MPI_File *)fd, param->offset, + buffer, length, MPI_BYTE, &status), + "cannot access explicit, collective"); + } else { + /* explicit, noncollective call */ + MPI_CHECK(Access_at(*(MPI_File *)fd, param->offset, buffer, + length, MPI_BYTE, &status), + "cannot access explicit, noncollective"); + } + } + } + return(length); +} /* IOR_Xfer_MPIIO() */ + + +/******************************************************************************/ +/* + * Perform fsync(). + */ + +void +IOR_Fsync_MPIIO(void * fd, IOR_param_t * param) +{ + ; +} /* IOR_Fsync_MPIIO() */ + + +/******************************************************************************/ +/* + * Close a file through the MPIIO interface. + */ + +void +IOR_Close_MPIIO(void * fd, + IOR_param_t * param) +{ + MPI_CHECK(MPI_File_close((MPI_File *)fd), "cannot close file"); + if ((param->useFileView == TRUE) && (param->fd_fppReadCheck == NULL)) { + /* + * need to free the datatype, so done in the close process + */ + MPI_CHECK(MPI_Type_free(¶m->fileType), + "cannot free MPI file datatype"); + MPI_CHECK(MPI_Type_free(¶m->transferType), + "cannot free MPI transfer datatype"); + } + free(fd); +} /* IOR_Close_MPIIO() */ + + +/******************************************************************************/ +/* + * Delete a file through the MPIIO interface. + */ + +void +IOR_Delete_MPIIO(char * testFileName, IOR_param_t * param) +{ + MPI_CHECK(MPI_File_delete(testFileName, (MPI_Info)MPI_INFO_NULL), + "cannot delete file"); +} /* IOR_Delete_MPIIO() */ + + +/******************************************************************************/ +/* + * Determine api version. + */ + +void +IOR_SetVersion_MPIIO(IOR_param_t *test) +{ + int version, subversion; + MPI_CHECK(MPI_Get_version(&version, &subversion), + "cannot get MPI version"); + sprintf(test->apiVersion, "%s (version=%d, subversion=%d)", + test->api, version, subversion); +} /* IOR_SetVersion_MPIIO() */ + + +/************************ L O C A L F U N C T I O N S ***********************/ + +/******************************************************************************/ +/* + * Seek to offset in file using the MPIIO interface. + */ + +static IOR_offset_t +SeekOffset_MPIIO(MPI_File fd, + IOR_offset_t offset, + IOR_param_t * param) +{ + int offsetFactor, + tasksPerFile; + IOR_offset_t tempOffset; + + tempOffset = offset; + + if (param->filePerProc) { + offsetFactor = 0; + tasksPerFile = 1; + } else { + offsetFactor = (rank + rankOffset) % param->numTasks; + tasksPerFile = param->numTasks; + } + if (param->useFileView) { + /* recall that offsets in a file view are + counted in units of transfer size */ + if (param->filePerProc) { + tempOffset = tempOffset / param->transferSize; + } else { + /* + * this formula finds a file view offset for a task + * from an absolute offset + */ + tempOffset = ((param->blockSize / param->transferSize) + * (tempOffset / (param->blockSize * tasksPerFile))) + + (((tempOffset % (param->blockSize * tasksPerFile)) + - (offsetFactor * param->blockSize)) + / param->transferSize); + } + } + MPI_CHECK(MPI_File_seek(fd, tempOffset, MPI_SEEK_SET), + "cannot seek offset"); + return(offset); +} /* SeekOffset_MPIIO() */ + + +/******************************************************************************/ +/* + * Use MPI_File_get_size() to return aggregate file size. + */ + +IOR_offset_t +IOR_GetFileSize_MPIIO(IOR_param_t * test, + MPI_Comm testComm, + char * testFileName) +{ + IOR_offset_t aggFileSizeFromStat, + tmpMin, tmpMax, tmpSum; + MPI_File fd; + + MPI_CHECK(MPI_File_open(testComm, testFileName, MPI_MODE_RDONLY, + MPI_INFO_NULL, &fd), + "cannot open file to get file size"); + MPI_CHECK(MPI_File_get_size(fd, &aggFileSizeFromStat), + "cannot get file size"); + MPI_CHECK(MPI_File_close(&fd), "cannot close file"); + + if (test->filePerProc == TRUE) { + MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1, + MPI_LONG_LONG_INT, MPI_SUM, testComm), + "cannot total data moved"); + aggFileSizeFromStat = tmpSum; + } else { + MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1, + MPI_LONG_LONG_INT, MPI_MIN, testComm), + "cannot total data moved"); + MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1, + MPI_LONG_LONG_INT, MPI_MAX, testComm), + "cannot total data moved"); + if (tmpMin != tmpMax) { + if (rank == 0) { + WARN("inconsistent file size by different tasks"); + } + /* incorrect, but now consistent across tasks */ + aggFileSizeFromStat = tmpMin; + } + } + + return(aggFileSizeFromStat); + +} /* IOR_GetFileSize_MPIIO() */ diff --git a/src/C/aiori-NCMPI.c b/src/C/aiori-NCMPI.c new file mode 100644 index 0000000..1c4bae5 --- /dev/null +++ b/src/C/aiori-NCMPI.c @@ -0,0 +1,406 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: aiori-NCMPI.c,v $ +* $Revision: 1.2 $ +* $Date: 2008/12/02 17:12:14 $ +* $Author: rklundt $ +* +* Purpose: +* Implementation of abstract I/O interface for Parallel NetCDF (NCMPI). +* +\******************************************************************************/ + +#include "aiori.h" /* abstract IOR interface */ +#include /* sys_errlist */ +#include /* only for fprintf() */ +#include +#include +#include + +#define NUM_DIMS 3 /* number of dimensions to data set */ + +/******************************************************************************/ +/* + * NCMPI_CHECK will display a custom error message and then exit the program + */ + +#define NCMPI_CHECK(NCMPI_RETURN, MSG) do { \ + char resultString[1024]; \ + \ + if (NCMPI_RETURN < 0) { \ + fprintf(stdout, "** error **\n"); \ + fprintf(stdout, "ERROR in %s (line %d): %s.\n", \ + __FILE__, __LINE__, MSG); \ + fprintf(stdout, "ERROR: %s.\n", ncmpi_strerror(NCMPI_RETURN)); \ + fprintf(stdout, "** exiting **\n"); \ + exit(-1); \ + } \ +} while(0) + +/**************************** P R O T O T Y P E S *****************************/ + +int GetFileMode(IOR_param_t *); +void SetHints (MPI_Info *, char *); +void ShowHints (MPI_Info *); + +/************************** D E C L A R A T I O N S ***************************/ + +extern int errno, /* error number */ + numTasksWorld, + rank, + rankOffset, + verbose; /* verbose output */ +extern MPI_Comm testComm; + +/***************************** F U N C T I O N S ******************************/ +/******************************************************************************/ +/* + * Create and open a file through the NCMPI interface. + */ + +void * +IOR_Create_NCMPI(char * testFileName, + IOR_param_t * param) +{ + int * fd; + int fd_mode; + MPI_Info mpiHints = MPI_INFO_NULL; + + /* Wei-keng Liao: read and set MPI file hints from hintsFile */ + SetHints(&mpiHints, param->hintsFileName); + if (rank == 0 && param->showHints) { + fprintf(stdout, "\nhints passed to MPI_File_open() {\n"); + ShowHints(&mpiHints); + fprintf(stdout, "}\n"); + } + + fd = (int *)malloc(sizeof(int)); + if (fd == NULL) ERR("Unable to malloc file descriptor"); + + fd_mode = GetFileMode(param); + NCMPI_CHECK(ncmpi_create(testComm, testFileName, fd_mode, + mpiHints, fd), "cannot create file"); + + /* Wei-keng Liao: print the MPI file hints currently used */ +/* WEL - add when ncmpi_get_file_info() is in current parallel-netcdf release + if (rank == 0 && param->showHints) { + MPI_CHECK(ncmpi_get_file_info(*fd, &mpiHints), + "cannot get file info"); + fprintf(stdout, "\nhints returned from opened file {\n"); + ShowHints(&mpiHints); + fprintf(stdout, "}\n"); + } +*/ + + /* Wei-keng Liao: free up the mpiHints object */ +/* WEL - this needs future fix from next release of PnetCDF + if (mpiHints != MPI_INFO_NULL) + MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info"); +*/ + + return(fd); +} /* IOR_Create_NCMPI() */ + + +/******************************************************************************/ +/* + * Open a file through the NCMPI interface. + */ + +void * +IOR_Open_NCMPI(char * testFileName, + IOR_param_t * param) +{ + int * fd; + int fd_mode; + MPI_Info mpiHints = MPI_INFO_NULL; + + /* Wei-keng Liao: read and set MPI file hints from hintsFile */ + SetHints(&mpiHints, param->hintsFileName); + if (rank == 0 && param->showHints) { + fprintf(stdout, "\nhints passed to MPI_File_open() {\n"); + ShowHints(&mpiHints); + fprintf(stdout, "}\n"); + } + + fd = (int *)malloc(sizeof(int)); + if (fd == NULL) ERR("Unable to malloc file descriptor"); + + fd_mode = GetFileMode(param); + NCMPI_CHECK(ncmpi_open(testComm, testFileName, fd_mode, + mpiHints, fd), "cannot open file"); + + /* Wei-keng Liao: print the MPI file hints currently used */ +/* WEL - add when ncmpi_get_file_info() is in current parallel-netcdf release + if (rank == 0 && param->showHints) { + MPI_CHECK(ncmpi_get_file_info(*fd, &mpiHints), + "cannot get file info"); + fprintf(stdout, "\nhints returned from opened file {\n"); + ShowHints(&mpiHints); + fprintf(stdout, "}\n"); + } +*/ + + /* Wei-keng Liao: free up the mpiHints object */ +/* WEL - this needs future fix from next release of PnetCDF + if (mpiHints != MPI_INFO_NULL) + MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info"); +*/ + + return(fd); +} /* IOR_Open_NCMPI() */ + + +/******************************************************************************/ +/* + * Write or read access to file using the NCMPI interface. + */ + +IOR_offset_t +IOR_Xfer_NCMPI(int access, + void * fd, + IOR_size_t * buffer, + IOR_offset_t length, + IOR_param_t * param) +{ + char * bufferPtr = (char *)buffer; + static int firstReadCheck = FALSE, + startDataSet; + int var_id, + dim_id[NUM_DIMS]; + MPI_Offset bufSize[NUM_DIMS], + offset[NUM_DIMS]; + IOR_offset_t segmentPosition; + int segmentNum, + transferNum; + + /* Wei-keng Liao: In IOR.c line 1979 says "block size must be a multiple + of transfer size." Hence, length should always == param->transferSize + below. I leave it here to double check. + */ + if (length != param->transferSize) { + char errMsg[256]; + sprintf(errMsg,"length(%lld) != param->transferSize(%lld)\n", + length, param->transferSize); + NCMPI_CHECK(-1, errMsg); + } + + /* determine by offset if need to start data set */ + if (param->filePerProc == TRUE) { + segmentPosition = (IOR_offset_t)0; + } else { + segmentPosition = (IOR_offset_t)((rank + rankOffset) % param->numTasks) + * param->blockSize; + } + if ((int)(param->offset - segmentPosition) == 0) { + startDataSet = TRUE; + /* + * this toggle is for the read check operation, which passes through + * this function twice; note that this function will open a data set + * only on the first read check and close only on the second + */ + if (access == READCHECK) { + if (firstReadCheck == TRUE) { + firstReadCheck = FALSE; + } else { + firstReadCheck = TRUE; + } + } + } + + if (startDataSet == TRUE && + (access != READCHECK || firstReadCheck == TRUE)) { + if (access == WRITE) { + int numTransfers = param->blockSize / param->transferSize; + + /* Wei-keng Liao: change 1D array to 3D array of dimensions: + [segmentCount*numTasksWorld][numTransfers][transferSize] + Requirement: none of these dimensions should be > 4G, + */ + NCMPI_CHECK(ncmpi_def_dim(*(int *)fd, "segments_times_np", + NC_UNLIMITED, &dim_id[0]), + "cannot define data set dimensions"); + NCMPI_CHECK(ncmpi_def_dim(*(int *)fd, "number_of_transfers", + numTransfers, &dim_id[1]), + "cannot define data set dimensions"); + NCMPI_CHECK(ncmpi_def_dim(*(int *)fd, "transfer_size", + param->transferSize, &dim_id[2]), + "cannot define data set dimensions"); + NCMPI_CHECK(ncmpi_def_var(*(int *)fd, "data_var", NC_BYTE, + NUM_DIMS, dim_id, &var_id), + "cannot define data set variables"); + NCMPI_CHECK(ncmpi_enddef(*(int *)fd), + "cannot close data set define mode"); + + } else { + NCMPI_CHECK(ncmpi_inq_varid(*(int *)fd, "data_var", &var_id), + "cannot retrieve data set variable"); + } + + if (param->collective == FALSE) { + NCMPI_CHECK(ncmpi_begin_indep_data(*(int *)fd), + "cannot enable independent data mode"); + } + + param->var_id = var_id; + startDataSet = FALSE; + } + + var_id = param->var_id; + + /* Wei-keng Liao: calculate the segment number */ + segmentNum = param->offset / (param->numTasks * param->blockSize); + + /* Wei-keng Liao: calculate the transfer number in each block */ + transferNum = param->offset % param->blockSize / param->transferSize; + + /* Wei-keng Liao: read/write the 3rd dim of the dataset, each is of + amount param->transferSize */ + bufSize[0] = 1; + bufSize[1] = 1; + bufSize[2] = param->transferSize; + + offset[0] = segmentNum * numTasksWorld + rank; + offset[1] = transferNum; + offset[2] = 0; + + /* access the file */ + if (access == WRITE) { /* WRITE */ + if (param->collective) { + NCMPI_CHECK(ncmpi_put_vara_all(*(int *)fd, var_id, offset, bufSize, + bufferPtr, length, MPI_BYTE), + "cannot write to data set"); + } else { + NCMPI_CHECK(ncmpi_put_vara(*(int *)fd, var_id, offset, bufSize, + bufferPtr, length, MPI_BYTE), + "cannot write to data set"); + } + } else { /* READ or CHECK */ + if (param->collective == TRUE) { + NCMPI_CHECK(ncmpi_get_vara_all(*(int *)fd, var_id, offset, bufSize, + bufferPtr, length, MPI_BYTE), + "cannot read from data set"); + } else { + NCMPI_CHECK(ncmpi_get_vara(*(int *)fd, var_id, offset, bufSize, + bufferPtr, length, MPI_BYTE), + "cannot read from data set"); + } + } + + return(length); +} /* IOR_Xfer_NCMPI() */ + + +/******************************************************************************/ +/* + * Perform fsync(). + */ + +void +IOR_Fsync_NCMPI(void * fd, IOR_param_t * param) +{ + ; +} /* IOR_Fsync_NCMPI() */ + + +/******************************************************************************/ +/* + * Close a file through the NCMPI interface. + */ + +void +IOR_Close_NCMPI(void * fd, + IOR_param_t * param) +{ + if (param->collective == FALSE) { + NCMPI_CHECK(ncmpi_end_indep_data(*(int *)fd), + "cannot disable independent data mode"); + } + NCMPI_CHECK(ncmpi_close(*(int *)fd), "cannot close file"); + free(fd); +} /* IOR_Close_NCMPI() */ + + +/******************************************************************************/ +/* + * Delete a file through the NCMPI interface. + */ + +void +IOR_Delete_NCMPI(char * testFileName, IOR_param_t * param) +{ + if (unlink(testFileName) != 0) WARN("cannot delete file"); +} /* IOR_Delete_NCMPI() */ + + +/******************************************************************************/ +/* + * Determine api version. + */ + +void +IOR_SetVersion_NCMPI(IOR_param_t * test) +{ + sprintf(test->apiVersion, "%s (%s)", + test->api, ncmpi_inq_libvers()); +} /* IOR_SetVersion_NCMPI() */ + + +/************************ L O C A L F U N C T I O N S ***********************/ + +/******************************************************************************/ +/* + * Return the correct file mode for NCMPI. + */ + +int +GetFileMode(IOR_param_t * param) +{ + int fd_mode = 0; + + /* set IOR file flags to NCMPI flags */ + /* -- file open flags -- */ + if (param->openFlags & IOR_RDONLY) {fd_mode |= NC_NOWRITE;} + if (param->openFlags & IOR_WRONLY) { + fprintf(stdout, "File write only not implemented in NCMPI\n"); + } + if (param->openFlags & IOR_RDWR) {fd_mode |= NC_WRITE;} + if (param->openFlags & IOR_APPEND) { + fprintf(stdout, "File append not implemented in NCMPI\n"); + } + if (param->openFlags & IOR_CREAT) {fd_mode |= NC_CLOBBER;} + if (param->openFlags & IOR_EXCL) { + fprintf(stdout, "Exclusive access not implemented in NCMPI\n"); + } + if (param->openFlags & IOR_TRUNC) { + fprintf(stdout, "File truncation not implemented in NCMPI\n"); + } + if (param->openFlags & IOR_DIRECT) { + fprintf(stdout, "O_DIRECT not implemented in NCMPI\n"); + } + + /* Wei-keng Liao: to enable > 4GB file size */ + fd_mode |= NC_64BIT_OFFSET; + + return(fd_mode); +} /* GetFileMode() */ + + +/******************************************************************************/ +/* + * Use MPIIO call to get file size. + */ + +IOR_offset_t +IOR_GetFileSize_NCMPI(IOR_param_t * test, + MPI_Comm testComm, + char * testFileName) +{ + return(IOR_GetFileSize_MPIIO(test, testComm, testFileName)); +} /* IOR_GetFileSize_NCMPI() */ diff --git a/src/C/aiori-POSIX.c b/src/C/aiori-POSIX.c new file mode 100644 index 0000000..e11a1cf --- /dev/null +++ b/src/C/aiori-POSIX.c @@ -0,0 +1,388 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: aiori-POSIX.c,v $ +* $Revision: 1.3 $ +* $Date: 2008/12/02 17:12:14 $ +* $Author: rklundt $ +* +* Purpose: +* Implementation of abstract I/O interface for POSIX. +* +\******************************************************************************/ +/********************* Modifications to IOR-2.10.1 **************************** +* hodson - 8/18/2008: * +* Added fsyncPerWrite option to do POSIX fsync after each POSIX write * +* More info on cannot delete file error message * +******************************************************************************/ + +#include "aiori.h" /* abstract IOR interface */ +#ifdef __linux__ +# include /* necessary for: */ +# define __USE_GNU /* O_DIRECT and */ +# include /* IO operations */ +# undef __USE_GNU +#endif /* __linux__ */ +#include /* sys_errlist */ +#include /* IO operations */ +#include /* only for fprintf() */ +#include +#include +#ifdef _MANUALLY_SET_LUSTRE_STRIPING +# include +#endif /* _MANUALLY_SET_LUSTRE_STRIPING */ + +#ifndef open64 /* necessary for TRU64 -- */ +# define open64 open /* unlikely, but may pose */ +#endif /* not open64 */ /* conflicting prototypes */ + +#ifndef lseek64 /* necessary for TRU64 -- */ +# define lseek64 lseek /* unlikely, but may pose */ +#endif /* not lseek64 */ /* conflicting prototypes */ + +#ifndef O_BINARY /* Required on Windows */ +# define O_BINARY 0 +#endif + + +/**************************** P R O T O T Y P E S *****************************/ + + +/************************** D E C L A R A T I O N S ***************************/ + +extern int errno; +extern int rank; +extern int rankOffset; +extern int verbose; +extern MPI_Comm testComm; + +/***************************** F U N C T I O N S ******************************/ + +/******************************************************************************/ +/* + * Creat and open a file through the POSIX interface. + */ + +void * +IOR_Create_POSIX(char * testFileName, + IOR_param_t * param) +{ + int fd_oflag = O_BINARY; + int *fd; + + fd = (int *)malloc(sizeof(int)); + if (fd == NULL) ERR("Unable to malloc file descriptor"); + + if (param->useO_DIRECT == TRUE) { +/* note that TRU64 needs O_DIRECTIO, SunOS uses directio(), + and everyone else needs O_DIRECT */ +#ifndef O_DIRECT +# ifndef O_DIRECTIO + WARN("cannot use O_DIRECT"); +# define O_DIRECT 000000 +# else /* O_DIRECTIO */ +# define O_DIRECT O_DIRECTIO +# endif /* not O_DIRECTIO */ +#endif /* not O_DIRECT */ + fd_oflag |= O_DIRECT; + } + +#ifndef _MANUALLY_SET_LUSTRE_STRIPING + /* If the lustre striping parameters are not the defaults */ + if (param->lustre_stripe_count != 0 + || param->lustre_stripe_size != 0 + || param->lustre_start_ost != -1 + || param->lustre_ignore_locks) { + ERR("This IOR was not compiled with Lustre support!"); + } + fd_oflag |= O_CREAT | O_RDWR; + *fd = open64(testFileName, fd_oflag, 0664); + if (*fd < 0) ERR("cannot open file"); +#else /* _MANUALLY_SET_LUSTRE_STRIPING */ + /* If the lustre striping parameters are not the defaults */ + if (param->lustre_stripe_count != 0 + || param->lustre_stripe_size != 0 + || param->lustre_start_ost != -1) { + + /* In the single-shared-file case, task 0 has to creat the + file with the Lustre striping options before any other processes + open the file */ + if (!param->filePerProc && rank != 0) { + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + fd_oflag |= O_RDWR; + *fd = open64(testFileName, fd_oflag, 0664); + if (*fd < 0) ERR("cannot open file"); + } else { + struct lov_user_md opts = { 0 }; + + /* Setup Lustre IOCTL striping pattern structure */ + opts.lmm_magic = LOV_USER_MAGIC; + opts.lmm_stripe_size = param->lustre_stripe_size; + opts.lmm_stripe_offset = param->lustre_start_ost; + opts.lmm_stripe_count = param->lustre_stripe_count; + + /* File needs to be opened O_EXCL because we cannot set + Lustre striping information on a pre-existing file. */ + fd_oflag |= O_CREAT | O_EXCL | O_RDWR | O_LOV_DELAY_CREATE; + *fd = open64(testFileName, fd_oflag, 0664); + if (*fd < 0) { + fprintf(stdout, "\nUnable to open '%s': %s\n", + testFileName, strerror(errno)); + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + } else if (ioctl(*fd, LL_IOC_LOV_SETSTRIPE, &opts)) { + char *errmsg = "stripe already set"; + if (errno != EEXIST && errno != EALREADY) + errmsg = strerror(errno); + fprintf(stdout, "\nError on ioctl for '%s' (%d): %s\n", + testFileName, *fd, errmsg); + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + } + if (!param->filePerProc) + MPI_CHECK(MPI_Barrier(testComm), "barrier error"); + } + } else { + fd_oflag |= O_CREAT | O_RDWR; + *fd = open64(testFileName, fd_oflag, 0664); + if (*fd < 0) ERR("cannot open file"); + } + + if (param->lustre_ignore_locks) { + int lustre_ioctl_flags = LL_FILE_IGNORE_LOCK; + if (ioctl(*fd, LL_IOC_SETFLAGS, &lustre_ioctl_flags) == -1) + ERR("cannot set ioctl"); + } +#endif /* not _MANUALLY_SET_LUSTRE_STRIPING */ + + return((void *)fd); +} /* IOR_Create_POSIX() */ + + +/******************************************************************************/ +/* + * Open a file through the POSIX interface. + */ + +void * +IOR_Open_POSIX(char * testFileName, + IOR_param_t * param) +{ + int fd_oflag = O_BINARY; + int *fd; + + fd = (int *)malloc(sizeof(int)); + if (fd == NULL) ERR("Unable to malloc file descriptor"); + + if (param->useO_DIRECT == TRUE) { +/* note that TRU64 needs O_DIRECTIO, SunOS uses directio(), + and everyone else needs O_DIRECT */ +#ifndef O_DIRECT +# ifndef O_DIRECTIO + WARN("cannot use O_DIRECT"); +# define O_DIRECT 000000 +# else /* O_DIRECTIO */ +# define O_DIRECT O_DIRECTIO +# endif /* not O_DIRECTIO */ +#endif /* not O_DIRECT */ + fd_oflag |= O_DIRECT; + } + + fd_oflag |= O_RDWR; + *fd = open64(testFileName, fd_oflag); + if (*fd < 0) ERR("cannot open file"); + +#ifdef _MANUALLY_SET_LUSTRE_STRIPING + if (param->lustre_ignore_locks) { + int lustre_ioctl_flags = LL_FILE_IGNORE_LOCK; + if (verbose >= VERBOSE_1) { + fprintf(stdout, "** Disabling lustre range locking **\n"); + } + if (ioctl(*fd, LL_IOC_SETFLAGS, &lustre_ioctl_flags) == -1) + ERR("cannot set ioctl"); + } +#endif /* _MANUALLY_SET_LUSTRE_STRIPING */ + + return((void *)fd); +} /* IOR_Open_POSIX() */ + + +/******************************************************************************/ +/* + * Write or read access to file using the POSIX interface. + */ + +IOR_offset_t +IOR_Xfer_POSIX(int access, + void * file, + IOR_size_t * buffer, + IOR_offset_t length, + IOR_param_t * param) +{ + int xferRetries = 0; + long long remaining = (long long)length; + char * ptr = (char *)buffer; + long long rc; + int fd; + + fd = *(int *)file; + + /* seek to offset */ + if (lseek64(fd, param->offset, SEEK_SET) == -1) + ERR("seek failed"); + + while (remaining > 0) { + /* write/read file */ + if (access == WRITE) { /* WRITE */ + if (verbose >= VERBOSE_4) { + fprintf(stdout, "task %d writing to offset %lld\n", + rank, param->offset + length - remaining); + } + rc = write(fd, ptr, remaining); + if (param->fsyncPerWrite == TRUE) IOR_Fsync_POSIX(&fd, param); + } else { /* READ or CHECK */ + if (verbose >= VERBOSE_4) { + fprintf(stdout, "task %d reading from offset %lld\n", + rank, param->offset + length - remaining); + } + rc = read(fd, ptr, remaining); + if (rc == 0) + ERR("hit EOF prematurely"); + } + if (rc == -1) + ERR("transfer failed"); + if (rc != remaining) { + fprintf(stdout, + "WARNING: Task %d requested transfer of %lld bytes,\n", + rank, remaining); + fprintf(stdout, + " but transferred %lld bytes at offset %lld\n", + rc, param->offset + length - remaining); + if (param->singleXferAttempt == TRUE) + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "barrier error"); + } + if (rc < remaining) { + if (xferRetries > MAX_RETRY) + ERR("too many retries -- aborting"); + if (xferRetries == 0) { + if (access == WRITE) { + WARN("This file system requires support of partial write()s"); + } else { + WARN("This file system requires support of partial read()s"); + } + fprintf(stdout, + "WARNING: Requested xfer of %lld bytes, but xferred %lld bytes\n", + remaining, rc); + } + if (verbose >= VERBOSE_2) { + fprintf(stdout, "Only transferred %lld of %lld bytes\n", + rc, remaining); + } + } + if (rc > remaining) /* this should never happen */ + ERR("too many bytes transferred!?!"); + remaining -= rc; + ptr += rc; + xferRetries++; + } + return(length); +} /* IOR_Xfer_POSIX() */ + + +/******************************************************************************/ +/* + * Perform fsync(). + */ + +void +IOR_Fsync_POSIX(void * fd, IOR_param_t * param) +{ + if (fsync(*(int *)fd) != 0) WARN("cannot perform fsync on file"); +} /* IOR_Fsync_POSIX() */ + + +/******************************************************************************/ +/* + * Close a file through the POSIX interface. + */ + +void +IOR_Close_POSIX(void *fd, + IOR_param_t * param) +{ + if (close(*(int *)fd) != 0) ERR("cannot close file"); + free(fd); +} /* IOR_Close_POSIX() */ + + +/******************************************************************************/ +/* + * Delete a file through the POSIX interface. + */ + +void +IOR_Delete_POSIX(char * testFileName, IOR_param_t * param) +{ + char errmsg[256]; + sprintf(errmsg,"[RANK %03d]:cannot delete file %s\n",rank,testFileName); + if (unlink(testFileName) != 0) WARN(errmsg); +} /* IOR_Delete_POSIX() */ + + +/******************************************************************************/ +/* + * Determine api version. + */ + +void +IOR_SetVersion_POSIX(IOR_param_t *test) +{ + strcpy(test->apiVersion, test->api); +} /* IOR_SetVersion_POSIX() */ + + +/******************************************************************************/ +/* + * Use POSIX stat() to return aggregate file size. + */ + +IOR_offset_t +IOR_GetFileSize_POSIX(IOR_param_t * test, + MPI_Comm testComm, + char * testFileName) +{ + struct stat stat_buf; + IOR_offset_t aggFileSizeFromStat, + tmpMin, tmpMax, tmpSum; + + if (stat(testFileName, &stat_buf) != 0) { + ERR("cannot get status of written file"); + } + aggFileSizeFromStat = stat_buf.st_size; + + if (test->filePerProc == TRUE) { + MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1, + MPI_LONG_LONG_INT, MPI_SUM, testComm), + "cannot total data moved"); + aggFileSizeFromStat = tmpSum; + } else { + MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1, + MPI_LONG_LONG_INT, MPI_MIN, testComm), + "cannot total data moved"); + MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1, + MPI_LONG_LONG_INT, MPI_MAX, testComm), + "cannot total data moved"); + if (tmpMin != tmpMax) { + if (rank == 0) { + WARN("inconsistent file size by different tasks"); + } + /* incorrect, but now consistent across tasks */ + aggFileSizeFromStat = tmpMin; + } + } + + return(aggFileSizeFromStat); +} /* IOR_GetFileSize_POSIX() */ diff --git a/src/C/aiori-noHDF5.c b/src/C/aiori-noHDF5.c new file mode 100644 index 0000000..8d52903 --- /dev/null +++ b/src/C/aiori-noHDF5.c @@ -0,0 +1,81 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: aiori-noHDF5.c,v $ +* $Revision: 1.2 $ +* $Date: 2008/12/02 17:12:14 $ +* $Author: rklundt $ +* +* Purpose: +* Empty HDF5 functions for when compiling without HDF5 support. +* +\******************************************************************************/ + +#include "aiori.h" + +void * +IOR_Create_HDF5(char * testFileName, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with HDF5 support"); + return 0; +} + +void * +IOR_Open_HDF5(char * testFileName, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with HDF5 support"); + return 0; +} + + +IOR_offset_t +IOR_Xfer_HDF5(int access, + void * fd, + IOR_size_t * buffer, + IOR_offset_t length, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with HDF5 support"); + return 0; +} + +void +IOR_Fsync_HDF5(void * fd, IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with HDF5 support"); +} + +void +IOR_Close_HDF5(void * fd, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with HDF5 support"); +} + +void +IOR_Delete_HDF5(char * testFileName, IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with HDF5 support"); +} + +void +IOR_SetVersion_HDF5(IOR_param_t *test) +{ + ERR("This copy of IOR was not compiled with HDF5 support"); +} + +IOR_offset_t +IOR_GetFileSize_HDF5(IOR_param_t * test, + MPI_Comm testComm, + char * testFileName) +{ + ERR("This copy of IOR was not compiled with HDF5 support"); + return 0; +} diff --git a/src/C/aiori-noMPIIO.c b/src/C/aiori-noMPIIO.c new file mode 100644 index 0000000..52b965a --- /dev/null +++ b/src/C/aiori-noMPIIO.c @@ -0,0 +1,81 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: aiori-noMPIIO.c,v $ +* $Revision: 1.1.1.1 $ +* $Date: 2007/10/15 23:36:54 $ +* $Author: rklundt $ +* +* Purpose: +* Empty MPIIO functions for when compiling without MPIIO support. +* +\******************************************************************************/ + +#include "aiori.h" + +void * +IOR_Create_MPIIO(char * testFileName, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with MPIIO support"); + return 0; +} + +void * +IOR_Open_MPIIO(char * testFileName, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with MPIIO support"); + return 0; +} + + +IOR_offset_t +IOR_Xfer_MPIIO(int access, + void * fd, + IOR_size_t * buffer, + IOR_offset_t length, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with MPIIO support"); + return 0; +} + +void +IOR_Fsync_MPIIO(void * fd, IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with MPIIO support"); +} + +void +IOR_Close_MPIIO(void * fd, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with MPIIO support"); +} + +void +IOR_Delete_MPIIO(char * testFileName, IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with MPIIO support"); +} + +void +IOR_SetVersion_MPIIO(IOR_param_t *test) +{ + ERR("This copy of IOR was not compiled with MPIIO support"); +} + +IOR_offset_t +IOR_GetFileSize_MPIIO(IOR_param_t * test, + MPI_Comm testComm, + char * testFileName) +{ + ERR("This copy of IOR was not compiled with MPIIO support"); + return 0; +} diff --git a/src/C/aiori-noNCMPI.c b/src/C/aiori-noNCMPI.c new file mode 100644 index 0000000..f7cf954 --- /dev/null +++ b/src/C/aiori-noNCMPI.c @@ -0,0 +1,81 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: aiori-noNCMPI.c,v $ +* $Revision: 1.2 $ +* $Date: 2008/12/02 17:12:14 $ +* $Author: rklundt $ +* +* Purpose: +* Empty NCMPI functions for when compiling without NCMPI support. +* +\******************************************************************************/ + +#include "aiori.h" + +void * +IOR_Create_NCMPI(char * testFileName, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with NCMPI support"); + return 0; +} + +void * +IOR_Open_NCMPI(char * testFileName, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with NCMPI support"); + return 0; +} + + +IOR_offset_t +IOR_Xfer_NCMPI(int access, + void * fd, + IOR_size_t * buffer, + IOR_offset_t length, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with NCMPI support"); + return 0; +} + +void +IOR_Fsync_NCMPI(void * fd, IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with NCMPI support"); +} + +void +IOR_Close_NCMPI(void * fd, + IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with NCMPI support"); +} + +void +IOR_Delete_NCMPI(char * testFileName, IOR_param_t * param) +{ + ERR("This copy of IOR was not compiled with NCMPI support"); +} + +void +IOR_SetVersion_NCMPI(IOR_param_t *test) +{ + ERR("This copy of IOR was not compiled with NCMPI support"); +} + +IOR_offset_t +IOR_GetFileSize_NCMPI(IOR_param_t * test, + MPI_Comm testComm, + char * testFileName) +{ + ERR("This copy of IOR was not compiled with NCMPI support"); + return 0; +} diff --git a/src/C/aiori.h b/src/C/aiori.h new file mode 100644 index 0000000..4ae08c9 --- /dev/null +++ b/src/C/aiori.h @@ -0,0 +1,234 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: aiori.h,v $ +* $Revision: 1.3 $ +* $Date: 2008/12/02 17:12:14 $ +* $Author: rklundt $ +* +* Purpose: +* This is a header file that contains the definitions and prototypes +* needed for the abstract I/O interfaces necessary for IOR. +* +\******************************************************************************/ +/********************* Modifications to IOR-2.10.1 ***************************** +* hodson - 8/18/2008: Added option flags for the following new options * +* int TestNum; * test reference number * +* double * writeVal; * array to write results - IOPS added * +* double * readVal; * array to read results - IOPS added * +* int taskPerNodeOffset; * task node offset for reading files * +* int reorderTasksRandom; * reorder tasks for random file read back * +* int reorderTasksRandomSeed; * reorder tasks for random file read seed * +* int fsyncPerWrite; * fsync() after each write * +*******************************************************************************/ + +#ifndef _AIORI_H +#define _AIORI_H + +#include "iordef.h" /* IOR Definitions */ +#include + +#ifndef MPI_FILE_NULL +# include +#endif /* not MPI_FILE_NULL */ + +#include + +/*************************** D E F I N I T I O N S ****************************/ + + /* -- file open flags -- */ +#define IOR_RDONLY 1 /* read only */ +#define IOR_WRONLY 2 /* write only */ +#define IOR_RDWR 4 /* read/write */ +#define IOR_APPEND 8 /* append */ +#define IOR_CREAT 16 /* create */ +#define IOR_TRUNC 32 /* truncate */ +#define IOR_EXCL 64 /* exclusive */ +#define IOR_DIRECT 128 /* bypass I/O buffers */ + /* -- file mode flags -- */ +#define IOR_IRWXU 1 /* read, write, execute perm: owner */ +#define IOR_IRUSR 2 /* read permission: owner */ +#define IOR_IWUSR 4 /* write permission: owner */ +#define IOR_IXUSR 8 /* execute permission: owner */ +#define IOR_IRWXG 16 /* read, write, execute perm: group */ +#define IOR_IRGRP 32 /* read permission: group */ +#define IOR_IWGRP 64 /* write permission: group */ +#define IOR_IXGRP 128 /* execute permission: group */ +#define IOR_IRWXO 256 /* read, write, execute perm: other */ +#define IOR_IROTH 512 /* read permission: other */ +#define IOR_IWOTH 1024 /* write permission: other */ +#define IOR_IXOTH 2048 /* execute permission: other */ + +/******************************************************************************/ +/* + * The parameter struct holds all of the "global" data to be passed, + * as well as results to be parsed. + * + * NOTE: If this is changed, also change: + * defaultParameters [defaults.h] + * DisplayUsage() [IOR.c] + * ShowTest() [IOR.c] + * DecodeDirective() [parse_options.c] + * ParseCommandLine() [parse_options.c] + * USER_GUIDE + */ + +typedef struct +{ + char debug[MAX_STR]; /* debug info string */ + unsigned int mode; /* file permissions */ + unsigned int openFlags; /* open flags */ + int TestNum; /* test reference number */ + char api[MAX_STR]; /* API for I/O */ + char apiVersion[MAX_STR]; /* API version */ + char platform[MAX_STR]; /* platform type */ + char testFileName[MAXPATHLEN]; /* full name for test */ + char testFileName_fppReadCheck[MAXPATHLEN];/* filename for fpp read check */ + char hintsFileName[MAXPATHLEN]; /* full name for hints file */ + char options[MAXPATHLEN]; /* options string */ + int numTasks; /* number of tasks for test */ + int nodes; /* number of nodes for test */ + int tasksPerNode; /* number of tasks per node */ + int repetitions; /* number of repetitions of test */ + int repCounter; /* rep counter */ + int multiFile; /* multiple files */ + int interTestDelay; /* delay between reps in seconds */ + double * writeVal[2]; /* array to write results */ + double * readVal[2]; /* array to read results */ + int open; /* flag for writing or reading */ + int readFile; /* read of existing file */ + int writeFile; /* write of file */ + int filePerProc; /* single file or file-per-process */ + int reorderTasks; /* reorder tasks for read back and check */ + int taskPerNodeOffset; /* task node offset for reading files */ + int reorderTasksRandom; /* reorder tasks for random file read back */ + int reorderTasksRandomSeed; /* reorder tasks for random file read seed */ + int checkWrite; /* check read after write */ + int checkRead; /* check read after read */ + int keepFile; /* don't delete the testfile on exit */ + int keepFileWithError; /* don't delete the testfile with errors */ + int errorFound; /* error found in data check */ + int quitOnError; /* quit code when error in check */ + int collective; /* collective I/O */ + IOR_offset_t segmentCount; /* number of segments (or HDF5 datasets) */ + IOR_offset_t blockSize; /* contiguous bytes to write per task */ + IOR_offset_t transferSize; /* size of transfer in bytes */ + IOR_offset_t offset; /* offset for read/write */ + IOR_offset_t * aggFileSizeFromCalc; /* calculated aggregate file size */ + IOR_offset_t * aggFileSizeFromStat; /* stat() aggregate file size */ + IOR_offset_t * aggFileSizeFromXfer; /* transfered aggregate file size */ + IOR_offset_t * aggFileSizeForBW; /* aggregate file size used for b/w */ + int preallocate; /* preallocate file size */ + int useFileView; /* use MPI_File_set_view */ + int useSharedFilePointer; /* use shared file pointer */ + int useStridedDatatype; /* put strided access into datatype */ + int useO_DIRECT; /* use O_DIRECT, bypassing I/O buffers */ + int showHints; /* show hints */ + int showHelp; /* show options and help */ + int uniqueDir; /* use unique directory for each fpp */ + int useExistingTestFile; /* do not delete test file before access */ + int storeFileOffset; /* use file offset as stored signature */ + int deadlineForStonewalling; /* max time in seconds to run any test phase */ + int maxTimeDuration; /* max time in minutes to run tests */ + int outlierThreshold; /* warn on outlier N seconds from mean */ + int verbose; /* verbosity */ + int setTimeStampSignature; /* set time stamp signature */ + unsigned int timeStampSignatureValue; /* value for time stamp signature */ + void * fd_fppReadCheck; /* additional fd for fpp read check */ + int randomSeed; /* random seed for write/read check */ + int randomOffset; /* access is to random offsets */ + MPI_Comm testComm; /* MPI communicator */ + + /* POSIX variables */ + int singleXferAttempt; /* do not retry transfer if incomplete */ + int fsyncPerWrite; /* fsync() after each write */ + int fsync; /* fsync() after write */ + + /* MPI variables */ + MPI_Datatype transferType; /* datatype for transfer */ + MPI_Datatype fileType; /* filetype for file view */ + + /* HDF5 variables */ + int individualDataSets; /* datasets not shared by all procs */ + int noFill; /* no fill in file creation */ + IOR_offset_t setAlignment; /* alignment in bytes */ + + /* NCMPI variables */ + int var_id; /* variable id handle for data set */ + + /* Lustre variables */ + int lustre_stripe_count; + int lustre_stripe_size; + int lustre_start_ost; + int lustre_ignore_locks; + +#if USE_UNDOC_OPT + int corruptFile; + int fillTheFileSystem; + int includeDeleteTime; + int multiReRead; + + /* NFS variables */ + char NFS_rootPath[MAXPATHLEN]; /* absolute path to NFS servers */ + char NFS_serverName[MAXPATHLEN]; /* prefix for NFS server name */ + int NFS_serverCount; /* number of NFS servers to be used */ +#endif /* USE_UNDOC_OPT */ + + int id; /* test's unique ID */ + int intraTestBarriers; /* barriers between open/op and op/close */ +} IOR_param_t; + + +/**************************** P R O T O T Y P E S *****************************/ + +/* functions for aiori-*.c */ +/* POSIX-specific functions */ +void * IOR_Create_POSIX (char *, IOR_param_t *); +void * IOR_Open_POSIX (char *, IOR_param_t *); +IOR_offset_t IOR_Xfer_POSIX (int, void *, IOR_size_t *, + IOR_offset_t, IOR_param_t *); +void IOR_Close_POSIX (void *, IOR_param_t *); +void IOR_Delete_POSIX (char *, IOR_param_t *); +void IOR_SetVersion_POSIX (IOR_param_t *); +void IOR_Fsync_POSIX (void *, IOR_param_t *); +IOR_offset_t IOR_GetFileSize_POSIX (IOR_param_t *, MPI_Comm, char *); + +/* MPIIO-specific functions */ +void * IOR_Create_MPIIO (char *, IOR_param_t *); +void * IOR_Open_MPIIO (char *, IOR_param_t *); +IOR_offset_t IOR_Xfer_MPIIO (int, void *, IOR_size_t *, + IOR_offset_t, IOR_param_t *); +void IOR_Close_MPIIO (void *, IOR_param_t *); +void IOR_Delete_MPIIO (char *, IOR_param_t *); +void IOR_SetVersion_MPIIO (IOR_param_t *); +void IOR_Fsync_MPIIO (void *, IOR_param_t *); +IOR_offset_t IOR_GetFileSize_MPIIO (IOR_param_t *, MPI_Comm, char *); + +/* HDF5-specific functions */ +void * IOR_Create_HDF5 (char *, IOR_param_t *); +void * IOR_Open_HDF5 (char *, IOR_param_t *); +IOR_offset_t IOR_Xfer_HDF5 (int, void *, IOR_size_t *, + IOR_offset_t, IOR_param_t *); +void IOR_Close_HDF5 (void *, IOR_param_t *); +void IOR_Delete_HDF5 (char *, IOR_param_t *); +void IOR_SetVersion_HDF5 (IOR_param_t *); +void IOR_Fsync_HDF5 (void *, IOR_param_t *); +IOR_offset_t IOR_GetFileSize_HDF5 (IOR_param_t *, MPI_Comm, char *); + +/* NCMPI-specific functions */ +void * IOR_Create_NCMPI (char *, IOR_param_t *); +void * IOR_Open_NCMPI (char *, IOR_param_t *); +IOR_offset_t IOR_Xfer_NCMPI (int, void *, IOR_size_t *, + IOR_offset_t, IOR_param_t *); +void IOR_Close_NCMPI (void *, IOR_param_t *); +void IOR_Delete_NCMPI (char *, IOR_param_t *); +void IOR_SetVersion_NCMPI (IOR_param_t *); +void IOR_Fsync_NCMPI (void *, IOR_param_t *); +IOR_offset_t IOR_GetFileSize_NCMPI (IOR_param_t *, MPI_Comm, char *); + +#endif /* not _AIORI_H */ diff --git a/src/C/cbif/Makefile b/src/C/cbif/Makefile new file mode 100644 index 0000000..5f65e29 --- /dev/null +++ b/src/C/cbif/Makefile @@ -0,0 +1,5 @@ +all: + $(CC) cbif.c -o cbif + +clean: + rm -f cbif diff --git a/src/C/cbif/cbif.c b/src/C/cbif/cbif.c new file mode 100644 index 0000000..966f5b0 --- /dev/null +++ b/src/C/cbif/cbif.c @@ -0,0 +1,135 @@ +/******************************************************************************\ +* * +* Copyright (c) 2006, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: cbif.c,v $ +* $Revision: 1.1.1.1 $ +* $Date: 2007/10/15 23:36:54 $ +* $Author: rklundt $ +* +* Purpose: +* Changes a specific Byte offset In File +* +\******************************************************************************/ + +#include +#include +#include +#include + +#define ERR(MSG) do { \ + fprintf(stdout, "** error **\n"); \ + fprintf(stdout, "ERROR in %s (line %d): %s.\n", \ + __FILE__, __LINE__, MSG); \ + fprintf(stdout, "ERROR: %s\n", strerror(errno)); \ + fprintf(stdout, "** exiting **\n"); \ + fflush(stdout); \ + exit(1); \ +} while (0) + +#define BYTE_BOUNDARY 8 + +int +main(int argc, char **argv) +{ + int i; + int fd; + int value; + unsigned char oldBuffer[BYTE_BOUNDARY]; + unsigned char newBuffer[BYTE_BOUNDARY]; + char *fileName; + long long int offset; + long long int alignedOffset; + long long int bufferIndex; + long long int indexMask = (long long int)(BYTE_BOUNDARY - 1); + long long int alignedMask = (long long int)(BYTE_BOUNDARY - 1) + ^ (long long int)(-1); + + /* check usage */ + if (argc != 3 && argc !=4) { + printf("Usage: %s []\n", argv[0]); + printf("Returns value of byte offset, or modifies to new value.\n"); + exit(1); + } + + /* gather arguments */ + fileName = argv[1]; + sscanf(argv[2], "%lld", &offset); + alignedOffset = offset & alignedMask; + bufferIndex = offset & indexMask; + if (argc == 4) { + sscanf(argv[3], "%d", &value); + if (value < 0 || value > 255) ERR("ERROR: must be 0-255"); + } + + /* open file */ + fd = open64(fileName, O_RDWR); + if (fd < 0) ERR("ERROR: Unable to open file"); + + /* seek to offset */ + if (lseek64(fd, alignedOffset, SEEK_SET) == -1) + ERR("ERROR: Unable to seek to file offset"); + + /* read into buffer */ + if (read(fd, &oldBuffer, BYTE_BOUNDARY) <= 0) + ERR("ERROR: Unable to read file offset"); + + if (argc == 4) { + /* write from buffer */ + /* update buffer with new value */ + memcpy(newBuffer, oldBuffer, BYTE_BOUNDARY); + newBuffer[bufferIndex] = (unsigned char)value; + + /* seek to offset */ + if (lseek64(fd, alignedOffset, SEEK_SET) == -1) + ERR("ERROR: Unable to seek to file offset"); + + /* write buffer */ + if (write(fd, &newBuffer, BYTE_BOUNDARY) != BYTE_BOUNDARY) + ERR("ERROR: Unable to write to file offset"); + } + + /* print data */ + /* print header */ + if (argc == 3) { + fprintf(stdout, "offset: original value: \n"); + fprintf(stdout, "------------------ ------------------\n"); + } else { + fprintf(stdout, + "offset: original value: new value: \n"); + fprintf(stdout, + "------------------ ------------------ ------------------\n"); + } + + /* next hex, multiple bytes */ + fprintf(stdout, "0x%016x 0x", alignedOffset); + for (i = 0; i < BYTE_BOUNDARY; i++) { + fprintf(stdout, "%02x", oldBuffer[i]); + } + if (argc == 4) { + fprintf(stdout, " 0x"); + for (i = 0; i < BYTE_BOUNDARY; i++) { + fprintf(stdout, "%02x", newBuffer[i]); + } + } + fprintf(stdout, "\n"); + + /* finally decimal, single byte */ + fprintf(stdout, "%-16lld %-3d (0x%02x)", offset, + oldBuffer[bufferIndex], oldBuffer[bufferIndex]); + if (argc == 4) { + fprintf(stdout, " %-3d (0x%02x)\n", + newBuffer[bufferIndex], newBuffer[bufferIndex]); + } + fprintf(stdout, "\n"); + + /* finished */ + fflush(stdout); + close(fd); + return(0); + +} /* main() */ diff --git a/src/C/defaults.h b/src/C/defaults.h new file mode 100644 index 0000000..cb2ff0c --- /dev/null +++ b/src/C/defaults.h @@ -0,0 +1,149 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: defaults.h,v $ +* $Revision: 1.3 $ +* $Date: 2008/12/02 17:12:14 $ +* $Author: rklundt $ +* +* Purpose: +* This is a header file that contains the default settings necessary for +* IOR. +* +\******************************************************************************/ +/********************* Modifications to IOR-2.10.1 ***************************** +* hodson - 8/18/2008: Added Default values for the following variables * +* int TestNum; * test reference number * +* int taskPerNodeOffset; * task node offset for reading files * +* int reorderTasksRandom; * reorder tasks for random file read back * +* int reorderTasksRandomSeed; * reorder tasks for random file read seed * +* int fsyncPerWrite; * fsync() after each write * +*******************************************************************************/ + +#ifndef _IOR_DEFAULTS_H +#define _IOR_DEFAULTS_H + +#include "aiori.h" + + +/*************************** D E F I N I T I O N S ****************************/ + +/******************************************************************************/ +/* + * Default parameter settings for a test script. This should be the minimum + * test runnable. + */ + +IOR_param_t defaultParameters = { + "", /* debug info string */ + IOR_IRUSR | /* file permissions */ + IOR_IWUSR | + IOR_IRGRP | + IOR_IWGRP, + IOR_RDWR | /* open flags POSIX/MPIIO */ + IOR_CREAT, + -1, /* test reference number */ + "POSIX", /* api */ + "", /* api version */ + "HOST(OSTYPE)", /* platform */ + "testFile", /* test file */ + "", /* filename for fpp read check */ + "", /* hints file */ + "", /* options */ + 0, /* numTasks */ + 1, /* nodes */ + 1, /* tasks per node */ + 1, /* repetitions */ + -1, /* rep counter */ + 0, /* multiple files */ + 0, /* intertest delay */ + NULL, NULL, /* write results array */ + NULL, NULL, /* read results array */ + WRITE, /* used in HDF5 for create(WRITE) and open(READ) */ + TRUE, /* read flag */ + TRUE, /* write flag */ + FALSE, /* file-per-proc flag */ + FALSE, /* reorder tasks */ + 1, /* task file offset for read */ + FALSE, /* reorder tasks random */ + 0, /* reorder tasks random seed offset value*/ + FALSE, /* check write */ + FALSE, /* check read */ + FALSE, /* keep test file on exit */ + FALSE, /* keep test file with errors */ + FALSE, /* error found in data check */ + FALSE, /* halt on error */ + FALSE, /* collective I/O */ + 1, /* segment count */ + 1048576, /* block size */ + 262144, /* transfer size */ + 0, /* offset */ + NULL, /* expected aggregate file size array */ + NULL, /* stat'ed aggregate file size array */ + NULL, /* xfered aggregate file size array */ + NULL, /* aggregate file size array used for b/w */ + FALSE, /* preallocate file size */ + FALSE, /* use file view */ + FALSE, /* use shared file pointer */ + FALSE, /* use strided datatype */ + FALSE, /* use O_DIRECT, bypassing I/O buffers */ + FALSE, /* show hints */ + FALSE, /* show help */ + FALSE, /* unique directory for each file-per-process */ + FALSE, /* do not delete test file before access */ + FALSE, /* use file offset as stored signature */ + 0, /* deadline in seconds for any test phase (0=off) */ + 0, /* max time in minutes to run tests (0=off) */ + 0, /* warn on outlier N seconds from mean (0=off) */ + 0, /* verbosity */ + 0, /* set time stamp signature */ + 0, /* time stamp signature value */ + NULL, /* additional fd for fpp read check */ + -1, /* random seed for write/read check */ + 0, /* access is to random, not sequential, offsets */ + MPI_COMM_WORLD, /* MPI communicator */ + + /* POSIX */ + FALSE, /* single transfer (no retry) */ + FALSE, /* fsync after each POSIX write */ + FALSE, /* fsync after POSIX write close */ + + /* MPI */ + 0, /* MPI transfer datatype */ + 0, /* MPI file view datatype */ + + /* HDF5 */ + FALSE, /* individual data sets */ + FALSE, /* no fill */ + 1, /* alignment */ + + /* NCMPI */ + 0, /* var_id handle for datasets */ + + /* Lustre */ + 0, /* lustre_stripe_count */ + 0, /* lustre_stripe_size */ + -1, /* lustre_start_ost */ + 0, /* lustre_ignore_locks */ + +#if USE_UNDOC_OPT + 0, /* corrupt file(s) */ + 0, /* fill the file system */ + 0, /* include delete time */ + 0, /* multiple rereads of file */ + + /* NFS */ + "", /* absolute path to NFS servers */ + "", /* prefix for NFS server name */ + 0, /* number of NFS servers to be used */ +#endif /* USE_UNDOC_OPT */ + + 0, /* test's unique ID */ + 0 /* intraTestBarriers */ +}; +#endif /* not _IOR_DEFAULTS_H */ diff --git a/src/C/iordef.h b/src/C/iordef.h new file mode 100644 index 0000000..7e0847e --- /dev/null +++ b/src/C/iordef.h @@ -0,0 +1,221 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: iordef.h,v $ +* $Revision: 1.4 $ +* $Date: 2010/08/02 16:43:14 $ +* $Author: rklundt $ +* +* Purpose: +* This is a header file that contains the definitions and macros +* needed for IOR. +* +\******************************************************************************/ + +#ifndef _IORDEF_H +#define _IORDEF_H + +#ifdef _WIN32 +# define _CRT_SECURE_NO_WARNINGS +# define _CRT_RAND_S +# pragma warning(4 : 4996) /* Don't complain about POSIX names */ +# pragma warning(4 : 4267) /* '=' : conversion from 'size_t' to 'int' */ +# pragma warning(4 : 4244) /* 'function' : conversion from 'IOR_offset_t' to 'int' */ + +# include +# include +# include +# include +# include +# include "win/getopt.h" + +# define MAXPATHLEN 1024 +# define F_OK 00 +# define W_OK 02 +# define R_OK 04 +# define X_OK 04 + +# define lseek _lseeki64 +# define fsync _commit +# define mkdir(dir, mode) _mkdir(dir) +# define strcasecmp _stricmp +# define strncasecmp _strnicmp +# define srandom srand +# define random() (rand() * (RAND_MAX+1) + rand()) /* Note: only 30 bits */ +# define sleep(X) Sleep((X)*1000) +# define getpagesize() 4096 +#else +# include /* MAXPATHLEN */ +# include +#endif +#include +#include +#include + +/************************** D E C L A R A T I O N S ***************************/ + +extern int numTasks, /* MPI variables */ + rank, + rankOffset, + verbose; /* verbose output */ + + +/*************************** D E F I N I T I O N S ****************************/ + +#define IOR_RELEASE "IOR-2.10.3" + +#ifndef FALSE +# define FALSE 0 +#endif /* not FALSE */ + +#ifndef TRUE +# define TRUE 1 +#endif /* not TRUE */ + +#ifndef NULL +# define NULL ((void *)0) +#endif /* not NULL */ + +#define KILOBYTE 1000 +#define MEGABYTE 1000000 +#define GIGABYTE 1000000000 + +#define KIBIBYTE (1 << 10) +#define MEBIBYTE (1 << 20) +#define GIBIBYTE (1 << 30) + +/* for displaying MiB or MB */ +#define BASE_TWO 0 +#define BASE_TEN 1 + +/* any write/read access in code */ +#define WRITE 0 +#define WRITECHECK 1 +#define READ 2 +#define READCHECK 3 +#define CHECK 4 + +/* verbosity settings */ +#define VERBOSE_0 0 +#define VERBOSE_1 1 +#define VERBOSE_2 2 +#define VERBOSE_3 3 +#define VERBOSE_4 4 +#define VERBOSE_5 5 + +#define MAX_STR 1024 /* max string length */ +#define MAX_HINTS 16 /* max number of hints */ +#define MAX_RETRY 10000 /* max retries for POSIX xfer */ + +#define DELIMITERS " \t\r\n=" /* ReadScript() */ +#define FILENAME_DELIMITER '@' /* ParseFileName() */ + +/* MACROs for debugging */ +#define HERE fprintf(stdout, "** LINE %d (TASK=%d) **\n", \ + __LINE__, rank); + +/* Other MACROs */ +#define USE_UNDOC_OPT TRUE /* USE UNDOCumented OPTion */ + +typedef long long int IOR_offset_t; +typedef long long int IOR_size_t; + + +/******************************** M A C R O S *********************************/ + +/******************************************************************************/ +/* + * WARN_RESET will display a custom error message and set value to default + */ +#define WARN_RESET(MSG, VAL) do { \ + test->VAL = defaultParameters.VAL; \ + if (rank == 0) { \ + fprintf(stdout, "WARNING: %s. Using value of %d.\n", \ + MSG, test->VAL); \ + } \ + fflush(stdout); \ +} while (0) + + +/******************************************************************************/ +/* + * WARN will display a custom error message as well as an error string from + * sys_errlist, but will not exit the program + */ + +#define WARN(MSG) do { \ + if (verbose > VERBOSE_2) { \ + fprintf(stdout, "WARNING: %s, in %s (line %d).\n", \ + MSG, __FILE__, __LINE__); \ + } else { \ + fprintf(stdout, "WARNING: %s.\n", MSG); \ + } \ + fflush(stdout); \ +} while (0) + + +/******************************************************************************/ +/* + * ERR will display a custom error message as well as an error string from + * sys_errlist and then exit the program + */ + +#define ERR(MSG) do { \ + fprintf(stdout, "** error **\n"); \ + fprintf(stdout, "ERROR in %s (line %d): %s.\n", \ + __FILE__, __LINE__, MSG); \ + fprintf(stdout, "ERROR: %s\n", strerror(errno)); \ + fprintf(stdout, "** exiting **\n"); \ + fflush(stdout); \ + MPI_Abort(MPI_COMM_WORLD, -1); \ +} while (0) + + +/******************************************************************************/ +/* + * MPI_CHECK will display a custom error message as well as an error string + * from the MPI_STATUS and then exit the program + */ + +#define MPI_CHECK(MPI_STATUS, MSG) do { \ + char resultString[MPI_MAX_ERROR_STRING]; \ + int resultLength; \ + \ + if (MPI_STATUS != MPI_SUCCESS) { \ + fprintf(stdout, "** error **\n"); \ + fprintf(stdout, "ERROR in %s (line %d): %s.\n", \ + __FILE__, __LINE__, MSG); \ + MPI_Error_string(MPI_STATUS, resultString, &resultLength); \ + fprintf(stdout, "MPI %s\n", resultString); \ + fprintf(stdout, "** exiting **\n"); \ + fflush(stdout); \ + MPI_Abort(MPI_COMM_WORLD, -1); \ + } \ +} while(0) + + +/******************************************************************************/ +/* + * System info for Windows. + */ + +#ifdef _WIN32 + +struct utsname { + char sysname [16]; + char nodename[257]; + char release [16]; + char version [16]; + char machine [16]; +}; + +extern int uname(struct utsname *name); + +#endif /* _WIN32 */ + +#endif /* not _IORDEF_H */ diff --git a/src/C/parse_options.c b/src/C/parse_options.c new file mode 100644 index 0000000..09fe0aa --- /dev/null +++ b/src/C/parse_options.c @@ -0,0 +1,434 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: parse_options.c,v $ +* $Revision: 1.3 $ +* $Date: 2008/12/02 17:12:14 $ +* $Author: rklundt $ +* +* Purpose: +* Parse commandline functions. +* +\******************************************************************************/ +/********************* Modifications to IOR-2.10.1 ***************************** +* hodson - 8/18/2008: Added parsing for the following variables * +* int TestNum; * test reference number * +* int taskPerNodeOffset; * task node offset for reading files * +* int reorderTasksRandom; * reorder tasks for random file read back * +* int reorderTasksRandomSeed; * reorder tasks for random file read seed * +* int fsyncPerWrite; * fsync() after each write * +*******************************************************************************/ + +#include "IOR.h" +#include "defaults.h" /* IOR defaults */ + +#include +#include +#include +#include + +IOR_param_t initialTestParams; + +/******************************************************************************/ +/* + * Check and correct all settings of each test in queue for correctness. + */ + +void CheckRunSettings(IOR_queue_t *tests) { + while (tests != NULL) { + /* If no write/read/check action requested, set both write and read */ + if (tests->testParameters.writeFile == FALSE + && tests->testParameters.readFile == FALSE + && tests->testParameters.checkWrite == FALSE + && tests->testParameters.checkRead == FALSE) { + tests->testParameters.readFile = TRUE; + tests->testParameters.writeFile = TRUE; + } + /* If numTasks set to 0, use all tasks */ + if (tests->testParameters.numTasks == 0) { + MPI_CHECK(MPI_Comm_size(MPI_COMM_WORLD, + &tests->testParameters.numTasks), + "MPI_Comm_size() error"); + } + tests = tests->nextTest; + } +} /* CheckRunSettings() */ + + +/******************************************************************************/ +/* + * Set flags from commandline string/value pairs. + */ + +void DecodeDirective(char *line, IOR_param_t *test) +{ + char option[MAX_STR]; + char value[MAX_STR]; + int rc; + + rc = sscanf(line, " %[^=# \t\r\n] = %[^# \t\r\n] ", option, value); + if (rc != 2 && rank == 0) { + fprintf(stdout, "Syntax error in configuration options: %s\n", line); + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + } + if (strcasecmp(option, "api") == 0) { + strcpy(test->api, value); + } else if (strcasecmp(option, "testnum") == 0) { + test->TestNum = atoi(value); + } else if (strcasecmp(option, "debug") == 0) { + strcpy(test->debug, value); + } else if (strcasecmp(option, "platform") == 0) { + strcpy(test->platform, value); + } else if (strcasecmp(option, "testfile") == 0) { + strcpy(test->testFileName, value); + } else if (strcasecmp(option, "hintsfilename") == 0) { + strcpy(test->hintsFileName, value); + } else if (strcasecmp(option, "deadlineforstonewalling") == 0) { + test->deadlineForStonewalling = atoi(value); + } else if (strcasecmp(option, "maxtimeduration") == 0) { + test->maxTimeDuration = atoi(value); + } else if (strcasecmp(option, "outlierthreshold") == 0) { + test->outlierThreshold = atoi(value); + } else if (strcasecmp(option, "nodes") == 0) { + test->nodes = atoi(value); + } else if (strcasecmp(option, "repetitions") == 0) { + test->repetitions = atoi(value); + } else if (strcasecmp(option, "intertestdelay") == 0) { + test->interTestDelay = atoi(value); + } else if (strcasecmp(option, "readfile") == 0) { + test->readFile = atoi(value); + } else if (strcasecmp(option, "writefile") == 0) { + test->writeFile = atoi(value); + } else if (strcasecmp(option, "fileperproc") == 0) { + test->filePerProc = atoi(value); + } else if (strcasecmp(option, "reordertasksconstant") == 0) { + test->reorderTasks = atoi(value); + } else if (strcasecmp(option, "taskpernodeoffset") == 0) { + test->taskPerNodeOffset = atoi(value); + } else if (strcasecmp(option, "reordertasksrandom") == 0) { + test->reorderTasksRandom = atoi(value); + } else if (strcasecmp(option, "reordertasksrandomSeed") == 0) { + test->reorderTasksRandomSeed = atoi(value); + } else if (strcasecmp(option, "checkwrite") == 0) { + test->checkWrite = atoi(value); + } else if (strcasecmp(option, "checkread") == 0) { + test->checkRead = atoi(value); + } else if (strcasecmp(option, "keepfile") == 0) { + test->keepFile = atoi(value); + } else if (strcasecmp(option, "keepfilewitherror") == 0) { + test->keepFileWithError = atoi(value); + } else if (strcasecmp(option, "multiFile") == 0) { + test->multiFile = atoi(value); + } else if (strcasecmp(option, "quitonerror") == 0) { + test->quitOnError = atoi(value); + } else if (strcasecmp(option, "segmentcount") == 0) { + test->segmentCount = StringToBytes(value); + } else if (strcasecmp(option, "blocksize") == 0) { + test->blockSize = StringToBytes(value); + } else if (strcasecmp(option, "transfersize") == 0) { + test->transferSize = StringToBytes(value); + } else if (strcasecmp(option, "setalignment") == 0) { + test->setAlignment = StringToBytes(value); + } else if (strcasecmp(option, "singlexferattempt") == 0) { + test->singleXferAttempt = atoi(value); + } else if (strcasecmp(option, "individualdatasets") == 0) { + test->individualDataSets = atoi(value); + } else if (strcasecmp(option, "intraTestBarriers") == 0) { + test->intraTestBarriers = atoi(value); + } else if (strcasecmp(option, "nofill") == 0) { + test->noFill = atoi(value); + } else if (strcasecmp(option, "verbose") == 0) { + test->verbose = atoi(value); + } else if (strcasecmp(option, "settimestampsignature") == 0) { + test->setTimeStampSignature = atoi(value); + } else if (strcasecmp(option, "collective") == 0) { + test->collective = atoi(value); + } else if (strcasecmp(option, "preallocate") == 0) { + test->preallocate = atoi(value); + } else if (strcasecmp(option, "storefileoffset") == 0) { + test->storeFileOffset = atoi(value); + } else if (strcasecmp(option, "usefileview") == 0) { + test->useFileView = atoi(value); + } else if (strcasecmp(option, "usesharedfilepointer") == 0) { + test->useSharedFilePointer = atoi(value); + } else if (strcasecmp(option, "useo_direct") == 0) { + test->useO_DIRECT = atoi(value); + } else if (strcasecmp(option, "usestrideddatatype") == 0) { + test->useStridedDatatype = atoi(value); + } else if (strcasecmp(option, "showhints") == 0) { + test->showHints = atoi(value); + } else if (strcasecmp(option, "showhelp") == 0) { + test->showHelp = atoi(value); + } else if (strcasecmp(option, "uniqueDir") == 0) { + test->uniqueDir = atoi(value); + } else if (strcasecmp(option, "useexistingtestfile") == 0) { + test->useExistingTestFile = atoi(value); + } else if (strcasecmp(option, "fsyncperwrite") == 0) { + test->fsyncPerWrite = atoi(value); + } else if (strcasecmp(option, "fsync") == 0) { + test->fsync = atoi(value); + } else if (strcasecmp(option, "randomoffset") == 0) { + test->randomOffset = atoi(value); + } else if (strcasecmp(option, "lustrestripecount") == 0) { + test->lustre_stripe_count = atoi(value); + } else if (strcasecmp(option, "lustrestripesize") == 0) { + test->lustre_stripe_size = StringToBytes(value); + } else if (strcasecmp(option, "lustrestartost") == 0) { + test->lustre_start_ost = atoi(value); + } else if (strcasecmp(option, "lustreignorelocks") == 0) { + test->lustre_ignore_locks = atoi(value); +#if USE_UNDOC_OPT + } else if (strcasecmp(option, "corruptFile") == 0) { + test->corruptFile = atoi(value); + } else if (strcasecmp(option, "fillTheFileSystem") == 0) { + test->fillTheFileSystem = atoi(value); + } else if (strcasecmp(option, "includeDeleteTime") == 0) { + test->includeDeleteTime = atoi(value); + } else if (strcasecmp(option, "multiReRead") == 0) { + test->multiReRead = atoi(value); + } else if (strcasecmp(option, "nfs_rootpath") == 0) { + strcpy(test->NFS_rootPath, value); + } else if (strcasecmp(option, "nfs_servername") == 0) { + strcpy(test->NFS_serverName, value); + } else if (strcasecmp(option, "nfs_servercount") == 0) { + test->NFS_serverCount = atoi(value); +#endif /* USE_UNDOC_OPT */ + } else if (strcasecmp(option, "numtasks") == 0) { + test->numTasks = atoi(value); + } else { + if (rank == 0) + fprintf(stdout, "Unrecognized parameter \"%s\"\n", option); + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + } +} /* DecodeDirective() */ + + +/******************************************************************************/ +/* + * Parse a single line, which may contain multiple comma-seperated directives + */ + +void +ParseLine(char *line, IOR_param_t *test) +{ + char *start, *end; + + start = line; + do { + end = strchr(start, ','); + if (end != NULL) + *end = '\0'; + DecodeDirective(start, test); + start = end + 1; + } while (end != NULL); + +} /* ParseLine() */ + + +/******************************************************************************/ +/* + * Determines if the string "haystack" contains only the string "needle", and + * possibly whitespace before and after needle. Function is case insensitive. + */ + +int +contains_only(char *haystack, char *needle) +{ + char *ptr, *end; + + end = haystack + strlen(haystack); + /* skip over leading shitspace */ + for (ptr = haystack; ptr < end; ptr++) { + if (!isspace(*ptr)) + break; + } + /* check for "needle" */ + if (strncasecmp(ptr, needle, strlen(needle)) != 0) + return 0; + /* make sure the rest of the line is only whitspace as well */ + for (ptr += strlen(needle); ptr < end; ptr++) { + if (!isspace(*ptr)) + return 0; + } + + return 1; +} /* contains_only() */ + + +/******************************************************************************/ +/* + * Read the configuration script, allocating and filling in the structure of + * global parameters. + */ + +IOR_queue_t * +ReadConfigScript(char * scriptName) +{ + int test_num = 0; + int runflag = 0; + char linebuf[MAX_STR]; + char empty[MAX_STR]; + FILE *file; + IOR_queue_t *head = NULL; + IOR_queue_t *tail = NULL; + IOR_queue_t *newTest = NULL; + + /* Initialize the first test */ + head = CreateNewTest(test_num++); + tail = head; + + /* open the script */ + file = fopen(scriptName, "r"); + if (file == NULL) + ERR("cannot open file"); + + /* search for the "IOR START" line */ + while(fgets(linebuf, MAX_STR, file) != NULL) { + if (contains_only(linebuf, "ior start")) { + break; + } + } + + /* Iterate over a block of IOR commands */ + while(fgets(linebuf, MAX_STR, file) != NULL) { + /* skip empty lines */ + if (sscanf(linebuf, "%s", empty) == -1) + continue; + /* skip lines containing only comments */ + if (sscanf(linebuf, " #%s", empty) == 1) + continue; + if (contains_only(linebuf, "ior stop")) { + break; + } else if (contains_only(linebuf, "run")) { + runflag = 1; + } else { + /* If this directive was preceded by a "run" line, then + create and initialize a new test structure */ + if (runflag) { + newTest = (IOR_queue_t *)malloc(sizeof(IOR_queue_t)); + if (newTest == NULL) + ERR("malloc failed"); + newTest->testParameters = tail->testParameters; + newTest->testParameters.id = test_num++; + tail->nextTest = newTest; + tail = newTest; + tail->nextTest = NULL; + runflag = 0; + } + ParseLine(linebuf, &tail->testParameters); + } + } + + /* close the script */ + if (fclose(file) != 0) + ERR("cannot close script file"); + + return(head); +} /* ReadConfigScript() */ + + +/******************************************************************************/ +/* + * Parse Commandline. + */ + +IOR_queue_t * +ParseCommandLine(int argc, char ** argv) +{ + static const char * opts + = "A:a:b:BcCQ:ZX:d:D:YeEf:FgG:hHi:j:J:IkKlmnN:o:O:pPqrRs:St:T:uU:vVwWxz"; + int c, i; + static IOR_queue_t *tests = NULL; + + /* suppress getopt() error message when a character is unrecognized */ + opterr = 0; + + initialTestParams = defaultParameters; + GetPlatformName(initialTestParams.platform); + initialTestParams.writeFile = initialTestParams.readFile = FALSE; + initialTestParams.checkWrite = initialTestParams.checkRead = FALSE; + + while ((c = getopt(argc, argv, opts)) != -1) { + switch (c) { + case 'A': initialTestParams.TestNum = atoi(optarg); break; + case 'a': strcpy(initialTestParams.api, optarg); break; + case 'b': initialTestParams.blockSize = + StringToBytes(optarg); break; + case 'B': initialTestParams.useO_DIRECT = TRUE; break; + case 'c': initialTestParams.collective = TRUE; break; + case 'C': initialTestParams.reorderTasks = TRUE; break; + case 'Q': initialTestParams.taskPerNodeOffset = + atoi(optarg); break; + case 'Z': initialTestParams.reorderTasksRandom = TRUE; break; + case 'X': initialTestParams.reorderTasksRandomSeed = + atoi(optarg); break; + case 'd': initialTestParams.interTestDelay = atoi(optarg); break; + case 'D': initialTestParams.deadlineForStonewalling = + atoi(optarg); break; + case 'Y': initialTestParams.fsyncPerWrite = TRUE; break; + case 'e': initialTestParams.fsync = TRUE; break; + case 'E': initialTestParams.useExistingTestFile = TRUE; break; + case 'f': tests = ReadConfigScript(optarg); break; + case 'F': initialTestParams.filePerProc = TRUE; break; + case 'g': initialTestParams.intraTestBarriers = TRUE; break; + case 'G': initialTestParams.setTimeStampSignature = + atoi(optarg); break; + case 'h': initialTestParams.showHelp = TRUE; break; + case 'H': initialTestParams.showHints = TRUE; break; + case 'i': initialTestParams.repetitions = atoi(optarg); break; + case 'I': initialTestParams.individualDataSets = TRUE; break; + case 'j': initialTestParams.outlierThreshold = + atoi(optarg); break; + case 'J': initialTestParams.setAlignment = + StringToBytes(optarg); break; + case 'k': initialTestParams.keepFile = TRUE; break; + case 'K': initialTestParams.keepFileWithError = TRUE; break; + case 'l': initialTestParams.storeFileOffset = TRUE; break; + case 'm': initialTestParams.multiFile = TRUE; break; + case 'n': initialTestParams.noFill = TRUE; break; + case 'N': initialTestParams.numTasks = atoi(optarg); break; + case 'o': strcpy(initialTestParams.testFileName, optarg); break; + case 'O': ParseLine(optarg, &initialTestParams); break; + case 'p': initialTestParams.preallocate = TRUE; break; + case 'P': initialTestParams.useSharedFilePointer = TRUE; break; + case 'q': initialTestParams.quitOnError = TRUE; break; + case 'r': initialTestParams.readFile = TRUE; break; + case 'R': initialTestParams.checkRead = TRUE; break; + case 's': initialTestParams.segmentCount = atoi(optarg); break; + case 'S': initialTestParams.useStridedDatatype = TRUE; break; + case 't': initialTestParams.transferSize = + StringToBytes(optarg); break; + case 'T': initialTestParams.maxTimeDuration = atoi(optarg);break; + case 'u': initialTestParams.uniqueDir = TRUE; break; + case 'U': strcpy(initialTestParams.hintsFileName, optarg); break; + case 'v': initialTestParams.verbose++; break; + case 'V': initialTestParams.useFileView = TRUE; break; + case 'w': initialTestParams.writeFile = TRUE; break; + case 'W': initialTestParams.checkWrite = TRUE; break; + case 'x': initialTestParams.singleXferAttempt = TRUE; break; + case 'z': initialTestParams.randomOffset = TRUE; break; + default: fprintf (stdout, "ParseCommandLine: unknown option `-%c'.\n", optopt); + } + } + + for (i = optind; i < argc; i++) + fprintf (stdout, "non-option argument: %s\n", argv[i]); + + /* If an IOR script was not used, initialize test queue to the defaults */ + if (tests == NULL) { + tests = (IOR_queue_t *) malloc (sizeof(IOR_queue_t)); + if (!tests) + ERR("malloc failed"); + tests->testParameters = initialTestParams; + tests->nextTest = NULL; + } + + CheckRunSettings(tests); + + return(tests); +} /* ParseCommandLine() */ diff --git a/src/C/utilities.c b/src/C/utilities.c new file mode 100644 index 0000000..eb68cc2 --- /dev/null +++ b/src/C/utilities.c @@ -0,0 +1,539 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* CVS info: +* $RCSfile: utilities.c,v $ +* $Revision: 1.3 $ +* $Date: 2008/12/02 17:12:14 $ +* $Author: rklundt $ +* +* Purpose: +* Additional utilities necessary for both MPIIO and HDF5. +* +\******************************************************************************/ +/**********************Modifications to IOR-2.10.1 ***************************** +* hodson, 8/18/2008: * +* Removed duplicate regex.h include (thanks to Brian Lucas) * +*******************************************************************************/ + +#include "aiori.h" /* abstract IOR interface */ +#include "IOR.h" /* IOR functions */ +#include /* sys_errlist */ +#include /* open() */ +#include /* pow() */ +#include /* only for fprintf() */ +#include +#include +#include +#include +#include +#ifndef _WIN32 +# include +# ifdef __sun /* SunOS does not support statfs(), instead uses statvfs() */ +# include +# else /* !__sun */ +# include +# endif /* __sun */ +# include /* gettimeofday() */ +#endif + +/************************** D E C L A R A T I O N S ***************************/ + +extern int errno, /* error number */ + numTasks, /* MPI variables */ + rank, + rankOffset, + verbose; /* verbose output */ + + +/***************************** F U N C T I O N S ******************************/ + +/******************************************************************************/ +/* + * Returns string containing the current time. + */ + +char * +CurrentTimeString(void) +{ + static time_t currentTime; + char * currentTimePtr; + + if ((currentTime = time(NULL)) == -1) ERR("cannot get current time"); + if ((currentTimePtr = ctime(¤tTime)) == NULL) { + ERR("cannot read current time"); + } + /* ctime string ends in \n */ + return (currentTimePtr); +} /* CurrentTimeString() */ + + +/******************************************************************************/ +/* + * Dump transfer buffer. + */ + +void +DumpBuffer(void *buffer, + size_t size) +{ + size_t i, j; + unsigned long long *dumpBuf = (unsigned long long *)buffer; + + for (i = 0; i < ((size/sizeof(IOR_size_t))/4); i++) { + for (j = 0; j < 4; j++) { + fprintf(stdout, "%016llx ", dumpBuf[4*i+j]); + } + fprintf(stdout, "\n"); + } + return; +} /* DumpBuffer() */ + + +/******************************************************************************/ +/* + * Sends all strings to root nodes and displays. + */ + +void +OutputToRoot(int numTasks, MPI_Comm comm, char * stringToDisplay) +{ + int i; + int swapNeeded = TRUE; + int pairsToSwap; + char ** stringArray; + char tmpString[MAX_STR]; + MPI_Status status; + + /* malloc string array */ + stringArray = (char **)malloc(sizeof(char *) * numTasks); + if (stringArray == NULL) ERR("out of memory"); + for (i = 0; i < numTasks; i++) { + stringArray[i] = (char *)malloc(sizeof(char) * MAX_STR); + if (stringArray[i] == NULL) ERR("out of memory"); + } + + strcpy(stringArray[rank], stringToDisplay); + + if (rank == 0) { + /* MPI_receive all strings */ + for (i = 1; i < numTasks; i++) { + MPI_CHECK(MPI_Recv(stringArray[i], MAX_STR, MPI_CHAR, + MPI_ANY_SOURCE, MPI_ANY_TAG, comm, &status), + "MPI_Recv() error"); + } + } else { + /* MPI_send string to root node */ + MPI_CHECK(MPI_Send(stringArray[rank], MAX_STR, MPI_CHAR, 0, 0, comm), + "MPI_Send() error"); + } + MPI_CHECK(MPI_Barrier(comm), "barrier error"); + + /* sort strings using bubblesort */ + if (rank == 0) { + pairsToSwap = numTasks-1; + while (swapNeeded) { + swapNeeded = FALSE; + for (i = 0; i < pairsToSwap; i++) { + if (strcmp(stringArray[i], stringArray[i+1]) > 0) { + strcpy(tmpString, stringArray[i]); + strcpy(stringArray[i], stringArray[i+1]); + strcpy(stringArray[i+1], tmpString); + swapNeeded = TRUE; + } + } + pairsToSwap--; + } + } + + /* display strings */ + if (rank == 0) { + for (i = 0; i < numTasks; i++) { + fprintf(stdout, "%s\n", stringArray[i]); + } + } + + /* free strings */ + for (i = 0; i < numTasks; i++) { + free(stringArray[i]); + } + free(stringArray); +} /* OutputToRoot() */ + + +/******************************************************************************/ +/* + * Set hints for MPIIO, HDF5, or NCMPI. + */ + +void +SetHints(MPI_Info * mpiHints, char * hintsFileName) +{ + char hintString[MAX_STR], + settingVal[MAX_STR], + valueVal[MAX_STR]; + extern char ** environ; + int i; + FILE * fd; + + /* + * This routine checks for hints from the environment and/or from the + * hints files. The hints are of the form: + * 'IOR_HINT____=', where is either 'MPI' + * or 'GPFS', is the full name of the hint to be set, and + * is the hint value. E.g., 'setenv IOR_HINT__MPI__IBM_largeblock_io true' + * or 'IOR_HINT__GPFS__hint=value' in the hints file. + */ + MPI_CHECK(MPI_Info_create(mpiHints), "cannot create info object"); + + /* get hints from environment */ + for (i = 0; environ[i] != NULL; i++) { + /* if this is an IOR_HINT, pass the hint to the info object */ + if (strncmp(environ[i], "IOR_HINT", strlen("IOR_HINT")) == 0) { + strcpy(hintString, environ[i]); + ExtractHint(settingVal, valueVal, hintString); + MPI_CHECK(MPI_Info_set(*mpiHints, settingVal, valueVal), + "cannot set info object"); + } + } + + /* get hints from hints file */ + if (strcmp(hintsFileName, "") != 0) { + + /* open the hint file */ + fd = fopen(hintsFileName, "r"); + if (fd == NULL) { + WARN("cannot open hints file"); + } else { + /* iterate over hints file */ + while(fgets(hintString, MAX_STR, fd) != NULL) { + if (strncmp(hintString, "IOR_HINT", strlen("IOR_HINT")) == 0) { + ExtractHint(settingVal, valueVal, hintString); + MPI_CHECK(MPI_Info_set(*mpiHints, settingVal, valueVal), + "cannot set info object"); + } + } + /* close the hints files */ + if (fclose(fd) != 0) ERR("cannot close hints file"); + } + } +} /* SetHints() */ + + +/******************************************************************************/ +/* + * Extract key/value pair from hint string. + */ + +void +ExtractHint(char * settingVal, + char * valueVal, + char * hintString) +{ + char * settingPtr, + * valuePtr, + * tmpPtr1, + * tmpPtr2; + + settingPtr = (char *)strtok(hintString, "="); + valuePtr = (char *)strtok(NULL, " \t\r\n"); + tmpPtr1 = settingPtr; + tmpPtr2 = (char *)strstr(settingPtr, "IOR_HINT__MPI__"); + if (tmpPtr1 == tmpPtr2) { + settingPtr += strlen("IOR_HINT__MPI__"); + + } else { + tmpPtr2 = (char *)strstr(settingPtr, "IOR_HINT__GPFS__"); + if (tmpPtr1 == tmpPtr2) { + settingPtr += strlen("IOR_HINT__GPFS__"); + fprintf(stdout, + "WARNING: Unable to set GPFS hints (not implemented.)\n"); + } + } + strcpy(settingVal, settingPtr); + strcpy(valueVal, valuePtr); +} /* ExtractHint() */ + + +/******************************************************************************/ +/* + * Show all hints (key/value pairs) in an MPI_Info object. + */ + +void ShowHints(MPI_Info * mpiHints) +{ + char key[MPI_MAX_INFO_VAL], + value[MPI_MAX_INFO_VAL]; + int flag, + i, + nkeys; + + MPI_CHECK(MPI_Info_get_nkeys(*mpiHints, &nkeys), + "cannot get info object keys"); + + for (i = 0; i < nkeys; i++) { + MPI_CHECK(MPI_Info_get_nthkey(*mpiHints, i, key), + "cannot get info object key"); + MPI_CHECK(MPI_Info_get(*mpiHints, key, MPI_MAX_INFO_VAL-1, + value, &flag), + "cannot get info object value"); + fprintf(stdout,"\t%s = %s\n", key, value); + } +} /* ShowHints() */ + + +/******************************************************************************/ +/* + * Takes a string of the form 64, 8m, 128k, 4g, etc. and converts to bytes. + */ + +IOR_offset_t +StringToBytes(char * size_str) +{ + IOR_offset_t size = 0; + char range; + int rc; + + rc = sscanf(size_str, "%lld%c", &size, &range); + if (rc == 2) { + switch ((int)range) { + case 'k': case 'K': size <<= 10; break; + case 'm': case 'M': size <<= 20; break; + case 'g': case 'G': size <<= 30; break; + } + } else if (rc == 0) { + size = -1; + } + return(size); +} /* StringToBytes() */ + + +/******************************************************************************/ +/* + * Displays size of file system and percent of data blocks and inodes used. + */ + +void +ShowFileSystemSize(char *fileSystem) +{ +#ifndef _WIN32 /* FIXME */ + int error; + char realPath[MAX_STR]; + char fileSystemUnitStr[MAX_STR] = "GiB"; + char inodeUnitStr[MAX_STR] = "Mi"; + long long int fileSystemUnitVal = 1024 * 1024 * 1024; + long long int inodeUnitVal = 1024 * 1024; + long long int totalFileSystemSize, + freeFileSystemSize, + totalInodes, + freeInodes; + double totalFileSystemSizeHR, + usedFileSystemPercentage, + usedInodePercentage; +#ifdef __sun /* SunOS does not support statfs(), instead uses statvfs() */ + struct statvfs statusBuffer; +#else /* !__sun */ + struct statfs statusBuffer; +#endif /* __sun */ + +#ifdef __sun + if (statvfs(fileSystem, &statusBuffer) != 0) { + ERR("unable to statvfs() file system"); + } +#else /* !__sun */ + if (statfs(fileSystem, &statusBuffer) != 0) { + ERR("unable to statfs() file system"); + } +#endif /* __sun */ + + /* data blocks */ +#ifdef __sun + totalFileSystemSize = statusBuffer.f_blocks * statusBuffer.f_frsize; + freeFileSystemSize = statusBuffer.f_bfree * statusBuffer.f_frsize; +#else /* !__sun */ + totalFileSystemSize = statusBuffer.f_blocks * statusBuffer.f_bsize; + freeFileSystemSize = statusBuffer.f_bfree * statusBuffer.f_bsize; +#endif /* __sun */ + + usedFileSystemPercentage = (1 - ((double)freeFileSystemSize + / (double)totalFileSystemSize)) * 100; + totalFileSystemSizeHR = (double)totalFileSystemSize + / (double)fileSystemUnitVal; + if (totalFileSystemSizeHR > 1024) { + totalFileSystemSizeHR = totalFileSystemSizeHR / 1024; + strcpy(fileSystemUnitStr, "TiB"); + } + + /* inodes */ + totalInodes = statusBuffer.f_files; + freeInodes = statusBuffer.f_ffree; + usedInodePercentage = (1 - ((double)freeInodes/(double)totalInodes)) * 100; + + /* show results */ + if (realpath(fileSystem, realPath) == NULL) { + ERR("unable to use realpath()"); + } + fprintf(stdout, "Path: %s\n", realPath); + fprintf(stdout, "FS: %.1f %s Used FS: %2.1f%% ", totalFileSystemSizeHR, + fileSystemUnitStr, usedFileSystemPercentage); + fprintf(stdout, "Inodes: %.1f %s Used Inodes: %2.1f%%\n", + (double)totalInodes / (double)inodeUnitVal, + inodeUnitStr, usedInodePercentage); + fflush(stdout); +#endif /* _WIN32 */ + + return; +} /* ShowFileSystemSize() */ + + +/******************************************************************************/ +/* + * Return match of regular expression -- 0 is failure, 1 is success. + */ + +int +Regex(char *string, char *pattern) +{ + int retValue = 0; +#ifndef _WIN32 /* Okay to always not match */ + regex_t regEx; + regmatch_t regMatch; + + regcomp(®Ex, pattern, REG_EXTENDED); + if (regexec(®Ex, string, 1, ®Match, 0) == 0) { + retValue = 1; + } + regfree(®Ex); +#endif + + return(retValue); +} /* Regex() */ + + +#if USE_UNDOC_OPT /* corruptFile */ +/******************************************************************************/ +/* + * Corrupt file to testing data checking options. + */ + +void CorruptFile(char *testFileName, + IOR_param_t *test, + int rep, + int access) +{ + IOR_offset_t tmpOff, range, eof; + char fileName[MAX_STR]; + + /* determine file name */ + strcpy(fileName, testFileName); + if (access == READCHECK && test->filePerProc) { + strcpy(fileName, test->testFileName_fppReadCheck); + } + + /* determine offset to modify */ + SeedRandGen(test->testComm); + eof = test->aggFileSizeFromCalc[rep] + / (test->filePerProc ? test->numTasks : 1); + if (access == WRITECHECK) { + range = eof - test->offset; + } else { /* READCHECK */ + range = test->transferSize; + } + tmpOff = (IOR_offset_t)((rand()/(float)RAND_MAX) * range) + test->offset; + + if (tmpOff >= eof) tmpOff = tmpOff / 2; + + /* corrupt at with */ + if (rank == 0 || test->filePerProc) { + ModifyByteInFile(fileName, tmpOff, 121); + } + + return; +} /* CorruptFile() */ + + +/******************************************************************************/ +/* + * Modify byte in file - used to testing write/read data checking. + */ + +void +ModifyByteInFile(char * fileName, + IOR_offset_t offset, + int byteValue) +{ + int fd; + char oldValue[1], + value[1]; + + value[0] = (char)byteValue; + + /* open file, show old value, update to new value */ + fd = open(fileName, O_RDWR); + lseek(fd, offset, SEEK_SET); + read(fd, oldValue, 1); + fprintf(stdout, + "** DEBUG: offset %lld in %s changed from %d to %d **\n", offset, + fileName, (unsigned char)oldValue[0], (unsigned char)value[0]); + lseek(fd, offset, SEEK_SET); + write(fd, value, 1); + close(fd); + + return; +} /* ModifyByteInFile() */ +#endif /* USE_UNDOC_OPT - corruptFile */ + + +/******************************************************************************/ +/* + * Seed random generator. + */ + +void +SeedRandGen(MPI_Comm testComm) +{ + unsigned int randomSeed; + + if (rank == 0) { +#ifdef _WIN32 + rand_s(&randomSeed); +#else + struct timeval randGenTimer; + gettimeofday(&randGenTimer, (struct timezone *)NULL); + randomSeed = randGenTimer.tv_usec; +#endif + } + MPI_CHECK(MPI_Bcast(&randomSeed, 1, MPI_INT, 0, + testComm), "cannot broadcast random seed value"); + srandom(randomSeed); + +} /* SeedRandGen() */ + + +/******************************************************************************/ +/* + * System info for Windows. + */ +#ifdef _WIN32 + +int uname(struct utsname *name) +{ + DWORD nodeNameSize = sizeof(name->nodename) - 1; + + memset(name, 0, sizeof(struct utsname)); + if (!GetComputerNameEx(ComputerNameDnsFullyQualified, name->nodename, &nodeNameSize)) + ERR("GetComputerNameEx failed"); + + strncpy(name->sysname, "Windows", sizeof(name->sysname)-1); + /* FIXME - these should be easy to fetch */ + strncpy(name->release, "-", sizeof(name->release)-1); + strncpy(name->version, "-", sizeof(name->version)-1); + strncpy(name->machine, "-", sizeof(name->machine)-1); + return 0; +} + +#endif /* _WIN32 */ \ No newline at end of file diff --git a/src/C/win/getopt.c b/src/C/win/getopt.c new file mode 100644 index 0000000..f0d727d --- /dev/null +++ b/src/C/win/getopt.c @@ -0,0 +1,1185 @@ +/* Getopt for GNU. + NOTE: getopt is now part of the C library, so if you don't know what + "Keep this file name-space clean" means, talk to drepper@gnu.org + before changing it! + Copyright (C) 1987,88,89,90,91,92,93,94,95,96,98,99,2000,2001 + Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +/* This tells Alpha OSF/1 not to define a getopt prototype in . + Ditto for AIX 3.2 and . */ + +#define _CRT_SECURE_NO_WARNINGS + +#ifndef _NO_PROTO +# define _NO_PROTO +#endif + +#ifdef HAVE_CONFIG_H +# include +#endif + +#if !defined __STDC__ || !__STDC__ +/* This is a separate conditional since some stdc systems + reject `defined (const)'. */ +# ifndef const +# define const +# endif +#endif + +#include +#include + +/* Comment out all this code if we are using the GNU C Library, and are not + actually compiling the library itself. This code is part of the GNU C + Library, but also included in many other GNU distributions. Compiling + and linking in this code is a waste when using the GNU C library + (especially if it is a shared library). Rather than having every GNU + program understand `configure --with-gnu-libc' and omit the object files, + it is simpler to just do this in the source for each such file. */ + +#define GETOPT_INTERFACE_VERSION 2 +#if !defined _LIBC && defined __GLIBC__ && __GLIBC__ >= 2 +# include +# if _GNU_GETOPT_INTERFACE_VERSION == GETOPT_INTERFACE_VERSION +# define ELIDE_CODE +# endif +#endif + +#ifndef ELIDE_CODE + + +/* This needs to come after some library #include + to get __GNU_LIBRARY__ defined. */ +#ifdef __GNU_LIBRARY__ +/* Don't include stdlib.h for non-GNU C libraries because some of them + contain conflicting prototypes for getopt. */ +# include +# include +#endif /* GNU C library. */ + +#ifdef VMS +# include +# if HAVE_STRING_H - 0 +# include +# endif +#endif + +#ifndef _ +/* This is for other GNU distributions with internationalized messages. */ +# if (HAVE_LIBINTL_H && ENABLE_NLS) || defined _LIBC +# include +# ifndef _ +# define _(msgid) gettext (msgid) +# endif +# else +# define _(msgid) (msgid) +# endif +# if defined _LIBC && defined USE_IN_LIBIO +# include +# endif +#endif + +/* This version of `getopt' appears to the caller like standard Unix `getopt' + but it behaves differently for the user, since it allows the user + to intersperse the options with the other arguments. + + As `getopt' works, it permutes the elements of ARGV so that, + when it is done, all the options precede everything else. Thus + all application programs are extended to handle flexible argument order. + + Setting the environment variable POSIXLY_CORRECT disables permutation. + Then the behavior is completely standard. + + GNU application programs can use a third alternative mode in which + they can distinguish the relative order of options and other arguments. */ + +#include "getopt.h" + +/* For communication from `getopt' to the caller. + When `getopt' finds an option that takes an argument, + the argument value is returned here. + Also, when `ordering' is RETURN_IN_ORDER, + each non-option ARGV-element is returned here. */ + +char *optarg; + +/* Index in ARGV of the next element to be scanned. + This is used for communication to and from the caller + and for communication between successive calls to `getopt'. + + On entry to `getopt', zero means this is the first call; initialize. + + When `getopt' returns -1, this is the index of the first of the + non-option elements that the caller should itself scan. + + Otherwise, `optind' communicates from one call to the next + how much of ARGV has been scanned so far. */ + +/* 1003.2 says this must be 1 before any call. */ +int optind = 1; + +/* Formerly, initialization of getopt depended on optind==0, which + causes problems with re-calling getopt as programs generally don't + know that. */ + +int __getopt_initialized; + +/* The next char to be scanned in the option-element + in which the last option character we returned was found. + This allows us to pick up the scan where we left off. + + If this is zero, or a null string, it means resume the scan + by advancing to the next ARGV-element. */ + +static char *nextchar; + +/* Callers store zero here to inhibit the error message + for unrecognized options. */ + +int opterr = 1; + +/* Set to an option character which was unrecognized. + This must be initialized on some systems to avoid linking in the + system's own getopt implementation. */ + +int optopt = '?'; + +/* Describe how to deal with options that follow non-option ARGV-elements. + + If the caller did not specify anything, + the default is REQUIRE_ORDER if the environment variable + POSIXLY_CORRECT is defined, PERMUTE otherwise. + + REQUIRE_ORDER means don't recognize them as options; + stop option processing when the first non-option is seen. + This is what Unix does. + This mode of operation is selected by either setting the environment + variable POSIXLY_CORRECT, or using `+' as the first character + of the list of option characters. + + PERMUTE is the default. We permute the contents of ARGV as we scan, + so that eventually all the non-options are at the end. This allows options + to be given in any order, even with programs that were not written to + expect this. + + RETURN_IN_ORDER is an option available to programs that were written + to expect options and other ARGV-elements in any order and that care about + the ordering of the two. We describe each non-option ARGV-element + as if it were the argument of an option with character code 1. + Using `-' as the first character of the list of option characters + selects this mode of operation. + + The special argument `--' forces an end of option-scanning regardless + of the value of `ordering'. In the case of RETURN_IN_ORDER, only + `--' can cause `getopt' to return -1 with `optind' != ARGC. */ + +static enum +{ + REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER +} ordering; + +/* Value of POSIXLY_CORRECT environment variable. */ +static char *posixly_correct; + +#ifdef __GNU_LIBRARY__ +/* We want to avoid inclusion of string.h with non-GNU libraries + because there are many ways it can cause trouble. + On some systems, it contains special magic macros that don't work + in GCC. */ +# include +# define my_index strchr +#else + +# if HAVE_STRING_H || WIN32 /* Pete Wilson mod 7/28/02 */ +# include +# else +# include +# endif + +/* Avoid depending on library functions or files + whose names are inconsistent. */ + +#ifndef getenv +//extern char *getenv (); +#endif + +static char * +my_index (str, chr) + const char *str; + int chr; +{ + while (*str) + { + if (*str == chr) + return (char *) str; + str++; + } + return 0; +} + +/* If using GCC, we can safely declare strlen this way. + If not using GCC, it is ok not to declare it. */ +#ifdef __GNUC__ +/* Note that Motorola Delta 68k R3V7 comes with GCC but not stddef.h. + That was relevant to code that was here before. */ +# if (!defined __STDC__ || !__STDC__) && !defined strlen +/* gcc with -traditional declares the built-in strlen to return int, + and has done so at least since version 2.4.5. -- rms. */ +extern int strlen (const char *); +# endif /* not __STDC__ */ +#endif /* __GNUC__ */ + +#endif /* not __GNU_LIBRARY__ */ + +/* Handle permutation of arguments. */ + +/* Describe the part of ARGV that contains non-options that have + been skipped. `first_nonopt' is the index in ARGV of the first of them; + `last_nonopt' is the index after the last of them. */ + +static int first_nonopt; +static int last_nonopt; + +#ifdef _LIBC +/* Stored original parameters. + XXX This is no good solution. We should rather copy the args so + that we can compare them later. But we must not use malloc(3). */ +extern int __libc_argc; +extern char **__libc_argv; + +/* Bash 2.0 gives us an environment variable containing flags + indicating ARGV elements that should not be considered arguments. */ + +# ifdef USE_NONOPTION_FLAGS +/* Defined in getopt_init.c */ +extern char *__getopt_nonoption_flags; + +static int nonoption_flags_max_len; +static int nonoption_flags_len; +# endif + +# ifdef USE_NONOPTION_FLAGS +# define SWAP_FLAGS(ch1, ch2) \ + if (nonoption_flags_len > 0) \ + { \ + char __tmp = __getopt_nonoption_flags[ch1]; \ + __getopt_nonoption_flags[ch1] = __getopt_nonoption_flags[ch2]; \ + __getopt_nonoption_flags[ch2] = __tmp; \ + } +# else +# define SWAP_FLAGS(ch1, ch2) +# endif +#else /* !_LIBC */ +# define SWAP_FLAGS(ch1, ch2) +#endif /* _LIBC */ + +/* Exchange two adjacent subsequences of ARGV. + One subsequence is elements [first_nonopt,last_nonopt) + which contains all the non-options that have been skipped so far. + The other is elements [last_nonopt,optind), which contains all + the options processed since those non-options were skipped. + + `first_nonopt' and `last_nonopt' are relocated so that they describe + the new indices of the non-options in ARGV after they are moved. */ + +#if defined __STDC__ && __STDC__ +static void exchange (char **); +#endif + +static void +exchange (argv) + char **argv; +{ + int bottom = first_nonopt; + int middle = last_nonopt; + int top = optind; + char *tem; + + /* Exchange the shorter segment with the far end of the longer segment. + That puts the shorter segment into the right place. + It leaves the longer segment in the right place overall, + but it consists of two parts that need to be swapped next. */ + +#if defined _LIBC && defined USE_NONOPTION_FLAGS + /* First make sure the handling of the `__getopt_nonoption_flags' + string can work normally. Our top argument must be in the range + of the string. */ + if (nonoption_flags_len > 0 && top >= nonoption_flags_max_len) + { + /* We must extend the array. The user plays games with us and + presents new arguments. */ + char *new_str = malloc (top + 1); + if (new_str == NULL) + nonoption_flags_len = nonoption_flags_max_len = 0; + else + { + memset (__mempcpy (new_str, __getopt_nonoption_flags, + nonoption_flags_max_len), + '\0', top + 1 - nonoption_flags_max_len); + nonoption_flags_max_len = top + 1; + __getopt_nonoption_flags = new_str; + } + } +#endif + + while (top > middle && middle > bottom) + { + if (top - middle > middle - bottom) + { + /* Bottom segment is the short one. */ + int len = middle - bottom; + register int i; + + /* Swap it with the top part of the top segment. */ + for (i = 0; i < len; i++) + { + tem = argv[bottom + i]; + argv[bottom + i] = argv[top - (middle - bottom) + i]; + argv[top - (middle - bottom) + i] = tem; + SWAP_FLAGS (bottom + i, top - (middle - bottom) + i); + } + /* Exclude the moved bottom segment from further swapping. */ + top -= len; + } + else + { + /* Top segment is the short one. */ + int len = top - middle; + register int i; + + /* Swap it with the bottom part of the bottom segment. */ + for (i = 0; i < len; i++) + { + tem = argv[bottom + i]; + argv[bottom + i] = argv[middle + i]; + argv[middle + i] = tem; + SWAP_FLAGS (bottom + i, middle + i); + } + /* Exclude the moved top segment from further swapping. */ + bottom += len; + } + } + + /* Update records for the slots the non-options now occupy. */ + + first_nonopt += (optind - last_nonopt); + last_nonopt = optind; +} + +/* Initialize the internal data when the first call is made. */ + +#if defined __STDC__ && __STDC__ +static const char *_getopt_initialize (int, char *const *, const char *); +#endif +static const char * +_getopt_initialize (argc, argv, optstring) + int argc; + char *const *argv; + const char *optstring; +{ + /* Start processing options with ARGV-element 1 (since ARGV-element 0 + is the program name); the sequence of previously skipped + non-option ARGV-elements is empty. */ + + first_nonopt = last_nonopt = optind; + + nextchar = NULL; + + posixly_correct = getenv ("POSIXLY_CORRECT"); + + /* Determine how to handle the ordering of options and nonoptions. */ + + if (optstring[0] == '-') + { + ordering = RETURN_IN_ORDER; + ++optstring; + } + else if (optstring[0] == '+') + { + ordering = REQUIRE_ORDER; + ++optstring; + } + else if (posixly_correct != NULL) + ordering = REQUIRE_ORDER; + else + ordering = PERMUTE; + +#if defined _LIBC && defined USE_NONOPTION_FLAGS + if (posixly_correct == NULL + && argc == __libc_argc && argv == __libc_argv) + { + if (nonoption_flags_max_len == 0) + { + if (__getopt_nonoption_flags == NULL + || __getopt_nonoption_flags[0] == '\0') + nonoption_flags_max_len = -1; + else + { + const char *orig_str = __getopt_nonoption_flags; + int len = nonoption_flags_max_len = strlen (orig_str); + if (nonoption_flags_max_len < argc) + nonoption_flags_max_len = argc; + __getopt_nonoption_flags = + (char *) malloc (nonoption_flags_max_len); + if (__getopt_nonoption_flags == NULL) + nonoption_flags_max_len = -1; + else + memset (__mempcpy (__getopt_nonoption_flags, orig_str, len), + '\0', nonoption_flags_max_len - len); + } + } + nonoption_flags_len = nonoption_flags_max_len; + } + else + nonoption_flags_len = 0; +#endif + + return optstring; +} + +/* Scan elements of ARGV (whose length is ARGC) for option characters + given in OPTSTRING. + + If an element of ARGV starts with '-', and is not exactly "-" or "--", + then it is an option element. The characters of this element + (aside from the initial '-') are option characters. If `getopt' + is called repeatedly, it returns successively each of the option characters + from each of the option elements. + + If `getopt' finds another option character, it returns that character, + updating `optind' and `nextchar' so that the next call to `getopt' can + resume the scan with the following option character or ARGV-element. + + If there are no more option characters, `getopt' returns -1. + Then `optind' is the index in ARGV of the first ARGV-element + that is not an option. (The ARGV-elements have been permuted + so that those that are not options now come last.) + + OPTSTRING is a string containing the legitimate option characters. + If an option character is seen that is not listed in OPTSTRING, + return '?' after printing an error message. If you set `opterr' to + zero, the error message is suppressed but we still return '?'. + + If a char in OPTSTRING is followed by a colon, that means it wants an arg, + so the following text in the same ARGV-element, or the text of the following + ARGV-element, is returned in `optarg'. Two colons mean an option that + wants an optional arg; if there is text in the current ARGV-element, + it is returned in `optarg', otherwise `optarg' is set to zero. + + If OPTSTRING starts with `-' or `+', it requests different methods of + handling the non-option ARGV-elements. + See the comments about RETURN_IN_ORDER and REQUIRE_ORDER, above. + + Long-named options begin with `--' instead of `-'. + Their names may be abbreviated as long as the abbreviation is unique + or is an exact match for some defined option. If they have an + argument, it follows the option name in the same ARGV-element, separated + from the option name by a `=', or else the in next ARGV-element. + When `getopt' finds a long-named option, it returns 0 if that option's + `flag' field is nonzero, the value of the option's `val' field + if the `flag' field is zero. + + The elements of ARGV aren't really const, because we permute them. + But we pretend they're const in the prototype to be compatible + with other systems. + + LONGOPTS is a vector of `struct option' terminated by an + element containing a name which is zero. + + LONGIND returns the index in LONGOPT of the long-named option found. + It is only valid when a long-named option has been found by the most + recent call. + + If LONG_ONLY is nonzero, '-' as well as '--' can introduce + long-named options. */ + +int +_getopt_internal (argc, argv, optstring, longopts, longind, long_only) + int argc; + char *const *argv; + const char *optstring; + const struct option *longopts; + int *longind; + int long_only; +{ + int print_errors = opterr; + if (optstring[0] == ':') + print_errors = 0; + + if (argc < 1) + return -1; + + optarg = NULL; + + if (optind == 0 || !__getopt_initialized) + { + if (optind == 0) + optind = 1; /* Don't scan ARGV[0], the program name. */ + optstring = _getopt_initialize (argc, argv, optstring); + __getopt_initialized = 1; + } + + /* Test whether ARGV[optind] points to a non-option argument. + Either it does not have option syntax, or there is an environment flag + from the shell indicating it is not an option. The later information + is only used when the used in the GNU libc. */ +#if defined _LIBC && defined USE_NONOPTION_FLAGS +# define NONOPTION_P (argv[optind][0] != '-' || argv[optind][1] == '\0' \ + || (optind < nonoption_flags_len \ + && __getopt_nonoption_flags[optind] == '1')) +#else +# define NONOPTION_P (argv[optind][0] != '-' || argv[optind][1] == '\0') +#endif + + if (nextchar == NULL || *nextchar == '\0') + { + /* Advance to the next ARGV-element. */ + + /* Give FIRST_NONOPT and LAST_NONOPT rational values if OPTIND has been + moved back by the user (who may also have changed the arguments). */ + if (last_nonopt > optind) + last_nonopt = optind; + if (first_nonopt > optind) + first_nonopt = optind; + + if (ordering == PERMUTE) + { + /* If we have just processed some options following some non-options, + exchange them so that the options come first. */ + + if (first_nonopt != last_nonopt && last_nonopt != optind) + exchange ((char **) argv); + else if (last_nonopt != optind) + first_nonopt = optind; + + /* Skip any additional non-options + and extend the range of non-options previously skipped. */ + + while (optind < argc && NONOPTION_P) + optind++; + last_nonopt = optind; + } + + /* The special ARGV-element `--' means premature end of options. + Skip it like a null option, + then exchange with previous non-options as if it were an option, + then skip everything else like a non-option. */ + + if (optind != argc && !strcmp (argv[optind], "--")) + { + optind++; + + if (first_nonopt != last_nonopt && last_nonopt != optind) + exchange ((char **) argv); + else if (first_nonopt == last_nonopt) + first_nonopt = optind; + last_nonopt = argc; + + optind = argc; + } + + /* If we have done all the ARGV-elements, stop the scan + and back over any non-options that we skipped and permuted. */ + + if (optind == argc) + { + /* Set the next-arg-index to point at the non-options + that we previously skipped, so the caller will digest them. */ + if (first_nonopt != last_nonopt) + optind = first_nonopt; + return -1; + } + + /* If we have come to a non-option and did not permute it, + either stop the scan or describe it to the caller and pass it by. */ + + if (NONOPTION_P) + { + if (ordering == REQUIRE_ORDER) + return -1; + optarg = argv[optind++]; + return 1; + } + + /* We have found another option-ARGV-element. + Skip the initial punctuation. */ + + nextchar = (argv[optind] + 1 + + (longopts != NULL && argv[optind][1] == '-')); + } + + /* Decode the current option-ARGV-element. */ + + /* Check whether the ARGV-element is a long option. + + If long_only and the ARGV-element has the form "-f", where f is + a valid short option, don't consider it an abbreviated form of + a long option that starts with f. Otherwise there would be no + way to give the -f short option. + + On the other hand, if there's a long option "fubar" and + the ARGV-element is "-fu", do consider that an abbreviation of + the long option, just like "--fu", and not "-f" with arg "u". + + This distinction seems to be the most useful approach. */ + + if (longopts != NULL + && (argv[optind][1] == '-' + || (long_only && (argv[optind][2] || !my_index (optstring, argv[optind][1]))))) + { + char *nameend; + const struct option *p; + const struct option *pfound = NULL; + int exact = 0; + int ambig = 0; + int indfound = -1; + int option_index; + + for (nameend = nextchar; *nameend && *nameend != '='; nameend++) + /* Do nothing. */ ; + + /* Test all long options for either exact match + or abbreviated matches. */ + for (p = longopts, option_index = 0; p->name; p++, option_index++) + if (!strncmp (p->name, nextchar, nameend - nextchar)) + { + if ((unsigned int) (nameend - nextchar) + == (unsigned int) strlen (p->name)) + { + /* Exact match found. */ + pfound = p; + indfound = option_index; + exact = 1; + break; + } + else if (pfound == NULL) + { + /* First nonexact match found. */ + pfound = p; + indfound = option_index; + } + else if (long_only + || pfound->has_arg != p->has_arg + || pfound->flag != p->flag + || pfound->val != p->val) + /* Second or later nonexact match found. */ + ambig = 1; + } + + if (ambig && !exact) + { + if (print_errors) + { +#if defined _LIBC && defined USE_IN_LIBIO + char *buf; + + __asprintf (&buf, _("%s: option `%s' is ambiguous\n"), + argv[0], argv[optind]); + + if (_IO_fwide (stderr, 0) > 0) + __fwprintf (stderr, L"%s", buf); + else + fputs (buf, stderr); + + free (buf); +#else + fprintf (stderr, _("%s: option `%s' is ambiguous\n"), + argv[0], argv[optind]); +#endif + } + nextchar += strlen (nextchar); + optind++; + optopt = 0; + return '?'; + } + + if (pfound != NULL) + { + option_index = indfound; + optind++; + if (*nameend) + { + /* Don't test has_arg with >, because some C compilers don't + allow it to be used on enums. */ + if (pfound->has_arg) + optarg = nameend + 1; + else + { + if (print_errors) + { +#if defined _LIBC && defined USE_IN_LIBIO + char *buf; +#endif + + if (argv[optind - 1][1] == '-') + { + /* --option */ +#if defined _LIBC && defined USE_IN_LIBIO + __asprintf (&buf, _("\ +%s: option `--%s' doesn't allow an argument\n"), + argv[0], pfound->name); +#else + fprintf (stderr, _("\ +%s: option `--%s' doesn't allow an argument\n"), + argv[0], pfound->name); +#endif + } + else + { + /* +option or -option */ +#if defined _LIBC && defined USE_IN_LIBIO + __asprintf (&buf, _("\ +%s: option `%c%s' doesn't allow an argument\n"), + argv[0], argv[optind - 1][0], + pfound->name); +#else + fprintf (stderr, _("\ +%s: option `%c%s' doesn't allow an argument\n"), + argv[0], argv[optind - 1][0], pfound->name); +#endif + } + +#if defined _LIBC && defined USE_IN_LIBIO + if (_IO_fwide (stderr, 0) > 0) + __fwprintf (stderr, L"%s", buf); + else + fputs (buf, stderr); + + free (buf); +#endif + } + + nextchar += strlen (nextchar); + + optopt = pfound->val; + return '?'; + } + } + else if (pfound->has_arg == 1) + { + if (optind < argc) + optarg = argv[optind++]; + else + { + if (print_errors) + { +#if defined _LIBC && defined USE_IN_LIBIO + char *buf; + + __asprintf (&buf, + _("%s: option `%s' requires an argument\n"), + argv[0], argv[optind - 1]); + + if (_IO_fwide (stderr, 0) > 0) + __fwprintf (stderr, L"%s", buf); + else + fputs (buf, stderr); + + free (buf); +#else + fprintf (stderr, + _("%s: option `%s' requires an argument\n"), + argv[0], argv[optind - 1]); +#endif + } + nextchar += strlen (nextchar); + optopt = pfound->val; + return optstring[0] == ':' ? ':' : '?'; + } + } + nextchar += strlen (nextchar); + if (longind != NULL) + *longind = option_index; + if (pfound->flag) + { + *(pfound->flag) = pfound->val; + return 0; + } + return pfound->val; + } + + /* Can't find it as a long option. If this is not getopt_long_only, + or the option starts with '--' or is not a valid short + option, then it's an error. + Otherwise interpret it as a short option. */ + if (!long_only || argv[optind][1] == '-' + || my_index (optstring, *nextchar) == NULL) + { + if (print_errors) + { +#if defined _LIBC && defined USE_IN_LIBIO + char *buf; +#endif + + if (argv[optind][1] == '-') + { + /* --option */ +#if defined _LIBC && defined USE_IN_LIBIO + __asprintf (&buf, _("%s: unrecognized option `--%s'\n"), + argv[0], nextchar); +#else + fprintf (stderr, _("%s: unrecognized option `--%s'\n"), + argv[0], nextchar); +#endif + } + else + { + /* +option or -option */ +#if defined _LIBC && defined USE_IN_LIBIO + __asprintf (&buf, _("%s: unrecognized option `%c%s'\n"), + argv[0], argv[optind][0], nextchar); +#else + fprintf (stderr, _("%s: unrecognized option `%c%s'\n"), + argv[0], argv[optind][0], nextchar); +#endif + } + +#if defined _LIBC && defined USE_IN_LIBIO + if (_IO_fwide (stderr, 0) > 0) + __fwprintf (stderr, L"%s", buf); + else + fputs (buf, stderr); + + free (buf); +#endif + } + nextchar = (char *) ""; + optind++; + optopt = 0; + return '?'; + } + } + + /* Look at and handle the next short option-character. */ + + { + char c = *nextchar++; + char *temp = my_index (optstring, c); + + /* Increment `optind' when we start to process its last character. */ + if (*nextchar == '\0') + ++optind; + + if (temp == NULL || c == ':') + { + if (print_errors) + { +#if defined _LIBC && defined USE_IN_LIBIO + char *buf; +#endif + + if (posixly_correct) + { + /* 1003.2 specifies the format of this message. */ +#if defined _LIBC && defined USE_IN_LIBIO + __asprintf (&buf, _("%s: illegal option -- %c\n"), + argv[0], c); +#else + fprintf (stderr, _("%s: illegal option -- %c\n"), argv[0], c); +#endif + } + else + { +#if defined _LIBC && defined USE_IN_LIBIO + __asprintf (&buf, _("%s: invalid option -- %c\n"), + argv[0], c); +#else + fprintf (stderr, _("%s: invalid option -- %c\n"), argv[0], c); +#endif + } + +#if defined _LIBC && defined USE_IN_LIBIO + if (_IO_fwide (stderr, 0) > 0) + __fwprintf (stderr, L"%s", buf); + else + fputs (buf, stderr); + + free (buf); +#endif + } + optopt = c; + return '?'; + } + /* Convenience. Treat POSIX -W foo same as long option --foo */ + if (temp[0] == 'W' && temp[1] == ';') + { + char *nameend; + const struct option *p; + const struct option *pfound = NULL; + int exact = 0; + int ambig = 0; + int indfound = 0; + int option_index; + + /* This is an option that requires an argument. */ + if (*nextchar != '\0') + { + optarg = nextchar; + /* If we end this ARGV-element by taking the rest as an arg, + we must advance to the next element now. */ + optind++; + } + else if (optind == argc) + { + if (print_errors) + { + /* 1003.2 specifies the format of this message. */ +#if defined _LIBC && defined USE_IN_LIBIO + char *buf; + + __asprintf (&buf, _("%s: option requires an argument -- %c\n"), + argv[0], c); + + if (_IO_fwide (stderr, 0) > 0) + __fwprintf (stderr, L"%s", buf); + else + fputs (buf, stderr); + + free (buf); +#else + fprintf (stderr, _("%s: option requires an argument -- %c\n"), + argv[0], c); +#endif + } + optopt = c; + if (optstring[0] == ':') + c = ':'; + else + c = '?'; + return c; + } + else + /* We already incremented `optind' once; + increment it again when taking next ARGV-elt as argument. */ + optarg = argv[optind++]; + + /* optarg is now the argument, see if it's in the + table of longopts. */ + + for (nextchar = nameend = optarg; *nameend && *nameend != '='; nameend++) + /* Do nothing. */ ; + + /* Test all long options for either exact match + or abbreviated matches. */ + for (p = longopts, option_index = 0; p->name; p++, option_index++) + if (!strncmp (p->name, nextchar, nameend - nextchar)) + { + if ((unsigned int) (nameend - nextchar) == strlen (p->name)) + { + /* Exact match found. */ + pfound = p; + indfound = option_index; + exact = 1; + break; + } + else if (pfound == NULL) + { + /* First nonexact match found. */ + pfound = p; + indfound = option_index; + } + else + /* Second or later nonexact match found. */ + ambig = 1; + } + if (ambig && !exact) + { + if (print_errors) + { +#if defined _LIBC && defined USE_IN_LIBIO + char *buf; + + __asprintf (&buf, _("%s: option `-W %s' is ambiguous\n"), + argv[0], argv[optind]); + + if (_IO_fwide (stderr, 0) > 0) + __fwprintf (stderr, L"%s", buf); + else + fputs (buf, stderr); + + free (buf); +#else + fprintf (stderr, _("%s: option `-W %s' is ambiguous\n"), + argv[0], argv[optind]); +#endif + } + nextchar += strlen (nextchar); + optind++; + return '?'; + } + if (pfound != NULL) + { + option_index = indfound; + if (*nameend) + { + /* Don't test has_arg with >, because some C compilers don't + allow it to be used on enums. */ + if (pfound->has_arg) + optarg = nameend + 1; + else + { + if (print_errors) + { +#if defined _LIBC && defined USE_IN_LIBIO + char *buf; + + __asprintf (&buf, _("\ +%s: option `-W %s' doesn't allow an argument\n"), + argv[0], pfound->name); + + if (_IO_fwide (stderr, 0) > 0) + __fwprintf (stderr, L"%s", buf); + else + fputs (buf, stderr); + + free (buf); +#else + fprintf (stderr, _("\ +%s: option `-W %s' doesn't allow an argument\n"), + argv[0], pfound->name); +#endif + } + + nextchar += strlen (nextchar); + return '?'; + } + } + else if (pfound->has_arg == 1) + { + if (optind < argc) + optarg = argv[optind++]; + else + { + if (print_errors) + { +#if defined _LIBC && defined USE_IN_LIBIO + char *buf; + + __asprintf (&buf, _("\ +%s: option `%s' requires an argument\n"), + argv[0], argv[optind - 1]); + + if (_IO_fwide (stderr, 0) > 0) + __fwprintf (stderr, L"%s", buf); + else + fputs (buf, stderr); + + free (buf); +#else + fprintf (stderr, + _("%s: option `%s' requires an argument\n"), + argv[0], argv[optind - 1]); +#endif + } + nextchar += strlen (nextchar); + return optstring[0] == ':' ? ':' : '?'; + } + } + nextchar += strlen (nextchar); + if (longind != NULL) + *longind = option_index; + if (pfound->flag) + { + *(pfound->flag) = pfound->val; + return 0; + } + return pfound->val; + } + nextchar = NULL; + return 'W'; /* Let the application handle it. */ + } + if (temp[1] == ':') + { + if (temp[2] == ':') + { + /* This is an option that accepts an argument optionally. */ + if (*nextchar != '\0') + { + optarg = nextchar; + optind++; + } + else + optarg = NULL; + nextchar = NULL; + } + else + { + /* This is an option that requires an argument. */ + if (*nextchar != '\0') + { + optarg = nextchar; + /* If we end this ARGV-element by taking the rest as an arg, + we must advance to the next element now. */ + optind++; + } + else if (optind == argc) + { + if (print_errors) + { + /* 1003.2 specifies the format of this message. */ +#if defined _LIBC && defined USE_IN_LIBIO + char *buf; + + __asprintf (&buf, + _("%s: option requires an argument -- %c\n"), + argv[0], c); + + if (_IO_fwide (stderr, 0) > 0) + __fwprintf (stderr, L"%s", buf); + else + fputs (buf, stderr); + + free (buf); +#else + fprintf (stderr, + _("%s: option requires an argument -- %c\n"), + argv[0], c); +#endif + } + optopt = c; + if (optstring[0] == ':') + c = ':'; + else + c = '?'; + } + else + /* We already incremented `optind' once; + increment it again when taking next ARGV-elt as argument. */ + optarg = argv[optind++]; + nextchar = NULL; + } + } + return c; + } +} + +int +getopt (argc, argv, optstring) + int argc; + char *const *argv; + const char *optstring; +{ + return _getopt_internal (argc, argv, optstring, + (const struct option *) 0, + (int *) 0, + 0); +} + +#endif /* Not ELIDE_CODE. */ + diff --git a/src/C/win/getopt.h b/src/C/win/getopt.h new file mode 100644 index 0000000..5dd8492 --- /dev/null +++ b/src/C/win/getopt.h @@ -0,0 +1,189 @@ +/* getopt.h */ +/* Declarations for getopt. + Copyright (C) 1989-1994, 1996-1999, 2001 Free Software + Foundation, Inc. This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute + it and/or modify it under the terms of the GNU Lesser + General Public License as published by the Free Software + Foundation; either version 2.1 of the License, or + (at your option) any later version. + + The GNU C Library is distributed in the hope that it will + be useful, but WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A + PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + + You should have received a copy of the GNU Lesser General + Public License along with the GNU C Library; if not, write + to the Free Software Foundation, Inc., 59 Temple Place, + Suite 330, Boston, MA 02111-1307 USA. */ + + + + + +#ifndef _GETOPT_H + +#ifndef __need_getopt +# define _GETOPT_H 1 +#endif + +/* If __GNU_LIBRARY__ is not already defined, either we are being used + standalone, or this is the first header included in the source file. + If we are being used with glibc, we need to include , but + that does not exist if we are standalone. So: if __GNU_LIBRARY__ is + not defined, include , which will pull in for us + if it's from glibc. (Why ctype.h? It's guaranteed to exist and it + doesn't flood the namespace with stuff the way some other headers do.) */ +#if !defined __GNU_LIBRARY__ +# include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* For communication from `getopt' to the caller. + When `getopt' finds an option that takes an argument, + the argument value is returned here. + Also, when `ordering' is RETURN_IN_ORDER, + each non-option ARGV-element is returned here. */ + +extern char *optarg; + +/* Index in ARGV of the next element to be scanned. + This is used for communication to and from the caller + and for communication between successive calls to `getopt'. + + On entry to `getopt', zero means this is the first call; initialize. + + When `getopt' returns -1, this is the index of the first of the + non-option elements that the caller should itself scan. + + Otherwise, `optind' communicates from one call to the next + how much of ARGV has been scanned so far. */ + +extern int optind; + +/* Callers store zero here to inhibit the error message `getopt' prints + for unrecognized options. */ + +extern int opterr; + +/* Set to an option character which was unrecognized. */ + +extern int optopt; + +#ifndef __need_getopt +/* Describe the long-named options requested by the application. + The LONG_OPTIONS argument to getopt_long or getopt_long_only is a vector + of `struct option' terminated by an element containing a name which is + zero. + + The field `has_arg' is: + no_argument (or 0) if the option does not take an argument, + required_argument (or 1) if the option requires an argument, + optional_argument (or 2) if the option takes an optional argument. + + If the field `flag' is not NULL, it points to a variable that is set + to the value given in the field `val' when the option is found, but + left unchanged if the option is not found. + + To have a long-named option do something other than set an `int' to + a compiled-in constant, such as set a value from `optarg', set the + option's `flag' field to zero and its `val' field to a nonzero + value (the equivalent single-letter option character, if there is + one). For long options that have a zero `flag' field, `getopt' + returns the contents of the `val' field. */ + +struct option +{ +# if (defined __STDC__ && __STDC__) || defined __cplusplus + const char *name; +# else + char *name; +# endif + /* has_arg can't be an enum because some compilers complain about + type mismatches in all the code that assumes it is an int. */ + int has_arg; + int *flag; + int val; +}; + +/* Names for the values of the `has_arg' field of `struct option'. */ + +# define no_argument 0 +# define required_argument 1 +# define optional_argument 2 +#endif /* need getopt */ + + +/* Get definitions and prototypes for functions to process the + arguments in ARGV (ARGC of them, minus the program name) for + options given in OPTS. + + Return the option character from OPTS just read. Return -1 when + there are no more options. For unrecognized options, or options + missing arguments, `optopt' is set to the option letter, and '?' is + returned. + + The OPTS string is a list of characters which are recognized option + letters, optionally followed by colons, specifying that that letter + takes an argument, to be placed in `optarg'. + + If a letter in OPTS is followed by two colons, its argument is + optional. This behavior is specific to the GNU `getopt'. + + The argument `--' causes premature termination of argument + scanning, explicitly telling `getopt' that there are no more + options. + + If OPTS begins with `--', then non-option arguments are treated as + arguments to the option '\0'. This behavior is specific to the GNU + `getopt'. */ + +#if (defined __STDC__ && __STDC__) || defined __cplusplus +# ifdef __cplusplus // __GNU_LIBRARY__ +/* Many other libraries have conflicting prototypes for getopt, with + differences in the consts, in stdlib.h. To avoid compilation + errors, only prototype getopt for the GNU C library. */ +extern int getopt (int ___argc, char *const *___argv, const char *__shortopts); +# else /* not __GNU_LIBRARY__ */ +extern int getopt (); +# endif /* __GNU_LIBRARY__ */ + +# ifndef __need_getopt +extern int getopt_long (int ___argc, char *const *___argv, + const char *__shortopts, + const struct option *__longopts, int *__longind); +extern int getopt_long_only (int ___argc, char *const *___argv, + const char *__shortopts, + const struct option *__longopts, int *__longind); + +/* Internal only. Users should not call this directly. */ +extern int _getopt_internal (int ___argc, char *const *___argv, + const char *__shortopts, + const struct option *__longopts, int *__longind, + int __long_only); +# endif +#else /* not __STDC__ */ +extern int getopt (); +# ifndef __need_getopt +extern int getopt_long (); +extern int getopt_long_only (); + +extern int _getopt_internal (); +# endif +#endif /* __STDC__ */ + +#ifdef __cplusplus +} +#endif + +/* Make sure we later can get all the definitions and declarations. */ +#undef __need_getopt + +#endif /* getopt.h */ + diff --git a/testing/IOR-tester.README b/testing/IOR-tester.README new file mode 100644 index 0000000..da57cf4 --- /dev/null +++ b/testing/IOR-tester.README @@ -0,0 +1,46 @@ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +\******************************************************************************/ + +The IOR-tester runs a series of tests to check and maintain the existing +functionality of the source code as code is modified. The IOR-tester creates +a default test, then modifies it to run test scripts. It runs a large number +of tests, most which are expected to pass, but some with an expectation of +failure. + +To run the code, modify the 'DefaultTest' dictionary in the source code to +reflect the test file location, the executable location, etc. Then, run +the code using './IOR-tester.py'. + +The expected-pass, pattern-independent tests include: + POSIX only: + o retry transfer + + MPIIO only: + o hints + o preallocation + + Both POSIX and MPIIO: + o repetition count + o intertest delay + o test file removal + o verbosity + +The expected-pass, pattern-dependent tests include: + POSIX: + o write-only, read-only, write/read, and write/read check + o fpp and single file + o segmented, strided + o zero-length, 4-byte, and larger file, block, and transfer sizes + MPIIO (same as POSIX, but using MPIIO access): + o noncollective + o noncollective, file view + o collective + o collective, file view + +The expected-fail tests include: + Both POSIX and MPIIO: + o repetition count diff --git a/testing/IOR-tester.py b/testing/IOR-tester.py new file mode 100755 index 0000000..4b0319f --- /dev/null +++ b/testing/IOR-tester.py @@ -0,0 +1,1611 @@ +#!/usr/local/bin/python +# +# Tester for IOR +# +#/*****************************************************************************\ +#* * +#* Copyright (c) 2003, The Regents of the University of California * +#* See the file COPYRIGHT for a complete copyright notice and license. * +#* * +#\*****************************************************************************/ +# +# CVS info: +# $RCSfile: IOR-tester.py,v $ +# $Revision: 1.1.1.1 $ +# $Date: 2007/10/15 23:36:54 $ +# $Author: rklundt $ + +import sys +import os.path +import string + +# definitions +RETURN_TOTAL_TESTS = -1 +TEST_SETTINGS = 0 +TEST_SUBSET_SIZE = 5 # number of tests in each job submission +IOR_SIZE_T = 8 +KIBIBYTE = 1024 +MEBIBYTE = KIBIBYTE * KIBIBYTE +GIBIBYTE = KIBIBYTE * MEBIBYTE +PASS = 1 +FAIL = 0 +TRUE = 1 +FALSE = 0 +scriptFileBase = './scriptFile' +executable = '../src/C/IOR' +#testDir = './tmp_test_dir' +testDir = '/p/glocal1/loewe/tmp_test_dir' + +debug = FALSE # debug mode = {FALSE, TRUE} + +################################################################################ +# class for default test parameters # +################################################################################ +class Test: + ###################### + # default parameters # + ###################### + def DefaultTest(self): + return { + # general + 'debug': 'debug info', + 'api': 'POSIX', + 'testFile': testDir + '/testFile.1', + 'hintsFileName': '', + 'repetitions': 1, + 'multiFile': 0, + 'interTestDelay': 0, + 'numTasks': 0, + 'readFile': 1, + 'writeFile': 1, + 'filePerProc': 0, + 'fsync': 0, + 'checkWrite': 1, + 'checkRead': 1, + 'keepFile': 1, + 'keepFileWithError': 1, + 'segmentCount': 1, + 'blockSize': MEBIBYTE, + 'transferSize': (MEBIBYTE / 4), + 'verbose': 0, + 'showHelp': 0, + 'reorderTasks': 1, # not default in code + 'quitOnError': 0, + 'useExistingTestFile': 0, + 'deadlineForStonewalling': 0, + 'maxTimeDuration': 0, + 'setTimeStampSignature': 0, + 'intraTestBarriers': 0, + 'storeFileOffset': 0, + 'randomOffset': 0, + # POSIX + 'singleXferAttempt': 0, + 'useO_DIRECT': 0, + # MPIIO + 'useFileView': 0, + 'preallocate': 0, + 'useSharedFilePointer': 0, # not working yet + 'useStridedDatatype': 0, # not working yet + # non-POSIX + 'showHints': 0, + 'collective': 0, + # HDF5 + 'setAlignment': 1, + 'noFill': 0, # in hdf5-1.6 or later version + 'individualDataSets': 0 # not working yet + } + + ################### + # tests to be run # + ################### + def Tests(self, expectation, testNumber): + +################################################################################ +################################################################################ +# # +# S T A R T O F T E S T S # +# # +################################################################################ +################################################################################ + POSIX_TESTS = [ + # + # pattern independent tests + # + + # POSIX, interTestDelay + [{'debug': 'POSIX interTestDelay', + 'interTestDelay': 0}], + [{'debug': 'POSIX interTestDelay', + 'interTestDelay': 1}], + + # POSIX, intraTestBarriers + [{'debug': 'POSIX intraTestBarriers', + 'intraTestBarriers': 1}], + + # POSIX, uniqueDir + [{'debug': 'POSIX uniqueDir', + 'filePerProc': 1, + 'uniqueDir': 1}], + + # POSIX, uniqueDir random + [{'debug': 'POSIX uniqueDir random', + 'filePerProc': 1, + 'randomOffset': 1, + 'checkRead': 0, + 'uniqueDir': 1}], + + # POSIX, fsync + [{'debug': 'POSIX fsync', + 'fsync': 1}], + + # POSIX, repetitions + [{'debug': 'POSIX repetitions', + 'repetitions': 1}], + [{'debug': 'POSIX repetitions', + 'repetitions': 3}], + + # POSIX, repetitions random + [{'debug': 'POSIX repetitions random', + 'randomOffset': 1, + 'checkRead': 0, + 'repetitions': 3}], + + # POSIX, multiFile + [{'debug': 'POSIX multiFile', + 'repetitions': 3, + 'multiFile': 1}], + + # POSIX, multiFile random + [{'debug': 'POSIX multiFile', + 'repetitions': 3, + 'randomOffset': 1, + 'checkRead': 0, + 'multiFile': 1}], + + # POSIX, writeFile-only + [{'debug': 'POSIX writeFile-only', + 'writeFile': 1, + 'readFile': 0, + 'checkRead': 0}], + + # POSIX, writeFile-only random + [{'debug': 'POSIX writeFile-only random', + 'writeFile': 1, + 'randomOffset': 1, + 'readFile': 0, + 'checkRead': 0}], + + # POSIX, readFile-only + [{'debug': 'POSIX readFile-only', + 'writeFile': 0, + 'checkWrite': 0, + 'readFile': 1}], + + # POSIX, readFile-only + [{'debug': 'POSIX readFile-only random', + 'writeFile': 0, + 'checkWrite': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'readFile': 1}], + + # POSIX, write/read check off + [{'debug': 'POSIX write-read check off', + 'checkWrite': 0, + 'checkRead': 0}], + + # POSIX, write/read check off random + [{'debug': 'POSIX write-read check off random', + 'checkWrite': 0, + 'randomOffset': 1, + 'checkRead': 0}], + + # POSIX, store file offset + [{'debug': 'POSIX store file offset', + 'storeFileOffset':1}], + + # POSIX, remove file + [{'debug': 'POSIX remove file', + 'keepFile': 0}], + + # POSIX, remove file random + [{'debug': 'POSIX remove file', + 'randomOffset': 1, + 'checkRead': 0, + 'keepFile': 0}], + + # POSIX, remove file with error + [{'debug': 'POSIX remove file with error', + 'keepFileWithError': 0}], + + # POSIX, remove file with error random + [{'debug': 'POSIX remove file with error random', + 'randomOffset': 1, + 'checkRead': 0, + 'keepFileWithError': 0}], + + + # POSIX, deadline for stonewalling + [{'debug': 'POSIX deadlineForStonewalling', + 'testFile': test.DefaultTest()['testFile'] + '.stonewall', + 'blockSize': GIBIBYTE, + 'readFile': 0, + 'checkWrite': 0, + 'checkRead': 0, + 'deadlineForStonewalling':1}], + + # POSIX, deadline for stonewalling random + [{'debug': 'POSIX deadlineForStonewalling random', + 'testFile': test.DefaultTest()['testFile'] + '.stonewall', + 'blockSize': GIBIBYTE, + 'readFile': 0, + 'checkWrite': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'deadlineForStonewalling':1}], + + # POSIX, max time duration + [{'debug': 'POSIX maxTimeDuration', + 'maxTimeDuration':1}], + + # POSIX, max time duration random + [{'debug': 'POSIX maxTimeDuration random', + 'randomOffset': 1, + 'checkRead': 0, + 'maxTimeDuration':1}], + + # POSIX, verbose + [{'debug': 'POSIX verbose 0', + 'verbose': 0}], + [{'debug': 'POSIX verbose 1', + 'verbose': 1}], + [{'debug': 'POSIX verbose 2', + 'verbose': 2}], + [{'debug': 'POSIX verbose 3', + 'verbose': 3}], + [{'debug': 'POSIX verbose 4', + 'verbose': 4, + 'blockSize': KIBIBYTE, + 'transferSize': (KIBIBYTE / 4)}], + + + # POSIX, multiple file names, ssf + [{'debug': 'POSIX multiple file names ssf', + 'testFile': testDir + '/f1@' + testDir + '/f2'}], + + # POSIX, multiple file names, ssf random + [{'debug': 'POSIX multiple file names ssf random', + 'randomOffset': 1, + 'checkRead': 0, + 'testFile': testDir + '/f1@' + testDir + '/f2'}], + + # POSIX, multiple file names, fpp + [{'debug': 'POSIX multiple file names fpp', + 'filePerProc': 1, + 'testFile': testDir + '/f1@' + testDir + '/f2'}], + + # POSIX, multiple file names, fpp random + [{'debug': 'POSIX multiple file names fpp random', + 'filePerProc': 1, + 'randomOffset': 1, + 'checkRead': 0, + 'testFile': testDir + '/f1@' + testDir + '/f2'}], + + # POSIX, corruptFile + [{'debug': 'POSIX corruptFile', + 'testFile': test.DefaultTest()['testFile'], + 'filePerProc': 0, + 'corruptFile': 1}], + + # POSIX, corruptFile random + [{'debug': 'POSIX corruptFile random', + 'testFile': test.DefaultTest()['testFile'], + 'filePerProc': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'corruptFile': 1}], + + # POSIX, corruptFile + [{'debug': 'POSIX corruptFile filePerProc', + 'filePerProc': 1, + 'corruptFile': 1}], + + # POSIX, corruptFile random + [{'debug': 'POSIX corruptFile filePerProc random', + 'filePerProc': 1, + 'randomOffset': 1, + 'checkRead': 0, + 'corruptFile': 1}], + + # POSIX, showHelp + [{'debug': 'POSIX showHelp', + 'showHelp': 1, + 'filePerProc': 0, + 'corruptFile': 0}], + + # POSIX, quitOnError + [{'debug': 'POSIX quitOnError', + 'quitOnError': 1}], + + # POSIX, quitOnError random + [{'debug': 'POSIX quitOnError random', + 'randomOffset': 1, + 'checkRead': 0, + 'quitOnError': 1}], + + # POSIX, singleXferAttempt + [{'debug': 'POSIX singleXferAttempt', + 'singleXferAttempt': 1}], + + # POSIX, singleXferAttempt random + [{'debug': 'POSIX singleXferAttempt random', + 'randomOffset': 1, + 'checkRead': 0, + 'singleXferAttempt': 1}], + + # POSIX, setTimeStampSignature + [{'debug': 'POSIX setTimeStampSignature', + 'setTimeStampSignature': 123}], + + # POSIX, setTimeStampSignature random + [{'debug': 'POSIX setTimeStampSignature random', + 'randomOffset': 1, + 'checkRead': 0, + 'setTimeStampSignature': 123}], + + # POSIX, useExistingTestFile [Note: don't follow HDF5 test] + [{'debug': 'POSIX useExistingTestFile', + 'useExistingTestFile': 1}], + + # POSIX, useExistingTestFile random [Note: don't follow HDF5 test] + [{'debug': 'POSIX useExistingTestFile', + 'randomOffset': 1, + 'checkRead': 0, + 'useExistingTestFile': 1}], + + # + # pattern dependent tests + # + + # POSIX, filePerProc + [{'debug': 'POSIX filePerProc', + 'filePerProc': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'POSIX filePerProc', + 'filePerProc': 1, + 'segmentCount': 1}], + + [{'debug': 'POSIX filePerProc', + 'filePerProc': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'POSIX filePerProc', + 'filePerProc': 1, + 'segmentCount': 3}], + + # POSIX, sharedFile + [{'debug': 'POSIX sharedFile', + 'filePerProc': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'POSIX sharedFile', + 'filePerProc': 0, + 'segmentCount': 1}], + + [{'debug': 'POSIX sharedFile', + 'filePerProc': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'POSIX sharedFile', + 'filePerProc': 0, + 'segmentCount': 3}], + + [{'debug': 'POSIX sharedFile numTasks', + 'filePerProc': 0, + 'numTasks': 2, + 'segmentCount': 3}], + + # + # pattern dependent tests, random + # + + # POSIX, filePerProc random + [{'debug': 'POSIX filePerProc random', + 'filePerProc': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 1}], + + [{'debug': 'POSIX filePerProc random', + 'filePerProc': 1, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 1}], + + [{'debug': 'POSIX filePerProc random', + 'filePerProc': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 3}], + + [{'debug': 'POSIX filePerProc random', + 'filePerProc': 1, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 3}], + + # POSIX, sharedFile random + [{'debug': 'POSIX sharedFile random', + 'filePerProc': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 1}], + + [{'debug': 'POSIX sharedFile random', + 'filePerProc': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 1}], + + [{'debug': 'POSIX sharedFile random', + 'filePerProc': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 3}], + + [{'debug': 'POSIX sharedFile random', + 'filePerProc': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 3}], + + [{'debug': 'POSIX sharedFile numTasks random', + 'filePerProc': 0, + 'numTasks': 2, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 3}] + ] + + MPIIO_TESTS = [ + # + # pattern independent tests + # + + # MPIIO, repetitions + [{'debug': 'MPIIO repetitions', + 'api': 'MPIIO', + 'repetitions': 1}], + [{'debug': 'MPIIO repetitions', + 'api': 'MPIIO', + 'repetitions': 3}], + [{'debug': 'MPIIO repetitions random', + 'api': 'MPIIO', + 'randomOffset': 1, + 'checkRead': 0, + 'repetitions': 3}], + + # MPIIO, multiFile + [{'debug': 'MPIIO multiFile', + 'api': 'MPIIO', + 'repetitions': 3, + 'multiFile': 1}], + + # MPIIO, multiFile random + [{'debug': 'MPIIO multiFile random', + 'api': 'MPIIO', + 'repetitions': 3, + 'randomOffset': 1, + 'checkRead': 0, + 'multiFile': 1}], + + # MPIIO, remove file + [{'debug': 'MPIIO remove file', + 'api': 'MPIIO', + 'keepFile': 0}], + + # MPIIO, remove file random + [{'debug': 'MPIIO remove file', + 'api': 'MPIIO', + 'randomOffset': 1, + 'checkRead': 0, + 'keepFile': 0}], + + # MPIIO, remove file with error + [{'debug': 'MPIIO remove file with error', + 'api': 'MPIIO', + 'keepFileWithError': 0}], + + # MPIIO, remove file with error random + [{'debug': 'MPIIO remove file with error random', + 'api': 'MPIIO', + 'randomOffset': 1, + 'checkRead': 0, + 'keepFileWithError': 0}], + + # MPIIO, deadline for stonewalling + [{'debug': 'MPIIO deadlineForStonewalling', + 'api': 'MPIIO', + 'testFile': test.DefaultTest()['testFile'] + '.stonewall', + 'blockSize': GIBIBYTE, + 'readFile': 0, + 'checkWrite': 0, + 'checkRead': 0, + 'deadlineForStonewalling':1}], + + # MPIIO, deadline for stonewalling random + [{'debug': 'MPIIO deadlineForStonewalling random', + 'api': 'MPIIO', + 'testFile': test.DefaultTest()['testFile'] + '.stonewall', + 'blockSize': GIBIBYTE, + 'readFile': 0, + 'checkWrite': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'deadlineForStonewalling':1}], + + # MPIIO, max time duration + [{'debug': 'MPIIO maxTimeDuration', + 'api': 'MPIIO', + 'maxTimeDuration':1}], + + # MPIIO, max time duration random + [{'debug': 'MPIIO maxTimeDuration random', + 'api': 'MPIIO', + 'randomOffset': 1, + 'checkRead': 0, + 'maxTimeDuration':1}], + + # MPIIO, quitOnError + [{'debug': 'MPIIO quitOnError', + 'api': 'MPIIO', + 'quitOnError': 1}], + + # MPIIO, quitOnError random + [{'debug': 'MPIIO quitOnError random', + 'api': 'MPIIO', + 'randomOffset': 1, + 'checkRead': 0, + 'quitOnError': 1}], + + # MPIIO, multiple file names, ssf + [{'debug': 'MPIIO multiple file names ssf', + 'api': 'MPIIO', + 'testFile': testDir + '/f1@' + testDir + '/f2'}], + + # MPIIO, multiple file names, ssf random + [{'debug': 'MPIIO multiple file names ssf random', + 'api': 'MPIIO', + 'randomOffset': 1, + 'checkRead': 0, + 'testFile': testDir + '/f1@' + testDir + '/f2'}], + + # MPIIO, multiple file names, fpp + [{'debug': 'MPIIO multiple file names fpp', + 'api': 'MPIIO', + 'filePerProc': 1, + 'testFile': testDir + '/f1@' + testDir + '/f2'}], + + # MPIIO, multiple file names, fpp random + [{'debug': 'MPIIO multiple file names fpp random', + 'api': 'MPIIO', + 'filePerProc': 1, + 'randomOffset': 1, + 'checkRead': 0, + 'testFile': testDir + '/f1@' + testDir + '/f2'}], + + # MPIIO, corruptFile + [{'debug': 'MPIIO corruptFile', + 'api': 'MPIIO', + 'testFile': test.DefaultTest()['testFile'], + 'filePerProc': 0, + 'corruptFile': 1}], + + # MPIIO, corruptFile random + [{'debug': 'MPIIO corruptFile random', + 'api': 'MPIIO', + 'testFile': test.DefaultTest()['testFile'], + 'filePerProc': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'corruptFile': 1}], + + # MPIIO, corruptFile + [{'debug': 'MPIIO corruptFile filePerProc', + 'api': 'MPIIO', + 'filePerProc': 1, + 'corruptFile': 1}], + + # MPIIO, corruptFile random + [{'debug': 'MPIIO corruptFile filePerProc random', + 'api': 'MPIIO', + 'filePerProc': 1, + 'randomOffset': 1, + 'checkRead': 0, + 'corruptFile': 1}], + + # MPIIO, useExistingTestFile + [{'debug': 'MPIIO useExistingTestFile', + 'api': 'MPIIO', + 'useExistingTestFile': 0, + 'filePerProc': 0, + 'corruptFile': 0}], + + # MPIIO, useExistingTestFile random + [{'debug': 'MPIIO useExistingTestFile random', + 'api': 'MPIIO', + 'useExistingTestFile': 0, + 'filePerProc': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'corruptFile': 0}], + + # MPIIO, preallocate + [{'debug': 'MPIIO preallocate', + 'api': 'MPIIO', + 'preallocate': 1}], + + # MPIIO, showHints + [{'debug': 'MPIIO showHints', + 'api': 'MPIIO', + 'showHints': 1}], + + # MPIIO, showHints w/hintsFileName + [{'debug': 'MPIIO showHints w/hintsFileName', + 'api': 'MPIIO', + 'hintsFileName': '/g/g0/loewe/IOR/test/hintsFile', + 'showHints': 1}], + + # MPIIO, setTimeStampSignature + [{'debug': 'MPIIO setTimeStampSignature', + 'api': 'MPIIO', + 'setTimeStampSignature': 123}], + + # MPIIO, setTimeStampSignature random + [{'debug': 'MPIIO setTimeStampSignature random', + 'api': 'MPIIO', + 'randomOffset': 1, + 'checkRead': 0, + 'setTimeStampSignature': 123}], + + # + # pattern dependent tests + # + + # MPIIO, independent + [{'debug': 'MPIIO independent', + 'api': 'MPIIO', + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent', + 'api': 'MPIIO', + 'collective': 0, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent', + 'api': 'MPIIO', + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent', + 'api': 'MPIIO', + 'collective': 0, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent numTasks', + 'api': 'MPIIO', + 'numTasks': 2, + 'collective': 0, + 'segmentCount': 3}], + + # MPIIO, independent random + [{'debug': 'MPIIO independent random', + 'api': 'MPIIO', + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent random', + 'api': 'MPIIO', + 'collective': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent random', + 'api': 'MPIIO', + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent random', + 'api': 'MPIIO', + 'collective': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent numTasks random', + 'api': 'MPIIO', + 'numTasks': 2, + 'collective': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 3}], + + # MPIIO, collective + [{'debug': 'MPIIO collective', + 'api': 'MPIIO', + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'MPIIO collective', + 'api': 'MPIIO', + 'collective': 1, + 'segmentCount': 1}], + + [{'debug': 'MPIIO collective', + 'api': 'MPIIO', + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'MPIIO collective', + 'api': 'MPIIO', + 'collective': 1, + 'segmentCount': 3}], + + [{'debug': 'MPIIO collective numTasks', + 'api': 'MPIIO', + 'numTasks': 2, + 'collective': 1, + 'segmentCount': 3}], + + # MPIIO, independent, useFileView + [{'debug': 'MPIIO independent useFileView', + 'api': 'MPIIO', + 'useFileView': 1, + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent useFileView', + 'api': 'MPIIO', + 'useFileView': 1, + 'collective': 0, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent useFileView', + 'api': 'MPIIO', + 'useFileView': 1, + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent useFileView', + 'api': 'MPIIO', + 'useFileView': 1, + 'collective': 0, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent useFileView numTasks', + 'api': 'MPIIO', + 'useFileView': 1, + 'numTasks': 2, + 'collective': 0, + 'segmentCount': 3}], + + # MPIIO, collective, useFileView + [{'debug': 'MPIIO collective useFileView', + 'api': 'MPIIO', + 'useFileView': 1, + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'MPIIO collective useFileView', + 'api': 'MPIIO', + 'useFileView': 1, + 'collective': 1, + 'segmentCount': 1}], + + [{'debug': 'MPIIO collective useFileView', + 'api': 'MPIIO', + 'useFileView': 1, + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'MPIIO collective useFileView', + 'api': 'MPIIO', + 'useFileView': 1, + 'collective': 1, + 'segmentCount': 3}], + + [{'debug': 'MPIIO collective useFileView numTasks', + 'api': 'MPIIO', + 'useFileView': 1, + 'numTasks': 2, + 'collective': 1, + 'segmentCount': 3}], + + # MPIIO, independent, filePerProc + [{'debug': 'MPIIO independent filePerProc', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent filePerProc', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 0, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent filePerProc', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent filePerProc', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 0, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent filePerProc numTasks', + 'api': 'MPIIO', + 'filePerProc': 1, + 'numTasks': 2, + 'collective': 0, + 'segmentCount': 3}], + + # MPIIO, independent, filePerProc random + [{'debug': 'MPIIO independent filePerProc random', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent filePerProc random', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent filePerProc random', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent filePerProc random', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent filePerProc numTasks random', + 'api': 'MPIIO', + 'filePerProc': 1, + 'numTasks': 2, + 'collective': 0, + 'randomOffset': 1, + 'checkRead': 0, + 'segmentCount': 3}], + + # MPIIO, collective, filePerProc + [{'debug': 'MPIIO collective filePerProc', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'MPIIO collective filePerProc', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 1, + 'segmentCount': 1}], + + [{'debug': 'MPIIO collective filePerProc', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'MPIIO collective filePerProc', + 'api': 'MPIIO', + 'filePerProc': 1, + 'collective': 1, + 'segmentCount': 3}], + + [{'debug': 'MPIIO collective filePerProc numTasks', + 'api': 'MPIIO', + 'filePerProc': 1, + 'numTasks': 2, + 'collective': 1, + 'segmentCount': 3}], + + # MPIIO, independent, filePerProc, useFileView + [{'debug': 'MPIIO independent filePerProc useFileView', + 'api': 'MPIIO', + 'filePerProc': 1, + 'useFileView': 1, + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent filePerProc useFileView', + 'api': 'MPIIO', + 'filePerProc': 1, + 'useFileView': 1, + 'collective': 0, + 'segmentCount': 1}], + + [{'debug': 'MPIIO independent filePerProc useFileView', + 'api': 'MPIIO', + 'filePerProc': 1, + 'useFileView': 1, + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent filePerProc useFileView', + 'api': 'MPIIO', + 'filePerProc': 1, + 'useFileView': 1, + 'collective': 0, + 'segmentCount': 3}], + + [{'debug': 'MPIIO independent filePerProc useFileView numTasks', + 'api': 'MPIIO', + 'filePerProc': 1, + 'numTasks': 2, + 'useFileView': 1, + 'collective': 0, + 'segmentCount': 3}], + + # MPIIO, collective, filePerProc, useFileView + [{'debug': 'MPIIO collective filePerProc useFileView', + 'api': 'MPIIO', + 'filePerProc': 1, + 'useFileView': 1, + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'MPIIO collective filePerProc useFileView', + 'api': 'MPIIO', + 'filePerProc': 1, + 'useFileView': 1, + 'collective': 1, + 'segmentCount': 1}], + + [{'debug': 'MPIIO collective filePerProc useFileView', + 'api': 'MPIIO', + 'filePerProc': 1, + 'useFileView': 1, + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'MPIIO collective filePerProc useFileView', + 'api': 'MPIIO', + 'filePerProc': 1, + 'useFileView': 1, + 'collective': 1, + 'segmentCount': 3}], + + [{'debug': 'MPIIO collective filePerProc useFileView numTasks', + 'api': 'MPIIO', + 'filePerProc': 1, + 'numTasks': 2, + 'useFileView': 1, + 'collective': 1, + 'segmentCount': 3}] + ] + + HDF5_TESTS = [ + # + # pattern independent tests + # + + # HDF5, repetitions + [{'debug': 'HDF5 repetitions', + 'api': 'HDF5', + 'repetitions': 1}], + [{'debug': 'HDF5 repetitions', + 'api': 'HDF5', + 'repetitions': 3}], + + # HDF5, multiFile + [{'debug': 'HDF5 multiFile', + 'api': 'HDF5', + 'repetitions': 3, + 'multiFile': 1}], + + # HDF5, useExistingTestFile [Note: this must follow HDF5 test] + [{'debug': 'HDF5 useExistingTestFile', + 'api': 'HDF5', + 'useExistingTestFile': 1}], + + # HDF5, remove file + [{'debug': 'HDF5 remove file', + 'api': 'HDF5', + 'keepFile': 0}], + + # HDF5, remove file with error + [{'debug': 'HDF5 remove file with error', + 'api': 'HDF5', + 'keepFileWithError': 0}], + + # HDF5, deadline for stonewalling + [{'debug': 'HDF5 deadlineForStonewalling', + 'api': 'HDF5', + 'testFile': test.DefaultTest()['testFile'] + '.stonewall', + 'blockSize': GIBIBYTE/4, + 'readFile': 0, + 'checkWrite': 0, + 'checkRead': 0, + 'deadlineForStonewalling':1}], + + # HDF5, max time duration + [{'debug': 'HDF5 maxTimeDuration', + 'api': 'HDF5', + 'maxTimeDuration':1}], + + # HDF5, quitOnError + [{'debug': 'HDF5 quitOnError', + 'api': 'HDF5', + 'quitOnError': 1}], + + # HDF5, multiple file names, ssf + [{'debug': 'HDF5 multiple file names ssf', + 'api': 'HDF5', + 'testFile': testDir + '/f1@' + testDir + '/f2'}], + + # HDF5, multiple file names, fpp + [{'debug': 'HDF5 multiple file names fpp', + 'api': 'HDF5', + 'filePerProc': 1, + 'testFile': testDir + '/f1@' + testDir + '/f2'}], + + # HDF5, corruptFile + [{'debug': 'HDF5 corruptFile', + 'api': 'HDF5', + 'testFile': test.DefaultTest()['testFile'], + 'filePerProc': 0, + 'corruptFile': 1}], + + # HDF5, corruptFile + [{'debug': 'HDF5 corruptFile filePerProc', + 'api': 'HDF5', + 'filePerProc': 1, + 'corruptFile': 1}], + + # HDF5, setTimeStampSignature + [{'debug': 'HDF5 setTimeStampSignature', + 'api': 'HDF5', + 'setTimeStampSignature': 123, + 'filePerProc': 0, + 'corruptFile': 0}], + + # HDF5, setAlignment + [{'debug': 'HDF5 setAlignment', + 'api': 'HDF5', + 'setAlignment': '4m'}], + + # HDF5, showHints + [{'debug': 'HDF5 showHints', + 'api': 'HDF5', + 'showHints': 0}], # WEL: omit this test until + # showHints works + + # HDF5, showHints w/hintsFileName + [{'debug': 'HDF5 showHints w/hintsFileName', + 'api': 'HDF5', + 'hintsFileName': '/g/g0/loewe/IOR/test/hintsFile', + 'showHints': 0}], # WEL: omit this test until + # showHints works + + # HDF5, noFill + [{'debug': 'HDF5 noFill', + 'api': 'HDF5', + 'noFill': 0}], # WEL: omit this test until + # noFill is standard + + # + # pattern dependent tests + # + + # HDF5, independent + [{'debug': 'HDF5 independent', + 'api': 'HDF5', + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'HDF5 independent', + 'api': 'HDF5', + 'collective': 0, + 'segmentCount': 1}], + + [{'debug': 'HDF5 independent', + 'api': 'HDF5', + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'HDF5 independent', + 'api': 'HDF5', + 'collective': 0, + 'segmentCount': 3}], + + [{'debug': 'HDF5 independent numTasks', + 'api': 'HDF5', + 'numTasks': 2, + 'collective': 0, + 'segmentCount': 3}], + + # HDF5, collective + [{'debug': 'HDF5 collective', + 'api': 'HDF5', + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'HDF5 collective', + 'api': 'HDF5', + 'collective': 1, + 'segmentCount': 1}], + + [{'debug': 'HDF5 collective', + 'api': 'HDF5', + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'HDF5 collective', + 'api': 'HDF5', + 'collective': 1, + 'segmentCount': 3}], + + [{'debug': 'HDF5 collective numTasks', + 'api': 'HDF5', + 'numTasks': 2, + 'collective': 1, + 'segmentCount': 3}] + ] + + NCMPI_TESTS = [ + # + # pattern independent tests + # + + # NCMPI, repetitions + [{'debug': 'NCMPI repetitions', + 'api': 'NCMPI', + 'repetitions': 1}], + [{'debug': 'NCMPI repetitions', + 'api': 'NCMPI', + 'repetitions': 3}], + + # NCMPI, multiFile + [{'debug': 'NCMPI multiFile', + 'api': 'NCMPI', + 'repetitions': 3, + 'multiFile': 1}], + + # NCMPI, deadline for stonewalling + [{'debug': 'NCMPI deadlineForStonewalling', + 'api': 'NCMPI', + 'testFile': test.DefaultTest()['testFile'] + '.stonewall', + 'blockSize': GIBIBYTE/4, + 'readFile': 0, + 'checkWrite': 0, + 'checkRead': 0, + 'deadlineForStonewalling':1}], + + # NCMPI, max time duration + [{'debug': 'NCMPI maxTimeDuration', + 'api': 'NCMPI', + 'maxTimeDuration':1}], + + # NCMPI, remove file + [{'debug': 'NCMPI remove file', + 'api': 'NCMPI', + 'keepFile': 0}], + + # NCMPI, remove file with error + [{'debug': 'NCMPI remove file with error', + 'api': 'NCMPI', + 'keepFileWithError': 0}], + + # NCMPI, quitOnError + [{'debug': 'NCMPI quitOnError', + 'api': 'NCMPI', + 'quitOnError': 1}], + + # NCMPI, multiple file names, ssf + [{'debug': 'NCMPI multiple file names ssf', + 'api': 'NCMPI', + 'testFile': testDir + '/f1@' + testDir + '/f2'}], + + # NCMPI, corruptFile + [{'debug': 'NCMPI corruptFile', + 'api': 'NCMPI', + 'testFile': test.DefaultTest()['testFile'], + 'corruptFile': 1}], + + # NCMPI, setTimeStampSignature + [{'debug': 'NCMPI setTimeStampSignature', + 'api': 'NCMPI', + 'setTimeStampSignature': 123, + 'corruptFile': 0}], + + # NCMPI, useExistingTestFile [Note: this must follow NCMPI test] + [{'debug': 'NCMPI useExistingTestFile', + 'api': 'NCMPI', + 'useExistingTestFile': 1}], + + # + # pattern dependent tests + # + + # NCMPI, independent + [{'debug': 'NCMPI independent', + 'api': 'NCMPI', + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'NCMPI independent', + 'api': 'NCMPI', + 'collective': 0, + 'segmentCount': 1}], + + [{'debug': 'NCMPI independent', + 'api': 'NCMPI', + 'collective': 0, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'NCMPI independent', + 'api': 'NCMPI', + 'collective': 0, + 'segmentCount': 3}], + + [{'debug': 'NCMPI independent numTasks', + 'api': 'NCMPI', + 'numTasks': 2, + 'collective': 0, + 'segmentCount': 3}], + + # NCMPI, collective + [{'debug': 'NCMPI collective', + 'api': 'NCMPI', + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 1}], + + [{'debug': 'NCMPI collective', + 'api': 'NCMPI', + 'collective': 1, + 'segmentCount': 1}], + + [{'debug': 'NCMPI collective', + 'api': 'NCMPI', + 'collective': 1, + 'blockSize': IOR_SIZE_T, + 'transferSize': IOR_SIZE_T, + 'segmentCount': 3}], + + [{'debug': 'NCMPI collective', + 'api': 'NCMPI', + 'collective': 1, + 'segmentCount': 3}], + + [{'debug': 'NCMPI collective numTasks', + 'api': 'NCMPI', + 'numTasks': 2, + 'collective': 1, + 'segmentCount': 3}] + ] + + PassTests = [] + if OS == "AIX": + PassTests = PassTests + POSIX_TESTS + PassTests = PassTests + MPIIO_TESTS + PassTests = PassTests + HDF5_TESTS + PassTests = PassTests + NCMPI_TESTS + elif OS == "Linux": + PassTests = PassTests + POSIX_TESTS + PassTests = PassTests + MPIIO_TESTS + #PassTests = PassTests + HDF5_TESTS + #PassTests = PassTests + NCMPI_TESTS + else: + PassTests = [ + [{'debug': 'failure to determine OS'}] + ] + + FailTests = [ + [{'debug': 'no tests should fail'}] + ] + +################################################################################ +################################################################################ +# # +# E N D O F T E S T S # +# # +################################################################################ +################################################################################ + # WEL debugging: if 0 == 0: PassTests = [ [{'debug': 'debug test'}] ] + + if expectation == PASS: + if testNumber == RETURN_TOTAL_TESTS: + return len(PassTests) + else: + return PassTests[testNumber] + else: # expectation == FAIL + if testNumber == RETURN_TOTAL_TESTS: + return len(FailTests) + else: + return FailTests[testNumber] + + + ################### + # run test script # + ################### + def RunScript(self, nodes, procs): + if OS == "AIX": + command = "poe " + executable + " -f " + scriptFile + \ + " -nodes " + str(nodes) + " -procs " + str(procs) + \ + " -rmpool systest -labelio no -retry wait" + elif OS == "Linux": + command = "srun -N " + str(procs) + " -n " + str(procs) + \ + " -ppdebug " + executable + " -f " + scriptFile + else: + command = "unable to run " + executable + " -f " + scriptFile + if debug == TRUE: + Flush2File(command) + else: + childIn, childOut = os.popen4(command) + childIn.close() + while 1: + line = childOut.readline() + if line == '': break + Flush2File(line[:-1]) + childOut.close() + return + + +########################## +# create subsets of list # +########################## +def ListSubsets(tests, size): + listOfSubsets = [] + start = end = 0 + totalTestsSize = len(tests) + for i in range(0, (totalTestsSize / size) + 1): + if end >= totalTestsSize: + break + end = end + size + listOfSubsets.append(tests[start:end]) + start = end + return listOfSubsets + + +################# +# flush to file # +################# +def Flush2File(string): + resultsFile.write(string + '\n') + resultsFile.flush() + + +################################## +# replace blanks with underscore # +################################## +def UnderScore(string): + uString = string + for i in range(0, len(uString)): + if uString[i] == ' ': + uString = uString[:i] + '_' + uString[i+1:] + return(uString) + + +############################# +# grep for keywords in file # +############################# +def grepForKeywords(keywords, resultsFileName): + # create pattern for grep + pattern = "\"" + for i in range(len(keywords)): + pattern = pattern + keywords[i] + if i < len(keywords)-1: + pattern = pattern + '|' + pattern = pattern + "\"" + + # grep for pattern in file + resultsFileNameTmp = resultsFileName + ".tmp" + cmd = "grep -i -E " + pattern + " " + resultsFileName \ + + " >> " + resultsFileNameTmp + " 2>&1" + os.system(cmd) + cmd = "cat " + resultsFileNameTmp + " >> " + resultsFileName + os.system(cmd) + cmd = "rm -f " + resultsFileNameTmp + os.system(cmd) + + +################################################################################ +# main # +################################################################################ +resultsFileName = "./test-results.txt-" + \ + os.popen("date +%m.%d.%y").read()[:-1] +resultsFile = open(resultsFileName, "w") +OS = os.popen("uname -s").read()[:-1] +test = Test() +testNumber = 0 + +#environment variables +nodes = 1 +proccnt = [1, 3] + +Flush2File("TESTING IOR C CODE") + +# loop through different processors counts +for proc in proccnt: + + # first run all expected-PASS test, then the FAILs + for testType in (PASS, FAIL): + + # test type info + if (testType == PASS): + Flush2File("\n*** STARTING EXPECTED P A S S TESTS (PROC=" \ + + str(proc) + ") ***") + else: + Flush2File("\n*** STARTING EXPECTED F A I L TESTS (PROC=" \ + + str(proc) + ") ***") + + # loop through all tests for test type + totalTests = range(test.Tests(testType, RETURN_TOTAL_TESTS)) + firstTest = TRUE + for testSubset in ListSubsets(totalTests, TEST_SUBSET_SIZE): + for i in testSubset: + if (firstTest == TRUE): + Flush2File("\n\n*** Setting up tests ***") + firstTest = FALSE + if (i % 10 == 0 and i != 0): + Flush2File("finished " + str(i) + " tests") + sys.stdout.flush() + # unless an expected fail test, only open a single script + # create script file name + if (testType == PASS): + scriptFile = scriptFileBase + '.' + str(proc) + '.PASS' + else: + scriptFile = scriptFileBase + '.' + str(proc) + '.FAIL' + + scriptFile = scriptFile + '-TESTS_' + str(testSubset[0:1][0]) \ + + '-' + str(testSubset[len(testSubset)-1:][0]) + if ((i % TEST_SUBSET_SIZE == 0) or (testType == FAIL)): + os.system("rm -f " + scriptFile) + script = open(scriptFile, "a") + script.write("IOR START" + "\n") + + # start with a default test, then modify + testValues = test.DefaultTest() + + # loop through all changes to the default script + for j in test.Tests(testType, i)[TEST_SETTINGS].keys(): + testValues[j] = test.Tests(testType, i)[TEST_SETTINGS][j] + + testNumber = testNumber + 1 + testValues['debug'] = UnderScore("Test No. " + \ + str(testNumber) + ": " + \ + testValues['debug']) + # write test information to script file + for entry in testValues.keys(): + if (str(testValues[entry]) != ''): + script.write("\t" + entry + "=" + + str(testValues[entry]) + "\n") + script.write("RUN" + "\n") + + # unless an expected fail test, only close a single script + if ((i == testSubset[len(testSubset)-1:][0]) \ + or (testType == FAIL)): + # create tail, close file + script.write("IOR STOP" + "\n") + script.close() + if (testType == FAIL): + Flush2File("finished 1 test") + else: + Flush2File("finished %d test%s" % ((i + 1), "s"[i==1:])) + firstTest = TRUE + + # display test info for failing test (note that the + # test fails before a description is displayed) + if testType == FAIL: + Flush2File("\n\t================================\n\n") + Flush2File("*** DEBUG MODE ***") + Flush2File("*** " + str(testValues['debug']) + \ + " ***") + Flush2File("") + + # run + os.system ("rm -rf " + testDir) + os.system("mkdir " + testDir) + test.RunScript(nodes, proc) + os.system ("rm -rf " + testDir) + if 0 == 0: os.system("rm -f " + scriptFile) # run scripts + +Flush2File("\nFINISHED TESTING IOR C CODE") +Flush2File("\nRESULTS:") +resultsFile.close() +grepForKeywords(["warn", "fail", "error", "Test_No"], resultsFileName) diff --git a/testing/hintsFile b/testing/hintsFile new file mode 100644 index 0000000..6931afb --- /dev/null +++ b/testing/hintsFile @@ -0,0 +1,2 @@ +IOR_HINT__MPI__unrecognizedHint=true +IOR_HINT__MPI__IBM_largeblock_io=true