diff --git a/misc/Makefile.in b/misc/Makefile.in index 83429401..e8cfdb5b 100644 --- a/misc/Makefile.in +++ b/misc/Makefile.in @@ -42,7 +42,8 @@ LPROGS= @E2INITRD_PROG@ TUNE2FS_OBJS= tune2fs.o util.o MKLPF_OBJS= mklost+found.o -MKE2FS_OBJS= mke2fs.o util.o profile.o prof_err.o default_profile.o +MKE2FS_OBJS= mke2fs.o util.o profile.o prof_err.o default_profile.o \ + mk_hugefiles.o CHATTR_OBJS= chattr.o LSATTR_OBJS= lsattr.o UUIDGEN_OBJS= uuidgen.o @@ -76,7 +77,7 @@ PROFILED_E2FREEFRAG_OBJS= profiled/e2freefrag.o PROFILED_E2UNDO_OBJS= profiled/e2undo.o PROFILED_E4DEFRAG_OBJS= profiled/e4defrag.o -SRCS= $(srcdir)/tune2fs.c $(srcdir)/mklost+found.c $(srcdir)/mke2fs.c \ +SRCS= $(srcdir)/tune2fs.c $(srcdir)/mklost+found.c $(srcdir)/mke2fs.c $(srcdir)/mk_hugefiles.c \ $(srcdir)/chattr.c $(srcdir)/lsattr.c $(srcdir)/dumpe2fs.c \ $(srcdir)/badblocks.c $(srcdir)/fsck.c $(srcdir)/util.c \ $(srcdir)/uuidgen.c $(srcdir)/blkid.c $(srcdir)/logsave.c \ @@ -647,7 +648,18 @@ mke2fs.o: $(srcdir)/mke2fs.c $(top_builddir)/lib/config.h \ $(srcdir)/util.h profile.h prof_err.h $(top_srcdir)/version.h \ $(srcdir)/nls-enable.h $(top_srcdir)/lib/quota/mkquota.h \ $(top_srcdir)/lib/quota/quotaio.h $(top_srcdir)/lib/quota/dqblk_v2.h \ - $(top_srcdir)/lib/quota/quotaio_tree.h $(top_srcdir)/lib/../e2fsck/dict.h + $(top_srcdir)/lib/quota/quotaio_tree.h $(top_srcdir)/lib/../e2fsck/dict.h \ + $(srcdir)/mke2fs.h +mk_hugefiles.o: $(srcdir)/mk_hugefiles.c $(top_builddir)/lib/config.h \ + $(top_builddir)/lib/dirpaths.h $(top_srcdir)/lib/ext2fs/ext2_fs.h \ + $(top_builddir)/lib/ext2fs/ext2_types.h $(top_srcdir)/lib/ext2fs/ext2fsP.h \ + $(top_srcdir)/lib/ext2fs/ext2fs.h $(top_srcdir)/lib/ext2fs/ext3_extents.h \ + $(top_srcdir)/lib/et/com_err.h $(top_srcdir)/lib/ext2fs/ext2_io.h \ + $(top_builddir)/lib/ext2fs/ext2_err.h \ + $(top_srcdir)/lib/ext2fs/ext2_ext_attr.h $(top_srcdir)/lib/ext2fs/bitops.h \ + $(top_srcdir)/lib/e2p/e2p.h $(top_srcdir)/lib/ext2fs/ext2fs.h \ + $(srcdir)/util.h profile.h prof_err.h $(srcdir)/nls-enable.h \ + $(srcdir)/mke2fs.h chattr.o: $(srcdir)/chattr.c $(top_builddir)/lib/config.h \ $(top_builddir)/lib/dirpaths.h $(top_srcdir)/lib/ext2fs/ext2_fs.h \ $(top_builddir)/lib/ext2fs/ext2_types.h $(top_srcdir)/lib/et/com_err.h \ diff --git a/misc/mk_hugefiles.c b/misc/mk_hugefiles.c new file mode 100644 index 00000000..12420af7 --- /dev/null +++ b/misc/mk_hugefiles.c @@ -0,0 +1,406 @@ +/* + * mk_hugefiles.c -- create huge files + */ + +#define _XOPEN_SOURCE 600 /* for inclusion of PATH_MAX in Solaris */ + +#include "config.h" +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif +#ifdef HAVE_GETOPT_H +#include +#else +extern char *optarg; +extern int optind; +#endif +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_STDLIB_H +#include +#endif +#ifdef HAVE_ERRNO_H +#include +#endif +#include +#include +#include +#include +#include +#include + +#include "ext2fs/ext2_fs.h" +#include "ext2fs/ext2fsP.h" +#include "et/com_err.h" +#include "uuid/uuid.h" +#include "e2p/e2p.h" +#include "ext2fs/ext2fs.h" +#include "util.h" +#include "profile.h" +#include "prof_err.h" +#include "nls-enable.h" +#include "mke2fs.h" + +static int uid; +static int gid; +static blk64_t num_blocks; +static blk64_t num_slack; +static unsigned long num_files; +static blk64_t goal; +static char *fn_prefix; +static int idx_digits; +static char *fn_buf; +static char *fn_numbuf; +int zero_hugefile = 1; + +static errcode_t create_directory(ext2_filsys fs, char *dir, + ext2_ino_t *ret_ino) + +{ + struct ext2_inode inode; + ext2_ino_t ino = EXT2_ROOT_INO; + ext2_ino_t newdir; + errcode_t retval = 0; + char *fn, *cp, *next; + + fn = malloc(strlen(dir) + 1); + if (fn == NULL) + return ENOMEM; + + strcpy(fn, dir); + cp = fn; + while(1) { + next = strchr(cp, '/'); + if (next) + *next++ = 0; + if (*cp) { + retval = ext2fs_new_inode(fs, ino, LINUX_S_IFDIR, + NULL, &newdir); + if (retval) + goto errout; + + retval = ext2fs_mkdir(fs, ino, newdir, cp); + if (retval) + goto errout; + + ino = newdir; + retval = ext2fs_read_inode(fs, ino, &inode); + if (retval) + goto errout; + + inode.i_uid = uid & 0xFFFF; + ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff); + inode.i_gid = gid & 0xFFFF; + ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff); + retval = ext2fs_write_inode(fs, ino, &inode); + if (retval) + goto errout; + } + if (next == NULL || *next == '\0') + break; + cp = next; + } +errout: + free(fn); + if (retval == 0) + *ret_ino = ino; + return retval; +} + +static errcode_t mk_hugefile(ext2_filsys fs, blk64_t num, + ext2_ino_t dir, unsigned long idx, ext2_ino_t *ino) + +{ + errcode_t retval; + blk64_t lblk, bend; + __u64 size; + blk64_t left; + blk64_t count = 0; + struct ext2_inode inode; + ext2_extent_handle_t handle; + + retval = ext2fs_new_inode(fs, 0, LINUX_S_IFREG, NULL, ino); + if (retval) + return retval; + + memset(&inode, 0, sizeof(struct ext2_inode)); + inode.i_mode = LINUX_S_IFREG | (0666 & ~fs->umask); + inode.i_links_count = 1; + inode.i_uid = uid & 0xFFFF; + ext2fs_set_i_uid_high(inode, (uid >> 16) & 0xffff); + inode.i_gid = gid & 0xFFFF; + ext2fs_set_i_gid_high(inode, (gid >> 16) & 0xffff); + + retval = ext2fs_write_new_inode(fs, *ino, &inode); + if (retval) + return retval; + + ext2fs_inode_alloc_stats2(fs, *ino, +1, 0); + + retval = ext2fs_extent_open2(fs, *ino, &inode, &handle); + if (retval) + return retval; + + lblk = 0; + left = num ? num : 1; + while (left) { + blk64_t pblk, end; + blk64_t n = left; + + retval = ext2fs_find_first_zero_block_bitmap2(fs->block_map, + goal, ext2fs_blocks_count(fs->super) - 1, &end); + if (retval) + goto errout; + goal = end; + + retval = ext2fs_find_first_set_block_bitmap2(fs->block_map, goal, + ext2fs_blocks_count(fs->super) - 1, &bend); + if (retval == ENOENT) { + bend = ext2fs_blocks_count(fs->super); + if (num == 0) + left = 0; + } + if (!num || bend - goal < left) + n = bend - goal; + pblk = goal; + if (num) + left -= n; + goal += n; + count += n; + ext2fs_block_alloc_stats_range(fs, pblk, n, +1); + + if (zero_hugefile) { + blk64_t ret_blk; + retval = ext2fs_zero_blocks2(fs, pblk, n, + &ret_blk, NULL); + + if (retval) + com_err(program_name, retval, + _("while zeroing block %llu " + "for hugefile"), ret_blk); + } + + while (n) { + blk64_t l = n; + struct ext2fs_extent newextent; + + if (l > EXT_INIT_MAX_LEN) + l = EXT_INIT_MAX_LEN; + + newextent.e_len = l; + newextent.e_pblk = pblk; + newextent.e_lblk = lblk; + newextent.e_flags = 0; + + retval = ext2fs_extent_insert(handle, + EXT2_EXTENT_INSERT_AFTER, &newextent); + if (retval) + return retval; + pblk += l; + lblk += l; + n -= l; + } + } + + retval = ext2fs_read_inode(fs, *ino, &inode); + if (retval) + goto errout; + + retval = ext2fs_iblk_add_blocks(fs, &inode, + count / EXT2FS_CLUSTER_RATIO(fs)); + if (retval) + goto errout; + size = (__u64) count * fs->blocksize; + inode.i_size = size & 0xffffffff; + inode.i_size_high = (size >> 32); + + retval = ext2fs_write_new_inode(fs, *ino, &inode); + if (retval) + goto errout; + + if (idx_digits) + sprintf(fn_numbuf, "%0*lu", idx_digits, idx); + else if (num_files > 1) + sprintf(fn_numbuf, "%lu", idx); + +retry: + retval = ext2fs_link(fs, dir, fn_buf, *ino, EXT2_FT_REG_FILE); + if (retval == EXT2_ET_DIR_NO_SPACE) { + retval = ext2fs_expand_dir(fs, dir); + if (retval) + goto errout; + goto retry; + } + + if (retval) + goto errout; + +errout: + if (handle) + ext2fs_extent_free(handle); + + return retval; +} + +static blk64_t calc_overhead(ext2_filsys fs, blk64_t num) +{ + blk64_t e_blocks, e_blocks2, e_blocks3, e_blocks4; + int extents_per_block; + int extents = (num + EXT_INIT_MAX_LEN - 1) / EXT_INIT_MAX_LEN; + + if (extents <= 4) + return 0; + + /* + * This calculation is due to the fact that we are inefficient + * in how handle extent splits when appending to the end of + * the extent tree. Sigh. We should fix this so that we can + * actually store 340 extents per 4k block, instead of only 170. + */ + extents_per_block = ((fs->blocksize - + sizeof(struct ext3_extent_header)) / + sizeof(struct ext3_extent)); + extents_per_block = (extents_per_block/ 2) - 1; + + e_blocks = (extents + extents_per_block - 1) / extents_per_block; + e_blocks2 = (e_blocks + extents_per_block - 1) / extents_per_block; + e_blocks3 = (e_blocks2 + extents_per_block - 1) / extents_per_block; + e_blocks4 = (e_blocks3 + extents_per_block - 1) / extents_per_block; + return e_blocks + e_blocks2 + e_blocks3 + e_blocks4; +} + +/* + * Find the place where we should start allocating blocks for the huge + * files. Leave free blocks at the beginning of the file + * system for things like metadata blocks. + */ +static blk64_t get_start_block(ext2_filsys fs, blk64_t slack) +{ + errcode_t retval; + blk64_t blk = fs->super->s_first_data_block, next; + blk64_t last_blk = ext2fs_blocks_count(fs->super) - 1; + + while (slack) { + retval = ext2fs_find_first_zero_block_bitmap2(fs->block_map, + blk, last_blk, &blk); + if (retval) + break; + + retval = ext2fs_find_first_set_block_bitmap2(fs->block_map, + blk, last_blk, &next); + if (retval) + next = last_blk; + next--; + + if (next - blk > slack) { + blk += slack; + break; + } + + slack -= (next - blk); + blk = next; + } + return blk; +} + +errcode_t mk_hugefiles(ext2_filsys fs) +{ + unsigned long i; + ext2_ino_t dir; + errcode_t retval; + blk64_t fs_blocks; + int d, dsize; + char *t; + + if (!get_bool_from_profile(fs_types, "make_hugefiles", 0)) + return 0; + + uid = get_int_from_profile(fs_types, "hugefiles_uid", 0); + gid = get_int_from_profile(fs_types, "hugefiles_gid", 0); + fs->umask = get_int_from_profile(fs_types, "hugefiles_umask", 077); + num_files = get_int_from_profile(fs_types, "num_hugefiles", 0); + t = get_string_from_profile(fs_types, "hugefiles_slack", "1M"); + num_slack = parse_num_blocks2(t, fs->super->s_log_block_size); + free(t); + t = get_string_from_profile(fs_types, "hugefiles_size", "0"); + num_blocks = parse_num_blocks2(t, fs->super->s_log_block_size); + free(t); + zero_hugefile = get_int_from_profile(fs_types, "zero_hugefiles", + zero_hugefile); + + t = get_string_from_profile(fs_types, "hugefiles_dir", "/"); + retval = create_directory(fs, t, &dir); + free(t); + if (retval) + return retval; + + fn_prefix = get_string_from_profile(fs_types, "hugefiles_name", + "hugefile"); + idx_digits = get_int_from_profile(fs_types, "hugefiles_digits", 5); + d = int_log10(num_files) + 1; + if (idx_digits > d) + d = idx_digits; + dsize = strlen(fn_prefix) + d + 16; + fn_buf = malloc(dsize); + if (!fn_buf) { + free(fn_prefix); + return ENOMEM; + } + strcpy(fn_buf, fn_prefix); + fn_numbuf = fn_buf + strlen(fn_prefix); + free(fn_prefix); + + fs_blocks = ext2fs_free_blocks_count(fs->super) -num_slack; + if (num_blocks == 0 && num_files == 0) + num_files = 1; + + if (num_files == 0 && num_blocks) { + num_files = fs_blocks / num_blocks; + fs_blocks -= (num_files / 16) + 1; + fs_blocks -= calc_overhead(fs, num_blocks) * num_files; + num_files = fs_blocks / num_blocks; + } + + if (num_blocks == 0 && num_files > 1) { + num_blocks = fs_blocks / num_files; + fs_blocks -= (num_files / 16) + 1; + fs_blocks -= calc_overhead(fs, num_blocks) * num_files; + num_blocks = fs_blocks / num_files; + } + + num_slack += calc_overhead(fs, num_blocks) * num_files; + num_slack += (num_files / 16) + 1; /* space for dir entries */ + goal = get_start_block(fs, num_slack); + + if (!quiet) { + if (zero_hugefile && verbose) + printf(_("Huge files will be zero'ed\n")); + printf(_("Creating %lu huge file(s) "), num_files); + if (num_blocks) + printf(_("with %llu blocks each"), num_blocks); + fputs(": ", stdout); + } + for (i=0; i < num_files; i++) { + ext2_ino_t ino; + + retval = mk_hugefile(fs, num_blocks, dir, i, &ino); + if (retval) { + com_err(program_name, retval, + _("while creating huge file %lu"), i); + goto errout; + } + } + if (!quiet) + fputs(_("done\n"), stdout); + +errout: + free(fn_buf); + return retval; +} diff --git a/misc/mke2fs.c b/misc/mke2fs.c index c3783baa..0ed02a55 100644 --- a/misc/mke2fs.c +++ b/misc/mke2fs.c @@ -62,6 +62,7 @@ extern int optind; #include "../version.h" #include "nls-enable.h" #include "quota/mkquota.h" +#include "mke2fs.h" #define STRIDE_LENGTH 8 @@ -76,13 +77,13 @@ extern int optind; extern int isatty(int); extern FILE *fpopen(const char *cmd, const char *mode); -static const char * program_name = "mke2fs"; +const char * program_name = "mke2fs"; static const char * device_name /* = NULL */; /* Command line options */ static int cflag; -static int verbose; -static int quiet; +int verbose; +int quiet; static int super_only; static int discard = 1; /* attempt to discard device before fs creation */ static int direct_io; @@ -108,7 +109,7 @@ static char *volume_label; static char *mount_dir; char *journal_device; static int sync_kludge; /* Set using the MKE2FS_SYNC env. option */ -static char **fs_types; +char **fs_types; static profile_t profile; @@ -143,7 +144,7 @@ static int int_log2(unsigned long long arg) return l; } -static int int_log10(unsigned long long arg) +int int_log10(unsigned long long arg) { int l; @@ -1243,7 +1244,7 @@ static char **parse_fs_type(const char *fs_type, return (list.list); } -static char *get_string_from_profile(char **types, const char *opt, +char *get_string_from_profile(char **types, const char *opt, const char *def_val) { char *ret = 0; @@ -1260,7 +1261,7 @@ static char *get_string_from_profile(char **types, const char *opt, return (ret); } -static int get_int_from_profile(char **types, const char *opt, int def_val) +int get_int_from_profile(char **types, const char *opt, int def_val) { int ret; char **cpp; @@ -1283,7 +1284,7 @@ static double get_double_from_profile(char **types, const char *opt, return ret; } -static int get_bool_from_profile(char **types, const char *opt, int def_val) +int get_bool_from_profile(char **types, const char *opt, int def_val) { int ret; char **cpp; @@ -2547,6 +2548,7 @@ int main (int argc, char *argv[]) "0s - skipping inode table wipe\n")); lazy_itable_init = 1; itable_zeroed = 1; + zero_hugefile = 0; } } @@ -2847,6 +2849,10 @@ no_journal: EXT4_FEATURE_RO_COMPAT_QUOTA)) create_quota_inodes(fs); + retval = mk_hugefiles(fs); + if (retval) + com_err(program_name, retval, "while creating huge files"); + if (!quiet) printf("%s", _("Writing superblocks and " "filesystem accounting information: ")); diff --git a/misc/mke2fs.conf.5.in b/misc/mke2fs.conf.5.in index f29e8544..39a1a403 100644 --- a/misc/mke2fs.conf.5.in +++ b/misc/mke2fs.conf.5.in @@ -420,6 +420,65 @@ system feature is enabled. It can be overridden via the .B \-C command line option to .BR mke2fs (8) +.TP +.I make_hugefiles +This boolean relation enables the creation of pre-allocated files as +part of formatting the file system. +.TP +.I hugefiles_uid +This relation controls the user ownership for all of the files and +directories created by the +.I make_hugefiles +feature. +.TP +.I hugefiles_gid +This relation controls the group ownership for all of the files and +directories created by the +.I make_hugefiles +feature. +.TP +.I hugefiles_umask +This relation specifies the umask used when creating the files and +directories by the +.I make_hugefiles +feature. +.TP +.I num_hugefiles +This relation specifies the number of huge files to be created. If this +relation is not specified, or is set to zero, and the +.I hugefiles_size +relation is non-zero, then +.I make_hugefiles +will create as many huge files as can fit to fill the entire file system. +.TP +.I hugefiles_slack +This relation specifies how much space should be reserved for other +files. +.TP +.I hugefiles_size +This relation specifies the size of the huge files. If this relation is +not specified, the default is to fill th efile system. +.TP +.I hugefiles_name +This relation specifies the base file name for the huge files. +.TP +.I hugefiles_digits +This relation specifies the (zero-padded) width of the field for the +huge file number. +.TP +.I zero_hugefiles +This boolean relation specifies whether or not zero blocks will be +written to the hugefiles while +.BR mke2fs(8) +is creating them. By default, zero blocks will be written to the huge +files to avoid stale data from being made available to potentially +untrusted user programs, unless the device supports a discard/trim +operation which will take care of zeroing the device blocks. By +.I zero_hugefiles +to false, this step will always be skipped, which can be useful if it is +known that the disk has been previously erased, or if the user programs +that will have access to the huge files are trusted to not reveal stale +data. .SH THE [devices] STANZA Each tag in the .I [devices] diff --git a/misc/mke2fs.h b/misc/mke2fs.h new file mode 100644 index 00000000..9fa6bfe1 --- /dev/null +++ b/misc/mke2fs.h @@ -0,0 +1,30 @@ +/* + * mke2fs.h + * + * Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, + * 2003, 2004, 2005 by Theodore Ts'o. + * + * %Begin-Header% + * This file may be redistributed under the terms of the GNU Public + * License. + * %End-Header% + */ + +/* mke2fs.c */ +extern const char * program_name; +extern int quiet; +extern int verbose; +extern int zero_hugefile; +extern char **fs_types; + +extern char *get_string_from_profile(char **types, const char *opt, + const char *def_val); +extern int get_int_from_profile(char **types, const char *opt, int def_val); +extern int get_bool_from_profile(char **types, const char *opt, int def_val); +extern int int_log10(unsigned long long arg); + +/* mk_hugefiles.c */ +extern errcode_t mk_hugefiles(ext2_filsys fs); + + +