From 9f543c9ca17699f9b4be343dfc558258907d60cb Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 4 Jan 2014 20:29:32 +0000 Subject: [PATCH] Add bmove.c as a basis for block moving during inode table grow --- Makefile | 4 +- bmove.c | 166 +++++++++++++++++++++++++++++++++++++++++++++++ ext2fsP.h | 146 +++++++++++++++++++++++++++++++++++++++++ realloc-inodes.c | 93 +++++++++++++++++++++++--- 4 files changed, 397 insertions(+), 12 deletions(-) create mode 100644 bmove.c create mode 100644 ext2fsP.h diff --git a/Makefile b/Makefile index 8df5f11..409273e 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,3 @@ all: realloc-inodes -realloc-inodes: realloc-inodes.c - gcc -Wall -o realloc-inodes -lcom_err -lext2fs realloc-inodes.c +realloc-inodes: realloc-inodes.c bmove.c ext2fsP.h + gcc -Wall -o realloc-inodes -lcom_err -lext2fs realloc-inodes.c bmove.c diff --git a/bmove.c b/bmove.c new file mode 100644 index 0000000..a1186bb --- /dev/null +++ b/bmove.c @@ -0,0 +1,166 @@ +/* + * bmove.c --- Move blocks around to make way for a particular + * filesystem structure. + * + * Copyright (C) 1997 Theodore Ts'o. + * + * %Begin-Header% + * This file may be redistributed under the terms of the GNU Library + * General Public License, version 2. + * %End-Header% + */ + +#include +#include +#if HAVE_UNISTD_H +#include +#endif +#if HAVE_SYS_TYPES_H +#include +#endif +#if HAVE_SYS_TIME_H +#include +#endif + +#include +#include "ext2fsP.h" + +struct process_block_struct { + ext2_ino_t ino; + struct ext2_inode * inode; + ext2fs_block_bitmap reserve; + ext2fs_block_bitmap alloc_map; + errcode_t error; + char *buf; + int add_dir; + int flags; +}; + +static int process_block(ext2_filsys fs, blk64_t *block_nr, + e2_blkcnt_t blockcnt, blk64_t ref_block, + int ref_offset, void *priv_data) +{ + struct process_block_struct *pb; + errcode_t retval; + int ret; + blk64_t block, orig; + + pb = (struct process_block_struct *) priv_data; + block = orig = *block_nr; + ret = 0; + + /* + * Let's see if this is one which we need to relocate + */ + if (ext2fs_test_block_bitmap2(pb->reserve, block)) { + do { + if (++block >= ext2fs_blocks_count(fs->super)) + block = fs->super->s_first_data_block; + if (block == orig) { + pb->error = EXT2_ET_BLOCK_ALLOC_FAIL; + return BLOCK_ABORT; + } + } while (ext2fs_test_block_bitmap2(pb->reserve, block) || + ext2fs_test_block_bitmap2(pb->alloc_map, block)); + + retval = io_channel_read_blk64(fs->io, orig, 1, pb->buf); + if (retval) { + pb->error = retval; + return BLOCK_ABORT; + } + retval = io_channel_write_blk64(fs->io, block, 1, pb->buf); + if (retval) { + pb->error = retval; + return BLOCK_ABORT; + } + *block_nr = block; + ext2fs_mark_block_bitmap2(pb->alloc_map, block); + ret = BLOCK_CHANGED; + if (pb->flags & EXT2_BMOVE_DEBUG) + printf("ino=%u, blockcnt=%lld, %llu->%llu\n", + (unsigned) pb->ino, blockcnt, + (unsigned long long) orig, + (unsigned long long) block); + } + if (pb->add_dir) { + retval = ext2fs_add_dir_block2(fs->dblist, pb->ino, + block, blockcnt); + if (retval) { + pb->error = retval; + ret |= BLOCK_ABORT; + } + } + return ret; +} + +errcode_t ext2fs_move_blocks(ext2_filsys fs, + ext2fs_block_bitmap reserve, + ext2fs_block_bitmap alloc_map, + int flags) +{ + ext2_ino_t ino; + struct ext2_inode inode; + errcode_t retval; + struct process_block_struct pb; + ext2_inode_scan scan; + char *block_buf; + + retval = ext2fs_open_inode_scan(fs, 0, &scan); + if (retval) + return retval; + + pb.reserve = reserve; + pb.error = 0; + pb.alloc_map = alloc_map ? alloc_map : fs->block_map; + pb.flags = flags; + + retval = ext2fs_get_array(4, fs->blocksize, &block_buf); + if (retval) + return retval; + pb.buf = block_buf + fs->blocksize * 3; + + /* + * If GET_DBLIST is set in the flags field, then we should + * gather directory block information while we're doing the + * block move. + */ + if (flags & EXT2_BMOVE_GET_DBLIST) { + if (fs->dblist) { + ext2fs_free_dblist(fs->dblist); + fs->dblist = NULL; + } + retval = ext2fs_init_dblist(fs, 0); + if (retval) + return retval; + } + + retval = ext2fs_get_next_inode(scan, &ino, &inode); + if (retval) + return retval; + + while (ino) { + if ((inode.i_links_count == 0) || + !ext2fs_inode_has_valid_blocks2(fs, &inode)) + goto next; + + pb.ino = ino; + pb.inode = &inode; + + pb.add_dir = (LINUX_S_ISDIR(inode.i_mode) && + flags & EXT2_BMOVE_GET_DBLIST); + + retval = ext2fs_block_iterate3(fs, ino, 0, block_buf, + process_block, &pb); + if (retval) + return retval; + if (pb.error) + return pb.error; + + next: + retval = ext2fs_get_next_inode(scan, &ino, &inode); + if (retval == EXT2_ET_BAD_BLOCK_IN_INODE_TABLE) + goto next; + } + return 0; +} + diff --git a/ext2fsP.h b/ext2fsP.h new file mode 100644 index 0000000..96f01af --- /dev/null +++ b/ext2fsP.h @@ -0,0 +1,146 @@ +/* + * ext2fsP.h --- private header file for ext2 library + * + * Copyright (C) 1997 Theodore Ts'o. + * + * %Begin-Header% + * This file may be redistributed under the terms of the GNU Library + * General Public License, version 2. + * %End-Header% + */ + +#include + +#define EXT2FS_MAX_NESTED_LINKS 8 + +/* + * Badblocks list + */ +struct ext2_struct_u32_list { + int magic; + int num; + int size; + __u32 *list; + int badblocks_flags; +}; + +struct ext2_struct_u32_iterate { + int magic; + ext2_u32_list bb; + int ptr; +}; + + +/* + * Directory block iterator definition + */ +struct ext2_struct_dblist { + int magic; + ext2_filsys fs; + unsigned long long size; + unsigned long long count; + int sorted; + struct ext2_db_entry2 * list; +}; + +/* + * For directory iterators + */ +struct dir_context { + ext2_ino_t dir; + int flags; + char *buf; + int (*func)(ext2_ino_t dir, + int entry, + struct ext2_dir_entry *dirent, + int offset, + int blocksize, + char *buf, + void *priv_data); + void *priv_data; + errcode_t errcode; +}; + +/* + * Inode cache structure + */ +struct ext2_inode_cache { + void * buffer; + blk_t buffer_blk; + int cache_last; + int cache_size; + int refcount; + struct ext2_inode_cache_ent *cache; +}; + +struct ext2_inode_cache_ent { + ext2_ino_t ino; + struct ext2_inode inode; +}; + +/* Function prototypes */ + +extern int ext2fs_process_dir_block(ext2_filsys fs, + blk64_t *blocknr, + e2_blkcnt_t blockcnt, + blk64_t ref_block, + int ref_offset, + void *priv_data); + +/* Generic numeric progress meter */ + +struct ext2fs_numeric_progress_struct { + __u64 max; + int log_max; + int skip_progress; +}; + +extern void ext2fs_numeric_progress_init(ext2_filsys fs, + struct ext2fs_numeric_progress_struct * progress, + const char *label, __u64 max); +extern void ext2fs_numeric_progress_update(ext2_filsys fs, + struct ext2fs_numeric_progress_struct * progress, + __u64 val); +extern void ext2fs_numeric_progress_close(ext2_filsys fs, + struct ext2fs_numeric_progress_struct * progress, + const char *message); + +/* + * 64-bit bitmap support + */ + +extern errcode_t ext2fs_alloc_generic_bmap(ext2_filsys fs, errcode_t magic, + int type, __u64 start, __u64 end, + __u64 real_end, + const char * description, + ext2fs_generic_bitmap *bmap); + +extern void ext2fs_free_generic_bmap(ext2fs_generic_bitmap bmap); + +extern errcode_t ext2fs_copy_generic_bmap(ext2fs_generic_bitmap src, + ext2fs_generic_bitmap *dest); + +extern errcode_t ext2fs_resize_generic_bmap(ext2fs_generic_bitmap bmap, + __u64 new_end, + __u64 new_real_end); +extern errcode_t ext2fs_fudge_generic_bmap_end(ext2fs_generic_bitmap bitmap, + errcode_t neq, + __u64 end, __u64 *oend); +extern int ext2fs_mark_generic_bmap(ext2fs_generic_bitmap bitmap, + __u64 arg); +extern int ext2fs_unmark_generic_bmap(ext2fs_generic_bitmap bitmap, + __u64 arg); +extern int ext2fs_test_generic_bmap(ext2fs_generic_bitmap bitmap, + __u64 arg); +extern errcode_t ext2fs_set_generic_bmap_range(ext2fs_generic_bitmap bitmap, + __u64 start, unsigned int num, + void *in); +extern errcode_t ext2fs_get_generic_bmap_range(ext2fs_generic_bitmap bitmap, + __u64 start, unsigned int num, + void *out); +extern void ext2fs_warn_bitmap32(ext2fs_generic_bitmap bitmap,const char *func); + +extern int ext2fs_mem_is_zero(const char *mem, size_t len); + +#define EXT2_BMOVE_GET_DBLIST 0x0001 +#define EXT2_BMOVE_DEBUG 0x0002 diff --git a/realloc-inodes.c b/realloc-inodes.c index b3a5833..7bfec4b 100644 --- a/realloc-inodes.c +++ b/realloc-inodes.c @@ -8,15 +8,13 @@ * 1) If shrinking - move inodes away from the end of each block group inode table * 1.1) move each inode to the new place, mark new place as occupied, unmark old one * 1.2) remember the old->new inode number mapping - * 2) If growing - move data away from extra blocks needed by growing inode tables - * 2.1) Iterate through all block groups and remember extra blocks needed for - * inode tables that are occupied. + * 2) If growing - move data away from extra blocks needed by growing inode tables: + * 2.1) Create a map of blocks that we want to free * 2.2) Iterate through all inodes and move remembered blocks. * It involves overwriting the whole file extent tree or block mapping... * If some of these blocks are in the bad block inode, we should either * abort the reallocation process, or move inode tables to another location * in a block group, possibly first defragmenting it... :-( - * 2.3) While copying, remember block number mapping and mark/unmark new/old blocks. * 3) Change all inode numbers in directory entries according to mappings from (1.2), * and then using a formula: new_num = 1 + ((old_num-1)/old_i_per_g)*new_i_per_g + ((old_num-1) % old_i_per_g) * 4) Move parts of inode tables so they are consecutive again if flex_bg feature is active @@ -45,6 +43,11 @@ #define _(a) (a) +errcode_t ext2fs_move_blocks(ext2_filsys fs, + ext2fs_block_bitmap reserve, + ext2fs_block_bitmap alloc_map, + int flags); + // "local data" for the inode reallocation process typedef struct { @@ -280,9 +283,75 @@ out: */ int extend_move_blocks(realloc_data *rd) { - // we'll probably need ext2fs_block_alloc_stats2(fs, new_blk, +1); - - return ENOSYS; + ext2fs_block_bitmap reserve_map; + blk64_t it_start, blk_diff, b_per_g; + dgrp_t grp, n_flex, n_grp, flex_count; + int retval, flexbg_size; + if (rd->new_inode_blocks_per_group == rd->fs->inode_blocks_per_group) + { + return 0; + } + blk_diff = rd->new_inode_blocks_per_group-rd->fs->inode_blocks_per_group; + b_per_g = EXT2_BLOCKS_PER_GROUP(rd->fs->super); + retval = ext2fs_allocate_block_bitmap(rd->fs, "reserved block map", &reserve_map); + if (retval) + { + return retval; + } + if (!rd->fs->block_map) + { + ext2fs_read_block_bitmap(rd->fs); + } + // Mark reserved blocks (those we want to free) + if (EXT2_HAS_INCOMPAT_FEATURE(rd->fs->super, EXT4_FEATURE_INCOMPAT_FLEX_BG) + && rd->fs->super->s_log_groups_per_flex) + { + // flex_bg + flexbg_size = 1 << rd->fs->super->s_log_groups_per_flex; + flex_count = (rd->fs->group_desc_count + flexbg_size - 1) / flexbg_size; + for (n_flex = 0; n_flex < flex_count; n_flex++) + { + n_grp = flexbg_size; + if (n_flex*flexbg_size+n_grp > rd->fs->group_desc_count) + { + n_grp = rd->fs->group_desc_count-n_flex*flexbg_size; + } + it_start = ext2fs_inode_table_loc(rd->fs, n_flex*flexbg_size); + // Check group boundaries + if ((it_start + rd->new_inode_blocks_per_group*n_grp - 1) / b_per_g + != (it_start + rd->fs->inode_blocks_per_group*n_grp - 1) / b_per_g) + { + retval = ENOSPC; + goto out; + } + it_start += rd->fs->inode_blocks_per_group*n_grp; + ext2fs_mark_block_bitmap_range2(reserve_map, it_start, blk_diff*n_grp); + } + } + else + { + // No flex_bg + for (grp = 0; grp < rd->fs->group_desc_count; grp++) + { + it_start = ext2fs_inode_table_loc(rd->fs, grp); + // Check group boundaries + if ((it_start + rd->new_inode_blocks_per_group - 1) / b_per_g + != (it_start + rd->fs->inode_blocks_per_group - 1) / b_per_g) + { + retval = ENOSPC; + goto out; + } + it_start += rd->fs->inode_blocks_per_group; + ext2fs_mark_block_bitmap_range2(reserve_map, it_start, rd->new_inode_blocks_per_group-rd->fs->inode_blocks_per_group); + } + } + // TODO Check the bad block inode + retval = ext2fs_move_blocks(rd->fs, reserve_map, rd->fs->block_map, 2); + ext2fs_mark_bb_dirty(rd->fs); + ext2fs_flush(rd->fs); +out: + ext2fs_free_block_bitmap(reserve_map); + return retval; } static int change_inode_numbers_callback(ext2_ino_t dir, int entry, @@ -327,13 +396,16 @@ int change_inode_numbers(realloc_data *rd) int change_super_and_bgd(realloc_data *rd) { blk64_t it_start, blk; - dgrp_t grp, n_flex, n_grp; + dgrp_t grp, n_flex, n_grp, flex_count; __u32 unus; __u32 i_per_g_diff = rd->new_inodes_per_group - EXT2_INODES_PER_GROUP(rd->fs->super); int flexbg_size = 0, i, retval = 0; void *buf = NULL; ext2fs_flush(rd->fs); - ext2fs_read_block_bitmap(rd->fs); + if (!rd->fs->block_map) + { + ext2fs_read_block_bitmap(rd->fs); + } if (rd->new_inode_blocks_per_group != rd->fs->inode_blocks_per_group) { // Move inode tables if flex_bg is active @@ -341,12 +413,13 @@ int change_super_and_bgd(realloc_data *rd) && rd->fs->super->s_log_groups_per_flex) { flexbg_size = 1 << rd->fs->super->s_log_groups_per_flex; + flex_count = (rd->fs->group_desc_count + flexbg_size - 1) / flexbg_size; retval = ext2fs_get_mem(EXT2_BLOCK_SIZE(rd->fs->super) * rd->new_inode_blocks_per_group * flexbg_size, &buf); if (retval) { goto out; } - for (n_flex = 0; n_flex < rd->fs->group_desc_count/flexbg_size; n_flex++) + for (n_flex = 0; n_flex < flex_count; n_flex++) { n_grp = flexbg_size; if (n_flex*flexbg_size+n_grp > rd->fs->group_desc_count)