diff --git a/e2fsck/ChangeLog b/e2fsck/ChangeLog index ec3980c0..d804721b 100644 --- a/e2fsck/ChangeLog +++ b/e2fsck/ChangeLog @@ -1,5 +1,12 @@ 2007-04-06 Theodore Tso + * dirinfo.c: If e2fsck.conf configures a scratch_files directory + which is available, and the number of directories exceeds + scratch_files.numdirs_threshold, then try to use the tdb + library to store the directory information abstraction. + This allows us to check very large filesystems without + needing as much physical memory. + * e2fsck.h, pass1.c, pass2.c: If e2fsck.conf configures a scratch_files directory which is available, and the number of directories exceeds scratch_files.numdirs_threshold, diff --git a/e2fsck/dirinfo.c b/e2fsck/dirinfo.c index 4e660758..e0232896 100644 --- a/e2fsck/dirinfo.c +++ b/e2fsck/dirinfo.c @@ -8,19 +8,63 @@ #undef DIRINFO_DEBUG #include "e2fsck.h" +#include +#include + +#include + struct dir_info_db { int count; int size; struct dir_info *array; struct dir_info *last_lookup; + char *tdb_fn; + TDB_CONTEXT *tdb; }; struct dir_info_iter { int i; + TDB_DATA tdb_iter; }; +struct dir_info_ent { + ext2_ino_t dotdot; /* Parent according to '..' */ + ext2_ino_t parent; /* Parent according to treewalk */ +}; + + static void e2fsck_put_dir_info(e2fsck_t ctx, struct dir_info *dir); +static void setup_tdb(e2fsck_t ctx, ext2_ino_t num_dirs) +{ + struct dir_info_db *db = ctx->dir_info; + errcode_t retval; + char *tdb_dir, uuid[40]; + int fd, threshold, enable; + + profile_get_string(ctx->profile, "scratch_files", "directory", 0, 0, + &tdb_dir); + profile_get_integer(ctx->profile, "scratch_files", + "numdirs_threshold", 0, 0, &threshold); + profile_get_boolean(ctx->profile, "scratch_files", + "dirinfo", 0, 1, &enable); + + if (!enable || !tdb_dir || access(tdb_dir, W_OK) || + (threshold && num_dirs <= threshold)) + return; + + retval = ext2fs_get_mem(strlen(tdb_dir) + 64, &db->tdb_fn); + if (retval) + return; + + uuid_unparse(ctx->fs->super->s_uuid, uuid); + sprintf(db->tdb_fn, "%s/%s-dirinfo-XXXXXX", tdb_dir, uuid); + fd = mkstemp(db->tdb_fn); + db->tdb = tdb_open(db->tdb_fn, 0, TDB_CLEAR_IF_FIRST, + O_RDWR | O_CREAT | O_TRUNC, 0600); + close(fd); +} + void setup_db(e2fsck_t ctx) { struct dir_info_db *db; @@ -33,22 +77,32 @@ void setup_db(e2fsck_t ctx) db->count = db->size = 0; db->array = 0; + ctx->dir_info = db; + retval = ext2fs_get_num_dirs(ctx->fs, &num_dirs); if (retval) num_dirs = 1024; /* Guess */ + + setup_tdb(ctx, num_dirs); + + if (db->tdb) { +#ifdef DIRINFO_DEBUG + printf("Note: using tdb!\n"); +#endif + return; + } + db->size = num_dirs + 10; db->array = (struct dir_info *) - e2fsck_allocate_memory(ctx, db->size + e2fsck_allocate_memory(ctx, db->size * sizeof (struct dir_info), "directory map"); - - ctx->dir_info = db; } /* * This subroutine is called during pass1 to create a directory info * entry. During pass1, the passed-in parent is 0; it will get filled - * in during pass2. + * in during pass2. */ void e2fsck_add_dir_info(e2fsck_t ctx, ext2_ino_t ino, ext2_ino_t parent) { @@ -64,7 +118,7 @@ void e2fsck_add_dir_info(e2fsck_t ctx, ext2_ino_t ino, ext2_ino_t parent) if (!ctx->dir_info) setup_db(ctx); db = ctx->dir_info; - + if (ctx->dir_info->count >= ctx->dir_info->size) { old_size = ctx->dir_info->size * sizeof(struct dir_info); ctx->dir_info->size += 10; @@ -81,7 +135,10 @@ void e2fsck_add_dir_info(e2fsck_t ctx, ext2_ino_t ino, ext2_ino_t parent) ent.parent = parent; ent.dotdot = parent; - e2fsck_put_dir_info(ctx, &ent); + if (db->tdb) { + e2fsck_put_dir_info(ctx, &ent); + return; + } /* * Normally, add_dir_info is called with each inode in @@ -98,12 +155,12 @@ void e2fsck_add_dir_info(e2fsck_t ctx, ext2_ino_t ino, ext2_ino_t parent) if (ctx->dir_info->array[i-1].ino < ino) break; dir = &ctx->dir_info->array[i]; - if (dir->ino != ino) + if (dir->ino != ino) for (j = ctx->dir_info->count++; j > i; j--) ctx->dir_info->array[j] = ctx->dir_info->array[j-1]; } else dir = &ctx->dir_info->array[ctx->dir_info->count++]; - + dir->ino = ino; dir->dotdot = parent; dir->parent = parent; @@ -117,6 +174,8 @@ static struct dir_info *e2fsck_get_dir_info(e2fsck_t ctx, ext2_ino_t ino) { struct dir_info_db *db = ctx->dir_info; int low, high, mid, ret; + struct dir_info_ent *buf; + static struct dir_info ret_dir_info; if (!db) return 0; @@ -125,6 +184,30 @@ static struct dir_info *e2fsck_get_dir_info(e2fsck_t ctx, ext2_ino_t ino) printf("e2fsck_get_dir_info %d...", ino); #endif + if (db->tdb) { + TDB_DATA key, data; + + key.dptr = (unsigned char *) &ino; + key.dsize = sizeof(ext2_ino_t); + + data = tdb_fetch(db->tdb, key); + if (!data.dptr) { + if (tdb_error(db->tdb) != TDB_ERR_NOEXIST) + printf("fetch failed: %s\n", + tdb_errorstr(db->tdb)); + return 0; + } + + buf = (struct dir_info_ent *) data.dptr; + ret_dir_info.ino = ino; + ret_dir_info.dotdot = buf->dotdot; + ret_dir_info.parent = buf->parent; +#ifdef DIRINFO_DEBUG + printf("(%d,%d,%d)\n", ino, buf->dotdot, buf->parent); +#endif + return &ret_dir_info; + } + if (db->last_lookup && db->last_lookup->ino == ino) return db->last_lookup; @@ -132,16 +215,16 @@ static struct dir_info *e2fsck_get_dir_info(e2fsck_t ctx, ext2_ino_t ino) high = ctx->dir_info->count-1; if (ino == ctx->dir_info->array[low].ino) { #ifdef DIRINFO_DEBUG - printf("(%d,%d,%d)\n", ino, - ctx->dir_info->array[low].dotdot, + printf("(%d,%d,%d)\n", ino, + ctx->dir_info->array[low].dotdot, ctx->dir_info->array[low].parent); #endif return &ctx->dir_info->array[low]; } if (ino == ctx->dir_info->array[high].ino) { #ifdef DIRINFO_DEBUG - printf("(%d,%d,%d)\n", ino, - ctx->dir_info->array[high].dotdot, + printf("(%d,%d,%d)\n", ino, + ctx->dir_info->array[high].dotdot, ctx->dir_info->array[high].parent); #endif return &ctx->dir_info->array[high]; @@ -153,8 +236,8 @@ static struct dir_info *e2fsck_get_dir_info(e2fsck_t ctx, ext2_ino_t ino) break; if (ino == ctx->dir_info->array[mid].ino) { #ifdef DIRINFO_DEBUG - printf("(%d,%d,%d)\n", ino, - ctx->dir_info->array[mid].dotdot, + printf("(%d,%d,%d)\n", ino, + ctx->dir_info->array[mid].dotdot, ctx->dir_info->array[mid].parent); #endif return &ctx->dir_info->array[mid]; @@ -169,10 +252,29 @@ static struct dir_info *e2fsck_get_dir_info(e2fsck_t ctx, ext2_ino_t ino) static void e2fsck_put_dir_info(e2fsck_t ctx, struct dir_info *dir) { + struct dir_info_db *db = ctx->dir_info; + struct dir_info_ent buf; + TDB_DATA key, data; + #ifdef DIRINFO_DEBUG printf("e2fsck_put_dir_info (%d, %d, %d)...", dir->ino, dir->dotdot, dir->parent); #endif + + if (!db->tdb) + return; + + buf.parent = dir->parent; + buf.dotdot = dir->dotdot; + + key.dptr = (unsigned char *) &dir->ino; + key.dsize = sizeof(ext2_ino_t); + data.dptr = (unsigned char *) &buf; + data.dsize = sizeof(buf); + + if (tdb_store(db->tdb, key, data, TDB_REPLACE) == -1) { + printf("store failed: %s\n", tdb_errorstr(db->tdb)); + } return; } @@ -182,6 +284,12 @@ static void e2fsck_put_dir_info(e2fsck_t ctx, struct dir_info *dir) void e2fsck_free_dir_info(e2fsck_t ctx) { if (ctx->dir_info) { + if (ctx->dir_info->tdb) + tdb_close(ctx->dir_info->tdb); + if (ctx->dir_info->tdb_fn) { + unlink(ctx->dir_info->tdb_fn); + free(ctx->dir_info->tdb_fn); + } ctx->dir_info->size = 0; ctx->dir_info->count = 0; ext2fs_free_mem(&ctx->dir_info); @@ -206,10 +314,14 @@ extern struct dir_info_iter *e2fsck_dir_info_iter_begin(e2fsck_t ctx) iter = e2fsck_allocate_memory(ctx, sizeof(struct dir_info_iter), "dir_info iterator"); memset(iter, 0, sizeof(iter)); + + if (db->tdb) + iter->tdb_iter = tdb_firstkey(db->tdb); + return iter; } -extern void e2fsck_dir_info_iter_end(e2fsck_t ctx, +extern void e2fsck_dir_info_iter_end(e2fsck_t ctx, struct dir_info_iter *iter) { ext2fs_free_mem(&iter); @@ -220,23 +332,48 @@ extern void e2fsck_dir_info_iter_end(e2fsck_t ctx, */ struct dir_info *e2fsck_dir_info_iter(e2fsck_t ctx, struct dir_info_iter *iter) { - if (!ctx->dir_info || !iter || - (iter->i >= ctx->dir_info->count)) + TDB_DATA data; + struct dir_info_db *db = ctx->dir_info; + struct dir_info_ent *buf; + static struct dir_info ret_dir_info; + + if (!ctx->dir_info || !iter) return 0; - + + if (db->tdb) { + if (iter->tdb_iter.dptr == 0) + return 0; + data = tdb_fetch(db->tdb, iter->tdb_iter); + if (!data.dptr) { + printf("iter fetch failed: %s\n", + tdb_errorstr(db->tdb)); + return 0; + } + buf = (struct dir_info_ent *) data.dptr; + ret_dir_info.ino = *((ext2_ino_t *) iter->tdb_iter.dptr); + ret_dir_info.dotdot = buf->dotdot; + ret_dir_info.parent = buf->parent; + iter->tdb_iter = tdb_nextkey(db->tdb, iter->tdb_iter); + return &ret_dir_info; + } + + if (iter->i >= ctx->dir_info->count) + return 0; + #ifdef DIRINFO_DEBUG - printf("iter(%d, %d, %d)...", ctx->dir_info->array[iter->i].ino, - ctx->dir_info->array[iter->i].dotdot, + printf("iter(%d, %d, %d)...", ctx->dir_info->array[iter->i].ino, + ctx->dir_info->array[iter->i].dotdot, ctx->dir_info->array[iter->i].parent); #endif - return(ctx->dir_info->array + iter->i++); + ctx->dir_info->last_lookup = ctx->dir_info->array + iter->i++; + return(ctx->dir_info->last_lookup); } /* * This function only sets the parent pointer, and requires that * dirinfo structure has already been created. */ -int e2fsck_dir_info_set_parent(e2fsck_t ctx, ext2_ino_t ino, +int e2fsck_dir_info_set_parent(e2fsck_t ctx, ext2_ino_t ino, ext2_ino_t parent) { struct dir_info *p; @@ -253,7 +390,7 @@ int e2fsck_dir_info_set_parent(e2fsck_t ctx, ext2_ino_t ino, * This function only sets the dot dot pointer, and requires that * dirinfo structure has already been created. */ -int e2fsck_dir_info_set_dotdot(e2fsck_t ctx, ext2_ino_t ino, +int e2fsck_dir_info_set_dotdot(e2fsck_t ctx, ext2_ino_t ino, ext2_ino_t dotdot) { struct dir_info *p; @@ -270,7 +407,7 @@ int e2fsck_dir_info_set_dotdot(e2fsck_t ctx, ext2_ino_t ino, * This function only sets the parent pointer, and requires that * dirinfo structure has already been created. */ -int e2fsck_dir_info_get_parent(e2fsck_t ctx, ext2_ino_t ino, +int e2fsck_dir_info_get_parent(e2fsck_t ctx, ext2_ino_t ino, ext2_ino_t *parent) { struct dir_info *p; @@ -286,7 +423,7 @@ int e2fsck_dir_info_get_parent(e2fsck_t ctx, ext2_ino_t ino, * This function only sets the dot dot pointer, and requires that * dirinfo structure has already been created. */ -int e2fsck_dir_info_get_dotdot(e2fsck_t ctx, ext2_ino_t ino, +int e2fsck_dir_info_get_dotdot(e2fsck_t ctx, ext2_ino_t ino, ext2_ino_t *dotdot) { struct dir_info *p;