From 0c265d4161a1fa3e98df49a79cc31b27b139e3ef Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:12:02 +1100 Subject: [PATCH 01/13] MemFSTest.ContentsOfEmptyFileSystem --- samples/memfs/memfs_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/samples/memfs/memfs_test.go b/samples/memfs/memfs_test.go index 97d2430..40fe749 100644 --- a/samples/memfs/memfs_test.go +++ b/samples/memfs/memfs_test.go @@ -13,6 +13,7 @@ import ( "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/samples/memfs" "github.com/jacobsa/gcsfuse/timeutil" + . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" "golang.org/x/net/context" ) @@ -85,7 +86,10 @@ func (t *MemFSTest) TearDown() { //////////////////////////////////////////////////////////////////////// func (t *MemFSTest) ContentsOfEmptyFileSystem() { - AssertTrue(false, "TODO") + entries, err := ioutil.ReadDir(t.mfs.Dir()) + + AssertEq(nil, err) + ExpectThat(entries, ElementsAre()) } func (t *MemFSTest) DoesFoo() { From 6f93f4eba94c4962b056987b430a34d3339c53ea Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:14:02 +1100 Subject: [PATCH 02/13] Implemented NewMemFS. --- samples/memfs/fs.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/samples/memfs/fs.go b/samples/memfs/fs.go index 0feac15..95524b6 100644 --- a/samples/memfs/fs.go +++ b/samples/memfs/fs.go @@ -44,6 +44,15 @@ type memFS struct { // Create a file system that stores data and metadata in memory. func NewMemFS( - clock timeutil.Clock) fuse.FileSystem { - panic("TODO(jacobsa): Implement NewMemFS.") + clock timeutil.Clock) (fs fuse.FileSystem) { + fs = &memFS{ + clock: clock, + } + + fs.(*memFS).mu = syncutil.NewInvariantMutex(fs.(*memFS).checkInvariants) + return +} + +func (fs *memFS) checkInvariants() { + panic("TODO") } From 2790095b6b83f7b329cbcfe2dd15bdd35bf26b8f Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:14:49 +1100 Subject: [PATCH 03/13] Implemented Init. --- samples/memfs/fs.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/samples/memfs/fs.go b/samples/memfs/fs.go index 95524b6..04ebc6e 100644 --- a/samples/memfs/fs.go +++ b/samples/memfs/fs.go @@ -8,6 +8,7 @@ import ( "github.com/jacobsa/fuse/fuseutil" "github.com/jacobsa/gcloud/syncutil" "github.com/jacobsa/gcsfuse/timeutil" + "golang.org/x/net/context" ) type memFS struct { @@ -56,3 +57,10 @@ func NewMemFS( func (fs *memFS) checkInvariants() { panic("TODO") } + +func (fs *memFS) Init( + ctx context.Context, + req *fuse.InitRequest) (resp *fuse.InitResponse, err error) { + resp = &fuse.InitResponse{} + return +} From 0d7c207f85daae68ff4bbf933fd9a53d6f380944 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:16:17 +1100 Subject: [PATCH 04/13] Added a TODO. --- samples/memfs/dir.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/samples/memfs/dir.go b/samples/memfs/dir.go index 073c490..7e59984 100644 --- a/samples/memfs/dir.go +++ b/samples/memfs/dir.go @@ -28,5 +28,8 @@ type memDir struct { // we use its indices for Dirent.Offset, which is exposed to the user who // might be calling readdir in a loop while concurrently modifying the // directory. Unused entries can, however, be reused. + // + // TODO(jacobsa): Add good tests exercising concurrent modifications while + // doing readdir, seekdir, etc. calls. entries []fuseutil.Dirent } From 2f98fdd29d7a8161df39e987093abcc0b01a39ce Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:17:52 +1100 Subject: [PATCH 05/13] Added a TODO. --- samples/memfs/inode.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/samples/memfs/inode.go b/samples/memfs/inode.go index 70c6f0c..91cad95 100644 --- a/samples/memfs/inode.go +++ b/samples/memfs/inode.go @@ -4,6 +4,10 @@ package memfs // Common attributes for files and directories. +// +// TODO(jacobsa): Add tests for interacting with a file/directory after it has +// been unlinked, including creating a new file. Make sure we don't screw up +// and reuse the inode while it is still in use. type inode struct { // The *memFile or *memDir for this inode, or nil if the inode is available // for reuse. From 9d4fcebedf1042df2ab24a68a9e207efe0173450 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:22:59 +1100 Subject: [PATCH 06/13] Implemented memFS.OpenDir. --- samples/memfs/fs.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/samples/memfs/fs.go b/samples/memfs/fs.go index 04ebc6e..ecb01c1 100644 --- a/samples/memfs/fs.go +++ b/samples/memfs/fs.go @@ -4,6 +4,8 @@ package memfs import ( + "fmt" + "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/fuseutil" "github.com/jacobsa/gcloud/syncutil" @@ -64,3 +66,25 @@ func (fs *memFS) Init( resp = &fuse.InitResponse{} return } + +func (fs *memFS) OpenDir( + ctx context.Context, + req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + // We don't mutate spontaneosuly, so if the VFS layer has asked for an + // inode that doesn't exist, something screwed up earlier (a lookup, a + // cache invalidation, etc.). + if req.Inode >= fuse.InodeID(len(fs.inodes)) { + panic(fmt.Sprintf("Inode out of range: %v vs. %v", req.Inode, len(fs.inodes))) + } + + var inode *inode = &fs.inodes[req.Inode] + if inode.impl == nil { + panic(fmt.Sprintf("Dead inode requested: %v", req.Inode)) + } + + // All is good. + return +} From f26fdfc80c568c33584ff98484b2afae746275ba Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:27:40 +1100 Subject: [PATCH 07/13] Implemented some of memFS.checkInvariants. --- samples/memfs/fs.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/samples/memfs/fs.go b/samples/memfs/fs.go index ecb01c1..ffe376d 100644 --- a/samples/memfs/fs.go +++ b/samples/memfs/fs.go @@ -5,6 +5,7 @@ package memfs import ( "fmt" + "reflect" "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/fuseutil" @@ -57,6 +58,35 @@ func NewMemFS( } func (fs *memFS) checkInvariants() { + // Check reserved inodes. + for i := 0; i < fuse.RootInodeID; i++ { + var inode *inode = &fs.inodes[i] + if inode.impl != nil { + panic(fmt.Sprintf("Non-nil impl for ID: %v", i)) + } + } + + // Check the root inode. + fs.inodes[fuse.RootInodeID].impl.(*dir) + + // Check inodes, building our own set of free IDs. + freeIDsEncountered := make(map[fuse.InodeID]struct{}) + for i := range fs.inodes { + var inode *inode = &fs.inodes[i] + if inode.impl == nil { + freeIDsEncountered[i] = struct{}{} + continue + } + + // Check for known types. + switch inode.impl.(type) { + case *memFile: + case *memDir: + default: + panic(fmt.Sprintf("Unknown inode type: %v", reflect.TypeOf(inode.impl))) + } + } + panic("TODO") } From a0a0963c49e14fb52d7bc7addfe7c18b7fa8699b Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:30:39 +1100 Subject: [PATCH 08/13] Made the RootInodeID constant untyped. --- file_system.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/file_system.go b/file_system.go index d8e7c54..d972428 100644 --- a/file_system.go +++ b/file_system.go @@ -4,6 +4,7 @@ package fuse import ( + "fmt" "os" "time" @@ -121,7 +122,26 @@ type InodeID uint64 // are minted by the file system, the FUSE VFS layer may send a request for // this ID without the file system ever having referenced it in a previous // response. -const RootInodeID InodeID = InodeID(bazilfuse.RootID) +const RootInodeID = 1 + +func init() { + // Make sure the constant above is correct. We do this at runtime rather than + // defining the constant in terms of bazilfuse.RootID for two reasons: + // + // 1. Users can more clearly see that the root ID is low and can therefore + // be used as e.g. an array index, with space reserved up to the root. + // + // 2. The constant can be untyped and can therefore more easily be used as + // an array index. + // + if RootInodeID != bazilfuse.RootID { + panic( + fmt.Sprintf( + "Oops, RootInodeID is wrong: %v vs. %v", + RootInodeID, + bazilfuse.RootID)) + } +} // Attributes for a file or directory inode. Corresponds to struct inode (cf. // http://goo.gl/tvYyQt). From 1b65aaf88752ef0fd83ad3b6965e18c6aa866435 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:37:01 +1100 Subject: [PATCH 09/13] Refactored invariant checking. --- samples/memfs/fs.go | 35 ++++++++++++++++++++++------------- samples/memfs/inode.go | 16 ++++++++++++++++ 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/samples/memfs/fs.go b/samples/memfs/fs.go index ffe376d..9193964 100644 --- a/samples/memfs/fs.go +++ b/samples/memfs/fs.go @@ -5,7 +5,6 @@ package memfs import ( "fmt" - "reflect" "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/fuseutil" @@ -58,6 +57,11 @@ func NewMemFS( } func (fs *memFS) checkInvariants() { + // Check general inode invariants. + for i := range fs.inodes { + fs.inodes[i].checkInvariants() + } + // Check reserved inodes. for i := 0; i < fuse.RootInodeID; i++ { var inode *inode = &fs.inodes[i] @@ -67,27 +71,32 @@ func (fs *memFS) checkInvariants() { } // Check the root inode. - fs.inodes[fuse.RootInodeID].impl.(*dir) + _ = fs.inodes[fuse.RootInodeID].impl.(*memDir) // Check inodes, building our own set of free IDs. freeIDsEncountered := make(map[fuse.InodeID]struct{}) - for i := range fs.inodes { + for i := fuse.RootInodeID + 1; i < len(fs.inodes); i++ { var inode *inode = &fs.inodes[i] if inode.impl == nil { - freeIDsEncountered[i] = struct{}{} + freeIDsEncountered[fuse.InodeID(i)] = struct{}{} continue } - - // Check for known types. - switch inode.impl.(type) { - case *memFile: - case *memDir: - default: - panic(fmt.Sprintf("Unknown inode type: %v", reflect.TypeOf(inode.impl))) - } } - panic("TODO") + // Check fs.freeInodes. + if len(fs.freeInodes) != len(freeIDsEncountered) { + panic( + fmt.Sprintf( + "Length mismatch: %v vs. %v", + len(fs.freeInodes), + len(freeIDsEncountered))) + } + + for _, id := range fs.freeInodes { + if _, ok := freeIDsEncountered[id]; !ok { + panic(fmt.Sprintf("Unexected free inode ID: %v", id)) + } + } } func (fs *memFS) Init( diff --git a/samples/memfs/inode.go b/samples/memfs/inode.go index 91cad95..d3ae224 100644 --- a/samples/memfs/inode.go +++ b/samples/memfs/inode.go @@ -3,6 +3,11 @@ package memfs +import ( + "fmt" + "reflect" +) + // Common attributes for files and directories. // // TODO(jacobsa): Add tests for interacting with a file/directory after it has @@ -15,3 +20,14 @@ type inode struct { // INVARIANT: impl is nil, or of type *memFile or *memDir impl interface{} } + +func (inode *inode) checkInvariants() { + switch inode.impl.(type) { + case nil: + case *memFile: + case *memDir: + default: + panic( + fmt.Sprintf("Unexpected inode impl type: %v", reflect.TypeOf(inode.impl))) + } +} From c38463a12226ba4906b311ab05f39f4b969b4347 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:40:28 +1100 Subject: [PATCH 10/13] Fixed broken invariants at construction. --- samples/memfs/dir.go | 6 ++++++ samples/memfs/fs.go | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/samples/memfs/dir.go b/samples/memfs/dir.go index 7e59984..5272e67 100644 --- a/samples/memfs/dir.go +++ b/samples/memfs/dir.go @@ -31,5 +31,11 @@ type memDir struct { // // TODO(jacobsa): Add good tests exercising concurrent modifications while // doing readdir, seekdir, etc. calls. + // + // INVIARANT: For each i < len(entries)-1, entries[i].Offset = i+1 entries []fuseutil.Dirent } + +func newDir() *memDir + +func (d *memDir) checkInvariants() diff --git a/samples/memfs/fs.go b/samples/memfs/fs.go index 9193964..ce8ea18 100644 --- a/samples/memfs/fs.go +++ b/samples/memfs/fs.go @@ -47,13 +47,20 @@ type memFS struct { // Create a file system that stores data and metadata in memory. func NewMemFS( - clock timeutil.Clock) (fs fuse.FileSystem) { - fs = &memFS{ - clock: clock, + clock timeutil.Clock) fuse.FileSystem { + // Set up the basic struct. + fs := &memFS{ + clock: clock, + inodes: make([]inode, fuse.RootInodeID+1), } - fs.(*memFS).mu = syncutil.NewInvariantMutex(fs.(*memFS).checkInvariants) - return + // Set up the root inode. + fs.inodes[fuse.RootInodeID].impl = newDir() + + // Set up invariant checking. + fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants) + + return fs } func (fs *memFS) checkInvariants() { From a6223c2344b9809ed647e33538d81624c65a6f39 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:44:16 +1100 Subject: [PATCH 11/13] Implemented missing memDir methods. --- samples/memfs/dir.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/samples/memfs/dir.go b/samples/memfs/dir.go index 5272e67..a2cc7fb 100644 --- a/samples/memfs/dir.go +++ b/samples/memfs/dir.go @@ -4,18 +4,14 @@ package memfs import ( + "fmt" + "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/fuseutil" "github.com/jacobsa/gcloud/syncutil" ) type memDir struct { - ///////////////////////// - // Constant data - ///////////////////////// - - inode fuse.InodeID - ///////////////////////// // Mutable state ///////////////////////// @@ -32,10 +28,21 @@ type memDir struct { // TODO(jacobsa): Add good tests exercising concurrent modifications while // doing readdir, seekdir, etc. calls. // - // INVIARANT: For each i < len(entries)-1, entries[i].Offset = i+1 + // INVARIANT: For each i, entries[i].Offset == i+1 entries []fuseutil.Dirent } -func newDir() *memDir +func newDir() (d *memDir) { + d = &memDir{} + d.mu = syncutil.NewInvariantMutex(d.checkInvariants) -func (d *memDir) checkInvariants() + return +} + +func (d *memDir) checkInvariants() { + for i, e := range d.entries { + if e.Offset != fuse.DirOffset(i+1) { + panic(fmt.Sprintf("Unexpected offset in entry: %v", e)) + } + } +} From 25a95e5074bfa4ec21c5c874cfab613db49d2b32 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:45:14 +1100 Subject: [PATCH 12/13] Fixed a crash. --- samples/memfs/fs.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/memfs/fs.go b/samples/memfs/fs.go index ce8ea18..c1930eb 100644 --- a/samples/memfs/fs.go +++ b/samples/memfs/fs.go @@ -116,6 +116,8 @@ func (fs *memFS) Init( func (fs *memFS) OpenDir( ctx context.Context, req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) { + resp = &fuse.OpenDirResponse{} + fs.mu.RLock() defer fs.mu.RUnlock() From 5bedddbd7d5036179f8c241d32f001ea599fb9fe Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 2 Mar 2015 14:52:29 +1100 Subject: [PATCH 13/13] Implemented memFS.ReadDir. --- samples/memfs/fs.go | 55 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/samples/memfs/fs.go b/samples/memfs/fs.go index c1930eb..406828f 100644 --- a/samples/memfs/fs.go +++ b/samples/memfs/fs.go @@ -113,26 +113,57 @@ func (fs *memFS) Init( return } +// Panic if not a live dir. +// +// LOCKS_EXCLUDED(fs.mu) +func (fs *memFS) getDirOrDie(inodeID fuse.InodeID) (d *memDir) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + if inodeID >= fuse.InodeID(len(fs.inodes)) { + panic(fmt.Sprintf("Inode out of range: %v vs. %v", inodeID, len(fs.inodes))) + } + + var inode *inode = &fs.inodes[inodeID] + d = inode.impl.(*memDir) + + return +} + func (fs *memFS) OpenDir( ctx context.Context, req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) { resp = &fuse.OpenDirResponse{} - fs.mu.RLock() - defer fs.mu.RUnlock() - // We don't mutate spontaneosuly, so if the VFS layer has asked for an // inode that doesn't exist, something screwed up earlier (a lookup, a // cache invalidation, etc.). - if req.Inode >= fuse.InodeID(len(fs.inodes)) { - panic(fmt.Sprintf("Inode out of range: %v vs. %v", req.Inode, len(fs.inodes))) - } + _ = fs.getDirOrDie(req.Inode) + + return +} + +func (fs *memFS) ReadDir( + ctx context.Context, + req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) { + resp = &fuse.ReadDirResponse{} + + // Grab the directory. + d := fs.getDirOrDie(req.Inode) + + d.mu.RLock() + defer d.mu.RUnlock() + + // Return the entries requested. + for i := int(req.Offset); i < len(d.entries); i++ { + resp.Data = fuseutil.AppendDirent(resp.Data, d.entries[i]) + + // Trim and stop early if we've exceeded the requested size. + if len(resp.Data) > req.Size { + resp.Data = resp.Data[:req.Size] + break + } + } - var inode *inode = &fs.inodes[req.Inode] - if inode.impl == nil { - panic(fmt.Sprintf("Dead inode requested: %v", req.Inode)) - } - - // All is good. return }