From 9b88ab0cc3ab37d3e414a6876c7955d5dc51c4ea Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 15:52:27 +1100 Subject: [PATCH 01/44] Began working on a generic mounting tool for samples/. For jacobsa/fuse#4. --- samples/mount_sample/mount.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 samples/mount_sample/mount.go diff --git a/samples/mount_sample/mount.go b/samples/mount_sample/mount.go new file mode 100644 index 0000000..c5a5d9d --- /dev/null +++ b/samples/mount_sample/mount.go @@ -0,0 +1,31 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A simple tool for mounting sample file systems, used by the tests in +// samples/. +package main + +import ( + "flag" + "os" +) + +var fType = flag.String( + "type", + "", + "The name of the samples/ sub-dir to be mounted.") + +func main() { + os.Exit(1) +} From 0bbe4cfa3d602c03ff3fc436c082e2a19e742880 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 15:54:20 +1100 Subject: [PATCH 02/44] Added flags for flushfs. --- samples/mount_sample/mount.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/samples/mount_sample/mount.go b/samples/mount_sample/mount.go index c5a5d9d..be63a1c 100644 --- a/samples/mount_sample/mount.go +++ b/samples/mount_sample/mount.go @@ -26,6 +26,16 @@ var fType = flag.String( "", "The name of the samples/ sub-dir to be mounted.") +var fFlushesFile = flag.String( + "flushfs.flushes_file", + "", + "Path to a file to which flushes should be reported, \\n-separated.") + +var fFsyncsFile = flag.String( + "flushfs.fsyncs_file", + "", + "Path to a file to which fsyncs should be reported, \\n-separated.") + func main() { os.Exit(1) } From 06fb8e2f1352950081b724994d1b5e62514cd227 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:00:12 +1100 Subject: [PATCH 03/44] Sketched main. --- samples/mount_sample/mount.go | 41 ++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/samples/mount_sample/mount.go b/samples/mount_sample/mount.go index be63a1c..9eb9828 100644 --- a/samples/mount_sample/mount.go +++ b/samples/mount_sample/mount.go @@ -18,13 +18,14 @@ package main import ( "flag" - "os" + "log" + + "github.com/jacobsa/fuse" + "golang.org/x/net/context" ) -var fType = flag.String( - "type", - "", - "The name of the samples/ sub-dir to be mounted.") +var fType = flag.String("type", "", "The name of the samples/ sub-dir.") +var fMountPoint = flag.String("mount_point", "", "Path to mount point.") var fFlushesFile = flag.String( "flushfs.flushes_file", @@ -36,6 +37,34 @@ var fFsyncsFile = flag.String( "", "Path to a file to which fsyncs should be reported, \\n-separated.") +func makeFS() (fs fuse.FileSystem, err error) + func main() { - os.Exit(1) + flag.Parse() + + // Create an appropriate file system. + fs, err := makeFS() + if err != nil { + log.Fatalf("makeFS: %v", err) + } + + // Mount the file system. + if *fMountPoint == "" { + log.Fatalf("You must set --mount_point.") + } + + mfs, err := fuse.Mount(*fMountPoint, fs, &fuse.MountConfig{}) + if err != nil { + log.Fatalf("Mount: %v", err) + } + + // Wait for it to be ready. + if err = mfs.WaitForReady(context.Background()); err != nil { + log.Fatalf("WaitForReady: %v", err) + } + + // Wait for it to be unmounted. + if err = mfs.Join(context.Background()); err != nil { + log.Fatalf("Join: %v", err) + } } From 215f15bdc79803d8b6bbda7a385fcd6c84695188 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:01:16 +1100 Subject: [PATCH 04/44] Sketched makeFS. --- samples/mount_sample/mount.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/samples/mount_sample/mount.go b/samples/mount_sample/mount.go index 9eb9828..ec36212 100644 --- a/samples/mount_sample/mount.go +++ b/samples/mount_sample/mount.go @@ -18,6 +18,7 @@ package main import ( "flag" + "fmt" "log" "github.com/jacobsa/fuse" @@ -37,7 +38,19 @@ var fFsyncsFile = flag.String( "", "Path to a file to which fsyncs should be reported, \\n-separated.") -func makeFS() (fs fuse.FileSystem, err error) +func makeFlushFS() (fs fuse.FileSystem, err error) + +func makeFS() (fs fuse.FileSystem, err error) { + switch *fType { + default: + err = fmt.Errorf("Unknown FS type: %v", *fType) + + case "flushfs": + fs, err = makeFlushFS() + } + + return +} func main() { flag.Parse() From ad0a833fa7adec3df55d01e85af18031d1e4fc0d Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:08:46 +1100 Subject: [PATCH 05/44] makeFlushFS --- samples/mount_sample/mount.go | 72 ++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/samples/mount_sample/mount.go b/samples/mount_sample/mount.go index ec36212..963186c 100644 --- a/samples/mount_sample/mount.go +++ b/samples/mount_sample/mount.go @@ -20,25 +20,79 @@ import ( "flag" "fmt" "log" + "os" + "github.com/jacobsa/bazilfuse" "github.com/jacobsa/fuse" + "github.com/jacobsa/fuse/samples/flushfs" "golang.org/x/net/context" ) var fType = flag.String("type", "", "The name of the samples/ sub-dir.") var fMountPoint = flag.String("mount_point", "", "Path to mount point.") -var fFlushesFile = flag.String( - "flushfs.flushes_file", - "", - "Path to a file to which flushes should be reported, \\n-separated.") +var fFlushesFile = flag.String("flushfs.flushes_file", "", "") +var fFsyncsFile = flag.String("flushfs.fsyncs_file", "", "") +var fFlushError = flag.Int("flushfs.flush_error", 0, "") +var fFsyncError = flag.Int("flushfs.fsync_error", 0, "") -var fFsyncsFile = flag.String( - "flushfs.fsyncs_file", - "", - "Path to a file to which fsyncs should be reported, \\n-separated.") +func makeFlushFS() (fs fuse.FileSystem, err error) { + // Check the flags. + if *fFlushesFile == "" || *fFsyncsFile == "" { + err = fmt.Errorf("You must set the flushfs flags.") + return + } -func makeFlushFS() (fs fuse.FileSystem, err error) + // Open the files. + flushes, err := os.OpenFile(*fFlushesFile, os.O_RDWR, 0) + if err != nil { + err = fmt.Errorf("Opening %s: %v", *fFlushesFile, err) + return + } + + fsyncs, err := os.OpenFile(*fFsyncsFile, os.O_RDWR, 0) + if err != nil { + err = fmt.Errorf("Opening %s: %v", *fFsyncsFile, err) + return + } + + // Set up errors. + var flushErr error + var fsyncErr error + + if *fFlushError != 0 { + flushErr = bazilfuse.Errno(*fFlushError) + } + + if *fFsyncError != 0 { + fsyncErr = bazilfuse.Errno(*fFsyncError) + } + + // Report flushes and fsyncs by writing the contents followed by a newline. + report := func(f *os.File, outErr error) func(string) error { + return func(s string) (err error) { + buf := []byte(s) + buf = append(buf, '\n') + + _, err = f.Write(buf) + if err != nil { + err = fmt.Errorf("Write: %v", err) + return + } + + err = outErr + return + } + } + + reportFlush := report(flushes, flushErr) + reportFsync := report(fsyncs, fsyncErr) + + // Create the file system. + fs, err = flushfs.NewFileSystem(reportFlush, reportFsync) + + return +} func makeFS() (fs fuse.FileSystem, err error) { switch *fType { From e997c8fc912ebc530f87591862b84ce1e5a290c3 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:11:01 +1100 Subject: [PATCH 06/44] Began categorizing tests. --- samples/flushfs/flush_fs_test.go | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 2421ede..6bcaedf 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -686,3 +686,37 @@ func (t *FlushFSTest) Mmap_CloseBeforeMunmap() { func (t *FlushFSTest) Directory() { AssertTrue(false, "TODO") } + +//////////////////////////////////////////////////////////////////////// +// No errors +//////////////////////////////////////////////////////////////////////// + +type NoErrorsTest struct { + flushFSTest +} + +func init() { RegisterTestSuite(&NoErrorsTest{}) } + +//////////////////////////////////////////////////////////////////////// +// Flush error +//////////////////////////////////////////////////////////////////////// + +type FlushErrorTest struct { + flushFSTest +} + +func init() { RegisterTestSuite(&FlushErrorTest{}) } + +func (t *FlushFSTest) SetUp(ti *TestInfo) + +//////////////////////////////////////////////////////////////////////// +// Fsync error +//////////////////////////////////////////////////////////////////////// + +type FsyncErrorTest struct { + flushFSTest +} + +func init() { RegisterTestSuite(&FsyncErrorTest{}) } + +func (t *FlushFSTest) SetUp(ti *TestInfo) From 7066fecc45fce312188890bb17877f9ba841e617 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:12:02 +1100 Subject: [PATCH 07/44] Began on helper. --- samples/flushfs/flush_fs_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 6bcaedf..124f06f 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -24,6 +24,7 @@ import ( "syscall" "testing" + "github.com/jacobsa/bazilfuse" "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/samples" "github.com/jacobsa/fuse/samples/flushfs" @@ -37,7 +38,7 @@ func TestFlushFS(t *testing.T) { RunTests(t) } // Boilerplate //////////////////////////////////////////////////////////////////////// -type FlushFSTest struct { +type flushFSTest struct { samples.SampleTest // File handles that are closed in TearDown if non-nil. @@ -55,9 +56,10 @@ type FlushFSTest struct { fsyncErr error } -func init() { RegisterTestSuite(&FlushFSTest{}) } - -func (t *FlushFSTest) SetUp(ti *TestInfo) { +func (t *flushFSTest) setUp( + ti *TestInfo, + flushErr bazilfuse.Errno, + fsyncErr bazilfuse.Errno) { var err error // Set up a file system. From f211ff829b3abeb5134a77501d8f09ef1a5c49c7 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:13:18 +1100 Subject: [PATCH 08/44] Renamed testing.go to in_process.go. --- samples/{testing.go => in_process.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename samples/{testing.go => in_process.go} (100%) diff --git a/samples/testing.go b/samples/in_process.go similarity index 100% rename from samples/testing.go rename to samples/in_process.go From ff9c68f93cb6e35aab67c5168f4838f67aa8ed87 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:15:53 +1100 Subject: [PATCH 09/44] Begain on a SubprocessTest helper. --- samples/subprocess.go | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 samples/subprocess.go diff --git a/samples/subprocess.go b/samples/subprocess.go new file mode 100644 index 0000000..c3cda3e --- /dev/null +++ b/samples/subprocess.go @@ -0,0 +1,46 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package samples + +import ( + "io" + + "github.com/jacobsa/fuse" + "golang.org/x/net/context" +) + +// A struct that implements common behavior needed by tests in the samples/ +// directory where the file system is mounted by a subprocess. Use it as an +// embedded field in your test fixture, calling its SetUp method from your +// SetUp method after setting the MountType and MountFlags fields. +type SubprocessTest struct { + // The type of the file system to mount. Must be recognized by mount_sample. + MountType string + + // Additional flags to be passed to the mount_sample tool. + MountFlags []string + + // A context object that can be used for long-running operations. + Ctx context.Context + + // The directory at which the file system is mounted. + Dir string + + // Anothing non-nil in this slice will be closed by TearDown. The test will + // fail if closing fails. + ToClose []io.Closer + + mfs *fuse.MountedFileSystem +} From 0a588af19f2ae90e183e42b1f0b31dd9794518e8 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:16:51 +1100 Subject: [PATCH 10/44] Fixed some comments. --- samples/in_process.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/in_process.go b/samples/in_process.go index bdb2b57..761133b 100644 --- a/samples/in_process.go +++ b/samples/in_process.go @@ -55,8 +55,8 @@ type SampleTest struct { mfs *fuse.MountedFileSystem } -// Mount the supplied file system and initialize the other exported fields of -// the struct. Panics on error. +// Mount t.FileSystem and initialize the other exported fields of the struct. +// Panics on error. // // REQUIRES: t.FileSystem has been set. func (t *SampleTest) SetUp(ti *ogletest.TestInfo) { @@ -66,7 +66,7 @@ func (t *SampleTest) SetUp(ti *ogletest.TestInfo) { } } -// Like Initialize, but doens't panic. +// Like SetUp, but doens't panic. func (t *SampleTest) initialize( fs fuse.FileSystem, config *fuse.MountConfig) (err error) { From 9ae10644938edb81b66cf45fa89d3ccc54d6cc79 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:23:22 +1100 Subject: [PATCH 11/44] Fixed a comment. --- samples/in_process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/in_process.go b/samples/in_process.go index 761133b..ac19b6e 100644 --- a/samples/in_process.go +++ b/samples/in_process.go @@ -90,7 +90,7 @@ func (t *SampleTest) initialize( return } - // Wait for it to be read. + // Wait for it to be ready. err = t.mfs.WaitForReady(t.Ctx) if err != nil { err = fmt.Errorf("WaitForReady: %v", err) From 5001868f24a962f69e04b7c5f8a2db92be2565f9 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:26:15 +1100 Subject: [PATCH 12/44] Sketched invokeMountSample. --- samples/subprocess.go | 116 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 2 deletions(-) diff --git a/samples/subprocess.go b/samples/subprocess.go index c3cda3e..66943dc 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -15,9 +15,16 @@ package samples import ( + "fmt" "io" + "io/ioutil" + "log" + "os/exec" + "strings" + "time" - "github.com/jacobsa/fuse" + "github.com/jacobsa/bazilfuse" + "github.com/jacobsa/ogletest" "golang.org/x/net/context" ) @@ -42,5 +49,110 @@ type SubprocessTest struct { // fail if closing fails. ToClose []io.Closer - mfs *fuse.MountedFileSystem + mountCmd *exec.Cmd +} + +// Mount the file system and initialize the other exported fields of the +// struct. Panics on error. +// +// REQUIRES: t.FileSystem has been set. +func (t *SubprocessTest) SetUp(ti *ogletest.TestInfo) { + err := t.initialize() + if err != nil { + panic(err) + } +} + +// Build the mount_sample tool if it has not yet been built for this process. +// Return a path to the binary. +func buildMountSample() (path string, err error) + +// Invoke mount_sample, returning a running command. +func invokeMountSample(path string, args []string) (cmd *exec.Cmd, err error) + +// Like SetUp, but doens't panic. +func (t *SubprocessTest) initialize() (err error) { + // Initialize the context. + t.Ctx = context.Background() + + // Set up a temporary directory. + t.Dir, err = ioutil.TempDir("", "sample_test") + if err != nil { + err = fmt.Errorf("TempDir: %v", err) + return + } + + // Build the mount_sample tool. + toolPath, err := buildMountSample() + if err != nil { + err = fmt.Errorf("buildMountSample: %v", err) + return + } + + // Invoke it. + args := []string{"--type", t.MountType} + args = append(args, t.MountFlags...) + + t.mountCmd, err = invokeMountSample(toolPath, args) + if err != nil { + err = fmt.Errorf("invokeMountSample: %v", err) + return + } + + // TODO(jacobsa): Probably need some sort of signalling (on stderr? write to + // a flag-controlled file?) when WaitForReady has returned. + + return +} + +// Unmount the file system and clean up. Panics on error. +func (t *SubprocessTest) TearDown() { + err := t.destroy() + if err != nil { + panic(err) + } +} + +// Like TearDown, but doesn't panic. +func (t *SubprocessTest) destroy() (err error) { + // Close what is necessary. + for _, c := range t.ToClose { + if c == nil { + continue + } + + ogletest.ExpectEq(nil, c.Close()) + } + + // Was the file system mounted? + if t.mountCmd == nil { + return + } + + // Unmount the file system. Try again on "resource busy" errors. + delay := 10 * time.Millisecond + for { + err = bazilfuse.Unmount(t.Dir) + if err == nil { + break + } + + if strings.Contains(err.Error(), "resource busy") { + log.Println("Resource busy error while unmounting; trying again") + time.Sleep(delay) + delay = time.Duration(1.3 * float64(delay)) + continue + } + + err = fmt.Errorf("Unmount: %v", err) + return + } + + // Wait for the subprocess. + if err = t.mountCmd.Wait(); err != nil { + err = fmt.Errorf("Cmd.Wait: %v", err) + return + } + + return } From ee8e869e357293d954206b4f797dc7e25c2559ed Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:31:52 +1100 Subject: [PATCH 13/44] buildMountSample --- samples/subprocess.go | 47 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/samples/subprocess.go b/samples/subprocess.go index 66943dc..a2ebbfa 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -20,7 +20,9 @@ import ( "io/ioutil" "log" "os/exec" + "path" "strings" + "sync" "time" "github.com/jacobsa/bazilfuse" @@ -63,9 +65,52 @@ func (t *SubprocessTest) SetUp(ti *ogletest.TestInfo) { } } +// Set by buildMountSample. +var mountSamplePath string +var mountSampleErr error +var mountSampleOnce sync.Once + // Build the mount_sample tool if it has not yet been built for this process. // Return a path to the binary. -func buildMountSample() (path string, err error) +func buildMountSample() (toolPath string, err error) { + // Build if we haven't yet. + mountSampleOnce.Do(func() { + // Create a temporary directory. + tempDir, err := ioutil.TempDir("", "") + if err != nil { + mountSampleErr = fmt.Errorf("TempDir: %v", err) + return + } + + mountSamplePath = path.Join(tempDir, "mount_sample") + + // Build the command. + cmd := exec.Command( + "go", + "build", + "github.com/jacobsa/fuse/samples/mount_sample", + "-o", + mountSamplePath) + + output, err := cmd.CombinedOutput() + if err != nil { + mountSampleErr = fmt.Errorf( + "mount_sample exited with %v, output:\n%s", + err, + string(output)) + + return + } + }) + + if mountSampleErr != nil { + err = mountSampleErr + return + } + + toolPath = mountSamplePath + return +} // Invoke mount_sample, returning a running command. func invokeMountSample(path string, args []string) (cmd *exec.Cmd, err error) From 27722cb25ab531dd0e2247677bcd71c4ec836c4c Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:32:50 +1100 Subject: [PATCH 14/44] invokeMountSample --- samples/subprocess.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/samples/subprocess.go b/samples/subprocess.go index a2ebbfa..52e3179 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -113,7 +113,15 @@ func buildMountSample() (toolPath string, err error) { } // Invoke mount_sample, returning a running command. -func invokeMountSample(path string, args []string) (cmd *exec.Cmd, err error) +func invokeMountSample(path string, args []string) (cmd *exec.Cmd, err error) { + cmd = exec.Command(path, args...) + if err = cmd.Start(); err != nil { + err = fmt.Errorf("Start: %v", err) + return + } + + return +} // Like SetUp, but doens't panic. func (t *SubprocessTest) initialize() (err error) { From 4ca6ed02c6b58436a74f42dde0f81cb6f13cdfbd Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:34:39 +1100 Subject: [PATCH 15/44] flushFSTest.setUp. --- samples/flushfs/flush_fs_test.go | 50 ++++++++++++-------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 124f06f..ec2e3ff 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -15,19 +15,18 @@ package flushfs_test import ( + "fmt" "io" "io/ioutil" "os" "path" "runtime" - "sync" "syscall" "testing" "github.com/jacobsa/bazilfuse" "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/samples" - "github.com/jacobsa/fuse/samples/flushfs" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" ) @@ -39,21 +38,11 @@ func TestFlushFS(t *testing.T) { RunTests(t) } //////////////////////////////////////////////////////////////////////// type flushFSTest struct { - samples.SampleTest + samples.SubprocessTest // File handles that are closed in TearDown if non-nil. f1 *os.File f2 *os.File - - mu sync.Mutex - - // GUARDED_BY(mu) - flushes []string - flushErr error - - // GUARDED_BY(mu) - fsyncs []string - fsyncErr error } func (t *flushFSTest) setUp( @@ -62,27 +51,26 @@ func (t *flushFSTest) setUp( fsyncErr bazilfuse.Errno) { var err error - // Set up a file system. - reportTo := func(slice *[]string, err *error) func(string) error { - return func(s string) error { - t.mu.Lock() - defer t.mu.Unlock() + // Set up files to receive flush and fsync reports. + panic("TODO") - *slice = append(*slice, s) - return *err - } + // Set up test config. + t.MountType = "flushfs" + t.MountFlags = []string{ + "--flushfs.flushes_file", + t.flushes.Name(), + + "--flushfs.fsyncs_file", + t.fsyncs.Name(), + + "--flushfs.flush_error", + fmt.Sprintf("%d", int(flushErr)), + + "--fsyncfs.fsync_error", + fmt.Sprintf("%d", int(fsyncErr)), } - t.FileSystem, err = flushfs.NewFileSystem( - reportTo(&t.flushes, &t.flushErr), - reportTo(&t.fsyncs, &t.fsyncErr)) - - if err != nil { - panic(err) - } - - // Mount it. - t.SampleTest.SetUp(ti) + t.SubprocessTest.SetUp(ti) } func (t *FlushFSTest) TearDown() { From 3924e25eb8b00c8f6aeebfd65a7d4f68dde23878 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:35:19 +1100 Subject: [PATCH 16/44] Refactored helpers. --- samples/flushfs/flush_fs_test.go | 34 ++++---------------------------- 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index ec2e3ff..438c6a2 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -94,41 +94,15 @@ func (t *FlushFSTest) TearDown() { // Return a copy of the current contents of t.flushes. // // LOCKS_EXCLUDED(t.mu) -func (t *FlushFSTest) getFlushes() (p []string) { - t.mu.Lock() - defer t.mu.Unlock() - - p = make([]string, len(t.flushes)) - copy(p, t.flushes) - return +func (t *flushFSTest) getFlushes() (p []string) { + panic("TODO") } // Return a copy of the current contents of t.fsyncs. // // LOCKS_EXCLUDED(t.mu) -func (t *FlushFSTest) getFsyncs() (p []string) { - t.mu.Lock() - defer t.mu.Unlock() - - p = make([]string, len(t.fsyncs)) - copy(p, t.fsyncs) - return -} - -// LOCKS_EXCLUDED(t.mu) -func (t *FlushFSTest) setFlushError(err error) { - t.mu.Lock() - defer t.mu.Unlock() - - t.flushErr = err -} - -// LOCKS_EXCLUDED(t.mu) -func (t *FlushFSTest) setFsyncError(err error) { - t.mu.Lock() - defer t.mu.Unlock() - - t.fsyncErr = err +func (t *flushFSTest) getFsyncs() (p []string) { + panic("TODO") } // Like syscall.Dup2, but correctly annotates the syscall as blocking. See here From f02cc98b33c89d33f90bc8b63764883b877c7a07 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:35:42 +1100 Subject: [PATCH 17/44] Fixed some build errors. --- samples/flushfs/flush_fs_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 438c6a2..39b8f9e 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -661,6 +661,8 @@ type NoErrorsTest struct { func init() { RegisterTestSuite(&NoErrorsTest{}) } +func (t *NoErrorsTest) SetUp(ti *TestInfo) + //////////////////////////////////////////////////////////////////////// // Flush error //////////////////////////////////////////////////////////////////////// @@ -671,7 +673,7 @@ type FlushErrorTest struct { func init() { RegisterTestSuite(&FlushErrorTest{}) } -func (t *FlushFSTest) SetUp(ti *TestInfo) +func (t *FlushErrorTest) SetUp(ti *TestInfo) //////////////////////////////////////////////////////////////////////// // Fsync error @@ -683,4 +685,4 @@ type FsyncErrorTest struct { func init() { RegisterTestSuite(&FsyncErrorTest{}) } -func (t *FlushFSTest) SetUp(ti *TestInfo) +func (t *FlushErrorTest) SetUp(ti *TestInfo) From 5dde9ea0b735046ea5d94e03fc37866830c4b915 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Mon, 23 Mar 2015 16:36:07 +1100 Subject: [PATCH 18/44] Fixed a silly mistake. --- samples/flushfs/flush_fs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 39b8f9e..564c118 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -685,4 +685,4 @@ type FsyncErrorTest struct { func init() { RegisterTestSuite(&FsyncErrorTest{}) } -func (t *FlushErrorTest) SetUp(ti *TestInfo) +func (t *FsyncErrorTest) SetUp(ti *TestInfo) From f0ae4d753eabc73dff63e4d8c9bc964161598dea Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 08:39:22 +1100 Subject: [PATCH 19/44] Recategorized tests. --- samples/flushfs/flush_fs_test.go | 216 +++++++++++++++---------------- 1 file changed, 106 insertions(+), 110 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 564c118..8e82a04 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -119,9 +119,17 @@ func dup2(oldfd int, newfd int) (err error) { } //////////////////////////////////////////////////////////////////////// -// Tests +// No errors //////////////////////////////////////////////////////////////////////// +type NoErrorsTest struct { + flushFSTest +} + +func init() { RegisterTestSuite(&NoErrorsTest{}) } + +func (t *NoErrorsTest) SetUp(ti *TestInfo) + func (t *FlushFSTest) CloseReports_ReadWrite() { var n int var off int64 @@ -304,24 +312,6 @@ func (t *FlushFSTest) CloseReports_MultipleTimes_OverlappingFileHandles() { AssertThat(t.getFsyncs(), ElementsAre()) } -func (t *FlushFSTest) CloseError() { - var err error - - // Open the file. - t.f1, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDWR, 0) - AssertEq(nil, err) - - // Configure a flush error. - t.setFlushError(fuse.ENOENT) - - // Close the file. - err = t.f1.Close() - t.f1 = nil - - AssertNe(nil, err) - ExpectThat(err, Error(HasSubstr("no such file"))) -} - func (t *FlushFSTest) FsyncReports() { var n int var err error @@ -361,23 +351,6 @@ func (t *FlushFSTest) FsyncReports() { AssertThat(t.getFsyncs(), ElementsAre("taco", "tacos")) } -func (t *FlushFSTest) FsyncError() { - var err error - - // Open the file. - t.f1, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDWR, 0) - AssertEq(nil, err) - - // Configure an fsync error. - t.setFsyncError(fuse.ENOENT) - - // Fsync. - err = t.f1.Sync() - - AssertNe(nil, err) - ExpectThat(err, Error(HasSubstr("no such file"))) -} - func (t *FlushFSTest) Dup() { var n int var err error @@ -442,45 +415,6 @@ func (t *FlushFSTest) Dup() { ExpectThat(t.getFsyncs(), ElementsAre()) } -func (t *FlushFSTest) Dup_FlushError() { - var err error - - // Open the file. - t.f1, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0) - AssertEq(nil, err) - - fd1 := t.f1.Fd() - - // Use dup(2) to get another copy. - fd2, err := syscall.Dup(int(fd1)) - AssertEq(nil, err) - - t.f2 = os.NewFile(uintptr(fd2), t.f1.Name()) - - // Configure a flush error. - t.setFlushError(fuse.ENOENT) - - // Close by the first handle. On OS X, where the semantics of file handles - // are different (cf. https://github.com/osxfuse/osxfuse/issues/199), this - // does not result in an error. - err = t.f1.Close() - t.f1 = nil - - if runtime.GOOS == "darwin" { - AssertEq(nil, err) - } else { - AssertNe(nil, err) - ExpectThat(err, Error(HasSubstr("no such file"))) - } - - // Close by the second handle. - err = t.f2.Close() - t.f2 = nil - - AssertNe(nil, err) - ExpectThat(err, Error(HasSubstr("no such file"))) -} - func (t *FlushFSTest) Dup2() { var n int var err error @@ -510,29 +444,6 @@ func (t *FlushFSTest) Dup2() { ExpectThat(t.getFsyncs(), ElementsAre()) } -func (t *FlushFSTest) Dup2_FlushError() { - var err error - - // Open the file. - t.f1, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0) - AssertEq(nil, err) - - // Open and unlink some temporary file. - t.f2, err = ioutil.TempFile("", "") - AssertEq(nil, err) - - err = os.Remove(t.f2.Name()) - AssertEq(nil, err) - - // Configure a flush error. - t.setFlushError(fuse.ENOENT) - - // Duplicate the temporary file descriptor on top of the file from our file - // system. We shouldn't see the flush error. - err = dup2(int(t.f2.Fd()), int(t.f1.Fd())) - ExpectEq(nil, err) -} - func (t *FlushFSTest) Mmap_MunmapBeforeClose() { var n int var err error @@ -651,18 +562,6 @@ func (t *FlushFSTest) Directory() { AssertTrue(false, "TODO") } -//////////////////////////////////////////////////////////////////////// -// No errors -//////////////////////////////////////////////////////////////////////// - -type NoErrorsTest struct { - flushFSTest -} - -func init() { RegisterTestSuite(&NoErrorsTest{}) } - -func (t *NoErrorsTest) SetUp(ti *TestInfo) - //////////////////////////////////////////////////////////////////////// // Flush error //////////////////////////////////////////////////////////////////////// @@ -675,6 +574,86 @@ func init() { RegisterTestSuite(&FlushErrorTest{}) } func (t *FlushErrorTest) SetUp(ti *TestInfo) +func (t *FlushFSTest) CloseError() { + var err error + + // Open the file. + t.f1, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDWR, 0) + AssertEq(nil, err) + + // Configure a flush error. + t.setFlushError(fuse.ENOENT) + + // Close the file. + err = t.f1.Close() + t.f1 = nil + + AssertNe(nil, err) + ExpectThat(err, Error(HasSubstr("no such file"))) +} + +func (t *FlushFSTest) Dup_FlushError() { + var err error + + // Open the file. + t.f1, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0) + AssertEq(nil, err) + + fd1 := t.f1.Fd() + + // Use dup(2) to get another copy. + fd2, err := syscall.Dup(int(fd1)) + AssertEq(nil, err) + + t.f2 = os.NewFile(uintptr(fd2), t.f1.Name()) + + // Configure a flush error. + t.setFlushError(fuse.ENOENT) + + // Close by the first handle. On OS X, where the semantics of file handles + // are different (cf. https://github.com/osxfuse/osxfuse/issues/199), this + // does not result in an error. + err = t.f1.Close() + t.f1 = nil + + if runtime.GOOS == "darwin" { + AssertEq(nil, err) + } else { + AssertNe(nil, err) + ExpectThat(err, Error(HasSubstr("no such file"))) + } + + // Close by the second handle. + err = t.f2.Close() + t.f2 = nil + + AssertNe(nil, err) + ExpectThat(err, Error(HasSubstr("no such file"))) +} + +func (t *FlushFSTest) Dup2_FlushError() { + var err error + + // Open the file. + t.f1, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0) + AssertEq(nil, err) + + // Open and unlink some temporary file. + t.f2, err = ioutil.TempFile("", "") + AssertEq(nil, err) + + err = os.Remove(t.f2.Name()) + AssertEq(nil, err) + + // Configure a flush error. + t.setFlushError(fuse.ENOENT) + + // Duplicate the temporary file descriptor on top of the file from our file + // system. We shouldn't see the flush error. + err = dup2(int(t.f2.Fd()), int(t.f1.Fd())) + ExpectEq(nil, err) +} + //////////////////////////////////////////////////////////////////////// // Fsync error //////////////////////////////////////////////////////////////////////// @@ -686,3 +665,20 @@ type FsyncErrorTest struct { func init() { RegisterTestSuite(&FsyncErrorTest{}) } func (t *FsyncErrorTest) SetUp(ti *TestInfo) + +func (t *FlushFSTest) FsyncError() { + var err error + + // Open the file. + t.f1, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDWR, 0) + AssertEq(nil, err) + + // Configure an fsync error. + t.setFsyncError(fuse.ENOENT) + + // Fsync. + err = t.f1.Sync() + + AssertNe(nil, err) + ExpectThat(err, Error(HasSubstr("no such file"))) +} From 01470b8c3458139891c83ce14008dbc32aae2b03 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 08:40:35 +1100 Subject: [PATCH 20/44] Fixed test names. --- samples/flushfs/flush_fs_test.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 8e82a04..c074809 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -130,7 +130,7 @@ func init() { RegisterTestSuite(&NoErrorsTest{}) } func (t *NoErrorsTest) SetUp(ti *TestInfo) -func (t *FlushFSTest) CloseReports_ReadWrite() { +func (t *NoErrorsTest) Close_ReadWrite() { var n int var off int64 var err error @@ -168,7 +168,7 @@ func (t *FlushFSTest) CloseReports_ReadWrite() { ExpectThat(t.getFsyncs(), ElementsAre()) } -func (t *FlushFSTest) CloseReports_ReadOnly() { +func (t *NoErrorsTest) Close_ReadOnly() { var err error // Open the file. @@ -189,7 +189,7 @@ func (t *FlushFSTest) CloseReports_ReadOnly() { ExpectThat(t.getFsyncs(), ElementsAre()) } -func (t *FlushFSTest) CloseReports_WriteOnly() { +func (t *NoErrorsTest) Close_WriteOnly() { var n int var err error @@ -216,7 +216,7 @@ func (t *FlushFSTest) CloseReports_WriteOnly() { ExpectThat(t.getFsyncs(), ElementsAre()) } -func (t *FlushFSTest) CloseReports_MultipleTimes_NonOverlappingFileHandles() { +func (t *NoErrorsTest) Close_MultipleTimes_NonOverlappingFileHandles() { var n int var err error @@ -263,7 +263,7 @@ func (t *FlushFSTest) CloseReports_MultipleTimes_NonOverlappingFileHandles() { AssertThat(t.getFsyncs(), ElementsAre()) } -func (t *FlushFSTest) CloseReports_MultipleTimes_OverlappingFileHandles() { +func (t *NoErrorsTest) Close_MultipleTimes_OverlappingFileHandles() { var n int var err error @@ -312,7 +312,7 @@ func (t *FlushFSTest) CloseReports_MultipleTimes_OverlappingFileHandles() { AssertThat(t.getFsyncs(), ElementsAre()) } -func (t *FlushFSTest) FsyncReports() { +func (t *NoErrorsTest) Fsync() { var n int var err error @@ -351,7 +351,7 @@ func (t *FlushFSTest) FsyncReports() { AssertThat(t.getFsyncs(), ElementsAre("taco", "tacos")) } -func (t *FlushFSTest) Dup() { +func (t *NoErrorsTest) Dup() { var n int var err error @@ -415,7 +415,7 @@ func (t *FlushFSTest) Dup() { ExpectThat(t.getFsyncs(), ElementsAre()) } -func (t *FlushFSTest) Dup2() { +func (t *NoErrorsTest) Dup2() { var n int var err error @@ -444,7 +444,7 @@ func (t *FlushFSTest) Dup2() { ExpectThat(t.getFsyncs(), ElementsAre()) } -func (t *FlushFSTest) Mmap_MunmapBeforeClose() { +func (t *NoErrorsTest) Mmap_MunmapBeforeClose() { var n int var err error @@ -504,7 +504,7 @@ func (t *FlushFSTest) Mmap_MunmapBeforeClose() { } } -func (t *FlushFSTest) Mmap_CloseBeforeMunmap() { +func (t *NoErrorsTest) Mmap_CloseBeforeMunmap() { var n int var err error @@ -558,7 +558,7 @@ func (t *FlushFSTest) Mmap_CloseBeforeMunmap() { ExpectThat(t.getFsyncs(), ElementsAre()) } -func (t *FlushFSTest) Directory() { +func (t *NoErrorsTest) Directory() { AssertTrue(false, "TODO") } @@ -574,7 +574,7 @@ func init() { RegisterTestSuite(&FlushErrorTest{}) } func (t *FlushErrorTest) SetUp(ti *TestInfo) -func (t *FlushFSTest) CloseError() { +func (t *FlushErrorTest) Close() { var err error // Open the file. @@ -592,7 +592,7 @@ func (t *FlushFSTest) CloseError() { ExpectThat(err, Error(HasSubstr("no such file"))) } -func (t *FlushFSTest) Dup_FlushError() { +func (t *FlushErrorTest) Dup() { var err error // Open the file. @@ -631,7 +631,7 @@ func (t *FlushFSTest) Dup_FlushError() { ExpectThat(err, Error(HasSubstr("no such file"))) } -func (t *FlushFSTest) Dup2_FlushError() { +func (t *FlushErrorTest) Dup2() { var err error // Open the file. @@ -666,7 +666,7 @@ func init() { RegisterTestSuite(&FsyncErrorTest{}) } func (t *FsyncErrorTest) SetUp(ti *TestInfo) -func (t *FlushFSTest) FsyncError() { +func (t *FsyncErrorTest) Fsync() { var err error // Open the file. From bfa3465cb95365e1ae55fc75abf3e18120136ca0 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 08:42:39 +1100 Subject: [PATCH 21/44] Open and close reporting files. --- samples/flushfs/flush_fs_test.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index c074809..4f2ee07 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -40,6 +40,10 @@ func TestFlushFS(t *testing.T) { RunTests(t) } type flushFSTest struct { samples.SubprocessTest + // Files to which mount_sample is writing reported flushes and fsyncs. + flushes *os.File + fsyncs *os.File + // File handles that are closed in TearDown if non-nil. f1 *os.File f2 *os.File @@ -52,7 +56,11 @@ func (t *flushFSTest) setUp( var err error // Set up files to receive flush and fsync reports. - panic("TODO") + t.flushes, err = ioutil.TempFile("", "") + AssertEq(nil, err) + + t.fsyncs, err = ioutil.TempFile("", "") + AssertEq(nil, err) // Set up test config. t.MountType = "flushfs" @@ -74,7 +82,15 @@ func (t *flushFSTest) setUp( } func (t *FlushFSTest) TearDown() { - // Close files if non-nil. + // Unlink reporting files. + os.Remove(t.flushes.Name()) + os.Remove(t.fsyncs.Name()) + + // Close reporting files. + t.flushes.Close() + t.fsyncs.Close() + + // Close test files if non-nil. if t.f1 != nil { ExpectEq(nil, t.f1.Close()) } From e7dc200b88ede6ac594372937a8cc303153da05c Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 08:43:03 +1100 Subject: [PATCH 22/44] Fixed some build errors. --- samples/flushfs/flush_fs_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 4f2ee07..78bc034 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -81,7 +81,7 @@ func (t *flushFSTest) setUp( t.SubprocessTest.SetUp(ti) } -func (t *FlushFSTest) TearDown() { +func (t *flushFSTest) TearDown() { // Unlink reporting files. os.Remove(t.flushes.Name()) os.Remove(t.fsyncs.Name()) @@ -100,7 +100,7 @@ func (t *FlushFSTest) TearDown() { } // Finish tearing down. - t.SampleTest.TearDown() + t.SubprocessTest.TearDown() } //////////////////////////////////////////////////////////////////////// From cab543fa8c64e83ce1da5c4a19c7ba3b8c7de20f Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 08:43:24 +1100 Subject: [PATCH 23/44] Fixed more build errors. --- samples/flushfs/flush_fs_test.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 78bc034..c428265 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -25,7 +25,6 @@ import ( "testing" "github.com/jacobsa/bazilfuse" - "github.com/jacobsa/fuse" "github.com/jacobsa/fuse/samples" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" @@ -597,9 +596,6 @@ func (t *FlushErrorTest) Close() { t.f1, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDWR, 0) AssertEq(nil, err) - // Configure a flush error. - t.setFlushError(fuse.ENOENT) - // Close the file. err = t.f1.Close() t.f1 = nil @@ -623,9 +619,6 @@ func (t *FlushErrorTest) Dup() { t.f2 = os.NewFile(uintptr(fd2), t.f1.Name()) - // Configure a flush error. - t.setFlushError(fuse.ENOENT) - // Close by the first handle. On OS X, where the semantics of file handles // are different (cf. https://github.com/osxfuse/osxfuse/issues/199), this // does not result in an error. @@ -661,9 +654,6 @@ func (t *FlushErrorTest) Dup2() { err = os.Remove(t.f2.Name()) AssertEq(nil, err) - // Configure a flush error. - t.setFlushError(fuse.ENOENT) - // Duplicate the temporary file descriptor on top of the file from our file // system. We shouldn't see the flush error. err = dup2(int(t.f2.Fd()), int(t.f1.Fd())) @@ -689,9 +679,6 @@ func (t *FsyncErrorTest) Fsync() { t.f1, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDWR, 0) AssertEq(nil, err) - // Configure an fsync error. - t.setFsyncError(fuse.ENOENT) - // Fsync. err = t.f1.Sync() From c090f136cd59dab5698a61ccdbe7768b5e3ed0b3 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 08:45:01 +1100 Subject: [PATCH 24/44] Implemented getFlushes and getFsyncs. --- samples/flushfs/flush_fs_test.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index c428265..d22d2ef 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -106,18 +106,26 @@ func (t *flushFSTest) TearDown() { // Helpers //////////////////////////////////////////////////////////////////////// +func readReports(f *os.File) (reports []string, err error) + // Return a copy of the current contents of t.flushes. -// -// LOCKS_EXCLUDED(t.mu) func (t *flushFSTest) getFlushes() (p []string) { - panic("TODO") + var err error + if p, err = readReports(t.flushes); err != nil { + panic(err) + } + + return } // Return a copy of the current contents of t.fsyncs. -// -// LOCKS_EXCLUDED(t.mu) func (t *flushFSTest) getFsyncs() (p []string) { - panic("TODO") + var err error + if p, err = readReports(t.fsyncs); err != nil { + panic(err) + } + + return } // Like syscall.Dup2, but correctly annotates the syscall as blocking. See here From ddfca9cb6b0a8b584152ba9bc0e49b2bd863635f Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 08:51:19 +1100 Subject: [PATCH 25/44] Implemented readReports. --- samples/flushfs/flush_fs_test.go | 35 +++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index d22d2ef..9811c8c 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -15,6 +15,8 @@ package flushfs_test import ( + "bufio" + "encoding/hex" "fmt" "io" "io/ioutil" @@ -106,7 +108,38 @@ func (t *flushFSTest) TearDown() { // Helpers //////////////////////////////////////////////////////////////////////// -func readReports(f *os.File) (reports []string, err error) +func readReports(f *os.File) (reports []string, err error) { + // Seek the file to the start. + _, err = f.Seek(0, 0) + if err != nil { + err = fmt.Errorf("Seek: %v", err) + return + } + + // We expect reports to end in a newline (including the final one). + reader := bufio.NewReader(f) + for { + var record []byte + record, err = reader.ReadBytes('\n') + if err == io.EOF { + if len(record) != 0 { + err = fmt.Errorf("Unexpected record:\n%s", hex.Dump(record)) + return + } + + err = nil + return + } + + if err != nil { + err = fmt.Errorf("ReadBytes: %v", err) + return + } + + // Strip the newline. + reports = append(reports, string(record[:len(record)-1])) + } +} // Return a copy of the current contents of t.flushes. func (t *flushFSTest) getFlushes() (p []string) { From f92de1a6e7bf67c23b3377df63aa2d985f19526e Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 08:52:05 +1100 Subject: [PATCH 26/44] NoErrorsTest.SetUp --- samples/flushfs/flush_fs_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 9811c8c..ff77bdd 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -184,7 +184,10 @@ type NoErrorsTest struct { func init() { RegisterTestSuite(&NoErrorsTest{}) } -func (t *NoErrorsTest) SetUp(ti *TestInfo) +func (t *NoErrorsTest) SetUp(ti *TestInfo) { + const noErr = 0 + t.flushFSTest.setUp(ti, noErr, noErr) +} func (t *NoErrorsTest) Close_ReadWrite() { var n int From 2665bd1342d4a99c36f250fc5e04ec4dc85c6ad9 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 08:52:56 +1100 Subject: [PATCH 27/44] Other SetUp methods. --- samples/flushfs/flush_fs_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index ff77bdd..88ec590 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -631,7 +631,10 @@ type FlushErrorTest struct { func init() { RegisterTestSuite(&FlushErrorTest{}) } -func (t *FlushErrorTest) SetUp(ti *TestInfo) +func (t *FlushErrorTest) SetUp(ti *TestInfo) { + const noErr = 0 + t.flushFSTest.SetUp(ti, bazilfuse.ENOENT, noErr) +} func (t *FlushErrorTest) Close() { var err error @@ -714,7 +717,10 @@ type FsyncErrorTest struct { func init() { RegisterTestSuite(&FsyncErrorTest{}) } -func (t *FsyncErrorTest) SetUp(ti *TestInfo) +func (t *FsyncErrorTest) SetUp(ti *TestInfo) { + const noErr = 0 + t.flushFSTest.SetUp(ti, noErr, bazilfuse.ENOENT) +} func (t *FsyncErrorTest) Fsync() { var err error From d12a25db1c6b3d7879bb3c9d99df80734d696a14 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 08:53:11 +1100 Subject: [PATCH 28/44] Fixed two build errors. --- samples/flushfs/flush_fs_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 88ec590..b528bea 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -633,7 +633,7 @@ func init() { RegisterTestSuite(&FlushErrorTest{}) } func (t *FlushErrorTest) SetUp(ti *TestInfo) { const noErr = 0 - t.flushFSTest.SetUp(ti, bazilfuse.ENOENT, noErr) + t.flushFSTest.setUp(ti, bazilfuse.ENOENT, noErr) } func (t *FlushErrorTest) Close() { @@ -719,7 +719,7 @@ func init() { RegisterTestSuite(&FsyncErrorTest{}) } func (t *FsyncErrorTest) SetUp(ti *TestInfo) { const noErr = 0 - t.flushFSTest.SetUp(ti, noErr, bazilfuse.ENOENT) + t.flushFSTest.setUp(ti, noErr, bazilfuse.ENOENT) } func (t *FsyncErrorTest) Fsync() { From 53d662ca0bcc189bac3823d9df3e52974d64630e Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 08:53:52 +1100 Subject: [PATCH 29/44] Fixed go build arg order. --- samples/subprocess.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/subprocess.go b/samples/subprocess.go index 52e3179..675f63d 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -88,9 +88,9 @@ func buildMountSample() (toolPath string, err error) { cmd := exec.Command( "go", "build", - "github.com/jacobsa/fuse/samples/mount_sample", "-o", - mountSamplePath) + mountSamplePath, + "github.com/jacobsa/fuse/samples/mount_sample") output, err := cmd.CombinedOutput() if err != nil { From dff6fe8c828039a40fb6a2f7857fe92ea1948e31 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 09:00:50 +1100 Subject: [PATCH 30/44] Share unmount code. --- samples/in_process.go | 27 ++++-------------------- samples/subprocess.go | 24 ++++------------------ samples/unmount.go | 48 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 43 deletions(-) create mode 100644 samples/unmount.go diff --git a/samples/in_process.go b/samples/in_process.go index ac19b6e..e4f2caf 100644 --- a/samples/in_process.go +++ b/samples/in_process.go @@ -18,8 +18,6 @@ import ( "fmt" "io" "io/ioutil" - "log" - "strings" "time" "github.com/googlecloudplatform/gcsfuse/timeutil" @@ -124,27 +122,10 @@ func (t *SampleTest) destroy() (err error) { return } - // Unmount the file system. Try again on "resource busy" errors. - delay := 10 * time.Millisecond - for { - err = t.mfs.Unmount() - if err == nil { - break - } - - if strings.Contains(err.Error(), "resource busy") { - log.Println("Resource busy error while unmounting; trying again") - time.Sleep(delay) - delay = time.Duration(1.3 * float64(delay)) - continue - } - - err = fmt.Errorf("MountedFileSystem.Unmount: %v", err) - return - } - - if err = t.mfs.Join(t.Ctx); err != nil { - err = fmt.Errorf("MountedFileSystem.Join: %v", err) + // Unmount the file system. + err = unmount(t.Dir) + if err != nil { + err = fmt.Errorf("unmount: %v", err) return } diff --git a/samples/subprocess.go b/samples/subprocess.go index 675f63d..a5d9edc 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -18,14 +18,10 @@ import ( "fmt" "io" "io/ioutil" - "log" "os/exec" "path" - "strings" "sync" - "time" - "github.com/jacobsa/bazilfuse" "github.com/jacobsa/ogletest" "golang.org/x/net/context" ) @@ -182,22 +178,10 @@ func (t *SubprocessTest) destroy() (err error) { return } - // Unmount the file system. Try again on "resource busy" errors. - delay := 10 * time.Millisecond - for { - err = bazilfuse.Unmount(t.Dir) - if err == nil { - break - } - - if strings.Contains(err.Error(), "resource busy") { - log.Println("Resource busy error while unmounting; trying again") - time.Sleep(delay) - delay = time.Duration(1.3 * float64(delay)) - continue - } - - err = fmt.Errorf("Unmount: %v", err) + // Unmount the file system. + err = unmount(t.Dir) + if err != nil { + err = fmt.Errorf("unmount: %v", err) return } diff --git a/samples/unmount.go b/samples/unmount.go new file mode 100644 index 0000000..37652e3 --- /dev/null +++ b/samples/unmount.go @@ -0,0 +1,48 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package samples + +import ( + "fmt" + "log" + "strings" + "time" + + "github.com/jacobsa/bazilfuse" +) + +// Unmount the file system mounted at the supplied directory. Try again on +// "resource busy" errors, which happen from time to time on OS X (due to weird +// requests from the Finder) and when tests don't or can't synchronize all +// events. +func unmount(dir string) (err error) { + delay := 10 * time.Millisecond + for { + err = bazilfuse.Unmount(dir) + if err == nil { + return + } + + if strings.Contains(err.Error(), "resource busy") { + log.Println("Resource busy error while unmounting; trying again") + time.Sleep(delay) + delay = time.Duration(1.3 * float64(delay)) + continue + } + + err = fmt.Errorf("Unmount: %v", err) + return + } +} From 39dfb0c8871acda49e0b7629bca70c43469ec5f5 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 09:07:02 +1100 Subject: [PATCH 31/44] Don't hide command termination errors behind unmount errors. --- samples/subprocess.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/samples/subprocess.go b/samples/subprocess.go index a5d9edc..e4b313a 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -18,6 +18,7 @@ import ( "fmt" "io" "io/ioutil" + "log" "os/exec" "path" "sync" @@ -173,17 +174,30 @@ func (t *SubprocessTest) destroy() (err error) { ogletest.ExpectEq(nil, c.Close()) } - // Was the file system mounted? + // If we didn't try to mount the file system, there's nothing further to do. if t.mountCmd == nil { return } - // Unmount the file system. - err = unmount(t.Dir) - if err != nil { - err = fmt.Errorf("unmount: %v", err) - return - } + // In the background, initiate an unmount. + unmountErrChan := make(chan error) + go func() { + unmountErrChan <- unmount(t.Dir) + }() + + // Make sure we wait for the unmount, even if we've already returned early in + // error. Return its error if we haven't seen any other error. + defer func() { + unmountErr := <-unmountErrChan + if unmountErr != nil { + if err != nil { + log.Println("unmount:", unmountErr) + return + } + + err = fmt.Errorf("unmount: %v", unmountErr) + } + }() // Wait for the subprocess. if err = t.mountCmd.Wait(); err != nil { From 258104ae329801b9194f9a76eeba73e87f596f27 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 09:22:39 +1100 Subject: [PATCH 32/44] Include stderr in exit errors. --- samples/subprocess.go | 46 ++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/samples/subprocess.go b/samples/subprocess.go index e4b313a..b9830d4 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -15,6 +15,7 @@ package samples import ( + "bytes" "fmt" "io" "io/ioutil" @@ -48,7 +49,9 @@ type SubprocessTest struct { // fail if closing fails. ToClose []io.Closer - mountCmd *exec.Cmd + mountCmd *exec.Cmd + mountStdout bytes.Buffer + mountStderr bytes.Buffer } // Mount the file system and initialize the other exported fields of the @@ -109,17 +112,6 @@ func buildMountSample() (toolPath string, err error) { return } -// Invoke mount_sample, returning a running command. -func invokeMountSample(path string, args []string) (cmd *exec.Cmd, err error) { - cmd = exec.Command(path, args...) - if err = cmd.Start(); err != nil { - err = fmt.Errorf("Start: %v", err) - return - } - - return -} - // Like SetUp, but doens't panic. func (t *SubprocessTest) initialize() (err error) { // Initialize the context. @@ -139,13 +131,22 @@ func (t *SubprocessTest) initialize() (err error) { return } - // Invoke it. - args := []string{"--type", t.MountType} + // Set up a command. + args := []string{ + toolPath, + "--type", + t.MountType, + } + args = append(args, t.MountFlags...) - t.mountCmd, err = invokeMountSample(toolPath, args) - if err != nil { - err = fmt.Errorf("invokeMountSample: %v", err) + t.mountCmd = exec.Command(toolPath, args...) + t.mountCmd.Stdout = &t.mountStdout + t.mountCmd.Stderr = &t.mountStderr + + // Start it. + if err = t.mountCmd.Start(); err != nil { + err = fmt.Errorf("mountCmd.Start: %v", err) return } @@ -201,7 +202,16 @@ func (t *SubprocessTest) destroy() (err error) { // Wait for the subprocess. if err = t.mountCmd.Wait(); err != nil { - err = fmt.Errorf("Cmd.Wait: %v", err) + if exitErr, ok := err.(*exec.ExitError); ok { + err = fmt.Errorf( + "mount_sample exited with %v. Stderr:\n%s", + exitErr, + t.mountStderr.String()) + + return + } + + err = fmt.Errorf("mountCmd.Wait: %v", err) return } From 47445387e7c4631e3ff4ec9896522256fed68508 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 09:25:43 +1100 Subject: [PATCH 33/44] Fixed an invocation bug. --- samples/subprocess.go | 1 - 1 file changed, 1 deletion(-) diff --git a/samples/subprocess.go b/samples/subprocess.go index b9830d4..8c1f9c0 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -133,7 +133,6 @@ func (t *SubprocessTest) initialize() (err error) { // Set up a command. args := []string{ - toolPath, "--type", t.MountType, } From 61f3528c80ea1064c1596fcd653ebb3bd33840b2 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 09:26:16 +1100 Subject: [PATCH 34/44] Fixed another invocation bug. --- samples/flushfs/flush_fs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index b528bea..f804284 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -75,7 +75,7 @@ func (t *flushFSTest) setUp( "--flushfs.flush_error", fmt.Sprintf("%d", int(flushErr)), - "--fsyncfs.fsync_error", + "--flushfs.fsync_error", fmt.Sprintf("%d", int(fsyncErr)), } From 6b6c7c92ca69e213517e735f80097c9be3bf9d0a Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 09:26:41 +1100 Subject: [PATCH 35/44] Added a sleep workaround for now. --- samples/subprocess.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/subprocess.go b/samples/subprocess.go index 8c1f9c0..834f257 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -23,6 +23,7 @@ import ( "os/exec" "path" "sync" + "time" "github.com/jacobsa/ogletest" "golang.org/x/net/context" @@ -151,6 +152,7 @@ func (t *SubprocessTest) initialize() (err error) { // TODO(jacobsa): Probably need some sort of signalling (on stderr? write to // a flag-controlled file?) when WaitForReady has returned. + time.Sleep(time.Second) return } From a4384665878fc25e07f7b2db10efff6c99a14729 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 09:27:09 +1100 Subject: [PATCH 36/44] Fixed another invocation problem. --- samples/subprocess.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/subprocess.go b/samples/subprocess.go index 834f257..6161ed6 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -136,6 +136,8 @@ func (t *SubprocessTest) initialize() (err error) { args := []string{ "--type", t.MountType, + "--mount_point", + t.Dir, } args = append(args, t.MountFlags...) From b62be9632cb58ea348e5483cf9eb60d6bc769d90 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 09:46:05 +1100 Subject: [PATCH 37/44] Pass report files by file descriptor. --- samples/flushfs/flush_fs_test.go | 11 +++++------ samples/mount_sample/mount.go | 21 ++++++--------------- samples/subprocess.go | 20 +++++++++++++++++++- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index f804284..4885357 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -66,12 +66,6 @@ func (t *flushFSTest) setUp( // Set up test config. t.MountType = "flushfs" t.MountFlags = []string{ - "--flushfs.flushes_file", - t.flushes.Name(), - - "--flushfs.fsyncs_file", - t.fsyncs.Name(), - "--flushfs.flush_error", fmt.Sprintf("%d", int(flushErr)), @@ -79,6 +73,11 @@ func (t *flushFSTest) setUp( fmt.Sprintf("%d", int(fsyncErr)), } + t.MountFiles = map[string]*os.File{ + "flushfs.flushes_file": t.flushes, + "flushfs.fsyncs_file": t.fsyncs, + } + t.SubprocessTest.SetUp(ti) } diff --git a/samples/mount_sample/mount.go b/samples/mount_sample/mount.go index 963186c..48f860d 100644 --- a/samples/mount_sample/mount.go +++ b/samples/mount_sample/mount.go @@ -31,30 +31,21 @@ import ( var fType = flag.String("type", "", "The name of the samples/ sub-dir.") var fMountPoint = flag.String("mount_point", "", "Path to mount point.") -var fFlushesFile = flag.String("flushfs.flushes_file", "", "") -var fFsyncsFile = flag.String("flushfs.fsyncs_file", "", "") +var fFlushesFile = flag.Uint64("flushfs.flushes_file", 0, "") +var fFsyncsFile = flag.Uint64("flushfs.fsyncs_file", 0, "") var fFlushError = flag.Int("flushfs.flush_error", 0, "") var fFsyncError = flag.Int("flushfs.fsync_error", 0, "") func makeFlushFS() (fs fuse.FileSystem, err error) { // Check the flags. - if *fFlushesFile == "" || *fFsyncsFile == "" { + if *fFlushesFile == 0 || *fFsyncsFile == 0 { err = fmt.Errorf("You must set the flushfs flags.") return } - // Open the files. - flushes, err := os.OpenFile(*fFlushesFile, os.O_RDWR, 0) - if err != nil { - err = fmt.Errorf("Opening %s: %v", *fFlushesFile, err) - return - } - - fsyncs, err := os.OpenFile(*fFsyncsFile, os.O_RDWR, 0) - if err != nil { - err = fmt.Errorf("Opening %s: %v", *fFsyncsFile, err) - return - } + // Set up the files. + flushes := os.NewFile(uintptr(*fFlushesFile), "(flushes file)") + fsyncs := os.NewFile(uintptr(*fFsyncsFile), "(fsyncs file)") // Set up errors. var flushErr error diff --git a/samples/subprocess.go b/samples/subprocess.go index 6161ed6..5db0ca6 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -20,6 +20,7 @@ import ( "io" "io/ioutil" "log" + "os" "os/exec" "path" "sync" @@ -40,6 +41,10 @@ type SubprocessTest struct { // Additional flags to be passed to the mount_sample tool. MountFlags []string + // A list of files to pass to mount_sample. The given string flag will be + // used to pass the file descriptor number. + MountFiles map[string]*os.File + // A context object that can be used for long-running operations. Ctx context.Context @@ -132,7 +137,7 @@ func (t *SubprocessTest) initialize() (err error) { return } - // Set up a command. + // Set up basic args for the subprocess. args := []string{ "--type", t.MountType, @@ -142,9 +147,22 @@ func (t *SubprocessTest) initialize() (err error) { args = append(args, t.MountFlags...) + // Set up inherited files and appropriate flags. + var extraFiles []*os.File + for flag, file := range t.MountFiles { + // Cf. os/exec.Cmd.ExtraFiles + fd := 3 + len(extraFiles) + + extraFiles = append(extraFiles, file) + args = append(args, "--"+flag) + args = append(args, fmt.Sprintf("%d", fd)) + } + + // Set up a command. t.mountCmd = exec.Command(toolPath, args...) t.mountCmd.Stdout = &t.mountStdout t.mountCmd.Stderr = &t.mountStderr + t.mountCmd.ExtraFiles = extraFiles // Start it. if err = t.mountCmd.Start(); err != nil { From 49455cd59c8fbf8aea8f2c1eded2cdcf6b4229d8 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 09:59:13 +1100 Subject: [PATCH 38/44] Wait for tool in a goroutine. --- samples/subprocess.go | 62 +++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/samples/subprocess.go b/samples/subprocess.go index 5db0ca6..fcfbcdd 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -55,9 +55,7 @@ type SubprocessTest struct { // fail if closing fails. ToClose []io.Closer - mountCmd *exec.Cmd - mountStdout bytes.Buffer - mountStderr bytes.Buffer + mountSampleErr <-chan error } // Mount the file system and initialize the other exported fields of the @@ -118,6 +116,35 @@ func buildMountSample() (toolPath string, err error) { return } +func waitForMountSample( + cmd *exec.Cmd, + errChan chan<- error, + stderr *bytes.Buffer) { + // However we exit, write the error to the channel. + var err error + defer func() { + errChan <- err + }() + + // Wait for the command. + err = cmd.Wait() + if err == nil { + return + } + + // Make exit errors nicer. + if exitErr, ok := err.(*exec.ExitError); ok { + err = fmt.Errorf( + "mount_sample exited with %v. Stderr:\n%s", + exitErr, + stderr.String()) + + return + } + + err = fmt.Errorf("Waiting for mount_sample: %v", err) +} + // Like SetUp, but doens't panic. func (t *SubprocessTest) initialize() (err error) { // Initialize the context. @@ -159,17 +186,22 @@ func (t *SubprocessTest) initialize() (err error) { } // Set up a command. - t.mountCmd = exec.Command(toolPath, args...) - t.mountCmd.Stdout = &t.mountStdout - t.mountCmd.Stderr = &t.mountStderr - t.mountCmd.ExtraFiles = extraFiles + var stderr bytes.Buffer + mountCmd := exec.Command(toolPath, args...) + mountCmd.Stderr = &stderr + mountCmd.ExtraFiles = extraFiles // Start it. - if err = t.mountCmd.Start(); err != nil { + if err = mountCmd.Start(); err != nil { err = fmt.Errorf("mountCmd.Start: %v", err) return } + // Launch a goroutine that waits for it and returns its status. + mountSampleErr := make(chan error, 1) + t.mountSampleErr = mountSampleErr + go waitForMountSample(mountCmd, mountSampleErr, &stderr) + // TODO(jacobsa): Probably need some sort of signalling (on stderr? write to // a flag-controlled file?) when WaitForReady has returned. time.Sleep(time.Second) @@ -197,7 +229,7 @@ func (t *SubprocessTest) destroy() (err error) { } // If we didn't try to mount the file system, there's nothing further to do. - if t.mountCmd == nil { + if t.mountSampleErr == nil { return } @@ -222,17 +254,7 @@ func (t *SubprocessTest) destroy() (err error) { }() // Wait for the subprocess. - if err = t.mountCmd.Wait(); err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - err = fmt.Errorf( - "mount_sample exited with %v. Stderr:\n%s", - exitErr, - t.mountStderr.String()) - - return - } - - err = fmt.Errorf("mountCmd.Wait: %v", err) + if err = <-t.mountSampleErr; err != nil { return } From 6729af8524e0861cd3522c59daa2174943f5dec4 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 10:08:17 +1100 Subject: [PATCH 39/44] Synchronize on ready with a pipe. --- samples/mount_sample/mount.go | 24 +++++++++++++++++++++ samples/subprocess.go | 40 ++++++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/samples/mount_sample/mount.go b/samples/mount_sample/mount.go index 48f860d..e7d0def 100644 --- a/samples/mount_sample/mount.go +++ b/samples/mount_sample/mount.go @@ -17,6 +17,7 @@ package main import ( + "errors" "flag" "fmt" "log" @@ -30,6 +31,7 @@ import ( var fType = flag.String("type", "", "The name of the samples/ sub-dir.") var fMountPoint = flag.String("mount_point", "", "Path to mount point.") +var fReadyFile = flag.Uint64("ready_file", 0, "FD to signal when ready.") var fFlushesFile = flag.Uint64("flushfs.flushes_file", 0, "") var fFsyncsFile = flag.Uint64("flushfs.fsyncs_file", 0, "") @@ -97,9 +99,25 @@ func makeFS() (fs fuse.FileSystem, err error) { return } +func getReadyFile() (f *os.File, err error) { + if *fReadyFile == 0 { + err = errors.New("You must set --ready_file.") + return + } + + f = os.NewFile(uintptr(*fReadyFile), "(ready file)") + return +} + func main() { flag.Parse() + // Grab the file to signal when ready. + readyFile, err := getReadyFile() + if err != nil { + log.Fatalf("getReadyFile:", err) + } + // Create an appropriate file system. fs, err := makeFS() if err != nil { @@ -121,6 +139,12 @@ func main() { log.Fatalf("WaitForReady: %v", err) } + // Signal that it is ready. + _, err = readyFile.Write([]byte("x")) + if err != nil { + log.Fatalf("readyFile.Write: %v", err) + } + // Wait for it to be unmounted. if err = mfs.Join(context.Background()); err != nil { log.Fatalf("Join: %v", err) diff --git a/samples/subprocess.go b/samples/subprocess.go index fcfbcdd..acfd97d 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -24,7 +24,6 @@ import ( "os/exec" "path" "sync" - "time" "github.com/jacobsa/ogletest" "golang.org/x/net/context" @@ -145,6 +144,16 @@ func waitForMountSample( err = fmt.Errorf("Waiting for mount_sample: %v", err) } +func waitForReady(readyReader *os.File, c chan<- struct{}) { + _, err := readyReader.Read(make([]byte, 1)) + if err != nil { + log.Printf("Readying from ready pipe: %v", err) + return + } + + c <- struct{}{} +} + // Like SetUp, but doens't panic. func (t *SubprocessTest) initialize() (err error) { // Initialize the context. @@ -174,6 +183,18 @@ func (t *SubprocessTest) initialize() (err error) { args = append(args, t.MountFlags...) + // Set up a pipe for the "ready" status. + readyReader, readyWriter, err := os.Pipe() + if err != nil { + err = fmt.Errorf("Pipe: %v", err) + return + } + + defer readyReader.Close() + defer readyWriter.Close() + + t.MountFiles["ready_file"] = readyWriter + // Set up inherited files and appropriate flags. var extraFiles []*os.File for flag, file := range t.MountFiles { @@ -199,12 +220,21 @@ func (t *SubprocessTest) initialize() (err error) { // Launch a goroutine that waits for it and returns its status. mountSampleErr := make(chan error, 1) - t.mountSampleErr = mountSampleErr go waitForMountSample(mountCmd, mountSampleErr, &stderr) - // TODO(jacobsa): Probably need some sort of signalling (on stderr? write to - // a flag-controlled file?) when WaitForReady has returned. - time.Sleep(time.Second) + // Wait for the tool to say the file system is ready. In parallel, watch for + // the tool to fail. + readyChan := make(chan struct{}, 1) + go waitForReady(readyReader, readyChan) + + select { + case <-readyChan: + case err = <-mountSampleErr: + return + } + + // TearDown is no responsible for joining. + t.mountSampleErr = mountSampleErr return } From b5e00243ba2cc19c714439ee295c721cbfba9936 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 10:24:08 +1100 Subject: [PATCH 40/44] Allow re-using the mount_sample binary across runs. --- samples/subprocess.go | 98 ++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/samples/subprocess.go b/samples/subprocess.go index acfd97d..288e185 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -16,6 +16,7 @@ package samples import ( "bytes" + "flag" "fmt" "io" "io/ioutil" @@ -29,6 +30,11 @@ import ( "golang.org/x/net/context" ) +var fToolPath = flag.String( + "mount_sample", + "", + "Path to the mount_sample tool. If unset, we will compile it.") + // A struct that implements common behavior needed by tests in the samples/ // directory where the file system is mounted by a subprocess. Use it as an // embedded field in your test fixture, calling its SetUp method from your @@ -68,50 +74,58 @@ func (t *SubprocessTest) SetUp(ti *ogletest.TestInfo) { } } -// Set by buildMountSample. -var mountSamplePath string -var mountSampleErr error -var mountSampleOnce sync.Once +// Private state for getToolPath. +var getToolPath_Path string +var getToolPath_Err error +var getToolPath_Once sync.Once -// Build the mount_sample tool if it has not yet been built for this process. -// Return a path to the binary. -func buildMountSample() (toolPath string, err error) { - // Build if we haven't yet. - mountSampleOnce.Do(func() { - // Create a temporary directory. - tempDir, err := ioutil.TempDir("", "") - if err != nil { - mountSampleErr = fmt.Errorf("TempDir: %v", err) - return - } - - mountSamplePath = path.Join(tempDir, "mount_sample") - - // Build the command. - cmd := exec.Command( - "go", - "build", - "-o", - mountSamplePath, - "github.com/jacobsa/fuse/samples/mount_sample") - - output, err := cmd.CombinedOutput() - if err != nil { - mountSampleErr = fmt.Errorf( - "mount_sample exited with %v, output:\n%s", - err, - string(output)) - - return - } - }) - - if mountSampleErr != nil { - err = mountSampleErr +// Implementation detail of getToolPath. +func getToolPathImpl() (toolPath string, err error) { + // Fast path: has the user set the flag? + if *fToolPath != "" { + toolPath = *fToolPath return } - toolPath = mountSamplePath + // Create a temporary directory. + tempDir, err := ioutil.TempDir("", "") + if err != nil { + err = fmt.Errorf("TempDir: %v", err) + return + } + + toolPath = path.Join(tempDir, "mount_sample") + + // Build the command. + cmd := exec.Command( + "go", + "build", + "-o", + toolPath, + "github.com/jacobsa/fuse/samples/mount_sample") + + output, err := cmd.CombinedOutput() + if err != nil { + err = fmt.Errorf( + "mount_sample exited with %v, output:\n%s", + err, + string(output)) + + return + } + + return +} + +// Build the mount_sample tool if it has not yet been built for this process. +// Return a path to the binary. +func getToolPath() (toolPath string, err error) { + // Build if we haven't yet. + getToolPath_Once.Do(func() { + getToolPath_Path, getToolPath_Err = getToolPathImpl() + }) + + toolPath, err = getToolPath_Path, getToolPath_Err return } @@ -167,9 +181,9 @@ func (t *SubprocessTest) initialize() (err error) { } // Build the mount_sample tool. - toolPath, err := buildMountSample() + toolPath, err := getToolPath() if err != nil { - err = fmt.Errorf("buildMountSample: %v", err) + err = fmt.Errorf("getToolPath: %v", err) return } From 5a7e50daa6e25a10619c31322181a9ec2745f83d Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 10:59:06 +1100 Subject: [PATCH 41/44] Use a helper to be more sure that temp files are cleaned up at exit. --- fsutil/fsutil.go | 50 ++++++++++++++++++++++++++++++++ samples/flushfs/flush_fs_test.go | 20 +++++-------- 2 files changed, 57 insertions(+), 13 deletions(-) create mode 100644 fsutil/fsutil.go diff --git a/fsutil/fsutil.go b/fsutil/fsutil.go new file mode 100644 index 0000000..152ef85 --- /dev/null +++ b/fsutil/fsutil.go @@ -0,0 +1,50 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fsutil + +import ( + "fmt" + "io/ioutil" + "os" + "path" +) + +// Create a temporary file with the same semantics as ioutil.TempFile, but +// ensure that it is unlinked before returning so that it does not persist +// after the process exits. +// +// Warning: this is not production-quality code, and should only be used for +// testing purposes. In particular, there is a race between creating and +// unlinking by name. +func AnonymousFile(dir string) (f *os.File, err error) { + // Choose a prefix based on the binary name. + prefix := path.Base(os.Args[0]) + + // Create the file. + f, err = ioutil.TempFile(dir, prefix) + if err != nil { + err = fmt.Errorf("TempFile: %v", err) + return + } + + // Unlink it. + err = os.Remove(f.Name()) + if err != nil { + err = fmt.Errorf("Remove: %v", err) + return + } + + return +} diff --git a/samples/flushfs/flush_fs_test.go b/samples/flushfs/flush_fs_test.go index 4885357..a5ffeb2 100644 --- a/samples/flushfs/flush_fs_test.go +++ b/samples/flushfs/flush_fs_test.go @@ -19,7 +19,6 @@ import ( "encoding/hex" "fmt" "io" - "io/ioutil" "os" "path" "runtime" @@ -27,6 +26,7 @@ import ( "testing" "github.com/jacobsa/bazilfuse" + "github.com/jacobsa/fuse/fsutil" "github.com/jacobsa/fuse/samples" . "github.com/jacobsa/oglematchers" . "github.com/jacobsa/ogletest" @@ -57,10 +57,10 @@ func (t *flushFSTest) setUp( var err error // Set up files to receive flush and fsync reports. - t.flushes, err = ioutil.TempFile("", "") + t.flushes, err = fsutil.AnonymousFile("") AssertEq(nil, err) - t.fsyncs, err = ioutil.TempFile("", "") + t.fsyncs, err = fsutil.AnonymousFile("") AssertEq(nil, err) // Set up test config. @@ -486,11 +486,8 @@ func (t *NoErrorsTest) Dup2() { AssertEq(nil, err) AssertEq(4, n) - // Open and unlink some temporary file. - t.f2, err = ioutil.TempFile("", "") - AssertEq(nil, err) - - err = os.Remove(t.f2.Name()) + // Create some anonymous temporary file. + t.f2, err = fsutil.AnonymousFile("") AssertEq(nil, err) // Duplicate the temporary file descriptor on top of the file from our file @@ -693,11 +690,8 @@ func (t *FlushErrorTest) Dup2() { t.f1, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0) AssertEq(nil, err) - // Open and unlink some temporary file. - t.f2, err = ioutil.TempFile("", "") - AssertEq(nil, err) - - err = os.Remove(t.f2.Name()) + // Create some anonymous temporary file. + t.f2, err = fsutil.AnonymousFile("") AssertEq(nil, err) // Duplicate the temporary file descriptor on top of the file from our file From 6cc47f10ed3b9abb237c00688751acd26465af7c Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 11:04:15 +1100 Subject: [PATCH 42/44] Unlink mount points on tear-down. --- samples/in_process.go | 7 +++++++ samples/subprocess.go | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/samples/in_process.go b/samples/in_process.go index e4f2caf..c0fb1da 100644 --- a/samples/in_process.go +++ b/samples/in_process.go @@ -18,6 +18,7 @@ import ( "fmt" "io" "io/ioutil" + "os" "time" "github.com/googlecloudplatform/gcsfuse/timeutil" @@ -129,5 +130,11 @@ func (t *SampleTest) destroy() (err error) { return } + // Unlink the mount point. + if err = os.Remove(t.Dir); err != nil { + err = fmt.Errorf("Unlinking mount point: %v", err) + return + } + return } diff --git a/samples/subprocess.go b/samples/subprocess.go index 288e185..b8464df 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -286,6 +286,7 @@ func (t *SubprocessTest) destroy() (err error) { // Make sure we wait for the unmount, even if we've already returned early in // error. Return its error if we haven't seen any other error. defer func() { + // Wait. unmountErr := <-unmountErrChan if unmountErr != nil { if err != nil { @@ -294,7 +295,11 @@ func (t *SubprocessTest) destroy() (err error) { } err = fmt.Errorf("unmount: %v", unmountErr) + return } + + // Attempt to unlink the mount point. + os.Remove(t.Dir) }() // Wait for the subprocess. From 12b184c011faea0c45376a453c03c96ac913f87c Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 11:27:18 +1100 Subject: [PATCH 43/44] Clean up after the tool binary. --- samples/subprocess.go | 87 +++++++++++++++++++++++++++++++++---------- 1 file changed, 68 insertions(+), 19 deletions(-) diff --git a/samples/subprocess.go b/samples/subprocess.go index b8464df..1b0bbe0 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -75,28 +75,36 @@ func (t *SubprocessTest) SetUp(ti *ogletest.TestInfo) { } // Private state for getToolPath. -var getToolPath_Path string -var getToolPath_Err error -var getToolPath_Once sync.Once +var getToolContents_Contents []byte +var getToolContents_Err error +var getToolContents_Once sync.Once // Implementation detail of getToolPath. -func getToolPathImpl() (toolPath string, err error) { +func getToolContentsImpl() (contents []byte, err error) { // Fast path: has the user set the flag? if *fToolPath != "" { - toolPath = *fToolPath + contents, err = ioutil.ReadFile(*fToolPath) + if err != nil { + err = fmt.Errorf("Reading mount_sample contents: %v", err) + return + } + return } - // Create a temporary directory. + // Create a temporary directory into which we will compile the tool. tempDir, err := ioutil.TempDir("", "") if err != nil { err = fmt.Errorf("TempDir: %v", err) return } - toolPath = path.Join(tempDir, "mount_sample") + toolPath := path.Join(tempDir, "mount_sample") - // Build the command. + // Ensure that we kill the temporary directory when we're finished here. + defer os.RemoveAll(tempDir) + + // Run "go build". cmd := exec.Command( "go", "build", @@ -114,18 +122,25 @@ func getToolPathImpl() (toolPath string, err error) { return } + // Slurp the tool contents. + contents, err = ioutil.ReadFile(toolPath) + if err != nil { + err = fmt.Errorf("ReadFile: %v", err) + return + } + return } // Build the mount_sample tool if it has not yet been built for this process. -// Return a path to the binary. -func getToolPath() (toolPath string, err error) { - // Build if we haven't yet. - getToolPath_Once.Do(func() { - getToolPath_Path, getToolPath_Err = getToolPathImpl() +// Return its contents. +func getToolContents() (contents []byte, err error) { + // Get hold of the binary contents, if we haven't yet. + getToolContents_Once.Do(func() { + getToolContents_Contents, getToolContents_Err = getToolContentsImpl() }) - toolPath, err = getToolPath_Path, getToolPath_Err + contents, err = getToolContents_Contents, getToolContents_Err return } @@ -180,10 +195,42 @@ func (t *SubprocessTest) initialize() (err error) { return } - // Build the mount_sample tool. - toolPath, err := getToolPath() + // Build/read the mount_sample tool. + toolContents, err := getToolContents() if err != nil { - err = fmt.Errorf("getToolPath: %v", err) + err = fmt.Errorf("getTooltoolContents: %v", err) + return + } + + // Create a temporary file to hold the contents of the tool. + toolFile, err := ioutil.TempFile("", "subprocess_test") + if err != nil { + err = fmt.Errorf("TempFile: %v", err) + return + } + + defer toolFile.Close() + + // Ensure that it is deleted when we leave. + toolPath := toolFile.Name() + defer os.Remove(toolPath) + + // Write out the tool contents and make them executable. + if _, err = toolFile.Write(toolContents); err != nil { + err = fmt.Errorf("toolFile.Write: %v", err) + return + } + + if err = toolFile.Chmod(0500); err != nil { + err = fmt.Errorf("toolFile.Chmod: %v", err) + return + } + + // Close the tool file to prevent "text file busy" errors below. + err = toolFile.Close() + toolFile = nil + if err != nil { + err = fmt.Errorf("toolFile.Close: %v", err) return } @@ -263,6 +310,8 @@ func (t *SubprocessTest) TearDown() { // Like TearDown, but doesn't panic. func (t *SubprocessTest) destroy() (err error) { + // Make sure we clean up after ourselves after everything else below. + // Close what is necessary. for _, c := range t.ToClose { if c == nil { @@ -298,8 +347,8 @@ func (t *SubprocessTest) destroy() (err error) { return } - // Attempt to unlink the mount point. - os.Remove(t.Dir) + // Clean up. + ogletest.ExpectEq(nil, os.Remove(t.Dir)) }() // Wait for the subprocess. From 4584e4de8f5f265b056013cc0ae982480301becd Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Tue, 24 Mar 2015 11:27:58 +1100 Subject: [PATCH 44/44] Be consistent about file names. --- samples/subprocess.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/subprocess.go b/samples/subprocess.go index 1b0bbe0..38733bf 100644 --- a/samples/subprocess.go +++ b/samples/subprocess.go @@ -93,7 +93,7 @@ func getToolContentsImpl() (contents []byte, err error) { } // Create a temporary directory into which we will compile the tool. - tempDir, err := ioutil.TempDir("", "") + tempDir, err := ioutil.TempDir("", "sample_test") if err != nil { err = fmt.Errorf("TempDir: %v", err) return @@ -203,7 +203,7 @@ func (t *SubprocessTest) initialize() (err error) { } // Create a temporary file to hold the contents of the tool. - toolFile, err := ioutil.TempFile("", "subprocess_test") + toolFile, err := ioutil.TempFile("", "sample_test") if err != nil { err = fmt.Errorf("TempFile: %v", err) return