Implemented file creation.

geesefs-0-30-9
Aaron Jacobs 2015-03-06 05:35:01 +11:00
commit d307babe25
7 changed files with 642 additions and 16 deletions

View File

@ -69,10 +69,32 @@ type FileSystem interface {
// Create a directory inode as a child of an existing directory inode. The
// kernel sends this in response to a mkdir(2) call.
//
// The kernel appears to verify the name doesn't already exist (mkdir calls
// mkdirat calls user_path_create calls filename_create, which verifies:
// http://goo.gl/FZpLu5). But volatile file systems and paranoid non-volatile
// file systems should check for the reasons described below on CreateFile.
MkDir(
ctx context.Context,
req *MkDirRequest) (*MkDirResponse, error)
// Create a file inode and open it.
//
// The kernel calls this method when the user asks to open a file with the
// O_CREAT flag and the kernel has observed that the file doesn't exist. (See
// for example lookup_open, http://goo.gl/PlqE9d).
//
// However it's impossible to tell for sure that all kernels make this check
// in all cases and the official fuse documentation is less than encouraging
// (" the file does not exist, first create it with the specified mode, and
// then open it"). Therefore file systems would be smart to be paranoid and
// check themselves, returning EEXIST when the file already exists. This of
// course particularly applies to file systems that are volatile from the
// kernel's point of view.
CreateFile(
ctx context.Context,
req *CreateFileRequest) (*CreateFileResponse, error)
///////////////////////////////////
// Inode destruction
///////////////////////////////////
@ -128,11 +150,45 @@ type FileSystem interface {
ctx context.Context,
req *OpenFileRequest) (*OpenFileResponse, error)
// Read data from a file previously opened with OpenFile.
// Read data from a file previously opened with CreateFile or OpenFile.
//
// Note that this method is not called for every call to read(2) by the end
// user; some reads may be served by the page cache. See notes on Write for
// more.
ReadFile(
ctx context.Context,
req *ReadFileRequest) (*ReadFileResponse, error)
// Write data to a file previously opened with CreateFile or OpenFile.
//
// When the user writes data using write(2), the write goes into the page
// cache and the page is marked dirty. Later the kernel may write back the
// page via the FUSE VFS layer, causing this method to be called:
//
// * The kernel calls address_space_operations::writepage when a dirty page
// needs to be written to backing store (see vfs.txt). Fuse sets this to
// fuse_writepage (see file.c).
//
// * fuse_writepage calls fuse_writepage_locked.
//
// * fuse_writepage_locked makes a write request to the userspace server.
//
// Note that writes *will* be received before a call to Flush when closing
// the file descriptor to which they were written:
//
// * fuse_flush calls write_inode_now, which appears to start a writeback
// in the background (it talks about a "flusher thread").
//
// * fuse_flush then calls fuse_sync_writes, which "[waits] for all pending
// writepages on the inode to finish".
//
// * Only then does fuse_flush finally send the flush request.
//
// TODO(jacobsa): Add links for all of the references above.
WriteFile(
ctx context.Context,
req *WriteFileRequest) (*WriteFileResponse, error)
// Release a previously-minted file handle. The kernel calls this when there
// are no more references to an open file: all file descriptors are closed
// and all memory mappings are unmapped.
@ -235,8 +291,8 @@ type RequestHeader struct {
}
// Information about a child inode within its parent directory. Shared by the
// responses for LookUpInode, MkDir, etc. Consumed by the kernel in order to
// set up a dcache entry.
// responses for LookUpInode, MkDir, CreateFile, etc. Consumed by the kernel in
// order to set up a dcache entry.
type ChildInodeEntry struct {
// The ID of the child inode. The file system must ensure that the returned
// inode ID remains valid until a later call to ForgetInode.
@ -247,6 +303,16 @@ type ChildInodeEntry struct {
Generation GenerationNumber
// Current attributes for the child inode.
//
// When creating a new inode, the file system is responsible for initializing
// and recording (where supported) attributes like time information,
// ownership information, etc.
//
// Ownership information in particular must be set to something reasonable or
// by default root will own everything and unprivileged users won't be able
// to do anything useful. In traditional file systems in the kernel, the
// function inode_init_owner (http://goo.gl/5qavg8) contains the
// standards-compliant logic for this.
Attributes InodeAttributes
// The FUSE VFS layer in the kernel maintains a cache of file attributes,
@ -377,18 +443,42 @@ type MkDirRequest struct {
type MkDirResponse struct {
// Information about the inode that was created.
//
// The file system is responsible for initializing and recording (where
// supported) attributes like time information, ownership information, etc.
//
// Ownership information in particular must be set to something reasonable or
// by default root will own everything and unprivileged users won't be able
// to do anything useful. In traditional file systems in the kernel, the
// function inode_init_owner (http://goo.gl/5qavg8) contains the
// standards-compliant logic for this.
Entry ChildInodeEntry
}
type CreateFileRequest struct {
Header RequestHeader
// The ID of parent directory inode within which to create the child file.
Parent InodeID
// The name of the child to create, and the mode with which to create it.
Name string
Mode os.FileMode
// Flags for the open operation.
Flags bazilfuse.OpenFlags
}
type CreateFileResponse struct {
// Information about the inode that was created.
Entry ChildInodeEntry
// An opaque ID that will be echoed in follow-up calls for this file using
// the same struct file in the kernel. In practice this usually means
// follow-up calls using the file descriptor returned by open(2).
//
// The handle may be supplied to the following methods:
//
// * ReadFile
// * WriteFile
// * ReleaseFileHandle
//
// The file system must ensure this ID remains valid until a later call to
// ReleaseFileHandle.
Handle HandleID
}
type RmDirRequest struct {
Header RequestHeader
@ -547,6 +637,7 @@ type OpenFileResponse struct {
// The handle may be supplied to the following methods:
//
// * ReadFile
// * WriteFile
// * ReleaseFileHandle
//
// The file system must ensure this ID remains valid until a later call to
@ -558,7 +649,7 @@ type ReadFileRequest struct {
Header RequestHeader
// The file inode that we are reading, and the handle previously returned by
// OpenFile when opening that inode.
// CreateFile or OpenFile when opening that inode.
Inode InodeID
Handle HandleID
@ -579,6 +670,48 @@ type ReadFileResponse struct {
Data []byte
}
type WriteFileRequest struct {
Header RequestHeader
// The file inode that we are modifying, and the handle previously returned
// by CreateFile or OpenFile when opening that inode.
Inode InodeID
Handle HandleID
// The offset at which to write the data below.
//
// The man page for pwrite(2) implies that aside from changing the file
// handle's offset, using pwrite is equivalent to using lseek(2) and then
// write(2). The man page for lseek(2) says the following:
//
// "The lseek() function allows the file offset to be set beyond the end of
// the file (but this does not change the size of the file). If data is later
// written at this point, subsequent reads of the data in the gap (a "hole")
// return null bytes (aq\0aq) until data is actually written into the gap."
//
// It is therefore reasonable to assume that the kernel is looking for
// the following semantics:
//
// * If the offset is less than or equal to the current size, extend the
// file as necessary to fit any data that goes past the end of the file.
//
// * If the offset is greater than the current size, extend the file
// with null bytes until it is not, then do the above.
//
Offset int64
// The data to write.
//
// The FUSE documentation requires that exactly the number of bytes supplied
// be written, except on error (http://goo.gl/KUpwwn). This appears to be
// because it uses file mmapping machinery (http://goo.gl/SGxnaN) to write a
// page at a time.
Data []byte
}
type WriteFileResponse struct {
}
type ReleaseFileHandleRequest struct {
Header RequestHeader

View File

@ -57,6 +57,12 @@ func (fs *NotImplementedFileSystem) MkDir(
return nil, fuse.ENOSYS
}
func (fs *NotImplementedFileSystem) CreateFile(
ctx context.Context,
req *fuse.CreateFileRequest) (*fuse.CreateFileResponse, error) {
return nil, fuse.ENOSYS
}
func (fs *NotImplementedFileSystem) RmDir(
ctx context.Context,
req *fuse.RmDirRequest) (*fuse.RmDirResponse, error) {
@ -93,6 +99,12 @@ func (fs *NotImplementedFileSystem) ReadFile(
return nil, fuse.ENOSYS
}
func (fs *NotImplementedFileSystem) WriteFile(
ctx context.Context,
req *fuse.WriteFileRequest) (*fuse.WriteFileResponse, error) {
return nil, fuse.ENOSYS
}
func (fs *NotImplementedFileSystem) ReleaseFileHandle(
ctx context.Context,
req *fuse.ReleaseFileHandleRequest) (*fuse.ReleaseFileHandleResponse, error) {

View File

@ -279,7 +279,7 @@ func (fs *memFS) MkDir(
parent := fs.getInodeForModifyingOrDie(req.Parent)
defer parent.mu.Unlock()
// Set up attributes from the child, using the credientials of the calling
// Set up attributes from the child, using the credentials of the calling
// process as owner (matching inode_init_owner, cf. http://goo.gl/5qavg8).
now := fs.clock.Now()
childAttrs := fuse.InodeAttributes{
@ -311,6 +311,52 @@ func (fs *memFS) MkDir(
return
}
func (fs *memFS) CreateFile(
ctx context.Context,
req *fuse.CreateFileRequest) (resp *fuse.CreateFileResponse, err error) {
resp = &fuse.CreateFileResponse{}
fs.mu.Lock()
defer fs.mu.Unlock()
// Grab the parent, which we will update shortly.
parent := fs.getInodeForModifyingOrDie(req.Parent)
defer parent.mu.Unlock()
// Set up attributes from the child, using the credentials of the calling
// process as owner (matching inode_init_owner, cf. http://goo.gl/5qavg8).
now := fs.clock.Now()
childAttrs := fuse.InodeAttributes{
Mode: req.Mode,
Atime: now,
Mtime: now,
Ctime: now,
Crtime: now,
Uid: req.Header.Uid,
Gid: req.Header.Gid,
}
// Allocate a child.
childID, child := fs.allocateInode(childAttrs)
defer child.mu.Unlock()
// Add an entry in the parent.
parent.AddChild(childID, req.Name, fuseutil.DT_File)
// Fill in the response entry.
resp.Entry.Child = childID
resp.Entry.Attributes = child.attributes
// We don't spontaneously mutate, so the kernel can cache as long as it wants
// (since it also handles invalidation).
resp.Entry.AttributesExpiration = fs.clock.Now().Add(365 * 24 * time.Hour)
resp.Entry.EntryExpiration = resp.Entry.EntryExpiration
// We have nothing interesting to put in the Handle field.
return
}
func (fs *memFS) RmDir(
ctx context.Context,
req *fuse.RmDirRequest) (resp *fuse.RmDirResponse, err error) {
@ -391,3 +437,42 @@ func (fs *memFS) ReadDir(
return
}
func (fs *memFS) OpenFile(
ctx context.Context,
req *fuse.OpenFileRequest) (resp *fuse.OpenFileResponse, err error) {
resp = &fuse.OpenFileResponse{}
fs.mu.RLock()
defer fs.mu.RUnlock()
// We don't mutate spontaneosuly, so if the VFS layer has asked for an
// inode that doesn't exist, something screwed up earlier (a lookup, a
// cache invalidation, etc.).
inode := fs.getInodeForReadingOrDie(req.Inode)
defer inode.mu.RUnlock()
if inode.dir {
panic("Found directory.")
}
return
}
func (fs *memFS) WriteFile(
ctx context.Context,
req *fuse.WriteFileRequest) (resp *fuse.WriteFileResponse, err error) {
resp = &fuse.WriteFileResponse{}
fs.mu.RLock()
defer fs.mu.RUnlock()
// Find the inode in question.
inode := fs.getInodeForModifyingOrDie(req.Inode)
defer inode.mu.Unlock()
// Serve the request.
_, err = inode.WriteAt(req.Data, req.Offset)
return
}

View File

@ -54,6 +54,7 @@ type inode struct {
// INVARIANT: No non-permission mode bits are set besides os.ModeDir
// INVARIANT: If dir, then os.ModeDir is set
// INVARIANT: If !dir, then os.ModeDir is not set
// INVARIANT: attributes.Size == len(contents)
attributes fuse.InodeAttributes // GUARDED_BY(mu)
// For directories, entries describing the children of the directory. Unused
@ -142,6 +143,15 @@ func (inode *inode) checkInvariants() {
panic("Non-nil entries in a file.")
}
}
// Check the size.
if inode.attributes.Size != uint64(len(inode.contents)) {
panic(
fmt.Sprintf(
"Unexpected size: %v vs. %v",
inode.attributes.Size,
len(inode.contents)))
}
}
// Return the index of the child within inode.entries, if it exists.
@ -279,3 +289,31 @@ func (inode *inode) ReadDir(offset int, size int) (data []byte, err error) {
return
}
// Write to the file's contents. See documentation for ioutil.WriterAt.
//
// REQUIRES: !inode.dir
// EXCLUSIVE_LOCKS_REQUIRED(inode.mu)
func (inode *inode) WriteAt(p []byte, off int64) (n int, err error) {
if inode.dir {
panic("WriteAt called on directory.")
}
// Ensure that the contents slice is long enough.
newLen := int(off) + len(p)
if len(inode.contents) < newLen {
padding := make([]byte, newLen-len(inode.contents))
inode.contents = append(inode.contents, padding...)
inode.attributes.Size = uint64(newLen)
}
// Copy in the data.
n = copy(inode.contents[off:], p)
// Sanity check.
if n != len(p) {
panic(fmt.Sprintf("Unexpected short copy: %v", n))
}
return
}

View File

@ -280,7 +280,7 @@ func (t *MemFSTest) Mkdir_IntermediateIsFile() {
err = os.Mkdir(dirName, 0754)
AssertNe(nil, err)
ExpectThat(err, Error(HasSubstr("TODO")))
ExpectThat(err, Error(HasSubstr("not a directory")))
}
func (t *MemFSTest) Mkdir_IntermediateIsNonExistent() {
@ -309,7 +309,45 @@ func (t *MemFSTest) Mkdir_PermissionDenied() {
}
func (t *MemFSTest) CreateNewFile_InRoot() {
AssertTrue(false, "TODO")
var err error
var fi os.FileInfo
var stat *syscall.Stat_t
fileName := path.Join(t.mfs.Dir(), "foo")
const contents = "Hello\x00world"
// Write a file.
createTime := t.clock.Now()
err = ioutil.WriteFile(fileName, []byte(contents), 0400)
AssertEq(nil, err)
// Simulate time advancing.
t.clock.AdvanceTime(time.Second)
// Stat it.
fi, err = os.Stat(fileName)
stat = fi.Sys().(*syscall.Stat_t)
AssertEq(nil, err)
ExpectEq("foo", fi.Name())
ExpectEq(len(contents), fi.Size())
ExpectEq(0400, fi.Mode())
ExpectEq(0, fi.ModTime().Sub(createTime))
ExpectFalse(fi.IsDir())
ExpectNe(0, stat.Ino)
ExpectEq(1, stat.Nlink)
ExpectEq(currentUid(), stat.Uid)
ExpectEq(currentGid(), stat.Gid)
ExpectEq(len(contents), stat.Size)
ExpectEq(0, timespecToTime(stat.Atimespec).Sub(createTime))
ExpectEq(0, timespecToTime(stat.Mtimespec).Sub(createTime))
ExpectEq(0, timespecToTime(stat.Ctimespec).Sub(createTime))
// Read it back.
slice, err := ioutil.ReadFile(fileName)
AssertEq(nil, err)
ExpectEq(contents, string(slice))
}
func (t *MemFSTest) CreateNewFile_InSubDir() {
@ -336,6 +374,10 @@ func (t *MemFSTest) UnlinkFile_NonExistent() {
AssertTrue(false, "TODO")
}
func (t *MemFSTest) UnlinkFile_StillOpen() {
AssertTrue(false, "TODO")
}
func (t *MemFSTest) Rmdir_NonEmpty() {
var err error
@ -468,3 +510,11 @@ func (t *MemFSTest) CaseSensitive() {
AssertThat(err, Error(HasSubstr("no such file or directory")))
}
}
func (t *MemFSTest) FileReadsAndWrites() {
AssertTrue(false, "TODO")
}
func (t *MemFSTest) FileReadsAndWrites_BeyondEOF() {
AssertTrue(false, "TODO")
}

253
samples/memfs/posix_test.go Normal file
View File

@ -0,0 +1,253 @@
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Tests for the behavior of os.File objects on plain old posix file systems,
// for use in verifying the intended behavior of memfs.
package memfs_test
import (
"io"
"io/ioutil"
"os"
"path"
"testing"
. "github.com/jacobsa/ogletest"
)
func TestPosix(t *testing.T) { RunTests(t) }
////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////
func getFileOffset(f *os.File) (offset int64, err error) {
const relativeToCurrent = 1
offset, err = f.Seek(0, relativeToCurrent)
return
}
////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////
type PosixTest struct {
// A temporary directory.
dir string
// Files to close when tearing down. Nil entries are skipped.
toClose []io.Closer
}
var _ SetUpInterface = &PosixTest{}
var _ TearDownInterface = &PosixTest{}
func init() { RegisterTestSuite(&PosixTest{}) }
func (t *PosixTest) SetUp(ti *TestInfo) {
var err error
// Create a temporary directory.
t.dir, err = ioutil.TempDir("", "posix_test")
if err != nil {
panic(err)
}
}
func (t *PosixTest) TearDown() {
// Close any files we opened.
for _, c := range t.toClose {
if c == nil {
continue
}
err := c.Close()
if err != nil {
panic(err)
}
}
// Remove the temporary directory.
err := os.RemoveAll(t.dir)
if err != nil {
panic(err)
}
}
////////////////////////////////////////////////////////////////////////
// Test functions
////////////////////////////////////////////////////////////////////////
func (t *PosixTest) WriteOverlapsEndOfFile() {
var err error
var n int
// Create a file.
f, err := os.Create(path.Join(t.dir, "foo"))
t.toClose = append(t.toClose, f)
AssertEq(nil, err)
// Make it 4 bytes long.
err = f.Truncate(4)
AssertEq(nil, err)
// Write the range [2, 6).
n, err = f.WriteAt([]byte("taco"), 2)
AssertEq(nil, err)
AssertEq(4, n)
// Read the full contents of the file.
contents, err := ioutil.ReadAll(f)
AssertEq(nil, err)
ExpectEq("\x00\x00taco", string(contents))
}
func (t *PosixTest) WriteStartsAtEndOfFile() {
var err error
var n int
// Create a file.
f, err := os.Create(path.Join(t.dir, "foo"))
t.toClose = append(t.toClose, f)
AssertEq(nil, err)
// Make it 2 bytes long.
err = f.Truncate(2)
AssertEq(nil, err)
// Write the range [2, 6).
n, err = f.WriteAt([]byte("taco"), 2)
AssertEq(nil, err)
AssertEq(4, n)
// Read the full contents of the file.
contents, err := ioutil.ReadAll(f)
AssertEq(nil, err)
ExpectEq("\x00\x00taco", string(contents))
}
func (t *PosixTest) WriteStartsPastEndOfFile() {
var err error
var n int
// Create a file.
f, err := os.Create(path.Join(t.dir, "foo"))
t.toClose = append(t.toClose, f)
AssertEq(nil, err)
// Write the range [2, 6).
n, err = f.WriteAt([]byte("taco"), 2)
AssertEq(nil, err)
AssertEq(4, n)
// Read the full contents of the file.
contents, err := ioutil.ReadAll(f)
AssertEq(nil, err)
ExpectEq("\x00\x00taco", string(contents))
}
func (t *PosixTest) WriteAtDoesntChangeOffset_NotAppendMode() {
var err error
var n int
// Create a file.
f, err := os.Create(path.Join(t.dir, "foo"))
t.toClose = append(t.toClose, f)
AssertEq(nil, err)
// Make it 16 bytes long.
err = f.Truncate(16)
AssertEq(nil, err)
// Seek to offset 4.
_, err = f.Seek(4, 0)
AssertEq(nil, err)
// Write the range [10, 14).
n, err = f.WriteAt([]byte("taco"), 2)
AssertEq(nil, err)
AssertEq(4, n)
// We should still be at offset 4.
offset, err := getFileOffset(f)
AssertEq(nil, err)
ExpectEq(4, offset)
}
func (t *PosixTest) WriteAtDoesntChangeOffset_AppendMode() {
var err error
var n int
// Create a file in append mode.
f, err := os.OpenFile(
path.Join(t.dir, "foo"),
os.O_RDWR|os.O_APPEND|os.O_CREATE,
0600)
t.toClose = append(t.toClose, f)
AssertEq(nil, err)
// Make it 16 bytes long.
err = f.Truncate(16)
AssertEq(nil, err)
// Seek to offset 4.
_, err = f.Seek(4, 0)
AssertEq(nil, err)
// Write the range [10, 14).
n, err = f.WriteAt([]byte("taco"), 2)
AssertEq(nil, err)
AssertEq(4, n)
// We should still be at offset 4.
offset, err := getFileOffset(f)
AssertEq(nil, err)
ExpectEq(4, offset)
}
func (t *PosixTest) ReadsPastEndOfFile() {
var err error
var n int
buf := make([]byte, 1024)
// Create a file.
f, err := os.Create(path.Join(t.dir, "foo"))
t.toClose = append(t.toClose, f)
AssertEq(nil, err)
// Give it some contents.
n, err = f.Write([]byte("taco"))
AssertEq(nil, err)
AssertEq(4, n)
// Read a range overlapping EOF.
n, err = f.ReadAt(buf[:4], 2)
AssertEq(io.EOF, err)
ExpectEq(2, n)
ExpectEq("co", string(buf[:n]))
// Read a range starting at EOF.
n, err = f.ReadAt(buf[:4], 4)
AssertEq(io.EOF, err)
ExpectEq(0, n)
ExpectEq("", string(buf[:n]))
// Read a range starting past EOF.
n, err = f.ReadAt(buf[:4], 100)
AssertEq(io.EOF, err)
ExpectEq(0, n)
ExpectEq("", string(buf[:n]))
}

View File

@ -195,6 +195,35 @@ func (s *server) handleFuseRequest(fuseReq bazilfuse.Request) {
s.logger.Println("Responding:", fuseResp)
typed.Respond(fuseResp)
case *bazilfuse.CreateRequest:
// Convert the request.
req := &CreateFileRequest{
Header: convertHeader(typed.Header),
Parent: InodeID(typed.Header.Node),
Name: typed.Name,
Mode: typed.Mode,
Flags: typed.Flags,
}
// Call the file system.
resp, err := s.fs.CreateFile(ctx, req)
if err != nil {
s.logger.Println("Responding:", err)
typed.RespondError(err)
return
}
// Convert the response.
fuseResp := &bazilfuse.CreateResponse{
OpenResponse: bazilfuse.OpenResponse{
Handle: bazilfuse.HandleID(resp.Handle),
},
}
convertChildInodeEntry(s.clock, &resp.Entry, &fuseResp.LookupResponse)
s.logger.Println("Responding:", fuseResp)
typed.Respond(fuseResp)
case *bazilfuse.RemoveRequest:
// We don't yet support files.
if !typed.Dir {
@ -326,6 +355,32 @@ func (s *server) handleFuseRequest(fuseReq bazilfuse.Request) {
typed.Respond(fuseResp)
}
case *bazilfuse.WriteRequest:
// Convert the request.
req := &WriteFileRequest{
Header: convertHeader(typed.Header),
Inode: InodeID(typed.Header.Node),
Handle: HandleID(typed.Handle),
Data: typed.Data,
Offset: typed.Offset,
}
// Call the file system.
_, err := s.fs.WriteFile(ctx, req)
if err != nil {
s.logger.Println("Responding:", err)
typed.RespondError(err)
return
}
// Convert the response.
fuseResp := &bazilfuse.WriteResponse{
Size: len(typed.Data),
}
s.logger.Println("Responding:", fuseResp)
typed.Respond(fuseResp)
default:
s.logger.Println("Unhandled type. Returning ENOSYS.")
typed.RespondError(ENOSYS)