From e61c7d94059ef4dafeb33109d715938101d7bede Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Thu, 13 Aug 2015 14:15:54 +1000 Subject: [PATCH 1/2] Allow the kernel to cache inode attributes. --- internal/fs/fs.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 6066add760..9720e5dbad 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -174,6 +174,12 @@ func NewServer(cfg *ServerConfig) (server fuse.Server, err error) { // fileSystem type //////////////////////////////////////////////////////////////////////// +// Any given object generation in GCS is immutable, and a new generation +// results in a new inode number. So every update from a remote system results +// in a new inode number, and it's therefore safe to allow the kernel to cache +// inode attributes forever. +const attrCacheDuration = 30 * 24 * time.Hour + // LOCK ORDERING // // Let FS be the file system lock. Define a strict partial order < as follows: @@ -846,7 +852,10 @@ func (fs *fileSystem) LookUpInode( // Fill out the response. op.Entry.Child = child.ID() - if op.Entry.Attributes, err = child.Attributes(ctx); err != nil { + op.Entry.AttributesExpiration = time.Now().Add(attrCacheDuration) + op.Entry.Attributes, err = child.Attributes(ctx) + + if err != nil { return } @@ -866,7 +875,9 @@ func (fs *fileSystem) GetInodeAttributes( defer in.Unlock() // Grab its attributes. + op.AttributesExpiration = time.Now().Add(attrCacheDuration) op.Attributes, err = in.Attributes(ctx) + if err != nil { return } @@ -919,7 +930,9 @@ func (fs *fileSystem) SetInodeAttributes( } // Fill in the response. + op.AttributesExpiration = time.Now().Add(attrCacheDuration) op.Attributes, err = in.Attributes(ctx) + if err != nil { err = fmt.Errorf("Attributes: %v", err) return @@ -988,6 +1001,7 @@ func (fs *fileSystem) MkDir( // Fill out the response. op.Entry.Child = child.ID() + op.Entry.AttributesExpiration = time.Now().Add(attrCacheDuration) op.Entry.Attributes, err = child.Attributes(ctx) if err != nil { @@ -1052,6 +1066,7 @@ func (fs *fileSystem) CreateFile( // Fill out the response. op.Entry.Child = child.ID() + op.Entry.AttributesExpiration = time.Now().Add(attrCacheDuration) op.Entry.Attributes, err = child.Attributes(ctx) if err != nil { @@ -1102,6 +1117,7 @@ func (fs *fileSystem) CreateSymlink( // Fill out the response. op.Entry.Child = child.ID() + op.Entry.AttributesExpiration = time.Now().Add(attrCacheDuration) op.Entry.Attributes, err = child.Attributes(ctx) if err != nil { From e621b72e15ebf82ac984afaf76c68ceea75734e6 Mon Sep 17 00:00:00 2001 From: Aaron Jacobs Date: Thu, 13 Aug 2015 04:32:53 +0000 Subject: [PATCH 2/2] Control inode attribute cache TTL with --stat-cache-ttl. See the notes on ServerConfig.InodeAttributeCacheTTL for details. --- docs/semantics.md | 5 +++ flags.go | 2 +- internal/fs/fs.go | 90 +++++++++++++++++++++++++++++++---------------- mount.go | 19 +++++----- 4 files changed, 75 insertions(+), 41 deletions(-) diff --git a/docs/semantics.md b/docs/semantics.md index fa63ce7379..96b01b499e 100644 --- a/docs/semantics.md +++ b/docs/semantics.md @@ -36,6 +36,11 @@ behavior is controlled by the `--stat-cache-ttl` flag, which can be set to a value like `10s` or `1.5h`. (The default is one minute.) Positive and negative stat results will be cached for the specified amount of time. +`--stat-cache-ttl` also controls the duration for which gcsfuse allows the +kernel to cache inode attributes. Caching these can help with file system +performance, since otherwise the kernel must send a request for inode attributes +to gcsfuse for each call to `write(2)`, `stat(2)`, and others. + **Warning**: Using stat caching breaks the consistency guarantees discussed in this document. It is safe only in the following situations: diff --git a/flags.go b/flags.go index 46810f3d8e..5681c01e66 100644 --- a/flags.go +++ b/flags.go @@ -133,7 +133,7 @@ func newApp() (app *cli.App) { cli.DurationFlag{ Name: "stat-cache-ttl", Value: time.Minute, - Usage: "How long to cache StatObject results from GCS.", + Usage: "How long to cache StatObject results and inode attributes.", }, cli.DurationFlag{ diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 9720e5dbad..4fa0f7030a 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -58,6 +58,18 @@ type ServerConfig struct { // See docs/semantics.md for more info. ImplicitDirectories bool + // How long to allow the kernel to cache inode attributes. + // + // Any given object generation in GCS is immutable, and a new generation + // results in a new inode number. So every update from a remote system results + // in a new inode number, and it's therefore safe to allow the kernel to cache + // inode attributes. + // + // The one exception to the above logic is that objects can be _deleted_, in + // which case stat::st_nlink changes. So choosing this value comes down to + // whether you care about that field being up to date. + InodeAttributeCacheTTL time.Duration + // If non-zero, each directory will maintain a cache from child name to // information about whether that name exists as a file and/or directory. // This may speed up calls to look up and stat inodes, especially when @@ -126,6 +138,7 @@ func NewServer(cfg *ServerConfig) (server fuse.Server, err error) { syncer: syncer, tempDir: cfg.TempDir, implicitDirs: cfg.ImplicitDirectories, + inodeAttributeCacheTTL: cfg.InodeAttributeCacheTTL, dirTypeCacheTTL: cfg.DirTypeCacheTTL, uid: cfg.Uid, gid: cfg.Gid, @@ -174,12 +187,6 @@ func NewServer(cfg *ServerConfig) (server fuse.Server, err error) { // fileSystem type //////////////////////////////////////////////////////////////////////// -// Any given object generation in GCS is immutable, and a new generation -// results in a new inode number. So every update from a remote system results -// in a new inode number, and it's therefore safe to allow the kernel to cache -// inode attributes forever. -const attrCacheDuration = 30 * 24 * time.Hour - // LOCK ORDERING // // Let FS be the file system lock. Define a strict partial order < as follows: @@ -217,9 +224,10 @@ type fileSystem struct { // Constant data ///////////////////////// - tempDir string - implicitDirs bool - dirTypeCacheTTL time.Duration + tempDir string + implicitDirs bool + inodeAttributeCacheTTL time.Duration + dirTypeCacheTTL time.Duration // The user and group owning everything in the file system. uid uint32 @@ -825,6 +833,30 @@ func (fs *fileSystem) unlockAndMaybeDisposeOfInode( fs.unlockAndDecrementLookupCount(in, 1) } +// Fetch attributes for the supplied inode and fill in an appropriate +// expiration time for them. +// +// LOCKS_REQUIRED(in) +func (fs *fileSystem) getAttributes( + ctx context.Context, + in inode.Inode) ( + attr fuseops.InodeAttributes, + expiration time.Time, + err error) { + // Call through. + attr, err = in.Attributes(ctx) + if err != nil { + return + } + + // Set up the expiration time. + if fs.inodeAttributeCacheTTL > 0 { + expiration = time.Now().Add(fs.inodeAttributeCacheTTL) + } + + return +} + //////////////////////////////////////////////////////////////////////// // fuse.FileSystem methods //////////////////////////////////////////////////////////////////////// @@ -851,9 +883,9 @@ func (fs *fileSystem) LookUpInode( defer fs.unlockAndMaybeDisposeOfInode(child, &err) // Fill out the response. - op.Entry.Child = child.ID() - op.Entry.AttributesExpiration = time.Now().Add(attrCacheDuration) - op.Entry.Attributes, err = child.Attributes(ctx) + e := &op.Entry + e.Child = child.ID() + e.Attributes, e.AttributesExpiration, err = fs.getAttributes(ctx, child) if err != nil { return @@ -875,9 +907,7 @@ func (fs *fileSystem) GetInodeAttributes( defer in.Unlock() // Grab its attributes. - op.AttributesExpiration = time.Now().Add(attrCacheDuration) - op.Attributes, err = in.Attributes(ctx) - + op.Attributes, op.AttributesExpiration, err = fs.getAttributes(ctx, in) if err != nil { return } @@ -930,11 +960,9 @@ func (fs *fileSystem) SetInodeAttributes( } // Fill in the response. - op.AttributesExpiration = time.Now().Add(attrCacheDuration) - op.Attributes, err = in.Attributes(ctx) - + op.Attributes, op.AttributesExpiration, err = fs.getAttributes(ctx, in) if err != nil { - err = fmt.Errorf("Attributes: %v", err) + err = fmt.Errorf("getAttributes: %v", err) return } @@ -1000,12 +1028,12 @@ func (fs *fileSystem) MkDir( defer fs.unlockAndMaybeDisposeOfInode(child, &err) // Fill out the response. - op.Entry.Child = child.ID() - op.Entry.AttributesExpiration = time.Now().Add(attrCacheDuration) - op.Entry.Attributes, err = child.Attributes(ctx) + e := &op.Entry + e.Child = child.ID() + e.Attributes, e.AttributesExpiration, err = fs.getAttributes(ctx, child) if err != nil { - err = fmt.Errorf("Attributes: %v", err) + err = fmt.Errorf("getAttributes: %v", err) return } @@ -1065,12 +1093,12 @@ func (fs *fileSystem) CreateFile( fs.mu.Unlock() // Fill out the response. - op.Entry.Child = child.ID() - op.Entry.AttributesExpiration = time.Now().Add(attrCacheDuration) - op.Entry.Attributes, err = child.Attributes(ctx) + e := &op.Entry + e.Child = child.ID() + e.Attributes, e.AttributesExpiration, err = fs.getAttributes(ctx, child) if err != nil { - err = fmt.Errorf("Attributes: %v", err) + err = fmt.Errorf("getAttributes: %v", err) return } @@ -1116,12 +1144,12 @@ func (fs *fileSystem) CreateSymlink( defer fs.unlockAndMaybeDisposeOfInode(child, &err) // Fill out the response. - op.Entry.Child = child.ID() - op.Entry.AttributesExpiration = time.Now().Add(attrCacheDuration) - op.Entry.Attributes, err = child.Attributes(ctx) + e := &op.Entry + e.Child = child.ID() + e.Attributes, e.AttributesExpiration, err = fs.getAttributes(ctx, child) if err != nil { - err = fmt.Errorf("Attributes: %v", err) + err = fmt.Errorf("getAttributes: %v", err) return } diff --git a/mount.go b/mount.go index 87d879b580..8fff259494 100644 --- a/mount.go +++ b/mount.go @@ -83,15 +83,16 @@ func mount( // Create a file system server. serverCfg := &fs.ServerConfig{ - Clock: timeutil.RealClock(), - Bucket: bucket, - TempDir: flags.TempDir, - ImplicitDirectories: flags.ImplicitDirs, - DirTypeCacheTTL: flags.TypeCacheTTL, - Uid: uid, - Gid: gid, - FilePerms: os.FileMode(flags.FileMode), - DirPerms: os.FileMode(flags.DirMode), + Clock: timeutil.RealClock(), + Bucket: bucket, + TempDir: flags.TempDir, + ImplicitDirectories: flags.ImplicitDirs, + InodeAttributeCacheTTL: flags.StatCacheTTL, + DirTypeCacheTTL: flags.TypeCacheTTL, + Uid: uid, + Gid: gid, + FilePerms: os.FileMode(flags.FileMode), + DirPerms: os.FileMode(flags.DirMode), AppendThreshold: 1 << 21, // 2 MiB, a total guess. TmpObjectPrefix: ".gcsfuse_tmp/",