From 267ae63276bfd5df2d4749b8aee85af6703c823d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zlatko=20=C4=8Calu=C5=A1i=C4=87?= Date: Sun, 6 Nov 2016 20:09:42 +0100 Subject: [PATCH] Remove fs package and dirty tricks it does The Linux kernel page cache ALWAYS knows better. Fighting it brings only worse performance. Usage of fadvise() is wrong 9 out of 10 times. Removing the whole fs package brings a nice 100% speedup when running costly prune command. And that is measured on localhost, the improvement could be much bigger when using network with higher latency. --- fs/doc.go | 2 - fs/file.go | 126 -------------------------------------------- fs/file_linux.go | 55 ------------------- fs/file_nonlinux.go | 15 ------ handlers.go | 6 +-- htpasswd.go | 5 +- 6 files changed, 4 insertions(+), 205 deletions(-) delete mode 100644 fs/doc.go delete mode 100644 fs/file.go delete mode 100644 fs/file_linux.go delete mode 100644 fs/file_nonlinux.go diff --git a/fs/doc.go b/fs/doc.go deleted file mode 100644 index f810d04..0000000 --- a/fs/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package fs implements an OS independent abstraction of a file system suitable for backup purposes. -package fs diff --git a/fs/file.go b/fs/file.go deleted file mode 100644 index bfb087b..0000000 --- a/fs/file.go +++ /dev/null @@ -1,126 +0,0 @@ -package fs - -import ( - "io" - "os" - "path/filepath" - "runtime" - "strings" -) - -// File is an open file on a file system. -type File interface { - io.Reader - io.Writer - io.Closer - - Fd() uintptr - Readdirnames(n int) ([]string, error) - Readdir(int) ([]os.FileInfo, error) - Seek(int64, int) (int64, error) - Stat() (os.FileInfo, error) -} - -// fixpath returns an absolute path on windows, so restic can open long file names. -func fixpath(name string) string { - if runtime.GOOS == "windows" { - abspath, err := filepath.Abs(name) - if err == nil { - // Check if \\?\UNC\ already exists. - if strings.HasPrefix(abspath, `\\?\UNC\`) { - return abspath - } - // Check if \\?\ already exists. - if strings.HasPrefix(abspath, `\\?\`) { - return abspath - } - // Check if path starts with \\. - if strings.HasPrefix(abspath, `\\`) { - return strings.Replace(abspath, `\\`, `\\?\UNC\`, 1) - } - // Normal path. - return `\\?\` + abspath - } - } - return name -} - -// Chmod changes the mode of the named file to mode. -func Chmod(name string, mode os.FileMode) error { - return os.Chmod(fixpath(name), mode) -} - -// Mkdir creates a new directory with the specified name and permission bits. If there is an error, it will be of type -// *PathError. -func Mkdir(name string, perm os.FileMode) error { - return os.Mkdir(fixpath(name), perm) -} - -// MkdirAll creates a directory named path, along with any necessary parents, and returns nil, or else returns an error. -// The permission bits perm are used for all directories that MkdirAll creates. If path is already a directory, -// MkdirAll does nothing and returns nil. -func MkdirAll(path string, perm os.FileMode) error { - return os.MkdirAll(fixpath(path), perm) -} - -// Readlink returns the destination of the named symbolic link. If there is an error, it will be of type *PathError. -func Readlink(name string) (string, error) { - return os.Readlink(fixpath(name)) -} - -// Remove removes the named file or directory. If there is an error, it will be of type *PathError. -func Remove(name string) error { - return os.Remove(fixpath(name)) -} - -// RemoveAll removes path and any children it contains. It removes everything it can but returns the first error it -// encounters. If the path does not exist, RemoveAll returns nil (no error). -func RemoveAll(path string) error { - return os.RemoveAll(fixpath(path)) -} - -// Rename renames (moves) oldpath to newpath. If newpath already exists, Rename replaces it. OS-specific restrictions -// may apply when oldpath and newpath are in different directories. If there is an error, it will be of type -// *LinkError. -func Rename(oldpath, newpath string) error { - return os.Rename(fixpath(oldpath), fixpath(newpath)) -} - -// Symlink creates newname as a symbolic link to oldname. If there is an error, it will be of type *LinkError. -func Symlink(oldname, newname string) error { - return os.Symlink(fixpath(oldname), fixpath(newname)) -} - -// Stat returns a FileInfo structure describing the named file. If there is an error, it will be of type *PathError. -func Stat(name string) (os.FileInfo, error) { - return os.Stat(fixpath(name)) -} - -// Lstat returns the FileInfo structure describing the named file. If the file is a symbolic link, the returned -// FileInfo describes the symbolic link. Lstat makes no attempt to follow the link. If there is an error, it will be -// of type *PathError. -func Lstat(name string) (os.FileInfo, error) { - return os.Lstat(fixpath(name)) -} - -// Create creates the named file with mode 0666 (before umask), truncating it if it already exists. If successful, -// methods on the returned File can be used for I/O; the associated file descriptor has mode O_RDWR. If there is an -// error, it will be of type *PathError. -func Create(name string) (*os.File, error) { - return os.Create(fixpath(name)) -} - -// OpenFile is the generalized open call; most users will use Open or Create instead. It opens the named file with -// specified flag (O_RDONLY etc.) and perm, (0666 etc.) if applicable. If successful, methods on the returned File can -// be used for I/O. If there is an error, it will be of type *PathError. -func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { - return os.OpenFile(fixpath(name), flag, perm) -} - -// Walk walks the file tree rooted at root, calling walkFn for each file or directory in the tree, including root. All -// errors that arise visiting files and directories are filtered by walkFn. The files are walked in lexical order, which -// makes the output deterministic but means that for very large directories Walk can be inefficient. Walk does not -// follow symbolic links. -func Walk(root string, walkFn filepath.WalkFunc) error { - return filepath.Walk(fixpath(root), walkFn) -} diff --git a/fs/file_linux.go b/fs/file_linux.go deleted file mode 100644 index d5e798b..0000000 --- a/fs/file_linux.go +++ /dev/null @@ -1,55 +0,0 @@ -package fs - -import ( - "os" - "syscall" - - "github.com/zcalusic/restic-server/errors" - "golang.org/x/sys/unix" -) - -// Open opens a file for reading, without updating the atime and without caching data on read. -func Open(name string) (File, error) { - file, err := os.OpenFile(name, os.O_RDONLY|syscall.O_NOATIME, 0) - if os.IsPermission(errors.Cause(err)) { - file, err = os.OpenFile(name, os.O_RDONLY, 0) - } - return &nonCachingFile{File: file}, err -} - -// nonCachingFile wraps an *os.File and calls fadvise() to instantly forget data that has been read or written. -type nonCachingFile struct { - *os.File - readOffset int64 -} - -func (f *nonCachingFile) Read(p []byte) (int, error) { - n, err := f.File.Read(p) - - if n > 0 { - ferr := unix.Fadvise(int(f.File.Fd()), f.readOffset, int64(n), unix.FADV_DONTNEED) - - f.readOffset += int64(n) - - if err == nil { - err = ferr - } - } - - return n, err -} - -// ClearCache syncs and then removes the file's content from the OS cache. -func ClearCache(file File) error { - f, ok := file.(*os.File) - if !ok { - panic("ClearCache called for file not *os.File") - } - - err := f.Sync() - if err != nil { - return err - } - - return unix.Fadvise(int(f.Fd()), 0, 0, unix.FADV_DONTNEED) -} diff --git a/fs/file_nonlinux.go b/fs/file_nonlinux.go deleted file mode 100644 index 8e0e86b..0000000 --- a/fs/file_nonlinux.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build !linux - -package fs - -import "os" - -// Open opens a file for reading. -func Open(name string) (File, error) { - return os.OpenFile(fixpath(name), os.O_RDONLY, 0) -} - -// ClearCache syncs and then removes the file's content from the OS cache. -func ClearCache(f File) error { - return nil -} diff --git a/handlers.go b/handlers.go index 0024c29..a834b59 100644 --- a/handlers.go +++ b/handlers.go @@ -10,8 +10,6 @@ import ( "path/filepath" "strings" "time" - - "github.com/zcalusic/restic-server/fs" ) // Context contains repository metadata. @@ -152,7 +150,7 @@ func GetBlob(c *Context) http.HandlerFunc { } path := filepath.Join(c.path, dir, name) - file, err := fs.Open(path) + file, err := os.Open(path) if err != nil { http.Error(w, "404 not found", 404) return @@ -172,7 +170,7 @@ func SaveBlob(c *Context) http.HandlerFunc { tmp := filepath.Join(c.path, "tmp", name) - tf, err := fs.OpenFile(tmp, os.O_CREATE|os.O_WRONLY, 0600) + tf, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY, 0600) if err != nil { http.Error(w, "500 internal server error", 500) return diff --git a/htpasswd.go b/htpasswd.go index 2e20d90..3f9c0a0 100644 --- a/htpasswd.go +++ b/htpasswd.go @@ -30,8 +30,7 @@ import ( "encoding/csv" "io" "log" - - "github.com/zcalusic/restic-server/fs" + "os" ) // Lookup passwords in a htpasswd file. The entries must have been created with -s for SHA encryption. @@ -44,7 +43,7 @@ type HtpasswdFile struct { // NewHtpasswdFromFile reads the users and passwords from a htpasswd file and returns them. If an error is encountered, // it is returned, together with a nil-Pointer for the HtpasswdFile. func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) { - r, err := fs.Open(path) + r, err := os.Open(path) if err != nil { return nil, err }