Remove fs package and dirty tricks it does

The Linux kernel page cache ALWAYS knows better.  Fighting it brings
only worse performance. Usage of fadvise() is wrong 9 out of 10 times.

Removing the whole fs package brings a nice 100% speedup when running
costly prune command. And that is measured on localhost, the improvement
could be much bigger when using network with higher latency.

fs/doc.go

@@ -1,2 +0,0 @@
-// Package fs implements an OS independent abstraction of a file system suitable for backup purposes.
-package fs

fs/file.go

@@ -1,126 +0,0 @@
-package fs
-
-import (
-	"io"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
-)
-
-// File is an open file on a file system.
-type File interface {
-	io.Reader
-	io.Writer
-	io.Closer
-
-	Fd() uintptr
-	Readdirnames(n int) ([]string, error)
-	Readdir(int) ([]os.FileInfo, error)
-	Seek(int64, int) (int64, error)
-	Stat() (os.FileInfo, error)
-}
-
-// fixpath returns an absolute path on windows, so restic can open long file names.
-func fixpath(name string) string {
-	if runtime.GOOS == "windows" {
-		abspath, err := filepath.Abs(name)
-		if err == nil {
-			// Check if \\?\UNC\ already exists.
-			if strings.HasPrefix(abspath, `\\?\UNC\`) {
-				return abspath
-			}
-			// Check if \\?\ already exists.
-			if strings.HasPrefix(abspath, `\\?\`) {
-				return abspath
-			}
-			// Check if path starts with \\.
-			if strings.HasPrefix(abspath, `\\`) {
-				return strings.Replace(abspath, `\\`, `\\?\UNC\`, 1)
-			}
-			// Normal path.
-			return `\\?\` + abspath
-		}
-	}
-	return name
-}
-
-// Chmod changes the mode of the named file to mode.
-func Chmod(name string, mode os.FileMode) error {
-	return os.Chmod(fixpath(name), mode)
-}
-
-// Mkdir creates a new directory with the specified name and permission bits.  If there is an error, it will be of type
-// *PathError.
-func Mkdir(name string, perm os.FileMode) error {
-	return os.Mkdir(fixpath(name), perm)
-}
-
-// MkdirAll creates a directory named path, along with any necessary parents, and returns nil, or else returns an error.
-// The permission bits perm are used for all directories that MkdirAll creates.  If path is already a directory,
-// MkdirAll does nothing and returns nil.
-func MkdirAll(path string, perm os.FileMode) error {
-	return os.MkdirAll(fixpath(path), perm)
-}
-
-// Readlink returns the destination of the named symbolic link.  If there is an error, it will be of type *PathError.
-func Readlink(name string) (string, error) {
-	return os.Readlink(fixpath(name))
-}
-
-// Remove removes the named file or directory.  If there is an error, it will be of type *PathError.
-func Remove(name string) error {
-	return os.Remove(fixpath(name))
-}
-
-// RemoveAll removes path and any children it contains.  It removes everything it can but returns the first error it
-// encounters.  If the path does not exist, RemoveAll returns nil (no error).
-func RemoveAll(path string) error {
-	return os.RemoveAll(fixpath(path))
-}
-
-// Rename renames (moves) oldpath to newpath.  If newpath already exists, Rename replaces it.  OS-specific restrictions
-// may apply when oldpath and newpath are in different directories.  If there is an error, it will be of type
-// *LinkError.
-func Rename(oldpath, newpath string) error {
-	return os.Rename(fixpath(oldpath), fixpath(newpath))
-}
-
-// Symlink creates newname as a symbolic link to oldname.  If there is an error, it will be of type *LinkError.
-func Symlink(oldname, newname string) error {
-	return os.Symlink(fixpath(oldname), fixpath(newname))
-}
-
-// Stat returns a FileInfo structure describing the named file.  If there is an error, it will be of type *PathError.
-func Stat(name string) (os.FileInfo, error) {
-	return os.Stat(fixpath(name))
-}
-
-// Lstat returns the FileInfo structure describing the named file.  If the file is a symbolic link, the returned
-// FileInfo describes the symbolic link.  Lstat makes no attempt to follow the link.  If there is an error, it will be
-// of type *PathError.
-func Lstat(name string) (os.FileInfo, error) {
-	return os.Lstat(fixpath(name))
-}
-
-// Create creates the named file with mode 0666 (before umask), truncating it if it already exists. If successful,
-// methods on the returned File can be used for I/O; the associated file descriptor has mode O_RDWR.  If there is an
-// error, it will be of type *PathError.
-func Create(name string) (*os.File, error) {
-	return os.Create(fixpath(name))
-}
-
-// OpenFile is the generalized open call; most users will use Open or Create instead.  It opens the named file with
-// specified flag (O_RDONLY etc.) and perm, (0666 etc.) if applicable.  If successful, methods on the returned File can
-// be used for I/O.  If there is an error, it will be of type *PathError.
-func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) {
-	return os.OpenFile(fixpath(name), flag, perm)
-}
-
-// Walk walks the file tree rooted at root, calling walkFn for each file or directory in the tree, including root. All
-// errors that arise visiting files and directories are filtered by walkFn. The files are walked in lexical order, which
-// makes the output deterministic but means that for very large directories Walk can be inefficient.  Walk does not
-// follow symbolic links.
-func Walk(root string, walkFn filepath.WalkFunc) error {
-	return filepath.Walk(fixpath(root), walkFn)
-}

fs/file_linux.go

@@ -1,55 +0,0 @@
-package fs
-
-import (
-	"os"
-	"syscall"
-
-	"github.com/zcalusic/restic-server/errors"
-	"golang.org/x/sys/unix"
-)
-
-// Open opens a file for reading, without updating the atime and without caching data on read.
-func Open(name string) (File, error) {
-	file, err := os.OpenFile(name, os.O_RDONLY|syscall.O_NOATIME, 0)
-	if os.IsPermission(errors.Cause(err)) {
-		file, err = os.OpenFile(name, os.O_RDONLY, 0)
-	}
-	return &nonCachingFile{File: file}, err
-}
-
-// nonCachingFile wraps an *os.File and calls fadvise() to instantly forget data that has been read or written.
-type nonCachingFile struct {
-	*os.File
-	readOffset int64
-}
-
-func (f *nonCachingFile) Read(p []byte) (int, error) {
-	n, err := f.File.Read(p)
-
-	if n > 0 {
-		ferr := unix.Fadvise(int(f.File.Fd()), f.readOffset, int64(n), unix.FADV_DONTNEED)
-
-		f.readOffset += int64(n)
-
-		if err == nil {
-			err = ferr
-		}
-	}
-
-	return n, err
-}
-
-// ClearCache syncs and then removes the file's content from the OS cache.
-func ClearCache(file File) error {
-	f, ok := file.(*os.File)
-	if !ok {
-		panic("ClearCache called for file not *os.File")
-	}
-
-	err := f.Sync()
-	if err != nil {
-		return err
-	}
-
-	return unix.Fadvise(int(f.Fd()), 0, 0, unix.FADV_DONTNEED)
-}

fs/file_nonlinux.go

@@ -1,15 +0,0 @@
-// +build !linux
-
-package fs
-
-import "os"
-
-// Open opens a file for reading.
-func Open(name string) (File, error) {
-	return os.OpenFile(fixpath(name), os.O_RDONLY, 0)
-}
-
-// ClearCache syncs and then removes the file's content from the OS cache.
-func ClearCache(f File) error {
-	return nil
-}

handlers.go

@@ -10,8 +10,6 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
-
-	"github.com/zcalusic/restic-server/fs"
 )
 
 // Context contains repository metadata.
@@ -152,7 +150,7 @@ func GetBlob(c *Context) http.HandlerFunc {
 		}
 		path := filepath.Join(c.path, dir, name)
 
-		file, err := fs.Open(path)
+		file, err := os.Open(path)
 		if err != nil {
 			http.Error(w, "404 not found", 404)
 			return
@@ -172,7 +170,7 @@ func SaveBlob(c *Context) http.HandlerFunc {
 
 		tmp := filepath.Join(c.path, "tmp", name)
 
-		tf, err := fs.OpenFile(tmp, os.O_CREATE|os.O_WRONLY, 0600)
+		tf, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY, 0600)
 		if err != nil {
 			http.Error(w, "500 internal server error", 500)
 			return

htpasswd.go

@@ -30,8 +30,7 @@ import (
 	"encoding/csv"
 	"io"
 	"log"
-
+	"os"
-	"github.com/zcalusic/restic-server/fs"
 )
 
 // Lookup passwords in a htpasswd file.  The entries must have been created with -s for SHA encryption.
@@ -44,7 +43,7 @@ type HtpasswdFile struct {
 // NewHtpasswdFromFile reads the users and passwords from a htpasswd file and returns them.  If an error is encountered,
 // it is returned, together with a nil-Pointer for the HtpasswdFile.
 func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
-	r, err := fs.Open(path)
+	r, err := os.Open(path)
 	if err != nil {
 		return nil, err
 	}