Make Server use the new repo.Handler

This contains all the glue to make Server use the new repo.Handler:

- Remove all old handlers
- Add ServeHTTP to make Server a single http.Handler
- Remove Goji routing and replace by net/http and custom routing logic

Additionally, this implements two-level backup repositories.
This commit is contained in:
Konrad Wojas 2020-05-04 01:28:13 +08:00 committed by Alexander Neumann
parent 55e549e92c
commit 1f593fafaf
8 changed files with 275 additions and 707 deletions

View file

@ -1,26 +1,18 @@
package restserver
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/miolini/datacounter"
"github.com/prometheus/client_golang/prometheus"
"goji.io/middleware"
"goji.io/pat"
"github.com/restic/rest-server/quota"
"github.com/restic/rest-server/repo"
)
// Server determines how a Mux's handlers behave.
// Server encapsulates the rest-server's settings and repo management logic
type Server struct {
Path string
Listen string
@ -34,25 +26,74 @@ type Server struct {
PrivateRepos bool
Prometheus bool
Debug bool
MaxRepoSize int64
htpasswdFile *HtpasswdFile
quotaManager *quota.Manager
}
func (s *Server) isHashed(dir string) bool {
return dir == "data"
// ServeHTTP makes this server an http.Handler. It handlers the administrative
// part of the request (figuring out the filesystem location, performing
// authentication, etc) and then passes it on to repo.Handler for actual
// REST API processing.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// First of all, check auth
username, ok := s.checkAuth(r)
if !ok {
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
}
// Perform the path parsing to determine the repo folder and remainder for the
// repo handler
folderPath, remainder := splitURLPath(r.URL.Path, 2) // FIXME: configurable
if !folderPathValid(folderPath) {
log.Printf("Invalid request path: %s", r.URL.Path)
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
// Check if the current user is allowed to access this path
if !s.NoAuth && s.PrivateRepos {
if len(folderPath) == 0 || folderPath[0] != username {
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
}
// Determine filesystem path for this repo
fsPath, err := join(s.Path, folderPath...)
if err != nil {
// We did not expect an error at this stage, because we just checked the path
log.Printf("Unexpected join error for path %q", r.URL.Path)
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
// Pass the request to the repo.Handler
opt := repo.Options{
AppendOnly: s.AppendOnly,
Debug: s.Debug,
QuotaManager: s.quotaManager, // may be nil
}
if s.Prometheus {
opt.BlobMetricFunc = makeBlobMetricFunc(username, folderPath)
}
repoHandler, err := repo.New(fsPath, opt)
if err != nil {
log.Printf("repo.New error: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
r.URL.Path = remainder // strip folderPath for next handler
repoHandler.ServeHTTP(w, r)
}
func valid(name string) bool {
// Based on net/http.Dir
// taken from net/http.Dir
if strings.Contains(name, "\x00") {
return false
}
// Path characters that are disallowed or unsafe under some operating systems
// are not allowed here.
// The most important one here is '/', since Goji does not decode '%2F' to '/'
// during routing, so we can end up with a '/' in the name here.
if strings.ContainsAny(name, "/\\:*?\"<>|") {
return false
}
if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) {
return false
}
@ -60,15 +101,17 @@ func valid(name string) bool {
return true
}
var validTypes = []string{"data", "index", "keys", "locks", "snapshots", "config"}
func (s *Server) isValidType(name string) bool {
for _, tpe := range validTypes {
func isValidType(name string) bool {
for _, tpe := range repo.ObjectTypes {
if name == tpe {
return true
}
}
for _, tpe := range repo.FileTypes {
if name == tpe {
return true
}
}
return false
}
@ -92,584 +135,44 @@ func join(base string, names ...string) (string, error) {
return filepath.Join(clean...), nil
}
// getRepo returns the repository location, relative to s.Path.
func (s *Server) getRepo(r *http.Request) string {
if strings.HasPrefix(fmt.Sprintf("%s", middleware.Pattern(r.Context())), "/:repo") {
return pat.Param(r, "repo")
// splitURLPath splits the URL path into a folderPath of the subrepo, and
// a remainder that can be passed to repo.Handler.
// Example: /foo/bar/locks/0123... will be split into:
// ["foo", "bar"] and "/locks/0123..."
func splitURLPath(urlPath string, maxDepth int) (folderPath []string, remainder string) {
if !strings.HasPrefix(urlPath, "/") {
// Really should start with "/"
return nil, urlPath
}
return "."
p := strings.SplitN(urlPath, "/", maxDepth+2)
// Skip the empty first one and the remainder in the last one
for _, name := range p[1 : len(p)-1] {
if isValidType(name) {
// We found a part that is a special repo file or dir
break
}
folderPath = append(folderPath, name)
}
// If the folder path is empty, the whole path is the remainder (do not strip '/')
if len(folderPath) == 0 {
return nil, urlPath
}
// Check that the urlPath starts with the reconstructed path, which should
// always be the case.
fullFolderPath := "/" + strings.Join(folderPath, "/")
if !strings.HasPrefix(urlPath, fullFolderPath) {
return nil, urlPath
}
return folderPath, urlPath[len(fullFolderPath):]
}
// getPath returns the path for a file type in the repo.
func (s *Server) getPath(r *http.Request, fileType string) (string, error) {
if !s.isValidType(fileType) {
return "", errors.New("invalid file type")
}
return join(s.Path, s.getRepo(r), fileType)
}
// getFilePath returns the path for a file in the repo.
func (s *Server) getFilePath(r *http.Request, fileType, name string) (string, error) {
if !s.isValidType(fileType) {
return "", errors.New("invalid file type")
}
if s.isHashed(fileType) {
if len(name) < 2 {
return "", errors.New("file name is too short")
}
return join(s.Path, s.getRepo(r), fileType, name[:2], name)
}
return join(s.Path, s.getRepo(r), fileType, name)
}
// getUser returns the username from the request, or an empty string if none.
func (s *Server) getUser(r *http.Request) string {
username, _, ok := r.BasicAuth()
if !ok {
return ""
}
return username
}
// getMetricLabels returns the prometheus labels from the request.
func (s *Server) getMetricLabels(r *http.Request) prometheus.Labels {
labels := prometheus.Labels{
"user": s.getUser(r),
"repo": s.getRepo(r),
"type": pat.Param(r, "type"),
}
return labels
}
// isUserPath checks if a request path is accessible by the user when using
// private repositories.
func isUserPath(username, path string) bool {
prefix := "/" + username
if !strings.HasPrefix(path, prefix) {
return false
}
return len(path) == len(prefix) || path[len(prefix)] == '/'
}
// AuthHandler wraps h with a http.HandlerFunc that performs basic authentication against the user/passwords pairs
// stored in f and returns the http.HandlerFunc.
func (s *Server) AuthHandler(f *HtpasswdFile, h http.Handler) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
username, password, ok := r.BasicAuth()
if !ok || !f.Validate(username, password) {
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
// resolve all relative elements in the path
urlPath := path.Clean(r.URL.Path)
if s.PrivateRepos && !isUserPath(username, urlPath) && urlPath != "/metrics" {
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
h.ServeHTTP(w, r)
}
}
// CheckConfig checks whether a configuration exists.
func (s *Server) CheckConfig(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("CheckConfig()")
}
cfg, err := s.getPath(r, "config")
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
st, err := os.Stat(cfg)
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
}
// GetConfig allows for a config to be retrieved.
func (s *Server) GetConfig(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("GetConfig()")
}
cfg, err := s.getPath(r, "config")
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
bytes, err := ioutil.ReadFile(cfg)
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
_, _ = w.Write(bytes)
}
// SaveConfig allows for a config to be saved.
func (s *Server) SaveConfig(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("SaveConfig()")
}
cfg, err := s.getPath(r, "config")
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
f, err := os.OpenFile(cfg, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil && os.IsExist(err) {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
return
}
_, err = io.Copy(f, r.Body)
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
err = f.Close()
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
_ = r.Body.Close()
}
// DeleteConfig removes a config.
func (s *Server) DeleteConfig(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("DeleteConfig()")
}
if s.AppendOnly {
http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
return
}
cfg, err := s.getPath(r, "config")
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
if err := os.Remove(cfg); err != nil {
if s.Debug {
log.Print(err)
}
if os.IsNotExist(err) {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
} else {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
}
return
}
}
const (
mimeTypeAPIV1 = "application/vnd.x.restic.rest.v1"
mimeTypeAPIV2 = "application/vnd.x.restic.rest.v2"
)
// ListBlobs lists all blobs of a given type in an arbitrary order.
func (s *Server) ListBlobs(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("ListBlobs()")
}
switch r.Header.Get("Accept") {
case mimeTypeAPIV2:
s.ListBlobsV2(w, r)
default:
s.ListBlobsV1(w, r)
}
}
// ListBlobsV1 lists all blobs of a given type in an arbitrary order.
func (s *Server) ListBlobsV1(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("ListBlobsV1()")
}
fileType := pat.Param(r, "type")
path, err := s.getPath(r, fileType)
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
items, err := ioutil.ReadDir(path)
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
var names []string
for _, i := range items {
if s.isHashed(fileType) {
subpath := filepath.Join(path, i.Name())
var subitems []os.FileInfo
subitems, err = ioutil.ReadDir(subpath)
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
for _, f := range subitems {
names = append(names, f.Name())
}
} else {
names = append(names, i.Name())
}
}
data, err := json.Marshal(names)
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", mimeTypeAPIV1)
_, _ = w.Write(data)
}
// Blob represents a single blob, its name and its size.
type Blob struct {
Name string `json:"name"`
Size int64 `json:"size"`
}
// ListBlobsV2 lists all blobs of a given type, together with their sizes, in an arbitrary order.
func (s *Server) ListBlobsV2(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("ListBlobsV2()")
}
fileType := pat.Param(r, "type")
path, err := s.getPath(r, fileType)
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
items, err := ioutil.ReadDir(path)
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
var blobs []Blob
for _, i := range items {
if s.isHashed(fileType) {
subpath := filepath.Join(path, i.Name())
var subitems []os.FileInfo
subitems, err = ioutil.ReadDir(subpath)
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
for _, f := range subitems {
blobs = append(blobs, Blob{Name: f.Name(), Size: f.Size()})
}
} else {
blobs = append(blobs, Blob{Name: i.Name(), Size: i.Size()})
}
}
data, err := json.Marshal(blobs)
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", mimeTypeAPIV2)
_, _ = w.Write(data)
}
// CheckBlob tests whether a blob exists.
func (s *Server) CheckBlob(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("CheckBlob()")
}
path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name"))
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
st, err := os.Stat(path)
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
}
// GetBlob retrieves a blob from the repository.
func (s *Server) GetBlob(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("GetBlob()")
}
path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name"))
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
file, err := os.Open(path)
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
wc := datacounter.NewResponseWriterCounter(w)
http.ServeContent(wc, r, "", time.Unix(0, 0), file)
if err = file.Close(); err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
if s.Prometheus {
labels := s.getMetricLabels(r)
metricBlobReadTotal.With(labels).Inc()
metricBlobReadBytesTotal.With(labels).Add(float64(wc.Count()))
}
}
// SaveBlob saves a blob to the repository.
func (s *Server) SaveBlob(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("SaveBlob()")
}
path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name"))
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
tf, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
if os.IsNotExist(err) {
// the error is caused by a missing directory, create it and retry
mkdirErr := os.MkdirAll(filepath.Dir(path), 0700)
if mkdirErr != nil {
log.Print(mkdirErr)
} else {
// try again
tf, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
}
}
if os.IsExist(err) {
http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
return
}
if err != nil {
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
// ensure this blob does not put us over the repo size limit (if there is one)
var outFile io.Writer = tf
if s.MaxRepoSize != 0 {
var errCode int
outFile, errCode, err = s.maxSizeWriter(r, tf)
if err != nil {
if s.Debug {
log.Println(err)
}
if errCode > 0 {
http.Error(w, http.StatusText(errCode), errCode)
}
return
}
}
written, err := io.Copy(outFile, r.Body)
if err != nil {
_ = tf.Close()
_ = os.Remove(path)
if s.MaxRepoSize > 0 {
s.incrementRepoSpaceUsage(-written)
}
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
if err := tf.Sync(); err != nil {
_ = tf.Close()
_ = os.Remove(path)
if s.MaxRepoSize > 0 {
s.incrementRepoSpaceUsage(-written)
}
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
if err := tf.Close(); err != nil {
_ = os.Remove(path)
if s.MaxRepoSize > 0 {
s.incrementRepoSpaceUsage(-written)
}
if s.Debug {
log.Print(err)
}
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
if s.Prometheus {
labels := s.getMetricLabels(r)
metricBlobWriteTotal.With(labels).Inc()
metricBlobWriteBytesTotal.With(labels).Add(float64(written))
}
}
// DeleteBlob deletes a blob from the repository.
func (s *Server) DeleteBlob(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("DeleteBlob()")
}
if s.AppendOnly && pat.Param(r, "type") != "locks" {
http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
return
}
path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name"))
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
var size int64
if s.Prometheus || s.MaxRepoSize > 0 {
stat, err := os.Stat(path)
if err == nil {
size = stat.Size()
}
}
if err := os.Remove(path); err != nil {
if s.Debug {
log.Print(err)
}
if os.IsNotExist(err) {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
} else {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
}
return
}
if s.MaxRepoSize > 0 {
s.incrementRepoSpaceUsage(-size)
}
if s.Prometheus {
labels := s.getMetricLabels(r)
metricBlobDeleteTotal.With(labels).Inc()
metricBlobDeleteBytesTotal.With(labels).Add(float64(size))
}
}
// CreateRepo creates repository directories.
func (s *Server) CreateRepo(w http.ResponseWriter, r *http.Request) {
if s.Debug {
log.Println("CreateRepo()")
}
repo, err := join(s.Path, s.getRepo(r))
if err != nil {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
if r.URL.Query().Get("create") != "true" {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
log.Printf("Creating repository directories in %s\n", repo)
if err := os.MkdirAll(repo, 0700); err != nil {
log.Print(err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
for _, d := range validTypes {
if d == "config" {
continue
}
if err := os.MkdirAll(filepath.Join(repo, d), 0700); err != nil {
log.Print(err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
}
for i := 0; i < 256; i++ {
if err := os.MkdirAll(filepath.Join(repo, "data", fmt.Sprintf("%02x", i)), 0700); err != nil {
log.Print(err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
// folderPathValid checks if a folderPath returned by splitURLPath is valid and
// safe.
func folderPathValid(folderPath []string) bool {
for _, name := range folderPath {
if name == "" || name == ".." || !valid(name) {
return false
}
}
return true
}