From 1f593fafaf0d8c86e891d3ecd2d7b8fcca28a9f0 Mon Sep 17 00:00:00 2001 From: Konrad Wojas Date: Mon, 4 May 2020 01:28:13 +0800 Subject: [PATCH] Make Server use the new repo.Handler This contains all the glue to make Server use the new repo.Handler: - Remove all old handlers - Add ServeHTTP to make Server a single http.Handler - Remove Goji routing and replace by net/http and custom routing logic Additionally, this implements two-level backup repositories. --- cmd/rest-server/main.go | 23 +- cmd/rest-server/main_test.go | 8 +- handlers.go | 707 ++++++----------------------------- handlers_test.go | 78 ++-- metrics.go | 31 +- mux.go | 88 +++-- quota/quota.go | 23 +- repo/repo.go | 24 +- 8 files changed, 275 insertions(+), 707 deletions(-) diff --git a/cmd/rest-server/main.go b/cmd/rest-server/main.go index 8e91358..30ce237 100644 --- a/cmd/rest-server/main.go +++ b/cmd/rest-server/main.go @@ -75,21 +75,6 @@ func tlsSettings() (bool, string, string, error) { return server.TLS, key, cert, nil } -func getHandler(server restserver.Server) (http.Handler, error) { - mux := restserver.NewHandler(server) - if server.NoAuth { - log.Println("Authentication disabled") - return mux, nil - } - - log.Println("Authentication enabled") - htpasswdFile, err := restserver.NewHtpasswdFromFile(filepath.Join(server.Path, ".htpasswd")) - if err != nil { - return nil, fmt.Errorf("cannot load .htpasswd (use --no-auth to disable): %v", err) - } - return server.AuthHandler(htpasswdFile, mux), nil -} - func runRoot(cmd *cobra.Command, args []string) error { if showVersion { fmt.Printf("rest-server %s compiled with %v on %v/%v\n", version, runtime.Version(), runtime.GOOS, runtime.GOARCH) @@ -112,7 +97,13 @@ func runRoot(cmd *cobra.Command, args []string) error { defer pprof.StopCPUProfile() } - handler, err := getHandler(server) + if server.NoAuth { + log.Println("Authentication disabled") + } else { + log.Println("Authentication enabled") + } + + handler, err := restserver.NewHandler(&server) if err != nil { log.Fatalf("error: %v", err) } diff --git a/cmd/rest-server/main_test.go b/cmd/rest-server/main_test.go index fe204df..685f4bf 100644 --- a/cmd/rest-server/main_test.go +++ b/cmd/rest-server/main_test.go @@ -86,14 +86,16 @@ func TestGetHandler(t *testing.T) { } }() + getHandler := restserver.NewHandler + // With NoAuth = false and no .htpasswd - _, err = getHandler(restserver.Server{Path: dir}) + _, err = getHandler(&restserver.Server{Path: dir}) if err == nil { t.Errorf("NoAuth=false: expected error, got nil") } // With NoAuth = true and no .htpasswd - _, err = getHandler(restserver.Server{NoAuth: true, Path: dir}) + _, err = getHandler(&restserver.Server{NoAuth: true, Path: dir}) if err != nil { t.Errorf("NoAuth=true: expected no error, got %v", err) } @@ -112,7 +114,7 @@ func TestGetHandler(t *testing.T) { }() // With NoAuth = false and with .htpasswd - _, err = getHandler(restserver.Server{Path: dir}) + _, err = getHandler(&restserver.Server{Path: dir}) if err != nil { t.Errorf("NoAuth=false with .htpasswd: expected no error, got %v", err) } diff --git a/handlers.go b/handlers.go index 5b51e15..d1ee6f4 100644 --- a/handlers.go +++ b/handlers.go @@ -1,26 +1,18 @@ package restserver import ( - "encoding/json" "errors" - "fmt" - "io" - "io/ioutil" "log" "net/http" - "os" "path" "path/filepath" "strings" - "time" - "github.com/miolini/datacounter" - "github.com/prometheus/client_golang/prometheus" - "goji.io/middleware" - "goji.io/pat" + "github.com/restic/rest-server/quota" + "github.com/restic/rest-server/repo" ) -// Server determines how a Mux's handlers behave. +// Server encapsulates the rest-server's settings and repo management logic type Server struct { Path string Listen string @@ -34,25 +26,74 @@ type Server struct { PrivateRepos bool Prometheus bool Debug bool + MaxRepoSize int64 + + htpasswdFile *HtpasswdFile + quotaManager *quota.Manager } -func (s *Server) isHashed(dir string) bool { - return dir == "data" +// ServeHTTP makes this server an http.Handler. It handlers the administrative +// part of the request (figuring out the filesystem location, performing +// authentication, etc) and then passes it on to repo.Handler for actual +// REST API processing. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // First of all, check auth + username, ok := s.checkAuth(r) + if !ok { + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + } + + // Perform the path parsing to determine the repo folder and remainder for the + // repo handler + folderPath, remainder := splitURLPath(r.URL.Path, 2) // FIXME: configurable + if !folderPathValid(folderPath) { + log.Printf("Invalid request path: %s", r.URL.Path) + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + // Check if the current user is allowed to access this path + if !s.NoAuth && s.PrivateRepos { + if len(folderPath) == 0 || folderPath[0] != username { + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + return + } + } + + // Determine filesystem path for this repo + fsPath, err := join(s.Path, folderPath...) + if err != nil { + // We did not expect an error at this stage, because we just checked the path + log.Printf("Unexpected join error for path %q", r.URL.Path) + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + + // Pass the request to the repo.Handler + opt := repo.Options{ + AppendOnly: s.AppendOnly, + Debug: s.Debug, + QuotaManager: s.quotaManager, // may be nil + } + if s.Prometheus { + opt.BlobMetricFunc = makeBlobMetricFunc(username, folderPath) + } + repoHandler, err := repo.New(fsPath, opt) + if err != nil { + log.Printf("repo.New error: %v", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + r.URL.Path = remainder // strip folderPath for next handler + repoHandler.ServeHTTP(w, r) } func valid(name string) bool { - // Based on net/http.Dir + // taken from net/http.Dir if strings.Contains(name, "\x00") { return false } - // Path characters that are disallowed or unsafe under some operating systems - // are not allowed here. - // The most important one here is '/', since Goji does not decode '%2F' to '/' - // during routing, so we can end up with a '/' in the name here. - if strings.ContainsAny(name, "/\\:*?\"<>|") { - return false - } if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) { return false } @@ -60,15 +101,17 @@ func valid(name string) bool { return true } -var validTypes = []string{"data", "index", "keys", "locks", "snapshots", "config"} - -func (s *Server) isValidType(name string) bool { - for _, tpe := range validTypes { +func isValidType(name string) bool { + for _, tpe := range repo.ObjectTypes { + if name == tpe { + return true + } + } + for _, tpe := range repo.FileTypes { if name == tpe { return true } } - return false } @@ -92,584 +135,44 @@ func join(base string, names ...string) (string, error) { return filepath.Join(clean...), nil } -// getRepo returns the repository location, relative to s.Path. -func (s *Server) getRepo(r *http.Request) string { - if strings.HasPrefix(fmt.Sprintf("%s", middleware.Pattern(r.Context())), "/:repo") { - return pat.Param(r, "repo") +// splitURLPath splits the URL path into a folderPath of the subrepo, and +// a remainder that can be passed to repo.Handler. +// Example: /foo/bar/locks/0123... will be split into: +// ["foo", "bar"] and "/locks/0123..." +func splitURLPath(urlPath string, maxDepth int) (folderPath []string, remainder string) { + if !strings.HasPrefix(urlPath, "/") { + // Really should start with "/" + return nil, urlPath } - - return "." + p := strings.SplitN(urlPath, "/", maxDepth+2) + // Skip the empty first one and the remainder in the last one + for _, name := range p[1 : len(p)-1] { + if isValidType(name) { + // We found a part that is a special repo file or dir + break + } + folderPath = append(folderPath, name) + } + // If the folder path is empty, the whole path is the remainder (do not strip '/') + if len(folderPath) == 0 { + return nil, urlPath + } + // Check that the urlPath starts with the reconstructed path, which should + // always be the case. + fullFolderPath := "/" + strings.Join(folderPath, "/") + if !strings.HasPrefix(urlPath, fullFolderPath) { + return nil, urlPath + } + return folderPath, urlPath[len(fullFolderPath):] } -// getPath returns the path for a file type in the repo. -func (s *Server) getPath(r *http.Request, fileType string) (string, error) { - if !s.isValidType(fileType) { - return "", errors.New("invalid file type") - } - return join(s.Path, s.getRepo(r), fileType) -} - -// getFilePath returns the path for a file in the repo. -func (s *Server) getFilePath(r *http.Request, fileType, name string) (string, error) { - if !s.isValidType(fileType) { - return "", errors.New("invalid file type") - } - - if s.isHashed(fileType) { - if len(name) < 2 { - return "", errors.New("file name is too short") - } - - return join(s.Path, s.getRepo(r), fileType, name[:2], name) - } - - return join(s.Path, s.getRepo(r), fileType, name) -} - -// getUser returns the username from the request, or an empty string if none. -func (s *Server) getUser(r *http.Request) string { - username, _, ok := r.BasicAuth() - if !ok { - return "" - } - return username -} - -// getMetricLabels returns the prometheus labels from the request. -func (s *Server) getMetricLabels(r *http.Request) prometheus.Labels { - labels := prometheus.Labels{ - "user": s.getUser(r), - "repo": s.getRepo(r), - "type": pat.Param(r, "type"), - } - return labels -} - -// isUserPath checks if a request path is accessible by the user when using -// private repositories. -func isUserPath(username, path string) bool { - prefix := "/" + username - if !strings.HasPrefix(path, prefix) { - return false - } - return len(path) == len(prefix) || path[len(prefix)] == '/' -} - -// AuthHandler wraps h with a http.HandlerFunc that performs basic authentication against the user/passwords pairs -// stored in f and returns the http.HandlerFunc. -func (s *Server) AuthHandler(f *HtpasswdFile, h http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - username, password, ok := r.BasicAuth() - if !ok || !f.Validate(username, password) { - http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) - return - } - - // resolve all relative elements in the path - urlPath := path.Clean(r.URL.Path) - if s.PrivateRepos && !isUserPath(username, urlPath) && urlPath != "/metrics" { - http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) - return - } - h.ServeHTTP(w, r) - } -} - -// CheckConfig checks whether a configuration exists. -func (s *Server) CheckConfig(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("CheckConfig()") - } - cfg, err := s.getPath(r, "config") - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - st, err := os.Stat(cfg) - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - return - } - - w.Header().Add("Content-Length", fmt.Sprint(st.Size())) -} - -// GetConfig allows for a config to be retrieved. -func (s *Server) GetConfig(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("GetConfig()") - } - cfg, err := s.getPath(r, "config") - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - bytes, err := ioutil.ReadFile(cfg) - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - return - } - - _, _ = w.Write(bytes) -} - -// SaveConfig allows for a config to be saved. -func (s *Server) SaveConfig(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("SaveConfig()") - } - cfg, err := s.getPath(r, "config") - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - f, err := os.OpenFile(cfg, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600) - if err != nil && os.IsExist(err) { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) - return - } - - _, err = io.Copy(f, r.Body) - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - err = f.Close() - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - _ = r.Body.Close() -} - -// DeleteConfig removes a config. -func (s *Server) DeleteConfig(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("DeleteConfig()") - } - - if s.AppendOnly { - http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) - return - } - - cfg, err := s.getPath(r, "config") - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - if err := os.Remove(cfg); err != nil { - if s.Debug { - log.Print(err) - } - if os.IsNotExist(err) { - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - } else { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - } - return - } -} - -const ( - mimeTypeAPIV1 = "application/vnd.x.restic.rest.v1" - mimeTypeAPIV2 = "application/vnd.x.restic.rest.v2" -) - -// ListBlobs lists all blobs of a given type in an arbitrary order. -func (s *Server) ListBlobs(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("ListBlobs()") - } - - switch r.Header.Get("Accept") { - case mimeTypeAPIV2: - s.ListBlobsV2(w, r) - default: - s.ListBlobsV1(w, r) - } -} - -// ListBlobsV1 lists all blobs of a given type in an arbitrary order. -func (s *Server) ListBlobsV1(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("ListBlobsV1()") - } - fileType := pat.Param(r, "type") - path, err := s.getPath(r, fileType) - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - items, err := ioutil.ReadDir(path) - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - return - } - - var names []string - for _, i := range items { - if s.isHashed(fileType) { - subpath := filepath.Join(path, i.Name()) - var subitems []os.FileInfo - subitems, err = ioutil.ReadDir(subpath) - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - return - } - for _, f := range subitems { - names = append(names, f.Name()) - } - } else { - names = append(names, i.Name()) - } - } - - data, err := json.Marshal(names) - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", mimeTypeAPIV1) - _, _ = w.Write(data) -} - -// Blob represents a single blob, its name and its size. -type Blob struct { - Name string `json:"name"` - Size int64 `json:"size"` -} - -// ListBlobsV2 lists all blobs of a given type, together with their sizes, in an arbitrary order. -func (s *Server) ListBlobsV2(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("ListBlobsV2()") - } - fileType := pat.Param(r, "type") - path, err := s.getPath(r, fileType) - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - items, err := ioutil.ReadDir(path) - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - return - } - - var blobs []Blob - for _, i := range items { - if s.isHashed(fileType) { - subpath := filepath.Join(path, i.Name()) - var subitems []os.FileInfo - subitems, err = ioutil.ReadDir(subpath) - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - return - } - for _, f := range subitems { - blobs = append(blobs, Blob{Name: f.Name(), Size: f.Size()}) - } - } else { - blobs = append(blobs, Blob{Name: i.Name(), Size: i.Size()}) - } - } - - data, err := json.Marshal(blobs) - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", mimeTypeAPIV2) - _, _ = w.Write(data) -} - -// CheckBlob tests whether a blob exists. -func (s *Server) CheckBlob(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("CheckBlob()") - } - - path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name")) - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - st, err := os.Stat(path) - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - return - } - - w.Header().Add("Content-Length", fmt.Sprint(st.Size())) -} - -// GetBlob retrieves a blob from the repository. -func (s *Server) GetBlob(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("GetBlob()") - } - - path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name")) - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - file, err := os.Open(path) - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - return - } - - wc := datacounter.NewResponseWriterCounter(w) - http.ServeContent(wc, r, "", time.Unix(0, 0), file) - - if err = file.Close(); err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - if s.Prometheus { - labels := s.getMetricLabels(r) - metricBlobReadTotal.With(labels).Inc() - metricBlobReadBytesTotal.With(labels).Add(float64(wc.Count())) - } -} - -// SaveBlob saves a blob to the repository. -func (s *Server) SaveBlob(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("SaveBlob()") - } - - path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name")) - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - tf, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600) - if os.IsNotExist(err) { - // the error is caused by a missing directory, create it and retry - mkdirErr := os.MkdirAll(filepath.Dir(path), 0700) - if mkdirErr != nil { - log.Print(mkdirErr) - } else { - // try again - tf, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600) - } - } - if os.IsExist(err) { - http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) - return - } - if err != nil { - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - // ensure this blob does not put us over the repo size limit (if there is one) - var outFile io.Writer = tf - if s.MaxRepoSize != 0 { - var errCode int - outFile, errCode, err = s.maxSizeWriter(r, tf) - if err != nil { - if s.Debug { - log.Println(err) - } - if errCode > 0 { - http.Error(w, http.StatusText(errCode), errCode) - } - return - } - } - - written, err := io.Copy(outFile, r.Body) - if err != nil { - _ = tf.Close() - _ = os.Remove(path) - if s.MaxRepoSize > 0 { - s.incrementRepoSpaceUsage(-written) - } - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) - return - } - - if err := tf.Sync(); err != nil { - _ = tf.Close() - _ = os.Remove(path) - if s.MaxRepoSize > 0 { - s.incrementRepoSpaceUsage(-written) - } - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - if err := tf.Close(); err != nil { - _ = os.Remove(path) - if s.MaxRepoSize > 0 { - s.incrementRepoSpaceUsage(-written) - } - if s.Debug { - log.Print(err) - } - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - if s.Prometheus { - labels := s.getMetricLabels(r) - metricBlobWriteTotal.With(labels).Inc() - metricBlobWriteBytesTotal.With(labels).Add(float64(written)) - } -} - -// DeleteBlob deletes a blob from the repository. -func (s *Server) DeleteBlob(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("DeleteBlob()") - } - - if s.AppendOnly && pat.Param(r, "type") != "locks" { - http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) - return - } - - path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name")) - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - var size int64 - if s.Prometheus || s.MaxRepoSize > 0 { - stat, err := os.Stat(path) - if err == nil { - size = stat.Size() - } - } - - if err := os.Remove(path); err != nil { - if s.Debug { - log.Print(err) - } - if os.IsNotExist(err) { - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - } else { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - } - return - } - - if s.MaxRepoSize > 0 { - s.incrementRepoSpaceUsage(-size) - } - if s.Prometheus { - labels := s.getMetricLabels(r) - metricBlobDeleteTotal.With(labels).Inc() - metricBlobDeleteBytesTotal.With(labels).Add(float64(size)) - } -} - -// CreateRepo creates repository directories. -func (s *Server) CreateRepo(w http.ResponseWriter, r *http.Request) { - if s.Debug { - log.Println("CreateRepo()") - } - - repo, err := join(s.Path, s.getRepo(r)) - if err != nil { - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - if r.URL.Query().Get("create") != "true" { - http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) - return - } - - log.Printf("Creating repository directories in %s\n", repo) - - if err := os.MkdirAll(repo, 0700); err != nil { - log.Print(err) - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - for _, d := range validTypes { - if d == "config" { - continue - } - - if err := os.MkdirAll(filepath.Join(repo, d), 0700); err != nil { - log.Print(err) - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - } - - for i := 0; i < 256; i++ { - if err := os.MkdirAll(filepath.Join(repo, "data", fmt.Sprintf("%02x", i)), 0700); err != nil { - log.Print(err) - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } +// folderPathValid checks if a folderPath returned by splitURLPath is valid and +// safe. +func folderPathValid(folderPath []string) bool { + for _, name := range folderPath { + if name == "" || name == ".." || !valid(name) { + return false + } } + return true } diff --git a/handlers_test.go b/handlers_test.go index 65cdf92..b3c799f 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -4,12 +4,14 @@ import ( "bytes" "crypto/rand" "encoding/hex" + "fmt" "io" "io/ioutil" "net/http" "net/http/httptest" "os" "path/filepath" + "reflect" "strings" "testing" ) @@ -46,27 +48,6 @@ func TestJoin(t *testing.T) { } } -func TestIsUserPath(t *testing.T) { - var tests = []struct { - username string - path string - result bool - }{ - {"foo", "/", false}, - {"foo", "/foo", true}, - {"foo", "/foo/", true}, - {"foo", "/foo/bar", true}, - {"foo", "/foobar", false}, - } - - for _, test := range tests { - result := isUserPath(test.username, test.path) - if result != test.result { - t.Errorf("isUserPath(%q, %q) was incorrect, got: %v, want: %v.", test.username, test.path, result, test.result) - } - } -} - // declare a few helper functions // wantFunc tests the HTTP response in res and calls t.Error() if something is incorrect. @@ -229,7 +210,7 @@ func TestResticHandler(t *testing.T) { }() // set append-only mode and configure path - mux := NewHandler(Server{ + mux, err := NewHandler(&Server{ AppendOnly: true, Path: tempdir, }) @@ -248,3 +229,56 @@ func TestResticHandler(t *testing.T) { }) } } + +func TestSplitURLPath(t *testing.T) { + var tests = []struct { + // Params + urlPath string + maxDepth int + // Expected result + folderPath []string + remainder string + }{ + {"/", 0, nil, "/"}, + {"/", 2, nil, "/"}, + {"/foo/bar/locks/0123", 0, nil, "/foo/bar/locks/0123"}, + {"/foo/bar/locks/0123", 1, []string{"foo"}, "/bar/locks/0123"}, + {"/foo/bar/locks/0123", 2, []string{"foo", "bar"}, "/locks/0123"}, + {"/foo/bar/locks/0123", 3, []string{"foo", "bar"}, "/locks/0123"}, + {"/foo/bar/zzz/locks/0123", 2, []string{"foo", "bar"}, "/zzz/locks/0123"}, + {"/foo/bar/zzz/locks/0123", 3, []string{"foo", "bar", "zzz"}, "/locks/0123"}, + {"/foo/bar/locks/", 2, []string{"foo", "bar"}, "/locks/"}, + {"/foo/bar/", 2, []string{"foo", "bar"}, "/"}, + {"/foo/bar", 2, []string{"foo"}, "/bar"}, + {"/locks/", 2, nil, "/locks/"}, + // This function only splits, it does not check the path components! + {"/../../locks/", 2, []string{"..", ".."}, "/locks/"}, + {"///locks/", 2, []string{"", ""}, "/locks/"}, + {"////locks/", 2, []string{"", ""}, "//locks/"}, + // Robustness against broken input + {"/", -42, nil, "/"}, + {"foo", 2, nil, "foo"}, + {"foo/bar", 2, nil, "foo/bar"}, + {"", 2, nil, ""}, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + folderPath, remainder := splitURLPath(test.urlPath, test.maxDepth) + + var fpEqual bool + if len(test.folderPath) == 0 && len(folderPath) == 0 { + fpEqual = true // this check allows for nil vs empty slice + } else { + fpEqual = reflect.DeepEqual(test.folderPath, folderPath) + } + if !fpEqual { + t.Errorf("wrong folderPath: want %v, got %v", test.folderPath, folderPath) + } + + if test.remainder != remainder { + t.Errorf("wrong remainder: want %v, got %v", test.remainder, remainder) + } + }) + } +} diff --git a/metrics.go b/metrics.go index 636eabb..fa64f0b 100644 --- a/metrics.go +++ b/metrics.go @@ -1,6 +1,11 @@ package restserver -import "github.com/prometheus/client_golang/prometheus" +import ( + "strings" + + "github.com/prometheus/client_golang/prometheus" + "github.com/restic/rest-server/repo" +) var metricLabelList = []string{"user", "repo", "type"} @@ -52,6 +57,30 @@ var metricBlobDeleteBytesTotal = prometheus.NewCounterVec( metricLabelList, ) +// makeBlobMetricFunc creates a metrics callback function that increments the +// Prometheus metrics. +func makeBlobMetricFunc(username string, folderPath []string) repo.BlobMetricFunc { + var f repo.BlobMetricFunc = func(objectType string, operation repo.BlobOperation, nBytes uint64) { + labels := prometheus.Labels{ + "user": username, + "repo": strings.Join(folderPath, ""), + "type": objectType, + } + switch operation { + case repo.BlobRead: + metricBlobReadTotal.With(labels).Inc() + metricBlobReadBytesTotal.With(labels).Add(float64(nBytes)) + case repo.BlobWrite: + metricBlobWriteTotal.With(labels).Inc() + metricBlobWriteBytesTotal.With(labels).Add(float64(nBytes)) + case repo.BlobDelete: + metricBlobDeleteTotal.With(labels).Inc() + metricBlobDeleteBytesTotal.With(labels).Add(float64(nBytes)) + } + } + return f +} + func init() { // These are always initialized, but only updated if Config.Prometheus is set prometheus.MustRegister(metricBlobWriteTotal) diff --git a/mux.go b/mux.go index 649abe6..9db037d 100644 --- a/mux.go +++ b/mux.go @@ -1,15 +1,15 @@ package restserver import ( + "fmt" "log" "net/http" "os" - - goji "goji.io" + "path/filepath" "github.com/gorilla/handlers" "github.com/prometheus/client_golang/prometheus/promhttp" - "goji.io/pat" + "github.com/restic/rest-server/quota" ) func (s *Server) debugHandler(next http.Handler) http.Handler { @@ -29,43 +29,51 @@ func (s *Server) logHandler(next http.Handler) http.Handler { return handlers.CombinedLoggingHandler(accessLog, next) } -// NewHandler returns the master HTTP multiplexer/router. -func NewHandler(server Server) *goji.Mux { - mux := goji.NewMux() - - if server.Debug { - mux.Use(server.debugHandler) +func (s *Server) checkAuth(r *http.Request) (username string, ok bool) { + if s.NoAuth { + return username, true } - - if server.Log != "" { - mux.Use(server.logHandler) + var password string + username, password, ok = r.BasicAuth() + if !ok || !s.htpasswdFile.Validate(username, password) { + return "", false } - - if server.Prometheus { - mux.Handle(pat.Get("/metrics"), promhttp.Handler()) - } - - mux.HandleFunc(pat.Head("/config"), server.CheckConfig) - mux.HandleFunc(pat.Head("/:repo/config"), server.CheckConfig) - mux.HandleFunc(pat.Get("/config"), server.GetConfig) - mux.HandleFunc(pat.Get("/:repo/config"), server.GetConfig) - mux.HandleFunc(pat.Post("/config"), server.SaveConfig) - mux.HandleFunc(pat.Post("/:repo/config"), server.SaveConfig) - mux.HandleFunc(pat.Delete("/config"), server.DeleteConfig) - mux.HandleFunc(pat.Delete("/:repo/config"), server.DeleteConfig) - mux.HandleFunc(pat.Get("/:type/"), server.ListBlobs) - mux.HandleFunc(pat.Get("/:repo/:type/"), server.ListBlobs) - mux.HandleFunc(pat.Head("/:type/:name"), server.CheckBlob) - mux.HandleFunc(pat.Head("/:repo/:type/:name"), server.CheckBlob) - mux.HandleFunc(pat.Get("/:type/:name"), server.GetBlob) - mux.HandleFunc(pat.Get("/:repo/:type/:name"), server.GetBlob) - mux.HandleFunc(pat.Post("/:type/:name"), server.SaveBlob) - mux.HandleFunc(pat.Post("/:repo/:type/:name"), server.SaveBlob) - mux.HandleFunc(pat.Delete("/:type/:name"), server.DeleteBlob) - mux.HandleFunc(pat.Delete("/:repo/:type/:name"), server.DeleteBlob) - mux.HandleFunc(pat.Post("/"), server.CreateRepo) - mux.HandleFunc(pat.Post("/:repo"), server.CreateRepo) - mux.HandleFunc(pat.Post("/:repo/"), server.CreateRepo) - - return mux + return username, true +} + +// NewHandler returns the master HTTP multiplexer/router. +func NewHandler(server *Server) (http.Handler, error) { + if !server.NoAuth { + var err error + server.htpasswdFile, err = NewHtpasswdFromFile(filepath.Join(server.Path, ".htpasswd")) + if err != nil { + return nil, fmt.Errorf("cannot load .htpasswd (use --no-auth to disable): %v", err) + } + } + + if server.MaxRepoSize > 0 { + log.Printf("Initializing quota (can take a while)...") + qm, err := quota.New(server.Path, server.MaxRepoSize) + if err != nil { + return nil, err + } + server.quotaManager = qm + log.Printf("Quota initialized, currenly using %.2f GiB", float64(qm.SpaceUsed()/1024/1024)) + } + + mux := http.NewServeMux() + if server.Prometheus { + // FIXME: need auth like in previous version? + mux.Handle("/metrics", promhttp.Handler()) + } + mux.Handle("/", server) + + var handler http.Handler = mux + if server.Debug { + handler = server.debugHandler(handler) + } + if server.Log != "" { + handler = server.logHandler(handler) + } + return handler, nil } diff --git a/quota/quota.go b/quota/quota.go index e037934..c36244e 100644 --- a/quota/quota.go +++ b/quota/quota.go @@ -30,7 +30,7 @@ type Manager struct { repoSize int64 // must be accessed using sync/atomic } -// maxSizeWriter limits the number of bytes written +// WrapWriter limits the number of bytes written // to the space that is currently available as given by // the server's MaxRepoSize. This type is safe for use // by multiple goroutines sharing the same *Server. @@ -40,11 +40,11 @@ type maxSizeWriter struct { } func (w maxSizeWriter) Write(p []byte) (n int, err error) { - if int64(len(p)) > w.m.repoSpaceRemaining() { + if int64(len(p)) > w.m.SpaceRemaining() { return 0, fmt.Errorf("repository has reached maximum size (%d bytes)", w.m.maxRepoSize) } n, err = w.Writer.Write(p) - w.m.incrementRepoSpaceUsage(int64(n)) + w.m.IncUsage(int64(n)) return n, err } @@ -58,9 +58,9 @@ func (m *Manager) updateSize() error { return nil } -// maxSizeWriter wraps w in a writer that enforces s.MaxRepoSize. +// WrapWriter wraps w in a writer that enforces s.MaxRepoSize. // If there is an error, a status code and the error are returned. -func (m *Manager) maxSizeWriter(req *http.Request, w io.Writer) (io.Writer, int, error) { +func (m *Manager) WrapWriter(req *http.Request, w io.Writer) (io.Writer, int, error) { currentSize := atomic.LoadInt64(&m.repoSize) // if content-length is set and is trustworthy, we can save some time @@ -84,10 +84,10 @@ func (m *Manager) maxSizeWriter(req *http.Request, w io.Writer) (io.Writer, int, return maxSizeWriter{Writer: w, m: m}, 0, nil } -// repoSpaceRemaining returns how much space is available in the repo +// SpaceRemaining returns how much space is available in the repo // according to s.MaxRepoSize. s.repoSize must already be set. // If there is no limit, -1 is returned. -func (m *Manager) repoSpaceRemaining() int64 { +func (m *Manager) SpaceRemaining() int64 { if m.maxRepoSize == 0 { return -1 } @@ -96,9 +96,14 @@ func (m *Manager) repoSpaceRemaining() int64 { return maxSize - currentSize } -// incrementRepoSpaceUsage increments the current repo size (which +// SpaceUsed returns how much space is used in the repo. +func (m *Manager) SpaceUsed() int64 { + return atomic.LoadInt64(&m.repoSize) +} + +// IncUsage increments the current repo size (which // must already be initialized). -func (m *Manager) incrementRepoSpaceUsage(by int64) { +func (m *Manager) IncUsage(by int64) { atomic.AddInt64(&m.repoSize, by) } diff --git a/repo/repo.go b/repo/repo.go index 5791633..e2591c4 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -14,6 +14,7 @@ import ( "time" "github.com/miolini/datacounter" + "github.com/restic/rest-server/quota" ) // Options are options for the Handler accepted by New @@ -24,9 +25,7 @@ type Options struct { FileMode os.FileMode BlobMetricFunc BlobMetricFunc - - // FIXME: This information is not persistent in the new setup - MaxRepoSize int64 + QuotaManager *quota.Manager } // DefaultDirMode is the file mode used for directory creation if not @@ -58,6 +57,7 @@ func New(path string, opt Options) (*Handler, error) { } // Handler handles all REST API requests for a single Restic backup repo +// Spec: https://restic.readthedocs.io/en/latest/100_references.html#rest-backend type Handler struct { path string // filesystem path of repo opt Options @@ -103,6 +103,7 @@ const ( // objectType: one of ObjectTypes // operation: one of the BlobOperations above // nBytes: the number of bytes affected, or 0 if not relevant +// TODO: Perhaps add http.Request for the username so that this can be cached? type BlobMetricFunc func(objectType string, operation BlobOperation, nBytes uint64) // ServeHTTP performs strict matching on the repo part of the URL path and @@ -208,13 +209,13 @@ func (h *Handler) sendMetric(objectType string, operation BlobOperation, nBytes // needSize tells you if we need the file size for metrics of quota accounting func (h *Handler) needSize() bool { - return h.opt.BlobMetricFunc != nil || h.opt.MaxRepoSize > 0 + return h.opt.BlobMetricFunc != nil || h.opt.QuotaManager != nil } // incrementRepoSpaceUsage increments the repo space usage if quota are enabled func (h *Handler) incrementRepoSpaceUsage(by int64) { - if h.opt.MaxRepoSize > 0 { - // FIXME: call the actual incrementRepoSpaceUsage + if h.opt.QuotaManager != nil { + h.opt.QuotaManager.IncUsage(by) } } @@ -222,15 +223,10 @@ func (h *Handler) incrementRepoSpaceUsage(by int64) { // as is if not. // If an error occurs, it returns both an error and the appropriate HTTP error code. func (h *Handler) wrapFileWriter(r *http.Request, w io.Writer) (io.Writer, int, error) { - var errCode int - if h.opt.MaxRepoSize > 0 { - // FIXME: optionally wrap with maxSizeWriter - // FIXME: return h.maxSizeWriter(r, tf) - // if err != nil && h.opt.Debug { - // log.Printf("wrapFileWriter: %v", err) - //} + if h.opt.QuotaManager == nil { + return w, 0, nil // unmodified } - return w, 0, nil + return h.opt.QuotaManager.WrapWriter(r, w) } // checkConfig checks whether a configuration exists.