package restserver import ( "encoding/json" "errors" "fmt" "io" "io/ioutil" "log" "net/http" "os" "path" "path/filepath" "strings" "time" "github.com/miolini/datacounter" "github.com/prometheus/client_golang/prometheus" "goji.io/middleware" "goji.io/pat" ) // Server determines how a Mux's handlers behave. type Server struct { Path string Listen string Log string CPUProfile string TLSKey string TLSCert string TLS bool NoAuth bool AppendOnly bool PrivateRepos bool Prometheus bool Debug bool MaxRepoSize int64 repoSize int64 // must be accessed using sync/atomic } func (s *Server) isHashed(dir string) bool { return dir == "data" } func valid(name string) bool { // Based on net/http.Dir if strings.Contains(name, "\x00") { return false } // Path characters that are disallowed or unsafe under some operating systems // are not allowed here. // The most important one here is '/', since Goji does not decode '%2F' to '/' // during routing, so we can end up with a '/' in the name here. if strings.ContainsAny(name, "/\\:*?\"<>|") { return false } if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) { return false } return true } var validTypes = []string{"data", "index", "keys", "locks", "snapshots", "config"} func (s *Server) isValidType(name string) bool { for _, tpe := range validTypes { if name == tpe { return true } } return false } // join takes a number of path names, sanitizes them, and returns them joined // with base for the current operating system to use (dirs separated by // filepath.Separator). The returned path is always either equal to base or a // subdir of base. func join(base string, names ...string) (string, error) { clean := make([]string, 0, len(names)+1) clean = append(clean, base) // taken from net/http.Dir for _, name := range names { if !valid(name) { return "", errors.New("invalid character in path") } clean = append(clean, filepath.FromSlash(path.Clean("/"+name))) } return filepath.Join(clean...), nil } // getRepo returns the repository location, relative to s.Path. func (s *Server) getRepo(r *http.Request) string { if strings.HasPrefix(fmt.Sprintf("%s", middleware.Pattern(r.Context())), "/:repo") { return pat.Param(r, "repo") } return "." } // getPath returns the path for a file type in the repo. func (s *Server) getPath(r *http.Request, fileType string) (string, error) { if !s.isValidType(fileType) { return "", errors.New("invalid file type") } return join(s.Path, s.getRepo(r), fileType) } // getFilePath returns the path for a file in the repo. func (s *Server) getFilePath(r *http.Request, fileType, name string) (string, error) { if !s.isValidType(fileType) { return "", errors.New("invalid file type") } if s.isHashed(fileType) { if len(name) < 2 { return "", errors.New("file name is too short") } return join(s.Path, s.getRepo(r), fileType, name[:2], name) } return join(s.Path, s.getRepo(r), fileType, name) } // getUser returns the username from the request, or an empty string if none. func (s *Server) getUser(r *http.Request) string { username, _, ok := r.BasicAuth() if !ok { return "" } return username } // getMetricLabels returns the prometheus labels from the request. func (s *Server) getMetricLabels(r *http.Request) prometheus.Labels { labels := prometheus.Labels{ "user": s.getUser(r), "repo": s.getRepo(r), "type": pat.Param(r, "type"), } return labels } // isUserPath checks if a request path is accessible by the user when using // private repositories. func isUserPath(username, path string) bool { prefix := "/" + username if !strings.HasPrefix(path, prefix) { return false } return len(path) == len(prefix) || path[len(prefix)] == '/' } // AuthHandler wraps h with a http.HandlerFunc that performs basic authentication against the user/passwords pairs // stored in f and returns the http.HandlerFunc. func (s *Server) AuthHandler(f *HtpasswdFile, h http.Handler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { username, password, ok := r.BasicAuth() if !ok || !f.Validate(username, password) { http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) return } if s.PrivateRepos && !isUserPath(username, r.URL.Path) && r.URL.Path != "/metrics" { http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) return } h.ServeHTTP(w, r) } } // CheckConfig checks whether a configuration exists. func (s *Server) CheckConfig(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("CheckConfig()") } cfg, err := s.getPath(r, "config") if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } st, err := os.Stat(cfg) if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return } w.Header().Add("Content-Length", fmt.Sprint(st.Size())) } // GetConfig allows for a config to be retrieved. func (s *Server) GetConfig(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("GetConfig()") } cfg, err := s.getPath(r, "config") if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } bytes, err := ioutil.ReadFile(cfg) if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return } _, _ = w.Write(bytes) } // SaveConfig allows for a config to be saved. func (s *Server) SaveConfig(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("SaveConfig()") } cfg, err := s.getPath(r, "config") if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } f, err := os.OpenFile(cfg, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600) if err != nil && os.IsExist(err) { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) return } _, err = io.Copy(f, r.Body) if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } err = f.Close() if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } _ = r.Body.Close() } // DeleteConfig removes a config. func (s *Server) DeleteConfig(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("DeleteConfig()") } if s.AppendOnly { http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) return } cfg, err := s.getPath(r, "config") if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } if err := os.Remove(cfg); err != nil { if s.Debug { log.Print(err) } if os.IsNotExist(err) { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) } else { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) } return } } const ( mimeTypeAPIV1 = "application/vnd.x.restic.rest.v1" mimeTypeAPIV2 = "application/vnd.x.restic.rest.v2" ) // ListBlobs lists all blobs of a given type in an arbitrary order. func (s *Server) ListBlobs(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("ListBlobs()") } switch r.Header.Get("Accept") { case mimeTypeAPIV2: s.ListBlobsV2(w, r) default: s.ListBlobsV1(w, r) } } // ListBlobsV1 lists all blobs of a given type in an arbitrary order. func (s *Server) ListBlobsV1(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("ListBlobsV1()") } fileType := pat.Param(r, "type") path, err := s.getPath(r, fileType) if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } items, err := ioutil.ReadDir(path) if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return } var names []string for _, i := range items { if s.isHashed(fileType) { subpath := filepath.Join(path, i.Name()) var subitems []os.FileInfo subitems, err = ioutil.ReadDir(subpath) if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return } for _, f := range subitems { names = append(names, f.Name()) } } else { names = append(names, i.Name()) } } data, err := json.Marshal(names) if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } w.Header().Set("Content-Type", mimeTypeAPIV1) _, _ = w.Write(data) } // Blob represents a single blob, its name and its size. type Blob struct { Name string `json:"name"` Size int64 `json:"size"` } // ListBlobsV2 lists all blobs of a given type, together with their sizes, in an arbitrary order. func (s *Server) ListBlobsV2(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("ListBlobsV2()") } fileType := pat.Param(r, "type") path, err := s.getPath(r, fileType) if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } items, err := ioutil.ReadDir(path) if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return } var blobs []Blob for _, i := range items { if s.isHashed(fileType) { subpath := filepath.Join(path, i.Name()) var subitems []os.FileInfo subitems, err = ioutil.ReadDir(subpath) if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return } for _, f := range subitems { blobs = append(blobs, Blob{Name: f.Name(), Size: f.Size()}) } } else { blobs = append(blobs, Blob{Name: i.Name(), Size: i.Size()}) } } data, err := json.Marshal(blobs) if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } w.Header().Set("Content-Type", mimeTypeAPIV2) _, _ = w.Write(data) } // CheckBlob tests whether a blob exists. func (s *Server) CheckBlob(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("CheckBlob()") } path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name")) if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } st, err := os.Stat(path) if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return } w.Header().Add("Content-Length", fmt.Sprint(st.Size())) } // GetBlob retrieves a blob from the repository. func (s *Server) GetBlob(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("GetBlob()") } path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name")) if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } file, err := os.Open(path) if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return } wc := datacounter.NewResponseWriterCounter(w) http.ServeContent(wc, r, "", time.Unix(0, 0), file) if err = file.Close(); err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } if s.Prometheus { labels := s.getMetricLabels(r) metricBlobReadTotal.With(labels).Inc() metricBlobReadBytesTotal.With(labels).Add(float64(wc.Count())) } } // tallySize counts the size of the contents of path. func tallySize(path string) (int64, error) { if path == "" { path = "." } var size int64 err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { if err != nil { return err } size += info.Size() return nil }) return size, err } // SaveBlob saves a blob to the repository. func (s *Server) SaveBlob(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("SaveBlob()") } path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name")) if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } tf, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600) if os.IsNotExist(err) { // the error is caused by a missing directory, create it and retry mkdirErr := os.MkdirAll(filepath.Dir(path), 0700) if mkdirErr != nil { log.Print(mkdirErr) } else { // try again tf, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600) } } if os.IsExist(err) { http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) return } if err != nil { if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } // ensure this blob does not put us over the repo size limit (if there is one) var outFile io.Writer = tf if s.MaxRepoSize != 0 { var errCode int outFile, errCode, err = s.maxSizeWriter(r, tf) if err != nil { if s.Debug { log.Println(err) } if errCode > 0 { http.Error(w, http.StatusText(errCode), errCode) } return } } written, err := io.Copy(outFile, r.Body) if err != nil { _ = tf.Close() _ = os.Remove(path) if s.MaxRepoSize > 0 { s.incrementRepoSpaceUsage(-written) } if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) return } if err := tf.Sync(); err != nil { _ = tf.Close() _ = os.Remove(path) if s.MaxRepoSize > 0 { s.incrementRepoSpaceUsage(-written) } if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } if err := tf.Close(); err != nil { _ = os.Remove(path) if s.MaxRepoSize > 0 { s.incrementRepoSpaceUsage(-written) } if s.Debug { log.Print(err) } http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } if s.Prometheus { labels := s.getMetricLabels(r) metricBlobWriteTotal.With(labels).Inc() metricBlobWriteBytesTotal.With(labels).Add(float64(written)) } } // DeleteBlob deletes a blob from the repository. func (s *Server) DeleteBlob(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("DeleteBlob()") } if s.AppendOnly && pat.Param(r, "type") != "locks" { http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) return } path, err := s.getFilePath(r, pat.Param(r, "type"), pat.Param(r, "name")) if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } var size int64 if s.Prometheus || s.MaxRepoSize > 0 { stat, err := os.Stat(path) if err == nil { size = stat.Size() } } if err := os.Remove(path); err != nil { if s.Debug { log.Print(err) } if os.IsNotExist(err) { http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) } else { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) } return } if s.MaxRepoSize > 0 { s.incrementRepoSpaceUsage(-size) } if s.Prometheus { labels := s.getMetricLabels(r) metricBlobDeleteTotal.With(labels).Inc() metricBlobDeleteBytesTotal.With(labels).Add(float64(size)) } } // CreateRepo creates repository directories. func (s *Server) CreateRepo(w http.ResponseWriter, r *http.Request) { if s.Debug { log.Println("CreateRepo()") } repo, err := join(s.Path, s.getRepo(r)) if err != nil { http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } if r.URL.Query().Get("create") != "true" { http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) return } log.Printf("Creating repository directories in %s\n", repo) if err := os.MkdirAll(repo, 0700); err != nil { log.Print(err) http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } for _, d := range validTypes { if d == "config" { continue } if err := os.MkdirAll(filepath.Join(repo, d), 0700); err != nil { log.Print(err) http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } } for i := 0; i < 256; i++ { if err := os.MkdirAll(filepath.Join(repo, "data", fmt.Sprintf("%02x", i)), 0700); err != nil { log.Print(err) http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return } } }