Stowage/rest-server
2
0
Fork 0
mirror of https://github.com/restic/rest-server.git synced 2025-10-19 15:43:21 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge 36c7652c9b into d39bc8e6cf

Browse source
This commit is contained in:
wojas 2021-08-09 09:15:25 +00:00 • committed by GitHub
commit e726225e80
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 257 additions and 139 deletions
Download patch file Download diff file Expand all files Collapse all files

132
cmd/rest-server/main.go
View file

@ -1,16 +1,18 @@
package main
import (
"errors"
"context"
"fmt"
"log"
"net/http"
"os"
"path/filepath"
"runtime"
"runtime/pprof"
"github.com/PowerDNS/go-tlsconfig"
"github.com/c2h5oh/datasize"
restserver "github.com/restic/rest-server"
"github.com/restic/rest-server/config"
"github.com/spf13/cobra"
)
@ -25,57 +27,37 @@ var cmdRoot = &cobra.Command{
//Version: fmt.Sprintf("rest-server %s compiled with %v on %v/%v\n", version, runtime.Version(), runtime.GOOS, runtime.GOARCH),
}
var server = restserver.Server{
Path: "/tmp/restic",
Listen: ":8000",
}
var (
showVersion bool
cpuProfile string
maxSizeBytes uint64
tlsEnabled bool
configFile string
flagConfig = config.Config{}
)
func init() {
flags := cmdRoot.Flags()
flags.StringVarP(&configFile, "config", "c", configFile, "path to YAML config file")
flags.StringVar(&cpuProfile, "cpu-profile", cpuProfile, "write CPU profile to file")
flags.BoolVar(&server.Debug, "debug", server.Debug, "output debug messages")
flags.StringVar(&server.Listen, "listen", server.Listen, "listen address")
flags.StringVar(&server.Log, "log", server.Log, "log HTTP requests in the combined log format")
flags.Int64Var(&server.MaxRepoSize, "max-size", server.MaxRepoSize, "the maximum size of the repository in bytes")
flags.StringVar(&server.Path, "path", server.Path, "data directory")
flags.BoolVar(&server.TLS, "tls", server.TLS, "turn on TLS support")
flags.StringVar(&server.TLSCert, "tls-cert", server.TLSCert, "TLS certificate path")
flags.StringVar(&server.TLSKey, "tls-key", server.TLSKey, "TLS key path")
flags.BoolVar(&server.NoAuth, "no-auth", server.NoAuth, "disable .htpasswd authentication")
flags.BoolVar(&server.AppendOnly, "append-only", server.AppendOnly, "enable append only mode")
flags.BoolVar(&server.PrivateRepos, "private-repos", server.PrivateRepos, "users can only access their private repo")
flags.BoolVar(&server.Prometheus, "prometheus", server.Prometheus, "enable Prometheus metrics")
flags.BoolVar(&server.Prometheus, "prometheus-no-auth", server.PrometheusNoAuth, "disable auth for Prometheus /metrics endpoint")
flags.BoolVar(&flagConfig.Debug, "debug", flagConfig.Debug, "output debug messages")
flags.StringVar(&flagConfig.Listen, "listen", flagConfig.Listen, "listen address")
flags.StringVar(&flagConfig.AccessLog, "log", flagConfig.AccessLog, "log HTTP requests in the combined log format")
flags.Uint64Var(&maxSizeBytes, "max-size", uint64(flagConfig.Quota.MaxSize), "the maximum size of the repository in bytes")
flags.StringVar(&flagConfig.Path, "path", flagConfig.Path, "data directory")
flags.BoolVar(&tlsEnabled, "tls", flagConfig.TLS.HasCertWithKey(), "turn on TLS support")
flags.StringVar(&flagConfig.TLS.CertFile, "tls-cert", flagConfig.TLS.CertFile, "TLS certificate path")
flags.StringVar(&flagConfig.TLS.KeyFile, "tls-key", flagConfig.TLS.KeyFile, "TLS key path")
flags.BoolVar(&flagConfig.Auth.Disabled, "no-auth", flagConfig.Auth.Disabled, "disable .htpasswd authentication")
flags.BoolVar(&flagConfig.AppendOnly, "append-only", flagConfig.AppendOnly, "enable append only mode")
flags.BoolVar(&flagConfig.PrivateRepos, "private-repos", flagConfig.PrivateRepos, "users can only access their private repo")
flags.BoolVar(&flagConfig.Metrics.Enabled, "prometheus", flagConfig.Metrics.Enabled, "enable Prometheus metrics")
flags.BoolVar(&flagConfig.Metrics.NoAuth, "prometheus-no-auth", flagConfig.Metrics.NoAuth, "disable auth for Prometheus /metrics endpoint")
flags.BoolVarP(&showVersion, "version", "V", showVersion, "output version and exit")
}
var version = "0.10.0-dev"
func tlsSettings() (bool, string, string, error) {
var key, cert string
if !server.TLS && (server.TLSKey != "" || server.TLSCert != "") {
return false, "", "", errors.New("requires enabled TLS")
} else if !server.TLS {
return false, "", "", nil
}
if server.TLSKey != "" {
key = server.TLSKey
} else {
key = filepath.Join(server.Path, "private_key")
}
if server.TLSCert != "" {
cert = server.TLSCert
} else {
cert = filepath.Join(server.Path, "public_key")
}
return server.TLS, key, cert, nil
}
func runRoot(cmd *cobra.Command, args []string) error {
if showVersion {
fmt.Printf("rest-server %s compiled with %v on %v/%v\n", version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
@ -84,7 +66,26 @@ func runRoot(cmd *cobra.Command, args []string) error {
log.SetFlags(0)
log.Printf("Data directory: %s", server.Path)
// Load config
conf := config.Default()
if configFile != "" {
if err := conf.LoadYAMLFile(configFile); err != nil {
return err
}
}
// Merge flag config
conf.Quota.MaxSize = datasize.ByteSize(maxSizeBytes)
conf.MergeFlags(flagConfig)
if conf.Debug {
log.Printf("Effective config:\n%s", conf.String())
}
if err := conf.Check(); err != nil {
return err
}
if tlsEnabled && !conf.TLS.HasCertWithKey() {
return fmt.Errorf("--tls set, but key and cert not configured")
}
if cpuProfile != "" {
f, err := os.Create(cpuProfile)
@ -98,40 +99,51 @@ func runRoot(cmd *cobra.Command, args []string) error {
defer pprof.StopCPUProfile()
}
if server.NoAuth {
log.Printf("Data directory: %s", conf.Path)
if conf.Auth.Disabled {
log.Println("Authentication disabled")
} else {
log.Println("Authentication enabled")
}
handler, err := restserver.NewHandler(&server)
if err != nil {
log.Fatalf("error: %v", err)
}
if server.PrivateRepos {
if conf.PrivateRepos {
log.Println("Private repositories enabled")
} else {
log.Println("Private repositories disabled")
}
enabledTLS, privateKey, publicKey, err := tlsSettings()
server, err := restserver.NewServer(*conf)
if err != nil {
return err
}
if !enabledTLS {
log.Printf("Starting server on %s\n", server.Listen)
err = http.ListenAndServe(server.Listen, handler)
} else {
log.Println("TLS enabled")
log.Printf("Private key: %s", privateKey)
log.Printf("Public key(certificate): %s", publicKey)
log.Printf("Starting server on %s\n", server.Listen)
err = http.ListenAndServeTLS(server.Listen, publicKey, privateKey, handler)
handler, err := restserver.NewHandler(server)
if err != nil {
return err
}
ctx := context.Background()
if !conf.TLS.HasCertWithKey() {
log.Printf("Starting server on %s\n", conf.Listen)
return http.ListenAndServe(conf.Listen, handler)
} else {
log.Println("TLS enabled")
log.Printf("Starting server on %s\n", conf.Listen)
manager, err := tlsconfig.NewManager(ctx, conf.TLS, tlsconfig.Options{
IsServer: true,
})
if err != nil {
return err
}
tlsConfig, err := manager.TLSConfig()
if err != nil {
return err
}
hs := http.Server{
Addr: conf.Listen,
Handler: handler,
TLSConfig: tlsConfig,
}
return hs.ListenAndServeTLS("", "") // Certificates are handled by TLSConfig
}
}
func main() {

65
cmd/rest-server/main_test.go
View file

@ -9,71 +9,6 @@ import (
restserver "github.com/restic/rest-server"
)
func TestTLSSettings(t *testing.T) {
type expected struct {
TLSKey string
TLSCert string
Error bool
}
type passed struct {
Path string
TLS bool
TLSKey string
TLSCert string
}
var tests = []struct {
passed passed
expected expected
}{
{passed{TLS: false}, expected{"", "", false}},
{passed{TLS: true}, expected{"/tmp/restic/private_key", "/tmp/restic/public_key", false}},
{passed{Path: "/tmp", TLS: true}, expected{"/tmp/private_key", "/tmp/public_key", false}},
{passed{Path: "/tmp", TLS: true, TLSKey: "/etc/restic/key", TLSCert: "/etc/restic/cert"}, expected{"/etc/restic/key", "/etc/restic/cert", false}},
{passed{Path: "/tmp", TLS: false, TLSKey: "/etc/restic/key", TLSCert: "/etc/restic/cert"}, expected{"", "", true}},
{passed{Path: "/tmp", TLS: false, TLSKey: "/etc/restic/key"}, expected{"", "", true}},
{passed{Path: "/tmp", TLS: false, TLSCert: "/etc/restic/cert"}, expected{"", "", true}},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
// defer func() { restserver.Server = defaultConfig }()
if test.passed.Path != "" {
server.Path = test.passed.Path
}
server.TLS = test.passed.TLS
server.TLSKey = test.passed.TLSKey
server.TLSCert = test.passed.TLSCert
gotTLS, gotKey, gotCert, err := tlsSettings()
if err != nil && !test.expected.Error {
t.Fatalf("tls_settings returned err (%v)", err)
}
if test.expected.Error {
if err == nil {
t.Fatalf("Error not returned properly (%v)", test)
} else {
return
}
}
if gotTLS != test.passed.TLS {
t.Errorf("TLS enabled, want (%v), got (%v)", test.passed.TLS, gotTLS)
}
wantKey := test.expected.TLSKey
if gotKey != wantKey {
t.Errorf("wrong TLSPrivPath path, want (%v), got (%v)", wantKey, gotKey)
}
wantCert := test.expected.TLSCert
if gotCert != wantCert {
t.Errorf("wrong TLSCertPath path, want (%v), got (%v)", wantCert, gotCert)
}
})
}
}
func TestGetHandler(t *testing.T) {
dir, err := ioutil.TempDir("", "rest-server-test")
if err != nil {

121
config/config.go Normal file
View file

@ -0,0 +1,121 @@
// Package config contains the configuration structures for rest-server
package config
import (
"fmt"
"io/ioutil"
"log"
"github.com/PowerDNS/go-tlsconfig"
"github.com/c2h5oh/datasize"
"gopkg.in/yaml.v2"
)
// Config is the config root object
type Config struct {
Path string `yaml:"path"`
AppendOnly bool `yaml:"append_only"`
PrivateRepos bool `yaml:"private_repos"`
Listen string `yaml:"listen"` // Address like ":8000"
TLS tlsconfig.Config `yaml:"tls"`
AccessLog string `yaml:"access_log"`
Debug bool `yaml:"debug"`
Quota Quota `yaml:"quota"`
Metrics Metrics `yaml:"metrics"`
Auth Auth `yaml:"auth"`
Users map[string]User `yaml:"users"`
}
// Quota configures disk usage quota enforcements
type Quota struct {
Scope string `yaml:"scope,omitempty"`
MaxSize datasize.ByteSize `yaml:"max_size"`
}
// Metrics configures Prometheus metrics
type Metrics struct {
Enabled bool `yaml:"enabled"`
NoAuth bool `yaml:"no_auth"`
}
// Auth configures authentication
type Auth struct {
Disabled bool `yaml:"disabled"`
Backend string `yaml:"backend,omitempty"`
HTPasswdFile string `yaml:"htpasswd_file"`
}
// User configures user overrides
type User struct {
AppendOnly *bool `yaml:"append_only,omitempty"`
PrivateRepos *bool `yaml:"private_repos,omitempty"`
}
// Check validates a Config instance
func (c Config) Check() error {
return nil
}
// String returns the config as a YAML string
func (c Config) String() string {
y, err := yaml.Marshal(c)
if err != nil {
log.Panicf("YAML marshal of config failed: %v", err) // Should never happen
}
return string(y)
}
// LoadYAML loads config from YAML. Any set value overwrites any existing value,
// but omitted keys are untouched.
func (c *Config) LoadYAML(yamlContents []byte) error {
return yaml.UnmarshalStrict(yamlContents, c)
}
// LoadYAML loads config from a YAML file. Any set value overwrites any existing value,
// but omitted keys are untouched.
func (c *Config) LoadYAMLFile(fpath string) error {
contents, err := ioutil.ReadFile(fpath)
if err != nil {
return fmt.Errorf("open yaml file: %w", err)
}
return c.LoadYAML(contents)
}
func mergeString(a, b string) string {
if b != "" {
return b
}
return a
}
// MergeFlags merges configuration set by commandline flags into the current Config
func (c *Config) MergeFlags(fc Config) {
c.Debug = c.Debug || fc.Debug
c.Listen = mergeString(c.Listen, fc.Listen)
c.AccessLog = mergeString(c.AccessLog, fc.AccessLog)
if fc.Quota.MaxSize > 0 {
c.Quota.MaxSize = fc.Quota.MaxSize
}
c.Path = mergeString(c.Path, fc.Path)
c.TLS.CertFile = mergeString(c.TLS.CertFile, fc.TLS.CertFile)
c.TLS.KeyFile = mergeString(c.TLS.KeyFile, fc.TLS.KeyFile)
c.Auth.Disabled = c.Auth.Disabled || fc.Auth.Disabled
c.AppendOnly = c.AppendOnly || fc.AppendOnly
c.PrivateRepos = c.PrivateRepos || fc.PrivateRepos
c.Metrics.Enabled = c.Metrics.Enabled || fc.Metrics.Enabled
c.Metrics.NoAuth = c.Metrics.NoAuth || fc.Metrics.NoAuth
}
// Default returns a Config with default settings
func Default() *Config {
return &Config{
Path: "/tmp/restic",
Listen: ":8000",
Users: make(map[string]User),
Auth: Auth{
Disabled: false,
Backend: "htpasswd",
HTPasswdFile: ".htpasswd",
},
}
}

4
go.mod
View file

@ -3,7 +3,9 @@ module github.com/restic/rest-server
go 1.14
require (
github.com/PowerDNS/go-tlsconfig v0.0.0-20201014142732-fe6ff56e2a95
github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a // indirect
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
github.com/golang/protobuf v1.0.0 // indirect
github.com/gorilla/handlers v1.3.0
github.com/inconshreveable/mousetrap v1.0.0 // indirect
@ -16,7 +18,7 @@ require (
github.com/spf13/cobra v0.0.1
github.com/spf13/pflag v1.0.0 // indirect
golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a // indirect
gopkg.in/yaml.v2 v2.4.0
)
replace goji.io v2.0.0+incompatible => github.com/goji/goji v2.0.0+incompatible

11
go.sum
View file

@ -1,5 +1,11 @@
github.com/PowerDNS/go-tlsconfig v0.0.0-20201014142732-fe6ff56e2a95 h1:jWxEVXkF1InUh1o5aCq4cc+ZjKKSwYsGV3yNK5Rpp6A=
github.com/PowerDNS/go-tlsconfig v0.0.0-20201014142732-fe6ff56e2a95/go.mod h1:Q+i/He4WS46khYyqBUWBASsayUrenws7sOh964AK7TY=
github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a h1:BtpsbiV638WQZwhA98cEZw2BsbnQJrbd0BI7tsy0W1c=
github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak=
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
github.com/go-logr/logr v0.2.1 h1:fV3MLmabKIZ383XifUjFSwcoGee0v9qgPp8wy5svibE=
github.com/go-logr/logr v0.2.1/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/golang/protobuf v1.0.0 h1:lsek0oXi8iFE9L+EXARyHIjU5rlWIhhTkjDz3vHhWWQ=
github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI=
@ -10,6 +16,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.0 h1:YNOwxxSJzSUARoD9KRZLz
github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miolini/datacounter v0.0.0-20171104152933-fd4e42a1d5e0 h1:clkDYGefEWUCwyCrwYn900sOaVGDpinPJgD0W6ebEjs=
github.com/miolini/datacounter v0.0.0-20171104152933-fd4e42a1d5e0/go.mod h1:P6fDJzlxN+cWYR09KbE9/ta+Y6JofX9tAUhJpWkWPaM=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/prometheus/client_golang v0.8.0 h1:1921Yw9Gc3iSc4VQh3PIoOqgPCZS7G/4xQNVUp8Mda8=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 h1:cLL6NowurKLMfCeQy4tIeph12XNQWgANCNvdyrOYKV4=
@ -26,3 +33,7 @@ golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4 h1:OfaUle5HH9Y0obNU74mlOZ
golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

44
handlers.go
View file

@ -8,19 +8,35 @@ import (
"path/filepath"
"strings"
"github.com/restic/rest-server/config"
"github.com/restic/rest-server/quota"
"github.com/restic/rest-server/repo"
)
// NewServer creates a new Server from a config.Config
func NewServer(c config.Config) (*Server, error) {
s := &Server{
Path: c.Path,
Log: c.AccessLog,
NoAuth: c.Auth.Disabled,
AppendOnly: c.AppendOnly,
PrivateRepos: c.PrivateRepos,
Prometheus: c.Metrics.Enabled,
PrometheusNoAuth: c.Metrics.NoAuth,
Debug: c.Debug,
MaxRepoSize: int64(c.Quota.MaxSize),
Config: c,
}
return s, nil
}
// Server encapsulates the rest-server's settings and repo management logic
type Server struct {
// Old attributes
// TODO: Remove these before 1.0 and directly use Config instead
Path string
Listen string
Log string
CPUProfile string
TLSKey string
TLSCert string
TLS bool
NoAuth bool
AppendOnly bool
PrivateRepos bool
@ -28,7 +44,9 @@ type Server struct {
PrometheusNoAuth bool
Debug bool
MaxRepoSize int64
PanicOnError bool
Config config.Config
htpasswdFile *HtpasswdFile
quotaManager *quota.Manager
@ -65,8 +83,20 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
// Allow per-user overrides
appendOnly := s.AppendOnly
privateRepos := s.PrivateRepos
if uc, ok := s.Config.Users[username]; ok {
if uc.AppendOnly != nil {
appendOnly = *uc.AppendOnly
}
if uc.PrivateRepos != nil {
privateRepos = *uc.PrivateRepos
}
}
// Check if the current user is allowed to access this path
if !s.NoAuth && s.PrivateRepos {
if !s.NoAuth && privateRepos {
if len(folderPath) == 0 || folderPath[0] != username {
httpDefaultError(w, http.StatusUnauthorized)
return
@ -84,7 +114,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Pass the request to the repo.Handler
opt := repo.Options{
AppendOnly: s.AppendOnly,
AppendOnly: appendOnly,
Debug: s.Debug,
QuotaManager: s.quotaManager, // may be nil
PanicOnError: s.PanicOnError,

11
mux.go
View file

@ -60,9 +60,16 @@ func (s *Server) wrapMetricsAuth(f http.HandlerFunc) http.HandlerFunc {
func NewHandler(server *Server) (http.Handler, error) {
if !server.NoAuth {
var err error
server.htpasswdFile, err = NewHtpasswdFromFile(filepath.Join(server.Path, ".htpasswd"))
htpasswd := server.Config.Auth.HTPasswdFile
if htpasswd == "" {
htpasswd = ".htpasswd"
}
if !filepath.IsAbs(htpasswd) {
htpasswd = filepath.Join(server.Path, htpasswd)
}
server.htpasswdFile, err = NewHtpasswdFromFile(htpasswd)
if err != nil {
return nil, fmt.Errorf("cannot load .htpasswd (use --no-auth to disable): %v", err)
return nil, fmt.Errorf("cannot load htpasswd file (use --no-auth to disable): %s: %v", htpasswd, err)
}
}