Add Prometheus metrics

Exposes a few metrics for Prometheus under /metrics if started with --prometheus.

Example:

    # HELP rest_server_blob_read_bytes_total Total number of bytes read from blobs
    # TYPE rest_server_blob_read_bytes_total counter
    rest_server_blob_read_bytes_total{repo="test",type="data"} 2.13557024e+09
    rest_server_blob_read_bytes_total{repo="test",type="index"} 1.198653e+06
    rest_server_blob_read_bytes_total{repo="test",type="keys"} 5388
    rest_server_blob_read_bytes_total{repo="test",type="locks"} 1975
    rest_server_blob_read_bytes_total{repo="test",type="snapshots"} 10018
    # HELP rest_server_blob_read_total Total number of blobs read
    # TYPE rest_server_blob_read_total counter
    rest_server_blob_read_total{repo="test",type="data"} 3985
    rest_server_blob_read_total{repo="test",type="index"} 21
    rest_server_blob_read_total{repo="test",type="keys"} 12
    rest_server_blob_read_total{repo="test",type="locks"} 12
    rest_server_blob_read_total{repo="test",type="snapshots"} 32
    # HELP rest_server_blob_write_bytes_total Total number of bytes written to blobs
    # TYPE rest_server_blob_write_bytes_total counter
    rest_server_blob_write_bytes_total{repo="test",type="data"} 1.063726179e+09
    rest_server_blob_write_bytes_total{repo="test",type="index"} 395586
    rest_server_blob_write_bytes_total{repo="test",type="locks"} 1975
    rest_server_blob_write_bytes_total{repo="test",type="snapshots"} 1933
    # HELP rest_server_blob_write_total Total number of blobs written
    # TYPE rest_server_blob_write_total counter
    rest_server_blob_write_total{repo="test",type="data"} 226
    rest_server_blob_write_total{repo="test",type="index"} 6
    rest_server_blob_write_total{repo="test",type="locks"} 12
    rest_server_blob_write_total{repo="test",type="snapshots"} 6

cmd/rest-server/main.go

@@ -30,6 +30,7 @@ func init() {
 	flags.StringVar(&restserver.Config.Path, "path", restserver.Config.Path, "data directory")
 	flags.BoolVar(&restserver.Config.TLS, "tls", restserver.Config.TLS, "turn on TLS support")
 	flags.BoolVar(&restserver.Config.AppendOnly, "append-only", restserver.Config.AppendOnly, "enable append only mode")
+	flags.BoolVar(&restserver.Config.Prometheus, "prometheus", restserver.Config.AppendOnly, "enable Prometheus metrics")
 }
 
 var version = "manually"

handlers.go

@@ -13,9 +13,10 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
-
 	"goji.io/middleware"
 	"goji.io/pat"
+	"github.com/miolini/datacounter"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 func isHashed(dir string) bool {
@@ -317,8 +318,15 @@ func GetBlob(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	http.ServeContent(w, r, "", time.Unix(0, 0), file)
+	wc := datacounter.NewResponseWriterCounter(w)
+	http.ServeContent(wc, r, "", time.Unix(0, 0), file)
 	file.Close()
+
+	if Config.Prometheus {
+		labels := prometheus.Labels{"repo": getRepo(r), "type": pat.Param(r, "type")}
+		metricBlobReadTotal.With(labels).Inc()
+		metricBlobReadBytesTotal.With(labels).Add(float64(wc.Count()))
+	}
 }
 
 // SaveBlob saves a blob to the repository.
@@ -342,7 +350,8 @@ func SaveBlob(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if _, err := io.Copy(tf, r.Body); err != nil {
+	written, err := io.Copy(tf, r.Body)
+	if err != nil {
 		tf.Close()
 		os.Remove(path)
 		if Config.Debug {
@@ -370,6 +379,12 @@ func SaveBlob(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 		return
 	}
+
+	if Config.Prometheus {
+		labels := prometheus.Labels{"repo": getRepo(r), "type": pat.Param(r, "type")}
+		metricBlobWriteTotal.With(labels).Inc()
+		metricBlobWriteBytesTotal.With(labels).Add(float64(written))
+	}
 }
 
 // DeleteBlob deletes a blob from the repository.

metrics.go

@@ -0,0 +1,44 @@
+package restserver
+
+import "github.com/prometheus/client_golang/prometheus"
+
+
+var metricBlobWriteTotal = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "rest_server_blob_write_total",
+		Help: "Total number of blobs written",
+	},
+	[]string{"repo", "type"},
+)
+
+var metricBlobWriteBytesTotal = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "rest_server_blob_write_bytes_total",
+		Help: "Total number of bytes written to blobs",
+	},
+	[]string{"repo", "type"},
+)
+
+var metricBlobReadTotal = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "rest_server_blob_read_total",
+		Help: "Total number of blobs read",
+	},
+	[]string{"repo", "type"},
+)
+
+var metricBlobReadBytesTotal = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "rest_server_blob_read_bytes_total",
+		Help: "Total number of bytes read from blobs",
+	},
+	[]string{"repo", "type"},
+)
+
+func init() {
+	// These are always initialized, but only updated if Config.Prometheus is set
+	prometheus.MustRegister(metricBlobWriteTotal)
+	prometheus.MustRegister(metricBlobWriteBytesTotal)
+	prometheus.MustRegister(metricBlobReadTotal)
+	prometheus.MustRegister(metricBlobReadBytesTotal)
+}

mux.go

@@ -8,6 +8,7 @@ import (
 	goji "goji.io"
 
 	"github.com/gorilla/handlers"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"goji.io/pat"
 )
 
@@ -19,6 +20,7 @@ var Config = struct {
 	CPUProfile string
 	Debug      bool
 	AppendOnly bool
+	Prometheus bool
 }{
 	Path:       "/tmp/restic",
 	Listen:     ":8000",
@@ -53,6 +55,10 @@ func NewMux() *goji.Mux {
 		mux.Use(logHandler)
 	}
 
+	if Config.Prometheus {
+		mux.Handle(pat.Get("/metrics"), promhttp.Handler())
+	}
+
 	mux.HandleFunc(pat.Head("/config"), CheckConfig)
 	mux.HandleFunc(pat.Head("/:repo/config"), CheckConfig)
 	mux.HandleFunc(pat.Get("/config"), GetConfig)