Add Prometheus metrics
Exposes a few metrics for Prometheus under /metrics if started with --prometheus.
Example:
# HELP rest_server_blob_read_bytes_total Total number of bytes read from blobs
# TYPE rest_server_blob_read_bytes_total counter
rest_server_blob_read_bytes_total{repo="test",type="data"} 2.13557024e+09
rest_server_blob_read_bytes_total{repo="test",type="index"} 1.198653e+06
rest_server_blob_read_bytes_total{repo="test",type="keys"} 5388
rest_server_blob_read_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_read_bytes_total{repo="test",type="snapshots"} 10018
# HELP rest_server_blob_read_total Total number of blobs read
# TYPE rest_server_blob_read_total counter
rest_server_blob_read_total{repo="test",type="data"} 3985
rest_server_blob_read_total{repo="test",type="index"} 21
rest_server_blob_read_total{repo="test",type="keys"} 12
rest_server_blob_read_total{repo="test",type="locks"} 12
rest_server_blob_read_total{repo="test",type="snapshots"} 32
# HELP rest_server_blob_write_bytes_total Total number of bytes written to blobs
# TYPE rest_server_blob_write_bytes_total counter
rest_server_blob_write_bytes_total{repo="test",type="data"} 1.063726179e+09
rest_server_blob_write_bytes_total{repo="test",type="index"} 395586
rest_server_blob_write_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_write_bytes_total{repo="test",type="snapshots"} 1933
# HELP rest_server_blob_write_total Total number of blobs written
# TYPE rest_server_blob_write_total counter
rest_server_blob_write_total{repo="test",type="data"} 226
rest_server_blob_write_total{repo="test",type="index"} 6
rest_server_blob_write_total{repo="test",type="locks"} 12
rest_server_blob_write_total{repo="test",type="snapshots"} 6
2017-10-24 23:03:50 +08:00
|
|
|
package restserver
|
|
|
|
|
2020-05-04 01:28:13 +08:00
|
|
|
import (
|
|
|
|
"strings"
|
|
|
|
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"github.com/restic/rest-server/repo"
|
|
|
|
)
|
Add Prometheus metrics
Exposes a few metrics for Prometheus under /metrics if started with --prometheus.
Example:
# HELP rest_server_blob_read_bytes_total Total number of bytes read from blobs
# TYPE rest_server_blob_read_bytes_total counter
rest_server_blob_read_bytes_total{repo="test",type="data"} 2.13557024e+09
rest_server_blob_read_bytes_total{repo="test",type="index"} 1.198653e+06
rest_server_blob_read_bytes_total{repo="test",type="keys"} 5388
rest_server_blob_read_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_read_bytes_total{repo="test",type="snapshots"} 10018
# HELP rest_server_blob_read_total Total number of blobs read
# TYPE rest_server_blob_read_total counter
rest_server_blob_read_total{repo="test",type="data"} 3985
rest_server_blob_read_total{repo="test",type="index"} 21
rest_server_blob_read_total{repo="test",type="keys"} 12
rest_server_blob_read_total{repo="test",type="locks"} 12
rest_server_blob_read_total{repo="test",type="snapshots"} 32
# HELP rest_server_blob_write_bytes_total Total number of bytes written to blobs
# TYPE rest_server_blob_write_bytes_total counter
rest_server_blob_write_bytes_total{repo="test",type="data"} 1.063726179e+09
rest_server_blob_write_bytes_total{repo="test",type="index"} 395586
rest_server_blob_write_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_write_bytes_total{repo="test",type="snapshots"} 1933
# HELP rest_server_blob_write_total Total number of blobs written
# TYPE rest_server_blob_write_total counter
rest_server_blob_write_total{repo="test",type="data"} 226
rest_server_blob_write_total{repo="test",type="index"} 6
rest_server_blob_write_total{repo="test",type="locks"} 12
rest_server_blob_write_total{repo="test",type="snapshots"} 6
2017-10-24 23:03:50 +08:00
|
|
|
|
2017-10-29 22:19:11 +08:00
|
|
|
var metricLabelList = []string{"user", "repo", "type"}
|
|
|
|
|
Add Prometheus metrics
Exposes a few metrics for Prometheus under /metrics if started with --prometheus.
Example:
# HELP rest_server_blob_read_bytes_total Total number of bytes read from blobs
# TYPE rest_server_blob_read_bytes_total counter
rest_server_blob_read_bytes_total{repo="test",type="data"} 2.13557024e+09
rest_server_blob_read_bytes_total{repo="test",type="index"} 1.198653e+06
rest_server_blob_read_bytes_total{repo="test",type="keys"} 5388
rest_server_blob_read_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_read_bytes_total{repo="test",type="snapshots"} 10018
# HELP rest_server_blob_read_total Total number of blobs read
# TYPE rest_server_blob_read_total counter
rest_server_blob_read_total{repo="test",type="data"} 3985
rest_server_blob_read_total{repo="test",type="index"} 21
rest_server_blob_read_total{repo="test",type="keys"} 12
rest_server_blob_read_total{repo="test",type="locks"} 12
rest_server_blob_read_total{repo="test",type="snapshots"} 32
# HELP rest_server_blob_write_bytes_total Total number of bytes written to blobs
# TYPE rest_server_blob_write_bytes_total counter
rest_server_blob_write_bytes_total{repo="test",type="data"} 1.063726179e+09
rest_server_blob_write_bytes_total{repo="test",type="index"} 395586
rest_server_blob_write_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_write_bytes_total{repo="test",type="snapshots"} 1933
# HELP rest_server_blob_write_total Total number of blobs written
# TYPE rest_server_blob_write_total counter
rest_server_blob_write_total{repo="test",type="data"} 226
rest_server_blob_write_total{repo="test",type="index"} 6
rest_server_blob_write_total{repo="test",type="locks"} 12
rest_server_blob_write_total{repo="test",type="snapshots"} 6
2017-10-24 23:03:50 +08:00
|
|
|
var metricBlobWriteTotal = prometheus.NewCounterVec(
|
|
|
|
prometheus.CounterOpts{
|
|
|
|
Name: "rest_server_blob_write_total",
|
|
|
|
Help: "Total number of blobs written",
|
|
|
|
},
|
2017-10-29 22:19:11 +08:00
|
|
|
metricLabelList,
|
Add Prometheus metrics
Exposes a few metrics for Prometheus under /metrics if started with --prometheus.
Example:
# HELP rest_server_blob_read_bytes_total Total number of bytes read from blobs
# TYPE rest_server_blob_read_bytes_total counter
rest_server_blob_read_bytes_total{repo="test",type="data"} 2.13557024e+09
rest_server_blob_read_bytes_total{repo="test",type="index"} 1.198653e+06
rest_server_blob_read_bytes_total{repo="test",type="keys"} 5388
rest_server_blob_read_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_read_bytes_total{repo="test",type="snapshots"} 10018
# HELP rest_server_blob_read_total Total number of blobs read
# TYPE rest_server_blob_read_total counter
rest_server_blob_read_total{repo="test",type="data"} 3985
rest_server_blob_read_total{repo="test",type="index"} 21
rest_server_blob_read_total{repo="test",type="keys"} 12
rest_server_blob_read_total{repo="test",type="locks"} 12
rest_server_blob_read_total{repo="test",type="snapshots"} 32
# HELP rest_server_blob_write_bytes_total Total number of bytes written to blobs
# TYPE rest_server_blob_write_bytes_total counter
rest_server_blob_write_bytes_total{repo="test",type="data"} 1.063726179e+09
rest_server_blob_write_bytes_total{repo="test",type="index"} 395586
rest_server_blob_write_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_write_bytes_total{repo="test",type="snapshots"} 1933
# HELP rest_server_blob_write_total Total number of blobs written
# TYPE rest_server_blob_write_total counter
rest_server_blob_write_total{repo="test",type="data"} 226
rest_server_blob_write_total{repo="test",type="index"} 6
rest_server_blob_write_total{repo="test",type="locks"} 12
rest_server_blob_write_total{repo="test",type="snapshots"} 6
2017-10-24 23:03:50 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
var metricBlobWriteBytesTotal = prometheus.NewCounterVec(
|
|
|
|
prometheus.CounterOpts{
|
|
|
|
Name: "rest_server_blob_write_bytes_total",
|
|
|
|
Help: "Total number of bytes written to blobs",
|
|
|
|
},
|
2017-10-29 22:19:11 +08:00
|
|
|
metricLabelList,
|
Add Prometheus metrics
Exposes a few metrics for Prometheus under /metrics if started with --prometheus.
Example:
# HELP rest_server_blob_read_bytes_total Total number of bytes read from blobs
# TYPE rest_server_blob_read_bytes_total counter
rest_server_blob_read_bytes_total{repo="test",type="data"} 2.13557024e+09
rest_server_blob_read_bytes_total{repo="test",type="index"} 1.198653e+06
rest_server_blob_read_bytes_total{repo="test",type="keys"} 5388
rest_server_blob_read_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_read_bytes_total{repo="test",type="snapshots"} 10018
# HELP rest_server_blob_read_total Total number of blobs read
# TYPE rest_server_blob_read_total counter
rest_server_blob_read_total{repo="test",type="data"} 3985
rest_server_blob_read_total{repo="test",type="index"} 21
rest_server_blob_read_total{repo="test",type="keys"} 12
rest_server_blob_read_total{repo="test",type="locks"} 12
rest_server_blob_read_total{repo="test",type="snapshots"} 32
# HELP rest_server_blob_write_bytes_total Total number of bytes written to blobs
# TYPE rest_server_blob_write_bytes_total counter
rest_server_blob_write_bytes_total{repo="test",type="data"} 1.063726179e+09
rest_server_blob_write_bytes_total{repo="test",type="index"} 395586
rest_server_blob_write_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_write_bytes_total{repo="test",type="snapshots"} 1933
# HELP rest_server_blob_write_total Total number of blobs written
# TYPE rest_server_blob_write_total counter
rest_server_blob_write_total{repo="test",type="data"} 226
rest_server_blob_write_total{repo="test",type="index"} 6
rest_server_blob_write_total{repo="test",type="locks"} 12
rest_server_blob_write_total{repo="test",type="snapshots"} 6
2017-10-24 23:03:50 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
var metricBlobReadTotal = prometheus.NewCounterVec(
|
|
|
|
prometheus.CounterOpts{
|
|
|
|
Name: "rest_server_blob_read_total",
|
|
|
|
Help: "Total number of blobs read",
|
|
|
|
},
|
2017-10-29 22:19:11 +08:00
|
|
|
metricLabelList,
|
Add Prometheus metrics
Exposes a few metrics for Prometheus under /metrics if started with --prometheus.
Example:
# HELP rest_server_blob_read_bytes_total Total number of bytes read from blobs
# TYPE rest_server_blob_read_bytes_total counter
rest_server_blob_read_bytes_total{repo="test",type="data"} 2.13557024e+09
rest_server_blob_read_bytes_total{repo="test",type="index"} 1.198653e+06
rest_server_blob_read_bytes_total{repo="test",type="keys"} 5388
rest_server_blob_read_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_read_bytes_total{repo="test",type="snapshots"} 10018
# HELP rest_server_blob_read_total Total number of blobs read
# TYPE rest_server_blob_read_total counter
rest_server_blob_read_total{repo="test",type="data"} 3985
rest_server_blob_read_total{repo="test",type="index"} 21
rest_server_blob_read_total{repo="test",type="keys"} 12
rest_server_blob_read_total{repo="test",type="locks"} 12
rest_server_blob_read_total{repo="test",type="snapshots"} 32
# HELP rest_server_blob_write_bytes_total Total number of bytes written to blobs
# TYPE rest_server_blob_write_bytes_total counter
rest_server_blob_write_bytes_total{repo="test",type="data"} 1.063726179e+09
rest_server_blob_write_bytes_total{repo="test",type="index"} 395586
rest_server_blob_write_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_write_bytes_total{repo="test",type="snapshots"} 1933
# HELP rest_server_blob_write_total Total number of blobs written
# TYPE rest_server_blob_write_total counter
rest_server_blob_write_total{repo="test",type="data"} 226
rest_server_blob_write_total{repo="test",type="index"} 6
rest_server_blob_write_total{repo="test",type="locks"} 12
rest_server_blob_write_total{repo="test",type="snapshots"} 6
2017-10-24 23:03:50 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
var metricBlobReadBytesTotal = prometheus.NewCounterVec(
|
|
|
|
prometheus.CounterOpts{
|
|
|
|
Name: "rest_server_blob_read_bytes_total",
|
|
|
|
Help: "Total number of bytes read from blobs",
|
|
|
|
},
|
2017-10-29 22:19:11 +08:00
|
|
|
metricLabelList,
|
|
|
|
)
|
|
|
|
|
|
|
|
var metricBlobDeleteTotal = prometheus.NewCounterVec(
|
|
|
|
prometheus.CounterOpts{
|
|
|
|
Name: "rest_server_blob_delete_total",
|
|
|
|
Help: "Total number of blobs deleted",
|
|
|
|
},
|
|
|
|
metricLabelList,
|
|
|
|
)
|
|
|
|
|
|
|
|
var metricBlobDeleteBytesTotal = prometheus.NewCounterVec(
|
|
|
|
prometheus.CounterOpts{
|
|
|
|
Name: "rest_server_blob_delete_bytes_total",
|
|
|
|
Help: "Total number of bytes of blobs deleted",
|
|
|
|
},
|
|
|
|
metricLabelList,
|
Add Prometheus metrics
Exposes a few metrics for Prometheus under /metrics if started with --prometheus.
Example:
# HELP rest_server_blob_read_bytes_total Total number of bytes read from blobs
# TYPE rest_server_blob_read_bytes_total counter
rest_server_blob_read_bytes_total{repo="test",type="data"} 2.13557024e+09
rest_server_blob_read_bytes_total{repo="test",type="index"} 1.198653e+06
rest_server_blob_read_bytes_total{repo="test",type="keys"} 5388
rest_server_blob_read_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_read_bytes_total{repo="test",type="snapshots"} 10018
# HELP rest_server_blob_read_total Total number of blobs read
# TYPE rest_server_blob_read_total counter
rest_server_blob_read_total{repo="test",type="data"} 3985
rest_server_blob_read_total{repo="test",type="index"} 21
rest_server_blob_read_total{repo="test",type="keys"} 12
rest_server_blob_read_total{repo="test",type="locks"} 12
rest_server_blob_read_total{repo="test",type="snapshots"} 32
# HELP rest_server_blob_write_bytes_total Total number of bytes written to blobs
# TYPE rest_server_blob_write_bytes_total counter
rest_server_blob_write_bytes_total{repo="test",type="data"} 1.063726179e+09
rest_server_blob_write_bytes_total{repo="test",type="index"} 395586
rest_server_blob_write_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_write_bytes_total{repo="test",type="snapshots"} 1933
# HELP rest_server_blob_write_total Total number of blobs written
# TYPE rest_server_blob_write_total counter
rest_server_blob_write_total{repo="test",type="data"} 226
rest_server_blob_write_total{repo="test",type="index"} 6
rest_server_blob_write_total{repo="test",type="locks"} 12
rest_server_blob_write_total{repo="test",type="snapshots"} 6
2017-10-24 23:03:50 +08:00
|
|
|
)
|
|
|
|
|
2020-05-04 01:28:13 +08:00
|
|
|
// makeBlobMetricFunc creates a metrics callback function that increments the
|
|
|
|
// Prometheus metrics.
|
|
|
|
func makeBlobMetricFunc(username string, folderPath []string) repo.BlobMetricFunc {
|
|
|
|
var f repo.BlobMetricFunc = func(objectType string, operation repo.BlobOperation, nBytes uint64) {
|
|
|
|
labels := prometheus.Labels{
|
|
|
|
"user": username,
|
2020-05-31 21:36:39 +08:00
|
|
|
"repo": strings.Join(folderPath, "/"),
|
2020-05-04 01:28:13 +08:00
|
|
|
"type": objectType,
|
|
|
|
}
|
|
|
|
switch operation {
|
|
|
|
case repo.BlobRead:
|
|
|
|
metricBlobReadTotal.With(labels).Inc()
|
|
|
|
metricBlobReadBytesTotal.With(labels).Add(float64(nBytes))
|
|
|
|
case repo.BlobWrite:
|
|
|
|
metricBlobWriteTotal.With(labels).Inc()
|
|
|
|
metricBlobWriteBytesTotal.With(labels).Add(float64(nBytes))
|
|
|
|
case repo.BlobDelete:
|
|
|
|
metricBlobDeleteTotal.With(labels).Inc()
|
|
|
|
metricBlobDeleteBytesTotal.With(labels).Add(float64(nBytes))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return f
|
|
|
|
}
|
|
|
|
|
Add Prometheus metrics
Exposes a few metrics for Prometheus under /metrics if started with --prometheus.
Example:
# HELP rest_server_blob_read_bytes_total Total number of bytes read from blobs
# TYPE rest_server_blob_read_bytes_total counter
rest_server_blob_read_bytes_total{repo="test",type="data"} 2.13557024e+09
rest_server_blob_read_bytes_total{repo="test",type="index"} 1.198653e+06
rest_server_blob_read_bytes_total{repo="test",type="keys"} 5388
rest_server_blob_read_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_read_bytes_total{repo="test",type="snapshots"} 10018
# HELP rest_server_blob_read_total Total number of blobs read
# TYPE rest_server_blob_read_total counter
rest_server_blob_read_total{repo="test",type="data"} 3985
rest_server_blob_read_total{repo="test",type="index"} 21
rest_server_blob_read_total{repo="test",type="keys"} 12
rest_server_blob_read_total{repo="test",type="locks"} 12
rest_server_blob_read_total{repo="test",type="snapshots"} 32
# HELP rest_server_blob_write_bytes_total Total number of bytes written to blobs
# TYPE rest_server_blob_write_bytes_total counter
rest_server_blob_write_bytes_total{repo="test",type="data"} 1.063726179e+09
rest_server_blob_write_bytes_total{repo="test",type="index"} 395586
rest_server_blob_write_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_write_bytes_total{repo="test",type="snapshots"} 1933
# HELP rest_server_blob_write_total Total number of blobs written
# TYPE rest_server_blob_write_total counter
rest_server_blob_write_total{repo="test",type="data"} 226
rest_server_blob_write_total{repo="test",type="index"} 6
rest_server_blob_write_total{repo="test",type="locks"} 12
rest_server_blob_write_total{repo="test",type="snapshots"} 6
2017-10-24 23:03:50 +08:00
|
|
|
func init() {
|
|
|
|
// These are always initialized, but only updated if Config.Prometheus is set
|
|
|
|
prometheus.MustRegister(metricBlobWriteTotal)
|
|
|
|
prometheus.MustRegister(metricBlobWriteBytesTotal)
|
|
|
|
prometheus.MustRegister(metricBlobReadTotal)
|
|
|
|
prometheus.MustRegister(metricBlobReadBytesTotal)
|
2017-10-29 22:19:11 +08:00
|
|
|
prometheus.MustRegister(metricBlobDeleteTotal)
|
|
|
|
prometheus.MustRegister(metricBlobDeleteBytesTotal)
|
Add Prometheus metrics
Exposes a few metrics for Prometheus under /metrics if started with --prometheus.
Example:
# HELP rest_server_blob_read_bytes_total Total number of bytes read from blobs
# TYPE rest_server_blob_read_bytes_total counter
rest_server_blob_read_bytes_total{repo="test",type="data"} 2.13557024e+09
rest_server_blob_read_bytes_total{repo="test",type="index"} 1.198653e+06
rest_server_blob_read_bytes_total{repo="test",type="keys"} 5388
rest_server_blob_read_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_read_bytes_total{repo="test",type="snapshots"} 10018
# HELP rest_server_blob_read_total Total number of blobs read
# TYPE rest_server_blob_read_total counter
rest_server_blob_read_total{repo="test",type="data"} 3985
rest_server_blob_read_total{repo="test",type="index"} 21
rest_server_blob_read_total{repo="test",type="keys"} 12
rest_server_blob_read_total{repo="test",type="locks"} 12
rest_server_blob_read_total{repo="test",type="snapshots"} 32
# HELP rest_server_blob_write_bytes_total Total number of bytes written to blobs
# TYPE rest_server_blob_write_bytes_total counter
rest_server_blob_write_bytes_total{repo="test",type="data"} 1.063726179e+09
rest_server_blob_write_bytes_total{repo="test",type="index"} 395586
rest_server_blob_write_bytes_total{repo="test",type="locks"} 1975
rest_server_blob_write_bytes_total{repo="test",type="snapshots"} 1933
# HELP rest_server_blob_write_total Total number of blobs written
# TYPE rest_server_blob_write_total counter
rest_server_blob_write_total{repo="test",type="data"} 226
rest_server_blob_write_total{repo="test",type="index"} 6
rest_server_blob_write_total{repo="test",type="locks"} 12
rest_server_blob_write_total{repo="test",type="snapshots"} 6
2017-10-24 23:03:50 +08:00
|
|
|
}
|