diff --git a/changelog/unreleased/issue-122 b/changelog/unreleased/issue-122 new file mode 100644 index 0000000..c98294e --- /dev/null +++ b/changelog/unreleased/issue-122 @@ -0,0 +1,8 @@ +Enhancement: Verify uploaded files + +rest-server now verifies that the hash of content of uploaded files matches +their filename. This ensures that transmission errors are detected and forces +restic to retry the upload. + +https://github.com/restic/rest-server/issues/122 +https://github.com/restic/rest-server/pull/130 diff --git a/handlers_test.go b/handlers_test.go index 46d4fed..b9edc2b 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -3,6 +3,7 @@ package restserver import ( "bytes" "crypto/rand" + "crypto/sha256" "encoding/hex" "fmt" "io" @@ -103,47 +104,58 @@ type TestRequest struct { // createOverwriteDeleteSeq returns a sequence which will create a new file at // path, and then try to overwrite and delete it. -func createOverwriteDeleteSeq(t testing.TB, path string) []TestRequest { +func createOverwriteDeleteSeq(t testing.TB, path string, data string) []TestRequest { // add a file, try to overwrite and delete it req := []TestRequest{ { req: newRequest(t, "GET", path, nil), want: []wantFunc{wantCode(http.StatusNotFound)}, }, - { - req: newRequest(t, "POST", path, strings.NewReader("foobar test config")), + } + + if !strings.HasSuffix(path, "/config") { + req = append(req, TestRequest{ + // broken upload must fail + req: newRequest(t, "POST", path, strings.NewReader(data+"broken")), + want: []wantFunc{wantCode(http.StatusBadRequest)}, + }) + } + + req = append(req, + TestRequest{ + req: newRequest(t, "POST", path, strings.NewReader(data)), want: []wantFunc{wantCode(http.StatusOK)}, }, - { + TestRequest{ req: newRequest(t, "GET", path, nil), want: []wantFunc{ wantCode(http.StatusOK), - wantBody("foobar test config"), + wantBody(data), }, }, - { - req: newRequest(t, "POST", path, strings.NewReader("other config")), + TestRequest{ + req: newRequest(t, "POST", path, strings.NewReader(data+"other stuff")), want: []wantFunc{wantCode(http.StatusForbidden)}, }, - { + TestRequest{ req: newRequest(t, "GET", path, nil), want: []wantFunc{ wantCode(http.StatusOK), - wantBody("foobar test config"), + wantBody(data), }, }, - { + TestRequest{ req: newRequest(t, "DELETE", path, nil), want: []wantFunc{wantCode(http.StatusForbidden)}, }, - { + TestRequest{ req: newRequest(t, "GET", path, nil), want: []wantFunc{ wantCode(http.StatusOK), - wantBody("foobar test config"), + wantBody(data), }, }, - } + ) return req } @@ -154,53 +166,59 @@ func TestResticHandler(t *testing.T) { if err != nil { t.Fatal(err) } - randomID := hex.EncodeToString(buf) + data := "random data file " + hex.EncodeToString(buf) + dataHash := sha256.Sum256([]byte(data)) + fileID := hex.EncodeToString(dataHash[:]) var tests = []struct { seq []TestRequest }{ - {createOverwriteDeleteSeq(t, "/config")}, - {createOverwriteDeleteSeq(t, "/data/"+randomID)}, + {createOverwriteDeleteSeq(t, "/config", data)}, + {createOverwriteDeleteSeq(t, "/data/"+fileID, data)}, { // ensure we can add and remove lock files []TestRequest{ { - req: newRequest(t, "GET", "/locks/"+randomID, nil), + req: newRequest(t, "GET", "/locks/"+fileID, nil), want: []wantFunc{wantCode(http.StatusNotFound)}, }, { - req: newRequest(t, "POST", "/locks/"+randomID, strings.NewReader("lock file")), + req: newRequest(t, "POST", "/locks/"+fileID, strings.NewReader(data+"broken")), + want: []wantFunc{wantCode(http.StatusBadRequest)}, + }, + { + req: newRequest(t, "POST", "/locks/"+fileID, strings.NewReader(data)), want: []wantFunc{wantCode(http.StatusOK)}, }, { - req: newRequest(t, "GET", "/locks/"+randomID, nil), + req: newRequest(t, "GET", "/locks/"+fileID, nil), want: []wantFunc{ wantCode(http.StatusOK), - wantBody("lock file"), + wantBody(data), }, }, { - req: newRequest(t, "POST", "/locks/"+randomID, strings.NewReader("other lock file")), + req: newRequest(t, "POST", "/locks/"+fileID, strings.NewReader(data+"other data")), want: []wantFunc{wantCode(http.StatusForbidden)}, }, { - req: newRequest(t, "DELETE", "/locks/"+randomID, nil), + req: newRequest(t, "DELETE", "/locks/"+fileID, nil), want: []wantFunc{wantCode(http.StatusOK)}, }, { - req: newRequest(t, "GET", "/locks/"+randomID, nil), + req: newRequest(t, "GET", "/locks/"+fileID, nil), want: []wantFunc{wantCode(http.StatusNotFound)}, }, }, }, // Test subrepos - {createOverwriteDeleteSeq(t, "/parent1/sub1/config")}, - {createOverwriteDeleteSeq(t, "/parent1/sub1/data/"+randomID)}, - {createOverwriteDeleteSeq(t, "/parent1/config")}, - {createOverwriteDeleteSeq(t, "/parent1/data/"+randomID)}, - {createOverwriteDeleteSeq(t, "/parent2/config")}, - {createOverwriteDeleteSeq(t, "/parent2/data/"+randomID)}, + {createOverwriteDeleteSeq(t, "/parent1/sub1/config", "foobar")}, + {createOverwriteDeleteSeq(t, "/parent1/sub1/data/"+fileID, data)}, + {createOverwriteDeleteSeq(t, "/parent1/config", "foobar")}, + {createOverwriteDeleteSeq(t, "/parent1/data/"+fileID, data)}, + {createOverwriteDeleteSeq(t, "/parent2/config", "foobar")}, + {createOverwriteDeleteSeq(t, "/parent2/data/"+fileID, data)}, } // setup rclone with a local backend in a temporary directory diff --git a/repo/repo.go b/repo/repo.go index adc5294..ee24ac6 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -1,6 +1,8 @@ package repo import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "io" @@ -569,7 +571,15 @@ func (h *Handler) saveBlob(w http.ResponseWriter, r *http.Request) { return } - written, err := io.Copy(outFile, r.Body) + // calculate hash for current request + hasher := sha256.New() + written, err := io.Copy(outFile, io.TeeReader(r.Body, hasher)) + + // reject if file content doesn't match file name + if err == nil && hex.EncodeToString(hasher.Sum(nil)) != objectID { + err = fmt.Errorf("file content does not match hash") + } + if err != nil { _ = tf.Close() _ = os.Remove(path)