encode,staticfiles: Content negotiation, precompressed files (#4045)

* encode: implement prefer setting

* encode: minimum_length configurable via caddyfile

* encode: configurable content-types which to encode

* file_server: support precompressed files

* encode: use ReponseMatcher for conditional encoding of content

* linting error & documentation of encode.PrecompressedOrder

* encode: allow just one response matcher

also change the namespace of the encoders back, I accidently changed to precompressed >.>
default matchers include a *  to match to any charset, that may be appended

* rounding of the PR

* added integration tests for new caddyfile directives
* improved various doc strings (punctuation and typos)
* added json tag for file_server precompress order and encode matcher

* file_server: add vary header, remove accept-ranges when serving precompressed files

* encode: move Suffix implementation to precompressed modules

modules/caddyhttp/encode/encode.go

@@ -23,6 +23,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
 	"sort"
 	"strconv"
@@ -43,12 +44,16 @@ type Encode struct {
 	// will be chosen based on the client's Accept-Encoding header.
 	EncodingsRaw caddy.ModuleMap `json:"encodings,omitempty" caddy:"namespace=http.encoders"`
 
-	// If the client has no strong preference, choose this encoding. TODO: Not yet implemented
-	// Prefer    []string `json:"prefer,omitempty"`
+	// If the client has no strong preference, choose these encodings in order.
+	Prefer []string `json:"prefer,omitempty"`
 
 	// Only encode responses that are at least this many bytes long.
 	MinLength int `json:"minimum_length,omitempty"`
 
+	// Only encode responses that match against this ResponseMmatcher.
+	// The default is a collection of text-based Content-Type headers.
+	Matcher *caddyhttp.ResponseMatcher `json:"match,omitempty"`
+
 	writerPools map[string]*sync.Pool // TODO: these pools do not get reused through config reloads...
 }
 
@@ -75,11 +80,46 @@ func (enc *Encode) Provision(ctx caddy.Context) error {
 	if enc.MinLength == 0 {
 		enc.MinLength = defaultMinLength
 	}
+
+	if enc.Matcher == nil {
+		// common text-based content types
+		enc.Matcher = &caddyhttp.ResponseMatcher{
+			Headers: http.Header{
+				"Content-Type": []string{
+					"text/*",
+					"application/json*",
+					"application/javascript*",
+					"application/xhtml+xml*",
+					"application/atom+xml*",
+					"application/rss+xml*",
+					"image/svg+xml*",
+				},
+			},
+		}
+	}
+
+	return nil
+}
+
+// Validate ensures that enc's configuration is valid.
+func (enc *Encode) Validate() error {
+	check := make(map[string]bool)
+	for _, encName := range enc.Prefer {
+		if _, ok := enc.writerPools[encName]; !ok {
+			return fmt.Errorf("encoding %s not enabled", encName)
+		}
+
+		if _, ok := check[encName]; ok {
+			return fmt.Errorf("encoding %s is duplicated in prefer", encName)
+		}
+		check[encName] = true
+	}
+
 	return nil
 }
 
 func (enc *Encode) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
-	for _, encName := range acceptedEncodings(r) {
+	for _, encName := range AcceptedEncodings(r, enc.Prefer) {
 		if _, ok := enc.writerPools[encName]; !ok {
 			continue // encoding not offered
 		}
@@ -150,6 +190,11 @@ func (rw *responseWriter) WriteHeader(status int) {
 	rw.statusCode = status
 }
 
+// Match determines, if encoding should be done based on the ResponseMatcher.
+func (enc *Encode) Match(rw *responseWriter) bool {
+	return enc.Matcher.Match(rw.statusCode, rw.Header())
+}
+
 // Write writes to the response. If the response qualifies,
 // it is encoded using the encoder, which is initialized
 // if not done so already.
@@ -240,7 +285,10 @@ func (rw *responseWriter) Close() error {
 
 // init should be called before we write a response, if rw.buf has contents.
 func (rw *responseWriter) init() {
-	if rw.Header().Get("Content-Encoding") == "" && rw.buf.Len() >= rw.config.MinLength {
+	if rw.Header().Get("Content-Encoding") == "" &&
+		rw.buf.Len() >= rw.config.MinLength &&
+		rw.config.Match(rw) {
+
 		rw.w = rw.config.writerPools[rw.encodingName].Get().(Encoder)
 		rw.w.Reset(rw.ResponseWriter)
 		rw.Header().Del("Content-Length") // https://github.com/golang/go/issues/14975
@@ -250,12 +298,14 @@ func (rw *responseWriter) init() {
 	rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
 }
 
-// acceptedEncodings returns the list of encodings that the
-// client supports, in descending order of preference. If
+// AcceptedEncodings returns the list of encodings that the
+// client supports, in descending order of preference.
+// The client preference via q-factor and the server
+// preference via Prefer setting are taken into account. If
 // the Sec-WebSocket-Key header is present then non-identity
 // encodings are not considered. See
 // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html.
-func acceptedEncodings(r *http.Request) []string {
+func AcceptedEncodings(r *http.Request, preferredOrder []string) []string {
 	acceptEncHeader := r.Header.Get("Accept-Encoding")
 	websocketKey := r.Header.Get("Sec-WebSocket-Key")
 	if acceptEncHeader == "" {
@@ -292,18 +342,29 @@ func acceptedEncodings(r *http.Request) []string {
 			continue
 		}
 
+		// set server preference
+		prefOrder := -1
+		for i, p := range preferredOrder {
+			if encName == p {
+				prefOrder = len(preferredOrder) - i
+				break
+			}
+		}
+
 		prefs = append(prefs, encodingPreference{
-			encoding: encName,
-			q:        qFactor,
+			encoding:    encName,
+			q:           qFactor,
+			preferOrder: prefOrder,
 		})
 	}
 
-	// sort preferences by descending q-factor
-	sort.Slice(prefs, func(i, j int) bool { return prefs[i].q > prefs[j].q })
-
-	// TODO: If no preference, or same pref for all encodings,
-	// and not websocket, use default encoding ordering (enc.Prefer)
-	// for those which are accepted by the client
+	// sort preferences by descending q-factor first, then by preferOrder
+	sort.Slice(prefs, func(i, j int) bool {
+		if math.Abs(prefs[i].q-prefs[j].q) < 0.00001 {
+			return prefs[i].preferOrder > prefs[j].preferOrder
+		}
+		return prefs[i].q > prefs[j].q
+	})
 
 	prefEncNames := make([]string, len(prefs))
 	for i := range prefs {
@@ -315,8 +376,9 @@ func acceptedEncodings(r *http.Request) []string {
 
 // encodingPreference pairs an encoding with its q-factor.
 type encodingPreference struct {
-	encoding string
-	q        float64
+	encoding    string
+	q           float64
+	preferOrder int
 }
 
 // Encoder is a type which can encode a stream of data.
@@ -332,6 +394,13 @@ type Encoding interface {
 	NewEncoder() Encoder
 }
 
+// Precompressed is a type which returns filename suffix of precompressed
+// file and Accept-Encoding header to use when serving this file.
+type Precompressed interface {
+	AcceptEncoding() string
+	Suffix() string
+}
+
 var bufPool = sync.Pool{
 	New: func() interface{} {
 		return new(bytes.Buffer)
@@ -344,6 +413,7 @@ const defaultMinLength = 512
 // Interface guards
 var (
 	_ caddy.Provisioner           = (*Encode)(nil)
+	_ caddy.Validator             = (*Encode)(nil)
 	_ caddyhttp.MiddlewareHandler = (*Encode)(nil)
 	_ caddyhttp.HTTPInterfaces    = (*responseWriter)(nil)
 )