Introduce packet conn wrappers (#7180)

* packet_conn_wrappers: Initial changes

* packet_conn_wrappers: Unwrap a packet conn only if there are no wrappers

---------

Co-authored-by: Matt Holt <mholt@users.noreply.github.com>

caddyconfig/httpcaddyfile/httptype.go

@@ -851,6 +851,20 @@ func (st *ServerType) serversFromPairings(
 				srv.ListenerWrappersRaw = append(srv.ListenerWrappersRaw, jsonListenerWrapper)
 			}
 
+			// Look for any config values that provide packet conn wrappers on the server block
+			for _, listenerConfig := range sblock.pile["packet_conn_wrapper"] {
+				packetConnWrapper, ok := listenerConfig.Value.(caddy.PacketConnWrapper)
+				if !ok {
+					return nil, fmt.Errorf("config for a packet conn wrapper did not provide a value that implements caddy.PacketConnWrapper")
+				}
+				jsonPacketConnWrapper := caddyconfig.JSONModuleObject(
+					packetConnWrapper,
+					"wrapper",
+					packetConnWrapper.(caddy.Module).CaddyModule().ID.Name(),
+					warnings)
+				srv.PacketConnWrappersRaw = append(srv.PacketConnWrappersRaw, jsonPacketConnWrapper)
+			}
+
 			// set up each handler directive, making sure to honor directive order
 			dirRoutes := sblock.pile["route"]
 			siteSubroute, err := buildSubroute(dirRoutes, groupCounter, true)

caddyconfig/httpcaddyfile/serveroptions.go

@@ -36,26 +36,27 @@ type serverOptions struct {
 	ListenerAddress string
 
 	// These will all map 1:1 to the caddyhttp.Server struct
-	Name                 string
+	Name                  string
-	ListenerWrappersRaw  []json.RawMessage
+	ListenerWrappersRaw   []json.RawMessage
-	ReadTimeout          caddy.Duration
+	PacketConnWrappersRaw []json.RawMessage
-	ReadHeaderTimeout    caddy.Duration
+	ReadTimeout           caddy.Duration
-	WriteTimeout         caddy.Duration
+	ReadHeaderTimeout     caddy.Duration
-	IdleTimeout          caddy.Duration
+	WriteTimeout          caddy.Duration
-	KeepAliveInterval    caddy.Duration
+	IdleTimeout           caddy.Duration
-	KeepAliveIdle        caddy.Duration
+	KeepAliveInterval     caddy.Duration
-	KeepAliveCount       int
+	KeepAliveIdle         caddy.Duration
-	MaxHeaderBytes       int
+	KeepAliveCount        int
-	EnableFullDuplex     bool
+	MaxHeaderBytes        int
-	Protocols            []string
+	EnableFullDuplex      bool
-	StrictSNIHost        *bool
+	Protocols             []string
-	TrustedProxiesRaw    json.RawMessage
+	StrictSNIHost         *bool
-	TrustedProxiesStrict int
+	TrustedProxiesRaw     json.RawMessage
-	TrustedProxiesUnix   bool
+	TrustedProxiesStrict  int
-	ClientIPHeaders      []string
+	TrustedProxiesUnix    bool
-	ShouldLogCredentials bool
+	ClientIPHeaders       []string
-	Metrics              *caddyhttp.Metrics
+	ShouldLogCredentials  bool
-	Trace                bool // TODO: EXPERIMENTAL
+	Metrics               *caddyhttp.Metrics
+	Trace                 bool // TODO: EXPERIMENTAL
 }
 
 func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
@@ -99,6 +100,26 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
 				serverOpts.ListenerWrappersRaw = append(serverOpts.ListenerWrappersRaw, jsonListenerWrapper)
 			}
 
+		case "packet_conn_wrappers":
+			for nesting := d.Nesting(); d.NextBlock(nesting); {
+				modID := "caddy.packetconns." + d.Val()
+				unm, err := caddyfile.UnmarshalModule(d, modID)
+				if err != nil {
+					return nil, err
+				}
+				packetConnWrapper, ok := unm.(caddy.PacketConnWrapper)
+				if !ok {
+					return nil, fmt.Errorf("module %s (%T) is not a packet conn wrapper", modID, unm)
+				}
+				jsonPacketConnWrapper := caddyconfig.JSONModuleObject(
+					packetConnWrapper,
+					"wrapper",
+					packetConnWrapper.(caddy.Module).CaddyModule().ID.Name(),
+					nil,
+				)
+				serverOpts.PacketConnWrappersRaw = append(serverOpts.PacketConnWrappersRaw, jsonPacketConnWrapper)
+			}
+
 		case "timeouts":
 			for nesting := d.Nesting(); d.NextBlock(nesting); {
 				switch d.Val() {
@@ -335,6 +356,7 @@ func applyServerOptions(
 
 		// set all the options
 		server.ListenerWrappersRaw = opts.ListenerWrappersRaw
+		server.PacketConnWrappersRaw = opts.PacketConnWrappersRaw
 		server.ReadTimeout = opts.ReadTimeout
 		server.ReadHeaderTimeout = opts.ReadHeaderTimeout
 		server.WriteTimeout = opts.WriteTimeout

listeners.go

@@ -511,7 +511,7 @@ func JoinNetworkAddress(network, host, port string) string {
 //
 // NOTE: This API is EXPERIMENTAL and may be changed or removed.
 // NOTE: user should close the returned listener twice, once to stop accepting new connections, the second time to free up the packet conn.
-func (na NetworkAddress) ListenQUIC(ctx context.Context, portOffset uint, config net.ListenConfig, tlsConf *tls.Config) (http3.QUICListener, error) {
+func (na NetworkAddress) ListenQUIC(ctx context.Context, portOffset uint, config net.ListenConfig, tlsConf *tls.Config, pcWrappers []PacketConnWrapper) (http3.QUICListener, error) {
 	lnKey := listenerKey("quic"+na.Network, na.JoinHostPort(portOffset))
 
 	sharedEarlyListener, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
@@ -523,12 +523,19 @@ func (na NetworkAddress) ListenQUIC(ctx context.Context, portOffset uint, config
 		ln := lnAny.(net.PacketConn)
 
 		h3ln := ln
-		for {
+		if len(pcWrappers) == 0 {
-			// retrieve the underlying socket, so quic-go can optimize.
+			for {
-			if unwrapper, ok := h3ln.(interface{ Unwrap() net.PacketConn }); ok {
+				// retrieve the underlying socket, so quic-go can optimize.
-				h3ln = unwrapper.Unwrap()
+				if unwrapper, ok := h3ln.(interface{ Unwrap() net.PacketConn }); ok {
-			} else {
+					h3ln = unwrapper.Unwrap()
-				break
+				} else {
+					break
+				}
+			}
+		} else {
+			// wrap packet conn before QUIC
+			for _, pcWrapper := range pcWrappers {
+				h3ln = pcWrapper.WrapPacketConn(h3ln)
 			}
 		}
 
@@ -775,6 +782,19 @@ type ListenerWrapper interface {
 	WrapListener(net.Listener) net.Listener
 }
 
+// PacketConnWrapper is a type that wraps a packet conn
+// so it can modify the input packet conn methods.
+// Modules that implement this interface are found
+// in the caddy.packetconns namespace. Usually, to
+// wrap a packet conn, you will define your own struct
+// type that embeds the input packet conn, then
+// implement your own methods that you want to wrap,
+// calling the underlying packet conn methods where
+// appropriate.
+type PacketConnWrapper interface {
+	WrapPacketConn(net.PacketConn) net.PacketConn
+}
+
 // listenerPool stores and allows reuse of active listeners.
 var listenerPool = NewUsagePool()
 

modules/caddyhttp/app.go

@@ -346,6 +346,20 @@ func (app *App) Provision(ctx caddy.Context) error {
 				srv.listenerWrappers = append([]caddy.ListenerWrapper{new(tlsPlaceholderWrapper)}, srv.listenerWrappers...)
 			}
 		}
+
+		// set up each packet conn modifier
+		if srv.PacketConnWrappersRaw != nil {
+			vals, err := ctx.LoadModule(srv, "PacketConnWrappersRaw")
+			if err != nil {
+				return fmt.Errorf("loading packet conn wrapper modules: %v", err)
+			}
+			// if any wrappers were configured, they come before the QUIC handshake;
+			// unlike TLS above, there is no QUIC placeholder
+			for _, val := range vals.([]any) {
+				srv.packetConnWrappers = append(srv.packetConnWrappers, val.(caddy.PacketConnWrapper))
+			}
+		}
+
 		// pre-compile the primary handler chain, and be sure to wrap it in our
 		// route handler so that important security checks are done, etc.
 		primaryRoute := emptyHandler

modules/caddyhttp/server.go

@@ -55,6 +55,10 @@ type Server struct {
 	// of the base listener. They are applied in the given order.
 	ListenerWrappersRaw []json.RawMessage `json:"listener_wrappers,omitempty" caddy:"namespace=caddy.listeners inline_key=wrapper"`
 
+	// A list of packet conn wrapper modules, which can modify the behavior
+	// of the base packet conn. They are applied in the given order.
+	PacketConnWrappersRaw []json.RawMessage `json:"packet_conn_wrappers,omitempty" caddy:"namespace=caddy.packetconns inline_key=wrapper"`
+
 	// How long to allow a read from a client's upload. Setting this
 	// to a short, non-zero value can mitigate slowloris attacks, but
 	// may also affect legitimately slow clients.
@@ -258,7 +262,8 @@ type Server struct {
 	primaryHandlerChain Handler
 	errorHandlerChain   Handler
 	listenerWrappers    []caddy.ListenerWrapper
-	listeners           []net.Listener       // stdlib http.Server will close these
+	packetConnWrappers  []caddy.PacketConnWrapper
+	listeners           []net.Listener
 	quicListeners       []http3.QUICListener // http3 now leave the quic.Listener management to us
 
 	tlsApp       *caddytls.TLS
@@ -625,7 +630,7 @@ func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error
 		return fmt.Errorf("starting HTTP/3 QUIC listener: %v", err)
 	}
 	addr.Network = h3net
-	h3ln, err := addr.ListenQUIC(s.ctx, 0, net.ListenConfig{}, tlsCfg)
+	h3ln, err := addr.ListenQUIC(s.ctx, 0, net.ListenConfig{}, tlsCfg, s.packetConnWrappers)
 	if err != nil {
 		return fmt.Errorf("starting HTTP/3 QUIC listener: %v", err)
 	}