From 5343f12c907a1531b8361b5c94a81bd3679d0011 Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Wed, 26 Jul 2023 17:25:24 +0200 Subject: [PATCH] Add an helper for proxied HTTP/2 The standard library's http.Server supports HTTP/2, but only for tls.Conn. This doesn't work when serving connections behind a reverse proxy which terminates TLS and uses the PROXY protocol. Supporting this requires some glue code, which the new helper provides. The example was tested with tlstunnel. Closes: https://github.com/pires/go-proxyproto/issues/90 --- examples/httpserver/httpserver.go | 6 +- go.mod | 5 + go.sum | 4 + helper/http2/http2.go | 183 ++++++++++++++++++++++++++++++ helper/http2/http2_test.go | 114 +++++++++++++++++++ helper/http2/listener.go | 67 +++++++++++ 6 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 go.sum create mode 100644 helper/http2/http2.go create mode 100644 helper/http2/http2_test.go create mode 100644 helper/http2/listener.go diff --git a/examples/httpserver/httpserver.go b/examples/httpserver/httpserver.go index b04f2c7..48452ca 100644 --- a/examples/httpserver/httpserver.go +++ b/examples/httpserver/httpserver.go @@ -7,6 +7,7 @@ import ( "time" "github.com/pires/go-proxyproto" + h2proxy "github.com/pires/go-proxyproto/helper/http2" ) // TODO: add httpclient example @@ -35,5 +36,8 @@ func main() { } defer proxyListener.Close() - server.Serve(proxyListener) + // Create an HTTP server which can handle proxied incoming connections for + // both HTTP/1 and HTTP/2. HTTP/2 support relies on TLS ALPN, the reverse + // proxy needs to be configured to accept "h2". + h2proxy.NewServer(&server, nil).Serve(proxyListener) } diff --git a/go.mod b/go.mod index 82539b3..902b0a7 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,8 @@ module github.com/pires/go-proxyproto go 1.18 + +require ( + golang.org/x/net v0.12.0 // indirect + golang.org/x/text v0.11.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8569e63 --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= diff --git a/helper/http2/http2.go b/helper/http2/http2.go new file mode 100644 index 0000000..24b59b0 --- /dev/null +++ b/helper/http2/http2.go @@ -0,0 +1,183 @@ +// Package http2 provides helpers for HTTP/2. +package http2 + +import ( + "crypto/tls" + "fmt" + "log" + "net" + "net/http" + "sync" + "time" + + "github.com/pires/go-proxyproto" + "golang.org/x/net/http2" +) + +const listenerRetryBaseDelay = 5 * time.Millisecond + +// Server is an HTTP server accepting both regular and proxied, both HTTP/1 and +// HTTP/2 connections. +// +// HTTP/2 is negotiated using TLS ALPN, either directly via a tls.Conn, either +// indirectly via the PROXY protocol. When the PROXY protocol is used, the +// TLS-terminating proxy in front of the server must be configured to accept +// the "h2" TLS ALPN protocol. +// +// The server is closed when the http.Server is. +type Server struct { + h1 *http.Server // regular HTTP/1 server + h2 *http2.Server // HTTP/2 server + h2Err error // HTTP/2 server setup error, if any + h1Listener h1Listener // pipe listener for the HTTP/1 server + + // The following fields are protected by the mutex + mu sync.Mutex + closed bool + listeners map[net.Listener]struct{} +} + +// NewServer creates a new HTTP server. +// +// A nil h2 is equivalent to a zero http2.Server. +func NewServer(h1 *http.Server, h2 *http2.Server) *Server { + if h2 == nil { + h2 = new(http2.Server) + } + srv := &Server{ + h1: h1, + h2: h2, + h2Err: http2.ConfigureServer(h1, h2), + listeners: make(map[net.Listener]struct{}), + } + srv.h1Listener = h1Listener{newPipeListener(), srv} + go func() { + // proxyListener.Accept never fails + _ = h1.Serve(srv.h1Listener) + }() + return srv +} + +func (srv *Server) errorLog() *log.Logger { + if srv.h1.ErrorLog != nil { + return srv.h1.ErrorLog + } + return log.Default() +} + +// Serve accepts incoming connections on the listener ln. +func (srv *Server) Serve(ln net.Listener) error { + if srv.h2Err != nil { + return srv.h2Err + } + + srv.mu.Lock() + ok := !srv.closed + if ok { + srv.listeners[ln] = struct{}{} + } + srv.mu.Unlock() + if !ok { + return http.ErrServerClosed + } + + defer func() { + srv.mu.Lock() + delete(srv.listeners, ln) + srv.mu.Unlock() + }() + + // net.Listener.Accept can fail for temporary failures, e.g. too many open + // files or other timeout conditions. In that case, wait and retry later. + // This mirrors what the net/http package does. + var delay time.Duration + for { + conn, err := ln.Accept() + if ne, ok := err.(net.Error); ok && ne.Timeout() { + if delay == 0 { + delay = listenerRetryBaseDelay + } else { + delay *= 2 + } + if max := 1 * time.Second; delay > max { + delay = max + } + srv.errorLog().Printf("listener %q: accept error (retrying in %v): %v", ln.Addr(), delay, err) + time.Sleep(delay) + } else if err != nil { + return fmt.Errorf("failed to accept connection: %w", err) + } + + delay = 0 + + go func() { + if err := srv.serveConn(conn); err != nil { + srv.errorLog().Printf("listener %q: %v", ln.Addr(), err) + } + }() + } +} + +func (srv *Server) serveConn(conn net.Conn) error { + var proto string + switch conn := conn.(type) { + case *tls.Conn: + proto = conn.ConnectionState().NegotiatedProtocol + case *proxyproto.Conn: + if proxyHeader := conn.ProxyHeader(); proxyHeader != nil { + tlvs, err := proxyHeader.TLVs() + if err != nil { + conn.Close() + return err + } + for _, tlv := range tlvs { + if tlv.Type == proxyproto.PP2_TYPE_ALPN { + proto = string(tlv.Value) + break + } + } + } + } + + // See https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids + switch proto { + case http2.NextProtoTLS, "h2c": + defer conn.Close() + opts := http2.ServeConnOpts{Handler: srv.h1.Handler} + srv.h2.ServeConn(conn, &opts) + return nil + case "", "http/1.0", "http/1.1": + return srv.h1Listener.ServeConn(conn) + default: + conn.Close() + return fmt.Errorf("unsupported protocol %q", proto) + } +} + +func (srv *Server) closeListeners() error { + srv.mu.Lock() + defer srv.mu.Unlock() + + srv.closed = true + + var err error + for ln := range srv.listeners { + if cerr := ln.Close(); cerr != nil { + err = cerr + } + } + return err +} + +// h1Listener is used to signal back http.Server's Close and Shutdown to the +// HTTP/2 server. +type h1Listener struct { + *pipeListener + srv *Server +} + +func (ln h1Listener) Close() error { + // pipeListener.Close never fails + _ = ln.pipeListener.Close() + return ln.srv.closeListeners() +} diff --git a/helper/http2/http2_test.go b/helper/http2/http2_test.go new file mode 100644 index 0000000..054f12d --- /dev/null +++ b/helper/http2/http2_test.go @@ -0,0 +1,114 @@ +package http2_test + +import ( + "errors" + "log" + "net" + "net/http" + "testing" + + "github.com/pires/go-proxyproto" + h2proxy "github.com/pires/go-proxyproto/helper/http2" + "golang.org/x/net/http2" +) + +func ExampleServer() { + ln, err := net.Listen("tcp", "localhost:80") + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + proxyLn := &proxyproto.Listener{ + Listener: ln, + } + + server := h2proxy.NewServer(&http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("Hello world!\n")) + }), + }, nil) + if err := server.Serve(proxyLn); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} + +func TestServer_h1(t *testing.T) { + addr, server := newTestServer(t) + defer server.Close() + + resp, err := http.Get("http://" + addr) + if err != nil { + t.Fatalf("failed to perform HTTP request: %v", err) + } + resp.Body.Close() +} + +func TestServer_h2(t *testing.T) { + addr, server := newTestServer(t) + defer server.Close() + + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatalf("failed to dial: %v", err) + } + defer conn.Close() + + proxyHeader := proxyproto.Header{ + Version: 2, + Command: proxyproto.LOCAL, + TransportProtocol: proxyproto.UNSPEC, + } + tlvs := []proxyproto.TLV{{ + Type: proxyproto.PP2_TYPE_ALPN, + Value: []byte("h2"), + }} + if err := proxyHeader.SetTLVs(tlvs); err != nil { + t.Fatalf("failed to set TLVs: %v", err) + } + if _, err := proxyHeader.WriteTo(conn); err != nil { + t.Fatalf("failed to write PROXY header: %v", err) + } + + h2Conn, err := new(http2.Transport).NewClientConn(conn) + if err != nil { + t.Fatalf("failed to create HTTP connection: %v", err) + } + + req, err := http.NewRequest(http.MethodGet, "http://"+addr, nil) + if err != nil { + t.Fatalf("failed to create HTTP request: %v", err) + } + + resp, err := h2Conn.RoundTrip(req) + if err != nil { + t.Fatalf("failed to perform HTTP request: %v", err) + } + resp.Body.Close() +} + +func newTestServer(t *testing.T) (addr string, server *http.Server) { + ln, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + + server = &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + }), + } + + h2Server := h2proxy.NewServer(server, nil) + done := make(chan error, 1) + go func() { + done <- h2Server.Serve(&proxyproto.Listener{Listener: ln}) + }() + + t.Cleanup(func() { + err := <-done + if err != nil && !errors.Is(err, net.ErrClosed) { + t.Fatalf("failed to serve: %v", err) + } + }) + + return ln.Addr().String(), server +} diff --git a/helper/http2/listener.go b/helper/http2/listener.go new file mode 100644 index 0000000..ce11bb9 --- /dev/null +++ b/helper/http2/listener.go @@ -0,0 +1,67 @@ +package http2 + +import ( + "net" + "sync" +) + +// pipeListener is a hack to workaround the lack of http.Server.ServeConn. +// See: https://github.com/golang/go/issues/36673 +type pipeListener struct { + ch chan net.Conn + closed bool + mu sync.Mutex +} + +func newPipeListener() *pipeListener { + return &pipeListener{ + ch: make(chan net.Conn, 64), + } +} + +func (ln *pipeListener) Accept() (net.Conn, error) { + conn, ok := <-ln.ch + if !ok { + return nil, net.ErrClosed + } + return conn, nil +} + +func (ln *pipeListener) Close() error { + ln.mu.Lock() + defer ln.mu.Unlock() + + if ln.closed { + return net.ErrClosed + } + ln.closed = true + close(ln.ch) + return nil +} + +// ServeConn enqueues a new connection. The connection will be returned in the +// next Accept call. +func (ln *pipeListener) ServeConn(conn net.Conn) error { + ln.mu.Lock() + defer ln.mu.Unlock() + + if ln.closed { + return net.ErrClosed + } + ln.ch <- conn + return nil +} + +func (ln *pipeListener) Addr() net.Addr { + return pipeAddr{} +} + +type pipeAddr struct{} + +func (pipeAddr) Network() string { + return "pipe" +} + +func (pipeAddr) String() string { + return "pipe" +}