improve proxy

Author: cedi

src/cmd/root.go

@@ -92,6 +92,12 @@ func init() {
 	if err != nil {
 		panic(fmt.Errorf("fatal binding flag: %w", err))
 	}
+
+	// Other config defaults
+	viper.SetDefault("proxy.maxIdleConns", 1000)
+	viper.SetDefault("proxy.maxIdleConnsPerHost", 100)
+	viper.SetDefault("proxy.timeout", "90s")
+	viper.SetDefault("proxy.compression", true)
 }
 
 func initConfig() {

src/config.yaml

@@ -1,6 +1,12 @@
 server:
   port: 8080
 
+proxy:
+  maxIdleConns: 1000
+  maxIdleConnsPerHost: 500
+  timeout: 30s
+  compression: false
+
 output:
   debug: true
   format: long

src/go.mod

@@ -5,6 +5,7 @@ go 1.24.0
 require (
 	github.com/fsnotify/fsnotify v1.8.0
 	github.com/gin-gonic/gin v1.10.0
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
 	github.com/sierrasoftworks/humane-errors-go v0.0.0-20241125132722-d032d7dd359e
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.20.1

src/go.sum

@@ -37,6 +37,8 @@ github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIx
 github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=

src/pkg/proxy/proxy.go

@@ -1,22 +1,28 @@
 package proxy
 
 import (
+	"context"
 	"fmt"
 	"github.com/SpechtLabs/StaticPages/pkg/config"
+	"github.com/golang/groupcache/singleflight"
 	"github.com/sierrasoftworks/humane-errors-go"
+	"github.com/spf13/viper"
 	"github.com/uptrace/opentelemetry-go-extra/otelzap"
 	"go.uber.org/zap"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"path"
 	"strings"
+	"sync"
+	"time"
 )
 
 type Proxy struct {
 	zapLog *otelzap.Logger
 	pages  map[string]*config.Page
 	proxy  *httputil.ReverseProxy
+	group  singleflight.Group
 }
 
 func NewProxy(zapLog *otelzap.Logger, pages []*config.Page) *Proxy {
@@ -37,62 +43,55 @@ func NewProxy(zapLog *otelzap.Logger, pages []*config.Page) *Proxy {
 	}
 
 	p.proxy = &httputil.ReverseProxy{
-		// Add a proxy director
-		Director: p.Director,
-
-		// Add error handler to log errors
-		ErrorHandler: p.ErrorHandler,
-
-		// Add response modifier to log response status
-		ModifyResponse: p.ModifyResponse,
+		Director:       p.Director,       // Add a proxy director
+		ErrorHandler:   p.ErrorHandler,   // Add error handler to log errors
+		ModifyResponse: p.ModifyResponse, // Add response modifier to log response status
+
+		// Allow transport configuration provided by user
+		Transport: &http.Transport{
+			MaxIdleConns:        viper.GetInt("proxy.maxIdleConns"),
+			MaxIdleConnsPerHost: viper.GetInt("proxy.maxIdleConnsPerHost"),
+			IdleConnTimeout:     viper.GetDuration("proxy.timeout"),
+			DisableCompression:  !viper.GetBool("proxy.compression"),
+		},
 	}
 
 	return p
 }
 
 func (p *Proxy) Director(req *http.Request) {
-	host := req.Host
+	if req.Context().Err() != nil {
+		p.zapLog.Ctx(req.Context()).Warn("request context canceled", zap.String("url", req.URL.String()), zap.String("path", req.URL.Path))
+		return
+	}
 
-	if strings.Contains(host, ":") {
-		host = strings.Split(host, ":")[0]
+	originalPath := req.URL.Path
+	requestUrl := req.Host
+	if strings.Contains(requestUrl, ":") {
+		requestUrl = strings.Split(requestUrl, ":")[0]
 	}
 
-	page, ok := p.pages[host]
+	page, ok := p.pages[requestUrl]
 	if !ok {
-		p.zapLog.Error("no page found for host", zap.String("host", host))
+		p.zapLog.Ctx(req.Context()).Error("no page found for requestUrl", zap.String("requestUrl", requestUrl))
 		return
 	}
 
-	targetURL, err := url.Parse(page.Proxy.URL.String())
+	backendUrl, err := url.Parse(page.Proxy.URL.String())
 	if err != nil {
-		p.zapLog.Error("invalid target URL", zap.Error(err), zap.String("url", page.Proxy.URL.String()))
+		p.zapLog.Ctx(req.Context()).Error("invalid target URL", zap.Error(err), zap.String("url", page.Proxy.URL.String()))
 		return
 	}
 
-	originalPath := req.URL.Path
-
-	// Create a clean path without double slashes
-	targetPath := path.Clean(fmt.Sprintf("/%s/%s",
-		page.Proxy.Path,
-		originalPath,
-	))
-
-	searchPath := append([]string{""}, page.Proxy.SearchPath...)
-
-	for _, lookupPath := range searchPath {
-		testTarget := path.Clean(fmt.Sprintf("/%s/%s",
-			targetPath,
-			lookupPath,
-		))
-
-		if resp, err := http.Head(targetURL.String() + testTarget); err == nil && resp.StatusCode < http.StatusBadRequest {
-			targetPath = testTarget
-			break
-		}
+	// Find the actual html document we are looking for
+	targetPath, err := p.lookupPath(req.Context(), page, requestUrl, backendUrl, path.Clean(fmt.Sprintf("/%s/%s", page.Proxy.Path, originalPath)))
+	if err != nil {
+		p.zapLog.Ctx(req.Context()).Error("no valid path found", zap.String("original_path", originalPath), zap.String("target_path", targetPath))
+		return
 	}
 
-	req.URL.Scheme = targetURL.Scheme
-	req.URL.Host = targetURL.Host
+	req.URL.Scheme = backendUrl.Scheme
+	req.URL.Host = backendUrl.Host
 	req.URL.Path = targetPath
 
 	// Clear the RequestURI as it's required for client requests
@@ -104,42 +103,50 @@ func (p *Proxy) Director(req *http.Request) {
 	}
 
 	req.Header.Set("X-Forwarded-Host", req.Host)
-	req.Header.Set("X-Origin-Host", targetURL.Host)
+	req.Header.Set("X-Origin-Host", backendUrl.Host)
 
 	// Log the request transformation
-	p.zapLog.Debug("transforming request",
+	p.zapLog.Ctx(req.Context()).Debug("transforming request",
 		zap.String("original_path", originalPath),
 		zap.String("target_path", targetPath),
-		zap.String("target_server", targetURL.String()),
+		zap.String("target_server", backendUrl.String()),
 		zap.String("target_url", req.URL.String()),
 	)
 }
 
 func (p *Proxy) ErrorHandler(w http.ResponseWriter, r *http.Request, err error) {
 	responseCode := http.StatusBadGateway
 
-	p.zapLog.Error("proxy error",
-		zap.String("error", err.Error()),
-		zap.String("url", r.URL.String()),
-	)
-
 	switch err.Error() {
 	case "context canceled":
 		responseCode = 499 // Nginx' non-standard code for when a client closes the connection
 	}
 
+	p.zapLog.Ctx(r.Context()).Error("proxy error",
+		zap.String("error", err.Error()),
+		zap.String("url", r.URL.String()),
+		zap.Int("status", responseCode),
+	)
+
 	http.Error(w, err.Error(), responseCode)
 }
 
 func (p *Proxy) ModifyResponse(r *http.Response) error {
 	if r.StatusCode >= 300 {
-		dump, _ := httputil.DumpResponse(r, true)
-
-		p.zapLog.Debug("received response",
-			zap.Int("status", r.StatusCode),
-			zap.String("url", r.Request.URL.String()),
-			zap.ByteString("url", dump),
-		)
+		if p.zapLog.Core().Enabled(zap.DebugLevel) {
+			dump, _ := httputil.DumpResponse(r, true)
+
+			p.zapLog.Ctx(r.Request.Context()).Debug("received response",
+				zap.Int("status", r.StatusCode),
+				zap.String("url", r.Request.URL.String()),
+				zap.ByteString("url", dump),
+			)
+		} else {
+			p.zapLog.Ctx(r.Request.Context()).Info("received response",
+				zap.Int("status", r.StatusCode),
+				zap.String("url", r.Request.URL.String()),
+			)
+		}
 	}
 
 	return nil
@@ -176,3 +183,58 @@ func (p *Proxy) Serve(addr string) humane.Error {
 
 	return nil
 }
+
+func (p *Proxy) probePath(ctx context.Context, url *url.URL, location string) (int, error) {
+	// create a http client with short timeout for fast failure
+	client := &http.Client{
+		Timeout: 2 * time.Second,
+	}
+
+	req, _ := http.NewRequestWithContext(ctx, http.MethodHead, url.String()+location, nil)
+	resp, err := client.Do(req)
+	if err != nil {
+		return http.StatusNotFound, err
+	}
+
+	return resp.StatusCode, err
+}
+
+func (p *Proxy) lookupPath(ctx context.Context, page *config.Page, sourceHost string, backendUrl *url.URL, targetPath string) (string, humane.Error) {
+	// Find the actual html document we are looking for
+	searchPath := append([]string{""}, page.Proxy.SearchPath...)
+	var wg sync.WaitGroup
+	validPath := make(chan string, len(searchPath))
+
+	for _, lookupPath := range searchPath {
+		// Probe each searchPath asynchronously
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			// Cache search path computation
+			cacheKey := fmt.Sprintf("%s-%s-%s", sourceHost, targetPath, lookupPath)
+			_, _ = p.group.Do(cacheKey, func() (interface{}, error) {
+				// Probe the path
+				testTarget := path.Clean(fmt.Sprintf("/%s/%s", targetPath, lookupPath))
+				statusCode, err := p.probePath(ctx, backendUrl, testTarget)
+				if err == nil && statusCode < http.StatusBadRequest {
+					validPath <- testTarget
+					return testTarget, nil
+				}
+
+				return nil, humane.Wrap(err, "Unable to probe path", "Make sure the path exists and is accessible.")
+			})
+		}()
+	}
+
+	go func() {
+		wg.Wait()
+		close(validPath)
+	}()
+
+	if validPath, ok := <-validPath; ok {
+		return validPath, nil
+	}
+
+	return "", humane.New("no valid path found", "Make sure the path exists and is accessible.")
+}