fix(core/gateway): option to limit directory size listing

Author: schomatis

assets/bindata.go

@@ -2,5 +2,5 @@
 package assets
 
 const (
-	BindataVersionHash = "512eb789cd905714e03f29d4e04de7549e8c9c3e"
+	BindataVersionHash = "e127d173ca7ba0b94d6b700016066bda84509e87"
 )

assets/bindata_version_hash.go

@@ -53,6 +53,11 @@
           {{ .Hash }}
         </div>
         {{ end }}
+        {{ if .WarnMaxDirectorySize }}
+        <div>
+          {{ .WarnMaxDirectorySize }}
+        </div>
+        {{ end }}
       </div>
       {{ if .Size }}
       <div class="no-linebreak flex-shrink-1 ml-auto">

assets/dir-index-html/dir-index.html

@@ -52,6 +52,11 @@
           {{ .Hash }}
         </div>
         {{ end }}
+        {{ if .WarnMaxDirectorySize }}
+        <div>
+          {{ .WarnMaxDirectorySize }}
+        </div>
+        {{ end }}
       </div>
       {{ if .Size }}
       <div class="no-linebreak flex-shrink-1 ml-auto">

assets/dir-index-html/src/dir-index.html

@@ -53,6 +53,12 @@ type Gateway struct {
 	//  }
 	PathPrefixes []string
 
+	// MaxDirectorySize is the maximum directory size that will be listed in the
+	// gateway. A directory being listed with a detected size above this threshold
+	// will be stopped to avoid performance downgrade while fetching entries'
+	// metadata. The default value of 0 means no limit will be applied.
+	MaxDirectorySize OptionalInteger `json:",omitempty"`
+
 	// FIXME: Not yet implemented
 	APICommands []string
 

config/gateway.go

@@ -302,7 +302,7 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se
 }
 
 func (api *UnixfsAPI) lsFromLinksAsync(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings) (<-chan coreiface.DirEntry, error) {
-	out := make(chan coreiface.DirEntry)
+	out := make(chan coreiface.DirEntry, uio.DefaultShardWidth)
 
 	go func() {
 		defer close(out)

core/coreapi/unixfs.go

@@ -16,9 +16,10 @@ import (
 )
 
 type GatewayConfig struct {
-	Headers      map[string][]string
-	Writable     bool
-	PathPrefixes []string
+	Headers          map[string][]string
+	Writable         bool
+	PathPrefixes     []string
+	MaxDirectorySize int
 }
 
 // A helper function to clean up a set of headers:
@@ -89,9 +90,10 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
 			}, headers[ACEHeadersName]...))
 
 		var gateway http.Handler = newGatewayHandler(GatewayConfig{
-			Headers:      headers,
-			Writable:     writable,
-			PathPrefixes: cfg.Gateway.PathPrefixes,
+			Headers:          headers,
+			Writable:         writable,
+			PathPrefixes:     cfg.Gateway.PathPrefixes,
+			MaxDirectorySize: int(cfg.Gateway.MaxDirectorySize.WithDefault(0)),
 		}, api)
 
 		gateway = otelhttp.NewHandler(gateway, "Gateway.Request")

core/corehttp/gateway.go

@@ -1,6 +1,7 @@
 package corehttp
 
 import (
+	"context"
 	"fmt"
 	"html"
 	"net/http"
@@ -9,6 +10,7 @@ import (
 	files "github.com/ipfs/go-ipfs-files"
 	"github.com/ipfs/go-ipfs/tracing"
 	ipath "github.com/ipfs/interface-go-ipfs-core/path"
+	"github.com/ipfs/interface-go-ipfs-core/options"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
@@ -17,8 +19,14 @@ import (
 func (i *gatewayHandler) serveUnixFs(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) {
 	ctx, span := tracing.Span(r.Context(), "Gateway", "ServeUnixFs", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
 	defer span.End()
+
+	// Decouple (potential) directory context to cancel it in case it's too big.
+	directoryContext, cancelDirContext := context.WithCancel(ctx)
+
 	// Handling UnixFS
-	dr, err := i.api.Unixfs().Get(ctx, resolvedPath)
+	// FIXME: We should be using `Unixfs().Ls()` not only here but as much as
+	//  possible in this function.
+	dr, err := i.api.Unixfs().Get(directoryContext, resolvedPath)
 	if err != nil {
 		webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound)
 		return
@@ -38,6 +46,38 @@ func (i *gatewayHandler) serveUnixFs(w http.ResponseWriter, r *http.Request, res
 		internalWebError(w, fmt.Errorf("unsupported UnixFs type"))
 		return
 	}
+
+	// Preemptively count directory entries to stop listing directories too big.
+	directoryTooBig := make(chan struct{})
+	if i.config.MaxDirectorySize > 0  {
+		dirEntryChan, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false))
+		if err != nil {
+			internalWebError(w, err)
+			return
+		}
+		directoryEntriesNumber := 0
+		go func() {
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				default:
+				}
+				select {
+				case <-ctx.Done():
+					return
+				case <-dirEntryChan:
+					directoryEntriesNumber++
+					if directoryEntriesNumber >= i.config.MaxDirectorySize {
+						close(directoryTooBig)
+						cancelDirContext()
+						return
+					}
+				}
+			}
+		}()
+	}
+
 	logger.Debugw("serving unixfs directory", "path", contentPath)
-	i.serveDirectory(w, r, resolvedPath, contentPath, dir, begin, logger)
+	i.serveDirectory(w, r, resolvedPath, contentPath, dir, begin, logger, directoryTooBig)
 }

core/corehttp/gateway_handler_unixfs.go

@@ -1,6 +1,7 @@
 package corehttp
 
 import (
+	"fmt"
 	"net/http"
 	"net/url"
 	gopath "path"
@@ -22,7 +23,7 @@ import (
 // serveDirectory returns the best representation of UnixFS directory
 //
 // It will return index.html if present, or generate directory listing otherwise.
-func (i *gatewayHandler) serveDirectory(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, begin time.Time, logger *zap.SugaredLogger) {
+func (i *gatewayHandler) serveDirectory(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, begin time.Time, logger *zap.SugaredLogger, directoryTooBig <-chan struct{}) {
 	ctx, span := tracing.Span(r.Context(), "Gateway", "ServeDirectory", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
 	defer span.End()
 
@@ -133,9 +134,16 @@ func (i *gatewayHandler) serveDirectory(w http.ResponseWriter, r *http.Request,
 		}
 		dirListing = append(dirListing, di)
 	}
+	var warnMaxDirectorySize string
 	if dirit.Err() != nil {
-		internalWebError(w, dirit.Err())
-		return
+		select {
+		case <-directoryTooBig:
+			warnMaxDirectorySize = fmt.Sprintf("Directories bigger than %d items can't be rendered fully. Try using the CLI: ipfs ls -s --size=false --resolve-type=false %s",
+				i.config.MaxDirectorySize, resolvedPath.Cid().String())
+		default:
+			internalWebError(w, dirit.Err())
+			return
+		}
 	}
 
 	// construct the correct back link
@@ -192,6 +200,7 @@ func (i *gatewayHandler) serveDirectory(w http.ResponseWriter, r *http.Request,
 		Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
 		BackLink:    backLink,
 		Hash:        hash,
+		WarnMaxDirectorySize: warnMaxDirectorySize,
 	}
 
 	logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash)

core/corehttp/gateway_handler_unixfs_dir.go

@@ -12,14 +12,15 @@ import (
 
 // structs for directory listing
 type listingTemplateData struct {
-	GatewayURL  string
-	DNSLink     bool
-	Listing     []directoryItem
-	Size        string
-	Path        string
-	Breadcrumbs []breadcrumb
-	BackLink    string
-	Hash        string
+	GatewayURL           string
+	DNSLink              bool
+	Listing              []directoryItem
+	Size                 string
+	Path                 string
+	Breadcrumbs          []breadcrumb
+	BackLink             string
+	Hash                 string
+	WarnMaxDirectorySize string
 }
 
 type directoryItem struct {