From ba11f70a7b57b8f7e4d739a3240e4e8aea5596f0 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Wed, 6 Apr 2022 17:30:44 -0300 Subject: [PATCH 1/8] fix(core/gateway): option to limit directory size listing --- assets/dir-index-html/dir-index.html | 5 +++ assets/dir-index-html/src/dir-index.html | 5 +++ config/gateway.go | 6 +++ core/coreapi/unixfs.go | 2 +- core/corehttp/gateway.go | 14 ++++--- core/corehttp/gateway_handler_unixfs.go | 43 ++++++++++++++++++++- core/corehttp/gateway_handler_unixfs_dir.go | 31 +++++++++------ core/corehttp/gateway_indexPage.go | 17 ++++---- 8 files changed, 95 insertions(+), 28 deletions(-) diff --git a/assets/dir-index-html/dir-index.html b/assets/dir-index-html/dir-index.html index ec00da79880..9a584136f82 100644 --- a/assets/dir-index-html/dir-index.html +++ b/assets/dir-index-html/dir-index.html @@ -53,6 +53,11 @@ {{ .Hash }} {{ end }} + {{ if .WarnMaxDirectorySize }} +
+ {{ .WarnMaxDirectorySize }} +
+ {{ end }} {{ if .Size }}
diff --git a/assets/dir-index-html/src/dir-index.html b/assets/dir-index-html/src/dir-index.html index f3dfd632878..6277b8d6212 100644 --- a/assets/dir-index-html/src/dir-index.html +++ b/assets/dir-index-html/src/dir-index.html @@ -52,6 +52,11 @@ {{ .Hash }}
{{ end }} + {{ if .WarnMaxDirectorySize }} +
+ {{ .WarnMaxDirectorySize }} +
+ {{ end }} {{ if .Size }}
diff --git a/config/gateway.go b/config/gateway.go index e85cda98688..ea2c5d419da 100644 --- a/config/gateway.go +++ b/config/gateway.go @@ -53,6 +53,12 @@ type Gateway struct { // } PathPrefixes []string + // MaxDirectorySize is the maximum directory size that will be listed in the + // gateway. A directory being listed with a detected size above this threshold + // will be stopped to avoid performance downgrade while fetching entries' + // metadata. The default value of 0 means no limit will be applied. + MaxDirectorySize OptionalInteger `json:",omitempty"` + // FIXME: Not yet implemented APICommands []string diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 5d3d7e80e30..3fd1fff6126 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -302,7 +302,7 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se } func (api *UnixfsAPI) lsFromLinksAsync(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings) (<-chan coreiface.DirEntry, error) { - out := make(chan coreiface.DirEntry) + out := make(chan coreiface.DirEntry, uio.DefaultShardWidth) go func() { defer close(out) diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index 2e794b53ffc..12a281f37a8 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -16,9 +16,10 @@ import ( ) type GatewayConfig struct { - Headers map[string][]string - Writable bool - PathPrefixes []string + Headers map[string][]string + Writable bool + PathPrefixes []string + MaxDirectorySize int } // A helper function to clean up a set of headers: @@ -89,9 +90,10 @@ func GatewayOption(writable bool, paths ...string) ServeOption { }, headers[ACEHeadersName]...)) var gateway http.Handler = newGatewayHandler(GatewayConfig{ - Headers: headers, - Writable: writable, - PathPrefixes: cfg.Gateway.PathPrefixes, + Headers: headers, + Writable: writable, + PathPrefixes: cfg.Gateway.PathPrefixes, + MaxDirectorySize: int(cfg.Gateway.MaxDirectorySize.WithDefault(0)), }, api) gateway = otelhttp.NewHandler(gateway, "Gateway.Request") diff --git a/core/corehttp/gateway_handler_unixfs.go b/core/corehttp/gateway_handler_unixfs.go index f91e2df3b37..05fe165f88b 100644 --- a/core/corehttp/gateway_handler_unixfs.go +++ b/core/corehttp/gateway_handler_unixfs.go @@ -9,6 +9,7 @@ import ( files "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipfs/tracing" + "github.com/ipfs/interface-go-ipfs-core/options" ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -18,8 +19,14 @@ import ( func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) { ctx, span := tracing.Span(ctx, "Gateway", "ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() + + // Decouple (potential) directory context to cancel it in case it's too big. + directoryContext, cancelDirContext := context.WithCancel(ctx) + // Handling UnixFS - dr, err := i.api.Unixfs().Get(ctx, resolvedPath) + // FIXME: We should be using `Unixfs().Ls()` not only here but as much as + // possible in this function. + dr, err := i.api.Unixfs().Get(directoryContext, resolvedPath) if err != nil { webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound) return @@ -39,6 +46,38 @@ func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter, internalWebError(w, fmt.Errorf("unsupported UnixFS type")) return } + + // Preemptively count directory entries to stop listing directories too big. + directoryTooBig := make(chan struct{}) + if i.config.MaxDirectorySize > 0 { + dirEntryChan, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false)) + if err != nil { + internalWebError(w, err) + return + } + directoryEntriesNumber := 0 + go func() { + for { + select { + case <-ctx.Done(): + return + default: + } + select { + case <-ctx.Done(): + return + case <-dirEntryChan: + directoryEntriesNumber++ + if directoryEntriesNumber >= i.config.MaxDirectorySize { + close(directoryTooBig) + cancelDirContext() + return + } + } + } + }() + } + logger.Debugw("serving unixfs directory", "path", contentPath) - i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger) + i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger, directoryTooBig) } diff --git a/core/corehttp/gateway_handler_unixfs_dir.go b/core/corehttp/gateway_handler_unixfs_dir.go index f462e52f8f6..ff766524c68 100644 --- a/core/corehttp/gateway_handler_unixfs_dir.go +++ b/core/corehttp/gateway_handler_unixfs_dir.go @@ -2,6 +2,7 @@ package corehttp import ( "context" + "fmt" "net/http" "net/url" gopath "path" @@ -24,7 +25,7 @@ import ( // serveDirectory returns the best representation of UnixFS directory // // It will return index.html if present, or generate directory listing otherwise. -func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, begin time.Time, logger *zap.SugaredLogger) { +func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, begin time.Time, logger *zap.SugaredLogger, directoryTooBig <-chan struct{}) { ctx, span := tracing.Span(ctx, "Gateway", "ServeDirectory", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() @@ -135,9 +136,16 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit } dirListing = append(dirListing, di) } + var warnMaxDirectorySize string if dirit.Err() != nil { - internalWebError(w, dirit.Err()) - return + select { + case <-directoryTooBig: + warnMaxDirectorySize = fmt.Sprintf("Directories bigger than %d items can't be rendered fully. Try using the CLI: ipfs ls -s --size=false --resolve-type=false %s", + i.config.MaxDirectorySize, resolvedPath.Cid().String()) + default: + internalWebError(w, dirit.Err()) + return + } } // construct the correct back link @@ -186,14 +194,15 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit // See comment above where originalUrlPath is declared. tplData := listingTemplateData{ - GatewayURL: gwURL, - DNSLink: dnslink, - Listing: dirListing, - Size: size, - Path: contentPath.String(), - Breadcrumbs: breadcrumbs(contentPath.String(), dnslink), - BackLink: backLink, - Hash: hash, + GatewayURL: gwURL, + DNSLink: dnslink, + Listing: dirListing, + Size: size, + Path: contentPath.String(), + Breadcrumbs: breadcrumbs(contentPath.String(), dnslink), + BackLink: backLink, + Hash: hash, + WarnMaxDirectorySize: warnMaxDirectorySize, } logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash) diff --git a/core/corehttp/gateway_indexPage.go b/core/corehttp/gateway_indexPage.go index fbea91649d3..284012cb6b3 100644 --- a/core/corehttp/gateway_indexPage.go +++ b/core/corehttp/gateway_indexPage.go @@ -12,14 +12,15 @@ import ( // structs for directory listing type listingTemplateData struct { - GatewayURL string - DNSLink bool - Listing []directoryItem - Size string - Path string - Breadcrumbs []breadcrumb - BackLink string - Hash string + GatewayURL string + DNSLink bool + Listing []directoryItem + Size string + Path string + Breadcrumbs []breadcrumb + BackLink string + Hash string + WarnMaxDirectorySize string } type directoryItem struct { From 11d364b2e398e57be967b879b1ddc0fdc772fd05 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 12 Apr 2022 03:57:49 +0200 Subject: [PATCH 2/8] feat(gw): HTMLDirListingLimit This is alternative take on the way we limit the HTML listing output. Instead of a hard cut-off, we list up to HTMLDirListingLimit. When a directory has more items than HTMLDirListingLimit we show additional header and footer informing user that only $HTMLDirListingLimit items are listed. This is a better UX. --- assets/dir-index-html/dir-index.html | 44 ++++++++++++++++++--- assets/dir-index-html/src/dir-index.html | 44 ++++++++++++++++++--- assets/dir-index-html/test/main.go | 17 ++++---- config/gateway.go | 11 +++--- core/corehttp/gateway.go | 16 ++++---- core/corehttp/gateway_handler_unixfs.go | 41 +------------------ core/corehttp/gateway_handler_unixfs_dir.go | 37 +++++++---------- core/corehttp/gateway_indexPage.go | 18 ++++----- docs/config.md | 14 +++++++ test/sharness/lib/test-lib.sh | 9 +++++ test/sharness/t0115-gateway-dir-listing.sh | 20 ++++++++++ 11 files changed, 168 insertions(+), 103 deletions(-) diff --git a/assets/dir-index-html/dir-index.html b/assets/dir-index-html/dir-index.html index 9a584136f82..2e0505f9fc9 100644 --- a/assets/dir-index-html/dir-index.html +++ b/assets/dir-index-html/dir-index.html @@ -31,7 +31,7 @@
About
Install
- +
@@ -53,11 +53,6 @@ {{ .Hash }}
{{ end }} - {{ if .WarnMaxDirectorySize }} -
- {{ .WarnMaxDirectorySize }} -
- {{ end }} {{ if .Size }}
@@ -77,6 +72,21 @@ + {{ if gt .HTMLDirListingLimit 0 }} + {{ if ge (len .Listing) .HTMLDirListingLimit }} + + + + +

Below listing shows only the first {{ .HTMLDirListingLimit }} items.

+

To see all CIDs from this directory, adjust Gateway.HTMLDirListingLimit, or use the CLI:

+

ipfs ls -s --size=false --resolve-type=false {{ .Hash }}

+ + + + + {{ end }} + {{ end }} {{ range .Listing }} @@ -95,6 +105,28 @@ {{ .Size }} {{ end }} + {{ if gt .HTMLDirListingLimit 0 }} + {{ if ge (len .Listing) .HTMLDirListingLimit }} + + + + + more + + + + + + + +

Directories bigger than {{ .HTMLDirListingLimit }} items can't be rendered fully on this gateway.

+

To see all CIDs from this directory, adjust Gateway.HTMLDirListingLimit, or use the CLI:

+

ipfs ls -s --size=false --resolve-type=false {{ .Hash }}

+ + + + + {{ end }} + {{ end }}
diff --git a/assets/dir-index-html/src/dir-index.html b/assets/dir-index-html/src/dir-index.html index 6277b8d6212..c4422dfcfad 100644 --- a/assets/dir-index-html/src/dir-index.html +++ b/assets/dir-index-html/src/dir-index.html @@ -30,7 +30,7 @@
About
Install
- +
@@ -52,11 +52,6 @@ {{ .Hash }} {{ end }} - {{ if .WarnMaxDirectorySize }} -
- {{ .WarnMaxDirectorySize }} -
- {{ end }} {{ if .Size }}
@@ -76,6 +71,21 @@ + {{ if gt .HTMLDirListingLimit 0 }} + {{ if ge (len .Listing) .HTMLDirListingLimit }} + + + + +

Below listing shows only the first {{ .HTMLDirListingLimit }} items.

+

To see all CIDs from this directory, adjust Gateway.HTMLDirListingLimit, or use the CLI:

+

ipfs ls -s --size=false --resolve-type=false {{ .Hash }}

+ + + + + {{ end }} + {{ end }} {{ range .Listing }} @@ -94,6 +104,28 @@ {{ .Size }} {{ end }} + {{ if gt .HTMLDirListingLimit 0 }} + {{ if ge (len .Listing) .HTMLDirListingLimit }} + + + + + more + + + + + + + +

Directories bigger than {{ .HTMLDirListingLimit }} items can't be rendered fully on this gateway.

+

To see all CIDs from this directory, adjust Gateway.HTMLDirListingLimit, or use the CLI:

+

ipfs ls -s --size=false --resolve-type=false {{ .Hash }}

+ + + + + {{ end }} + {{ end }}
diff --git a/assets/dir-index-html/test/main.go b/assets/dir-index-html/test/main.go index c02523a9f40..5b8504a0053 100644 --- a/assets/dir-index-html/test/main.go +++ b/assets/dir-index-html/test/main.go @@ -12,14 +12,15 @@ const templateFile = "../dir-index.html" // Copied from go-ipfs/core/corehttp/gateway_indexPage.go type listingTemplateData struct { - GatewayURL string - DNSLink bool - Listing []directoryItem - Size string - Path string - Breadcrumbs []breadcrumb - BackLink string - Hash string + GatewayURL string + DNSLink bool + Listing []directoryItem + Size string + Path string + Breadcrumbs []breadcrumb + BackLink string + Hash string + HTMLDirListingLimit int } type directoryItem struct { diff --git a/config/gateway.go b/config/gateway.go index ea2c5d419da..5e54eb295b4 100644 --- a/config/gateway.go +++ b/config/gateway.go @@ -53,11 +53,12 @@ type Gateway struct { // } PathPrefixes []string - // MaxDirectorySize is the maximum directory size that will be listed in the - // gateway. A directory being listed with a detected size above this threshold - // will be stopped to avoid performance downgrade while fetching entries' - // metadata. The default value of 0 means no limit will be applied. - MaxDirectorySize OptionalInteger `json:",omitempty"` + // HTMLDirListingLimit is the maximum number of directory items that will + // be listed in HTML generated by the gateway. HTML output for directory + // above this threshold will be truncated after this many items to avoid + // performance downgrade while fetching entries' metadata. Setting this to + // value of 0 means no limit will be applied. + HTMLDirListingLimit *OptionalInteger `json:",omitempty"` // FIXME: Not yet implemented APICommands []string diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index 12a281f37a8..bab33f1e4a2 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -16,10 +16,10 @@ import ( ) type GatewayConfig struct { - Headers map[string][]string - Writable bool - PathPrefixes []string - MaxDirectorySize int + Headers map[string][]string + Writable bool + PathPrefixes []string + HTMLDirListingLimit int } // A helper function to clean up a set of headers: @@ -90,10 +90,10 @@ func GatewayOption(writable bool, paths ...string) ServeOption { }, headers[ACEHeadersName]...)) var gateway http.Handler = newGatewayHandler(GatewayConfig{ - Headers: headers, - Writable: writable, - PathPrefixes: cfg.Gateway.PathPrefixes, - MaxDirectorySize: int(cfg.Gateway.MaxDirectorySize.WithDefault(0)), + Headers: headers, + Writable: writable, + PathPrefixes: cfg.Gateway.PathPrefixes, + HTMLDirListingLimit: int(cfg.Gateway.HTMLDirListingLimit.WithDefault(2500)), }, api) gateway = otelhttp.NewHandler(gateway, "Gateway.Request") diff --git a/core/corehttp/gateway_handler_unixfs.go b/core/corehttp/gateway_handler_unixfs.go index 05fe165f88b..b318a641a09 100644 --- a/core/corehttp/gateway_handler_unixfs.go +++ b/core/corehttp/gateway_handler_unixfs.go @@ -9,7 +9,6 @@ import ( files "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-ipfs/tracing" - "github.com/ipfs/interface-go-ipfs-core/options" ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -20,13 +19,8 @@ func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter, ctx, span := tracing.Span(ctx, "Gateway", "ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() - // Decouple (potential) directory context to cancel it in case it's too big. - directoryContext, cancelDirContext := context.WithCancel(ctx) - // Handling UnixFS - // FIXME: We should be using `Unixfs().Ls()` not only here but as much as - // possible in this function. - dr, err := i.api.Unixfs().Get(directoryContext, resolvedPath) + dr, err := i.api.Unixfs().Get(ctx, resolvedPath) if err != nil { webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound) return @@ -47,37 +41,6 @@ func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter, return } - // Preemptively count directory entries to stop listing directories too big. - directoryTooBig := make(chan struct{}) - if i.config.MaxDirectorySize > 0 { - dirEntryChan, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false)) - if err != nil { - internalWebError(w, err) - return - } - directoryEntriesNumber := 0 - go func() { - for { - select { - case <-ctx.Done(): - return - default: - } - select { - case <-ctx.Done(): - return - case <-dirEntryChan: - directoryEntriesNumber++ - if directoryEntriesNumber >= i.config.MaxDirectorySize { - close(directoryTooBig) - cancelDirContext() - return - } - } - } - }() - } - logger.Debugw("serving unixfs directory", "path", contentPath) - i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger, directoryTooBig) + i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger) } diff --git a/core/corehttp/gateway_handler_unixfs_dir.go b/core/corehttp/gateway_handler_unixfs_dir.go index ff766524c68..5c0ec4c22a7 100644 --- a/core/corehttp/gateway_handler_unixfs_dir.go +++ b/core/corehttp/gateway_handler_unixfs_dir.go @@ -2,7 +2,6 @@ package corehttp import ( "context" - "fmt" "net/http" "net/url" gopath "path" @@ -25,7 +24,7 @@ import ( // serveDirectory returns the best representation of UnixFS directory // // It will return index.html if present, or generate directory listing otherwise. -func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, begin time.Time, logger *zap.SugaredLogger, directoryTooBig <-chan struct{}) { +func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, begin time.Time, logger *zap.SugaredLogger) { ctx, span := tracing.Span(ctx, "Gateway", "ServeDirectory", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() @@ -112,7 +111,12 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit // storage for directory listing var dirListing []directoryItem dirit := dir.Entries() + itemCount := 0 for dirit.Next() { + itemCount++ + if itemCount > i.config.HTMLDirListingLimit { + break + } size := "?" if s, err := dirit.Node().Size(); err == nil { // Size may not be defined/supported. Continue anyways. @@ -136,17 +140,6 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit } dirListing = append(dirListing, di) } - var warnMaxDirectorySize string - if dirit.Err() != nil { - select { - case <-directoryTooBig: - warnMaxDirectorySize = fmt.Sprintf("Directories bigger than %d items can't be rendered fully. Try using the CLI: ipfs ls -s --size=false --resolve-type=false %s", - i.config.MaxDirectorySize, resolvedPath.Cid().String()) - default: - internalWebError(w, dirit.Err()) - return - } - } // construct the correct back link // https://github.com/ipfs/go-ipfs/issues/1365 @@ -194,15 +187,15 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit // See comment above where originalUrlPath is declared. tplData := listingTemplateData{ - GatewayURL: gwURL, - DNSLink: dnslink, - Listing: dirListing, - Size: size, - Path: contentPath.String(), - Breadcrumbs: breadcrumbs(contentPath.String(), dnslink), - BackLink: backLink, - Hash: hash, - WarnMaxDirectorySize: warnMaxDirectorySize, + GatewayURL: gwURL, + DNSLink: dnslink, + Listing: dirListing, + Size: size, + Path: contentPath.String(), + Breadcrumbs: breadcrumbs(contentPath.String(), dnslink), + BackLink: backLink, + Hash: hash, + HTMLDirListingLimit: i.config.HTMLDirListingLimit, } logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash) diff --git a/core/corehttp/gateway_indexPage.go b/core/corehttp/gateway_indexPage.go index 284012cb6b3..1e3da88c64c 100644 --- a/core/corehttp/gateway_indexPage.go +++ b/core/corehttp/gateway_indexPage.go @@ -12,15 +12,15 @@ import ( // structs for directory listing type listingTemplateData struct { - GatewayURL string - DNSLink bool - Listing []directoryItem - Size string - Path string - Breadcrumbs []breadcrumb - BackLink string - Hash string - WarnMaxDirectorySize string + GatewayURL string + DNSLink bool + Listing []directoryItem + Size string + Path string + Breadcrumbs []breadcrumb + BackLink string + Hash string + HTMLDirListingLimit int } type directoryItem struct { diff --git a/docs/config.md b/docs/config.md index 519a7c10546..693f5d562d5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -51,6 +51,7 @@ config file at runtime. - [`Gateway.NoDNSLink`](#gatewaynodnslink) - [`Gateway.HTTPHeaders`](#gatewayhttpheaders) - [`Gateway.RootRedirect`](#gatewayrootredirect) + - [`Gateway.HTMLDirListingLimit`](#gatewayhtmldirlistinglimit) - [`Gateway.Writable`](#gatewaywritable) - [`Gateway.PathPrefixes`](#gatewaypathprefixes) - [`Gateway.PublicGateways`](#gatewaypublicgateways) @@ -648,6 +649,19 @@ Default: `""` Type: `string` (url) +### `Gateway.HTMLDirListingLimit` + +The maximum number of directory items that will be listed in HTML generated by +the gateway. HTML output for directory above this threshold will be truncated +after this many items to avoid performance downgrade while fetching entries' +metadata. + +Setting this to `0` means no limit will be applied. + +Default: `2500` + +Type: `optionalInteger` + ### `Gateway.Writable` A boolean to configure whether the gateway is writeable or not. diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index 38f12a0250c..3b6d97862f0 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -388,6 +388,15 @@ test_should_contain() { fi } +test_not_should_contain() { + test "$#" = 2 || error "bug in the test script: not 2 parameters to test_not_should_contain" + if grep -q "$1" "$2" + then + echo "'$2' contains undesired value '$1'" + return 1 + fi +} + test_str_contains() { find=$1 shift diff --git a/test/sharness/t0115-gateway-dir-listing.sh b/test/sharness/t0115-gateway-dir-listing.sh index 0fc86ed7904..d4512170121 100755 --- a/test/sharness/t0115-gateway-dir-listing.sh +++ b/test/sharness/t0115-gateway-dir-listing.sh @@ -135,6 +135,26 @@ test_expect_success "dnslink gw: hash column should be a CID link to cid.ipfs.io test_should_contain "" list_response ' +## ============================================================================ +## Test dir listing of a big directory +## ============================================================================ + +ipfs config --json Gateway.HTMLDirListingLimit 2 +# restart daemon to apply config changes +test_kill_ipfs_daemon +test_launch_ipfs_daemon + +test_expect_success "dir listing should be limited to Gateway.HTMLDirListingLimit" ' + curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ > list_response && + test_should_contain "Index of" list_response && + test_should_contain "Below listing shows only the first 2 items." list_response && + test_should_contain "api" list_response && + test_should_contain "ipfs" list_response && + test_not_should_contain "ipns" list_response && + test_should_contain "Directories bigger than 2 items can'\''t be rendered fully on this gateway." list_response && + test_should_contain "To see all CIDs from this directory, adjust Gateway.HTMLDirListingLimit, or use the CLI" list_response +' + ## ============================================================================ ## End of tests, cleanup ## ============================================================================ From 713fc5d7f6eb68ce8727d08e599d77faeb764ae1 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 12 Apr 2022 05:48:30 +0200 Subject: [PATCH 3/8] fix: 0 disables Gateway.HTMLDirListingLimit --- core/corehttp/gateway_handler_unixfs_dir.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/corehttp/gateway_handler_unixfs_dir.go b/core/corehttp/gateway_handler_unixfs_dir.go index 5c0ec4c22a7..58882fcc9a8 100644 --- a/core/corehttp/gateway_handler_unixfs_dir.go +++ b/core/corehttp/gateway_handler_unixfs_dir.go @@ -114,7 +114,7 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit itemCount := 0 for dirit.Next() { itemCount++ - if itemCount > i.config.HTMLDirListingLimit { + if i.config.HTMLDirListingLimit > 0 && itemCount > i.config.HTMLDirListingLimit { break } size := "?" From 9ded4b5dfe87fb9927eb4dae68e555d2a757540a Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 15 Apr 2022 00:22:28 +0200 Subject: [PATCH 4/8] refactor: Gateway.FastDirIndexThreshold see explainer in docs/config.md --- assets/dir-index-html/dir-index.html | 37 ------------ assets/dir-index-html/src/dir-index.html | 37 ------------ assets/dir-index-html/test/main.go | 18 +++--- config/gateway.go | 15 ++--- core/corehttp/gateway.go | 16 ++--- core/corehttp/gateway_handler_unixfs_dir.go | 67 ++++++++++++--------- core/corehttp/gateway_indexPage.go | 18 +++--- docs/config.md | 18 +++--- test/sharness/lib/test-lib.sh | 4 +- test/sharness/t0115-gateway-dir-listing.sh | 26 ++++---- 10 files changed, 99 insertions(+), 157 deletions(-) diff --git a/assets/dir-index-html/dir-index.html b/assets/dir-index-html/dir-index.html index 2e0505f9fc9..49ac2bb1f1a 100644 --- a/assets/dir-index-html/dir-index.html +++ b/assets/dir-index-html/dir-index.html @@ -72,21 +72,6 @@ - {{ if gt .HTMLDirListingLimit 0 }} - {{ if ge (len .Listing) .HTMLDirListingLimit }} - - - - -

Below listing shows only the first {{ .HTMLDirListingLimit }} items.

-

To see all CIDs from this directory, adjust Gateway.HTMLDirListingLimit, or use the CLI:

-

ipfs ls -s --size=false --resolve-type=false {{ .Hash }}

- - - - - {{ end }} - {{ end }} {{ range .Listing }} @@ -105,28 +90,6 @@ {{ .Size }} {{ end }} - {{ if gt .HTMLDirListingLimit 0 }} - {{ if ge (len .Listing) .HTMLDirListingLimit }} - - - - + more - - - - - - - -

Directories bigger than {{ .HTMLDirListingLimit }} items can't be rendered fully on this gateway.

-

To see all CIDs from this directory, adjust Gateway.HTMLDirListingLimit, or use the CLI:

-

ipfs ls -s --size=false --resolve-type=false {{ .Hash }}

- - - - - {{ end }} - {{ end }} diff --git a/assets/dir-index-html/src/dir-index.html b/assets/dir-index-html/src/dir-index.html index c4422dfcfad..376c4cd7705 100644 --- a/assets/dir-index-html/src/dir-index.html +++ b/assets/dir-index-html/src/dir-index.html @@ -71,21 +71,6 @@ - {{ if gt .HTMLDirListingLimit 0 }} - {{ if ge (len .Listing) .HTMLDirListingLimit }} - - - - -

Below listing shows only the first {{ .HTMLDirListingLimit }} items.

-

To see all CIDs from this directory, adjust Gateway.HTMLDirListingLimit, or use the CLI:

-

ipfs ls -s --size=false --resolve-type=false {{ .Hash }}

- - - - - {{ end }} - {{ end }} {{ range .Listing }} @@ -104,28 +89,6 @@ {{ .Size }} {{ end }} - {{ if gt .HTMLDirListingLimit 0 }} - {{ if ge (len .Listing) .HTMLDirListingLimit }} - - - - + more - - - - - - - -

Directories bigger than {{ .HTMLDirListingLimit }} items can't be rendered fully on this gateway.

-

To see all CIDs from this directory, adjust Gateway.HTMLDirListingLimit, or use the CLI:

-

ipfs ls -s --size=false --resolve-type=false {{ .Hash }}

- - - - - {{ end }} - {{ end }} diff --git a/assets/dir-index-html/test/main.go b/assets/dir-index-html/test/main.go index 5b8504a0053..43b4a098101 100644 --- a/assets/dir-index-html/test/main.go +++ b/assets/dir-index-html/test/main.go @@ -12,15 +12,15 @@ const templateFile = "../dir-index.html" // Copied from go-ipfs/core/corehttp/gateway_indexPage.go type listingTemplateData struct { - GatewayURL string - DNSLink bool - Listing []directoryItem - Size string - Path string - Breadcrumbs []breadcrumb - BackLink string - Hash string - HTMLDirListingLimit int + GatewayURL string + DNSLink bool + Listing []directoryItem + Size string + Path string + Breadcrumbs []breadcrumb + BackLink string + Hash string + FastDirIndexThreshold int } type directoryItem struct { diff --git a/config/gateway.go b/config/gateway.go index 5e54eb295b4..a12a8f0d9a9 100644 --- a/config/gateway.go +++ b/config/gateway.go @@ -53,14 +53,15 @@ type Gateway struct { // } PathPrefixes []string - // HTMLDirListingLimit is the maximum number of directory items that will - // be listed in HTML generated by the gateway. HTML output for directory - // above this threshold will be truncated after this many items to avoid - // performance downgrade while fetching entries' metadata. Setting this to - // value of 0 means no limit will be applied. - HTMLDirListingLimit *OptionalInteger `json:",omitempty"` + // FastDirIndexThreshold is the minimal number of items in a directory that + // triggers Gateway to switch to an inexpensive HTML listing generation. + // This allows for listing big directories with tens of thousands of items + // can be returned as soon as the root node of dag-pb directory is + // resolved, without reading any additional metadata from child nodes. + // Setting to 0 will enable fast listings for all directories. + FastDirIndexThreshold *OptionalInteger `json:",omitempty"` - // FIXME: Not yet implemented + // FIXME: Not yet implemented: https://github.com/ipfs/go-ipfs/issues/8059 APICommands []string // NoFetch configures the gateway to _not_ fetch blocks in response to diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index bab33f1e4a2..a4ae5383179 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -16,10 +16,10 @@ import ( ) type GatewayConfig struct { - Headers map[string][]string - Writable bool - PathPrefixes []string - HTMLDirListingLimit int + Headers map[string][]string + Writable bool + PathPrefixes []string + FastDirIndexThreshold int } // A helper function to clean up a set of headers: @@ -90,10 +90,10 @@ func GatewayOption(writable bool, paths ...string) ServeOption { }, headers[ACEHeadersName]...)) var gateway http.Handler = newGatewayHandler(GatewayConfig{ - Headers: headers, - Writable: writable, - PathPrefixes: cfg.Gateway.PathPrefixes, - HTMLDirListingLimit: int(cfg.Gateway.HTMLDirListingLimit.WithDefault(2500)), + Headers: headers, + Writable: writable, + PathPrefixes: cfg.Gateway.PathPrefixes, + FastDirIndexThreshold: int(cfg.Gateway.FastDirIndexThreshold.WithDefault(100)), }, api) gateway = otelhttp.NewHandler(gateway, "Gateway.Request") diff --git a/core/corehttp/gateway_handler_unixfs_dir.go b/core/corehttp/gateway_handler_unixfs_dir.go index 58882fcc9a8..e9794d8692b 100644 --- a/core/corehttp/gateway_handler_unixfs_dir.go +++ b/core/corehttp/gateway_handler_unixfs_dir.go @@ -15,6 +15,7 @@ import ( "github.com/ipfs/go-ipfs/tracing" path "github.com/ipfs/go-path" "github.com/ipfs/go-path/resolver" + options "github.com/ipfs/interface-go-ipfs-core/options" ipath "github.com/ipfs/interface-go-ipfs-core/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -110,37 +111,45 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit // storage for directory listing var dirListing []directoryItem - dirit := dir.Entries() - itemCount := 0 - for dirit.Next() { - itemCount++ - if i.config.HTMLDirListingLimit > 0 && itemCount > i.config.HTMLDirListingLimit { - break - } - size := "?" - if s, err := dirit.Node().Size(); err == nil { - // Size may not be defined/supported. Continue anyways. - size = humanize.Bytes(uint64(s)) - } - resolved, err := i.api.ResolvePath(ctx, ipath.Join(resolvedPath, dirit.Name())) - if err != nil { + // Optimization 1: + // List children without fetching their root blocks (fast, but no size info) + results, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false)) + if err != nil { + internalWebError(w, err) + return + } + for link := range results { + if link.Err != nil { internalWebError(w, err) return } - hash := resolved.Cid().String() - - // See comment above where originalUrlPath is declared. + hash := link.Cid.String() di := directoryItem{ - Size: size, - Name: dirit.Name(), - Path: gopath.Join(originalUrlPath, dirit.Name()), + Size: "", // no size because we did not fetch child nodes + Name: link.Name, + Path: gopath.Join(originalUrlPath, link.Name), Hash: hash, ShortHash: shortHash(hash), } dirListing = append(dirListing, di) } + // Optimization 2: fetch sizes only for dirs below FastDirIndexThreshold + if len(dirListing) < i.config.FastDirIndexThreshold { + dirit := dir.Entries() + linkNo := 0 + for dirit.Next() { + size := "?" + if s, err := dirit.Node().Size(); err == nil { + // Size may not be defined/supported. Continue anyways. + size = humanize.Bytes(uint64(s)) + } + dirListing[linkNo].Size = size + linkNo++ + } + } + // construct the correct back link // https://github.com/ipfs/go-ipfs/issues/1365 var backLink string = originalUrlPath @@ -187,15 +196,15 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit // See comment above where originalUrlPath is declared. tplData := listingTemplateData{ - GatewayURL: gwURL, - DNSLink: dnslink, - Listing: dirListing, - Size: size, - Path: contentPath.String(), - Breadcrumbs: breadcrumbs(contentPath.String(), dnslink), - BackLink: backLink, - Hash: hash, - HTMLDirListingLimit: i.config.HTMLDirListingLimit, + GatewayURL: gwURL, + DNSLink: dnslink, + Listing: dirListing, + Size: size, + Path: contentPath.String(), + Breadcrumbs: breadcrumbs(contentPath.String(), dnslink), + BackLink: backLink, + Hash: hash, + FastDirIndexThreshold: i.config.FastDirIndexThreshold, } logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash) diff --git a/core/corehttp/gateway_indexPage.go b/core/corehttp/gateway_indexPage.go index 1e3da88c64c..6cc548cdc4d 100644 --- a/core/corehttp/gateway_indexPage.go +++ b/core/corehttp/gateway_indexPage.go @@ -12,15 +12,15 @@ import ( // structs for directory listing type listingTemplateData struct { - GatewayURL string - DNSLink bool - Listing []directoryItem - Size string - Path string - Breadcrumbs []breadcrumb - BackLink string - Hash string - HTMLDirListingLimit int + GatewayURL string + DNSLink bool + Listing []directoryItem + Size string + Path string + Breadcrumbs []breadcrumb + BackLink string + Hash string + FastDirIndexThreshold int } type directoryItem struct { diff --git a/docs/config.md b/docs/config.md index 693f5d562d5..5fc00e73d67 100644 --- a/docs/config.md +++ b/docs/config.md @@ -51,7 +51,7 @@ config file at runtime. - [`Gateway.NoDNSLink`](#gatewaynodnslink) - [`Gateway.HTTPHeaders`](#gatewayhttpheaders) - [`Gateway.RootRedirect`](#gatewayrootredirect) - - [`Gateway.HTMLDirListingLimit`](#gatewayhtmldirlistinglimit) + - [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold) - [`Gateway.Writable`](#gatewaywritable) - [`Gateway.PathPrefixes`](#gatewaypathprefixes) - [`Gateway.PublicGateways`](#gatewaypublicgateways) @@ -649,16 +649,18 @@ Default: `""` Type: `string` (url) -### `Gateway.HTMLDirListingLimit` +### `Gateway.FastDirIndexThreshold` -The maximum number of directory items that will be listed in HTML generated by -the gateway. HTML output for directory above this threshold will be truncated -after this many items to avoid performance downgrade while fetching entries' -metadata. +The minimal number of items in a directory that triggers Gateway +to switch to an inexpensive HTML listing generation. -Setting this to `0` means no limit will be applied. +This allows for listing big directories with tens of thousands of items +can be returned as soon as the root node of dag-pb directory is +resolved, without reading any additional metadata from child nodes. -Default: `2500` +Setting to 0 will enable fast listings for all directories. + +Default: `100` Type: `optionalInteger` diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index 3b6d97862f0..0757c323cf9 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -388,8 +388,8 @@ test_should_contain() { fi } -test_not_should_contain() { - test "$#" = 2 || error "bug in the test script: not 2 parameters to test_not_should_contain" +test_should_not_contain() { + test "$#" = 2 || error "bug in the test script: not 2 parameters to test_should_not_contain" if grep -q "$1" "$2" then echo "'$2' contains undesired value '$1'" diff --git a/test/sharness/t0115-gateway-dir-listing.sh b/test/sharness/t0115-gateway-dir-listing.sh index d4512170121..91ab8afe1fa 100755 --- a/test/sharness/t0115-gateway-dir-listing.sh +++ b/test/sharness/t0115-gateway-dir-listing.sh @@ -28,7 +28,9 @@ test_expect_success "Add the test directory" ' echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt && echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt && DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) && - FILE_CID=$(ipfs files stat /ipfs/$DIR_CID/ą/ę/file-źł.txt | head -1) + FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) && + FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size) + echo "$FILE_CID / $FILE_SIZE" ' ## ============================================================================ @@ -139,20 +141,22 @@ test_expect_success "dnslink gw: hash column should be a CID link to cid.ipfs.io ## Test dir listing of a big directory ## ============================================================================ -ipfs config --json Gateway.HTMLDirListingLimit 2 +test_expect_success "dir listing should resolve child sizes if under Gateway.FastDirIndexThreshold" ' + curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę/ | tee list_response && + test_should_contain "/ipfs/${FILE_CID}?filename" list_response && + test_should_contain ">${FILE_SIZE} B" list_response +' + +# force fast dir index for all responses +ipfs config --json Gateway.FastDirIndexThreshold 0 # restart daemon to apply config changes test_kill_ipfs_daemon test_launch_ipfs_daemon -test_expect_success "dir listing should be limited to Gateway.HTMLDirListingLimit" ' - curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ > list_response && - test_should_contain "Index of" list_response && - test_should_contain "Below listing shows only the first 2 items." list_response && - test_should_contain "api" list_response && - test_should_contain "ipfs" list_response && - test_not_should_contain "ipns" list_response && - test_should_contain "Directories bigger than 2 items can'\''t be rendered fully on this gateway." list_response && - test_should_contain "To see all CIDs from this directory, adjust Gateway.HTMLDirListingLimit, or use the CLI" list_response +test_expect_success "dir listing should not resolve child sizes beyond Gateway.FastDirIndexThreshold" ' + curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę/ | tee list_response && + test_should_contain "/ipfs/${FILE_CID}?filename" list_response && + test_should_not_contain ">${FILE_SIZE} B" list_response ' ## ============================================================================ From 13ba0f7d7ccb78b77b420465e012383df8fd416e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 15 Apr 2022 20:23:54 +0200 Subject: [PATCH 5/8] refactor: prealoc slices --- core/corehttp/gateway_handler.go | 2 +- core/corehttp/gateway_handler_unixfs_dir.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 0c34cc9b175..156f924fdb9 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -766,8 +766,8 @@ func (i *gatewayHandler) buildIpfsRootsHeader(contentPath string, r *http.Reques the last root (responsible for specific article) may not change at all. */ var sp strings.Builder - var pathRoots []string pathSegments := strings.Split(contentPath[6:], "/") + pathRoots := make([]string, 0, len(pathSegments)) sp.WriteString(contentPath[:5]) // /ipfs or /ipns for _, root := range pathSegments { if root == "" { diff --git a/core/corehttp/gateway_handler_unixfs_dir.go b/core/corehttp/gateway_handler_unixfs_dir.go index e9794d8692b..3eda85e1094 100644 --- a/core/corehttp/gateway_handler_unixfs_dir.go +++ b/core/corehttp/gateway_handler_unixfs_dir.go @@ -109,9 +109,6 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit return } - // storage for directory listing - var dirListing []directoryItem - // Optimization 1: // List children without fetching their root blocks (fast, but no size info) results, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false)) @@ -119,6 +116,10 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit internalWebError(w, err) return } + + // storage for directory listing + dirListing := make([]directoryItem, 0, len(results)) + for link := range results { if link.Err != nil { internalWebError(w, err) From 736d6de0a16b96343045f69ab4a4cb1865ed2b07 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 15 Apr 2022 20:41:18 +0200 Subject: [PATCH 6/8] docs: Gateway.FastDirIndexThreshold --- config/gateway.go | 10 +++++----- docs/config.md | 9 ++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/config/gateway.go b/config/gateway.go index a12a8f0d9a9..63c0a6d2a20 100644 --- a/config/gateway.go +++ b/config/gateway.go @@ -53,11 +53,11 @@ type Gateway struct { // } PathPrefixes []string - // FastDirIndexThreshold is the minimal number of items in a directory that - // triggers Gateway to switch to an inexpensive HTML listing generation. - // This allows for listing big directories with tens of thousands of items - // can be returned as soon as the root node of dag-pb directory is - // resolved, without reading any additional metadata from child nodes. + // FastDirIndexThreshold is the minimum number of items in a directory + // before the Gateway switches to a shallow, faster listing which only + // requires the root node. This allows for listing big directories fast, + // without the linear slowdown caused by reading size metadata from child + // nodes. // Setting to 0 will enable fast listings for all directories. FastDirIndexThreshold *OptionalInteger `json:",omitempty"` diff --git a/docs/config.md b/docs/config.md index 5fc00e73d67..b20abb2d199 100644 --- a/docs/config.md +++ b/docs/config.md @@ -651,12 +651,11 @@ Type: `string` (url) ### `Gateway.FastDirIndexThreshold` -The minimal number of items in a directory that triggers Gateway -to switch to an inexpensive HTML listing generation. +The minimum number of items in a directory before the Gateway switches +to a shallow, faster listing which only requires the root node. -This allows for listing big directories with tens of thousands of items -can be returned as soon as the root node of dag-pb directory is -resolved, without reading any additional metadata from child nodes. +This allows for listing big directories fast, without the linear slowdown caused +by reading size metadata from child nodes. Setting to 0 will enable fast listings for all directories. From bd8f07c09e2e88a8d9bc2b74510137e6249ac166 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 15 Apr 2022 21:03:37 +0200 Subject: [PATCH 7/8] refactor: core/corehttp/gateway_handler.go https://github.com/ipfs/go-ipfs/pull/8853#discussion_r851437088 --- core/corehttp/gateway_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 156f924fdb9..0c34cc9b175 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -766,8 +766,8 @@ func (i *gatewayHandler) buildIpfsRootsHeader(contentPath string, r *http.Reques the last root (responsible for specific article) may not change at all. */ var sp strings.Builder + var pathRoots []string pathSegments := strings.Split(contentPath[6:], "/") - pathRoots := make([]string, 0, len(pathSegments)) sp.WriteString(contentPath[:5]) // /ipfs or /ipns for _, root := range pathSegments { if root == "" { From 6272f8d2ef70333eee53c50338b7be2bb0cf0e38 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 20 Apr 2022 16:47:59 +0200 Subject: [PATCH 8/8] docs: apply suggestions from code review Co-authored-by: Alan Shaw --- config/gateway.go | 2 +- docs/config.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/gateway.go b/config/gateway.go index 63c0a6d2a20..486089b7123 100644 --- a/config/gateway.go +++ b/config/gateway.go @@ -53,7 +53,7 @@ type Gateway struct { // } PathPrefixes []string - // FastDirIndexThreshold is the minimum number of items in a directory + // FastDirIndexThreshold is the maximum number of items in a directory // before the Gateway switches to a shallow, faster listing which only // requires the root node. This allows for listing big directories fast, // without the linear slowdown caused by reading size metadata from child diff --git a/docs/config.md b/docs/config.md index b20abb2d199..77f34abe400 100644 --- a/docs/config.md +++ b/docs/config.md @@ -651,10 +651,10 @@ Type: `string` (url) ### `Gateway.FastDirIndexThreshold` -The minimum number of items in a directory before the Gateway switches +The maximum number of items in a directory before the Gateway switches to a shallow, faster listing which only requires the root node. -This allows for listing big directories fast, without the linear slowdown caused +This allows for fast listings of big directories, without the linear slowdown caused by reading size metadata from child nodes. Setting to 0 will enable fast listings for all directories.