Skip to content

Commit 25cc85f

Browse files
schomatislidelAlan Shaw
authored
feat(gateway): Gateway.FastDirIndexThreshold (#8853)
* fix(core/gateway): option to limit directory size listing * feat(gw): HTMLDirListingLimit This is alternative take on the way we limit the HTML listing output. Instead of a hard cut-off, we list up to HTMLDirListingLimit. When a directory has more items than HTMLDirListingLimit we show additional header and footer informing user that only $HTMLDirListingLimit items are listed. This is a better UX. * fix: 0 disables Gateway.HTMLDirListingLimit * refactor: Gateway.FastDirIndexThreshold see explainer in docs/config.md * refactor: prealoc slices * docs: Gateway.FastDirIndexThreshold * refactor: core/corehttp/gateway_handler.go #8853 (comment) * docs: apply suggestions from code review Co-authored-by: Alan Shaw <[email protected]> Co-authored-by: Marcin Rataj <[email protected]> Co-authored-by: Alan Shaw <[email protected]>
1 parent 4f7d4bc commit 25cc85f

File tree

12 files changed

+128
-54
lines changed

12 files changed

+128
-54
lines changed

‎assets/dir-index-html/dir-index.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
<div class="menu-item-narrow"><a href="https://ipfs.io" target="_blank" rel="noopener noreferrer">About</a></div>
3232
<div class="menu-item-narrow"><a href="https://ipfs.io#install" target="_blank" rel="noopener noreferrer">Install</a></div>
3333
<div>
34-
<a href="https://github.com/ipfs/dir-index-html/issues/" target="_blank" rel="noopener noreferrer">
34+
<a href="https://github.com/ipfs/go-ipfs/issues/new/choose" target="_blank" rel="noopener noreferrer" title="Report a bug">
3535
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 18.4 21"><circle cx="7.5" cy="4.8" r="1"/><circle cx="11.1" cy="4.8" r="1"/><path d="M12.7 8.4c-0.5-1.5-1.9-2.5-3.5-2.5 -1.6 0-3 1-3.5 2.5H12.7z"/><path d="M8.5 9.7H5c-0.5 0.8-0.7 1.7-0.7 2.7 0 2.6 1.8 4.8 4.2 5.2V9.7z"/><path d="M13.4 9.7H9.9v7.9c2.4-0.4 4.2-2.5 4.2-5.2C14.1 11.4 13.9 10.5 13.4 9.7z"/><circle cx="15.7" cy="12.9" r="1"/><circle cx="15.1" cy="15.4" r="1"/><circle cx="15.3" cy="10.4" r="1"/><circle cx="2.7" cy="12.9" r="1"/><circle cx="3.3" cy="15.4" r="1"/><circle cx="3.1" cy="10.4" r="1"/></svg>
3636
</a>
3737
</div>

‎assets/dir-index-html/src/dir-index.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
<div class="menu-item-narrow"><a href="https://ipfs.io" target="_blank" rel="noopener noreferrer">About</a></div>
3131
<div class="menu-item-narrow"><a href="https://ipfs.io#install" target="_blank" rel="noopener noreferrer">Install</a></div>
3232
<div>
33-
<a href="https://github.com/ipfs/dir-index-html/issues/" target="_blank" rel="noopener noreferrer">
33+
<a href="https://github.com/ipfs/go-ipfs/issues/new/choose" target="_blank" rel="noopener noreferrer" title="Report a bug">
3434
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 18.4 21"><circle cx="7.5" cy="4.8" r="1"/><circle cx="11.1" cy="4.8" r="1"/><path d="M12.7 8.4c-0.5-1.5-1.9-2.5-3.5-2.5 -1.6 0-3 1-3.5 2.5H12.7z"/><path d="M8.5 9.7H5c-0.5 0.8-0.7 1.7-0.7 2.7 0 2.6 1.8 4.8 4.2 5.2V9.7z"/><path d="M13.4 9.7H9.9v7.9c2.4-0.4 4.2-2.5 4.2-5.2C14.1 11.4 13.9 10.5 13.4 9.7z"/><circle cx="15.7" cy="12.9" r="1"/><circle cx="15.1" cy="15.4" r="1"/><circle cx="15.3" cy="10.4" r="1"/><circle cx="2.7" cy="12.9" r="1"/><circle cx="3.3" cy="15.4" r="1"/><circle cx="3.1" cy="10.4" r="1"/></svg>
3535
</a>
3636
</div>

‎assets/dir-index-html/test/main.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,15 @@ const templateFile = "../dir-index.html"
1212

1313
// Copied from go-ipfs/core/corehttp/gateway_indexPage.go
1414
type listingTemplateData struct {
15-
GatewayURL string
16-
DNSLink bool
17-
Listing []directoryItem
18-
Size string
19-
Path string
20-
Breadcrumbs []breadcrumb
21-
BackLink string
22-
Hash string
15+
GatewayURL string
16+
DNSLink bool
17+
Listing []directoryItem
18+
Size string
19+
Path string
20+
Breadcrumbs []breadcrumb
21+
BackLink string
22+
Hash string
23+
FastDirIndexThreshold int
2324
}
2425

2526
type directoryItem struct {

‎config/gateway.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,15 @@ type Gateway struct {
5353
// }
5454
PathPrefixes []string
5555

56-
// FIXME: Not yet implemented
56+
// FastDirIndexThreshold is the maximum number of items in a directory
57+
// before the Gateway switches to a shallow, faster listing which only
58+
// requires the root node. This allows for listing big directories fast,
59+
// without the linear slowdown caused by reading size metadata from child
60+
// nodes.
61+
// Setting to 0 will enable fast listings for all directories.
62+
FastDirIndexThreshold *OptionalInteger `json:",omitempty"`
63+
64+
// FIXME: Not yet implemented: https://github.com/ipfs/go-ipfs/issues/8059
5765
APICommands []string
5866

5967
// NoFetch configures the gateway to _not_ fetch blocks in response to

‎core/coreapi/unixfs.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se
302302
}
303303

304304
func (api *UnixfsAPI) lsFromLinksAsync(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings) (<-chan coreiface.DirEntry, error) {
305-
out := make(chan coreiface.DirEntry)
305+
out := make(chan coreiface.DirEntry, uio.DefaultShardWidth)
306306

307307
go func() {
308308
defer close(out)

‎core/corehttp/gateway.go‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,10 @@ import (
1616
)
1717

1818
type GatewayConfig struct {
19-
Headers map[string][]string
20-
Writable bool
21-
PathPrefixes []string
19+
Headers map[string][]string
20+
Writable bool
21+
PathPrefixes []string
22+
FastDirIndexThreshold int
2223
}
2324

2425
// A helper function to clean up a set of headers:
@@ -89,9 +90,10 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
8990
}, headers[ACEHeadersName]...))
9091

9192
var gateway http.Handler = newGatewayHandler(GatewayConfig{
92-
Headers: headers,
93-
Writable: writable,
94-
PathPrefixes: cfg.Gateway.PathPrefixes,
93+
Headers: headers,
94+
Writable: writable,
95+
PathPrefixes: cfg.Gateway.PathPrefixes,
96+
FastDirIndexThreshold: int(cfg.Gateway.FastDirIndexThreshold.WithDefault(100)),
9597
}, api)
9698

9799
gateway = otelhttp.NewHandler(gateway, "Gateway.Request")

‎core/corehttp/gateway_handler_unixfs.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) {
1919
ctx, span := tracing.Span(ctx, "Gateway", "ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
2020
defer span.End()
21+
2122
// Handling UnixFS
2223
dr, err := i.api.Unixfs().Get(ctx, resolvedPath)
2324
if err != nil {
@@ -39,6 +40,7 @@ func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter,
3940
internalWebError(w, fmt.Errorf("unsupported UnixFS type"))
4041
return
4142
}
43+
4244
logger.Debugw("serving unixfs directory", "path", contentPath)
4345
i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger)
4446
}

‎core/corehttp/gateway_handler_unixfs_dir.go‎

Lines changed: 39 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/ipfs/go-ipfs/tracing"
1616
path "github.com/ipfs/go-path"
1717
"github.com/ipfs/go-path/resolver"
18+
options "github.com/ipfs/interface-go-ipfs-core/options"
1819
ipath "github.com/ipfs/interface-go-ipfs-core/path"
1920
"go.opentelemetry.io/otel/attribute"
2021
"go.opentelemetry.io/otel/trace"
@@ -102,36 +103,46 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
102103
return
103104
}
104105

106+
// Optimization 1:
107+
// List children without fetching their root blocks (fast, but no size info)
108+
results, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false))
109+
if err != nil {
110+
internalWebError(w, err)
111+
return
112+
}
113+
105114
// storage for directory listing
106-
var dirListing []directoryItem
107-
dirit := dir.Entries()
108-
for dirit.Next() {
109-
size := "?"
110-
if s, err := dirit.Node().Size(); err == nil {
111-
// Size may not be defined/supported. Continue anyways.
112-
size = humanize.Bytes(uint64(s))
113-
}
115+
dirListing := make([]directoryItem, 0, len(results))
114116

115-
resolved, err := i.api.ResolvePath(ctx, ipath.Join(resolvedPath, dirit.Name()))
116-
if err != nil {
117+
for link := range results {
118+
if link.Err != nil {
117119
internalWebError(w, err)
118120
return
119121
}
120-
hash := resolved.Cid().String()
121-
122-
// See comment above where originalUrlPath is declared.
122+
hash := link.Cid.String()
123123
di := directoryItem{
124-
Size: size,
125-
Name: dirit.Name(),
126-
Path: gopath.Join(originalUrlPath, dirit.Name()),
124+
Size: "", // no size because we did not fetch child nodes
125+
Name: link.Name,
126+
Path: gopath.Join(originalUrlPath, link.Name),
127127
Hash: hash,
128128
ShortHash: shortHash(hash),
129129
}
130130
dirListing = append(dirListing, di)
131131
}
132-
if dirit.Err() != nil {
133-
internalWebError(w, dirit.Err())
134-
return
132+
133+
// Optimization 2: fetch sizes only for dirs below FastDirIndexThreshold
134+
if len(dirListing) < i.config.FastDirIndexThreshold {
135+
dirit := dir.Entries()
136+
linkNo := 0
137+
for dirit.Next() {
138+
size := "?"
139+
if s, err := dirit.Node().Size(); err == nil {
140+
// Size may not be defined/supported. Continue anyways.
141+
size = humanize.Bytes(uint64(s))
142+
}
143+
dirListing[linkNo].Size = size
144+
linkNo++
145+
}
135146
}
136147

137148
// construct the correct back link
@@ -180,14 +191,15 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
180191

181192
// See comment above where originalUrlPath is declared.
182193
tplData := listingTemplateData{
183-
GatewayURL: gwURL,
184-
DNSLink: dnslink,
185-
Listing: dirListing,
186-
Size: size,
187-
Path: contentPath.String(),
188-
Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
189-
BackLink: backLink,
190-
Hash: hash,
194+
GatewayURL: gwURL,
195+
DNSLink: dnslink,
196+
Listing: dirListing,
197+
Size: size,
198+
Path: contentPath.String(),
199+
Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
200+
BackLink: backLink,
201+
Hash: hash,
202+
FastDirIndexThreshold: i.config.FastDirIndexThreshold,
191203
}
192204

193205
logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash)

‎core/corehttp/gateway_indexPage.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,15 @@ import (
1212

1313
// structs for directory listing
1414
type listingTemplateData struct {
15-
GatewayURL string
16-
DNSLink bool
17-
Listing []directoryItem
18-
Size string
19-
Path string
20-
Breadcrumbs []breadcrumb
21-
BackLink string
22-
Hash string
15+
GatewayURL string
16+
DNSLink bool
17+
Listing []directoryItem
18+
Size string
19+
Path string
20+
Breadcrumbs []breadcrumb
21+
BackLink string
22+
Hash string
23+
FastDirIndexThreshold int
2324
}
2425

2526
type directoryItem struct {

‎docs/config.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ config file at runtime.
5151
- [`Gateway.NoDNSLink`](#gatewaynodnslink)
5252
- [`Gateway.HTTPHeaders`](#gatewayhttpheaders)
5353
- [`Gateway.RootRedirect`](#gatewayrootredirect)
54+
- [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold)
5455
- [`Gateway.Writable`](#gatewaywritable)
5556
- [`Gateway.PathPrefixes`](#gatewaypathprefixes)
5657
- [`Gateway.PublicGateways`](#gatewaypublicgateways)
@@ -646,6 +647,20 @@ Default: `""`
646647

647648
Type: `string` (url)
648649

650+
### `Gateway.FastDirIndexThreshold`
651+
652+
The maximum number of items in a directory before the Gateway switches
653+
to a shallow, faster listing which only requires the root node.
654+
655+
This allows for fast listings of big directories, without the linear slowdown caused
656+
by reading size metadata from child nodes.
657+
658+
Setting to 0 will enable fast listings for all directories.
659+
660+
Default: `100`
661+
662+
Type: `optionalInteger`
663+
649664
### `Gateway.Writable`
650665

651666
A boolean to configure whether the gateway is writeable or not.

0 commit comments

Comments
 (0)