Skip to content

Commit 11d364b

Browse files
lidellidel
committed
feat(gw): HTMLDirListingLimit
This is alternative take on the way we limit the HTML listing output. Instead of a hard cut-off, we list up to HTMLDirListingLimit. When a directory has more items than HTMLDirListingLimit we show additional header and footer informing user that only $HTMLDirListingLimit items are listed. This is a better UX.
1 parent ba11f70 commit 11d364b

11 files changed

Lines changed: 168 additions & 103 deletions

File tree

‎assets/dir-index-html/dir-index.html‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
<div class="menu-item-narrow"><a href="https://ipfs.io" target="_blank" rel="noopener noreferrer">About</a></div>
3232
<div class="menu-item-narrow"><a href="https://ipfs.io#install" target="_blank" rel="noopener noreferrer">Install</a></div>
3333
<div>
34-
<a href="https://github.com/ipfs/dir-index-html/issues/" target="_blank" rel="noopener noreferrer">
34+
<a href="https://github.com/ipfs/go-ipfs/issues/new/choose" target="_blank" rel="noopener noreferrer" title="Report a bug">
3535
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 18.4 21"><circle cx="7.5" cy="4.8" r="1"/><circle cx="11.1" cy="4.8" r="1"/><path d="M12.7 8.4c-0.5-1.5-1.9-2.5-3.5-2.5 -1.6 0-3 1-3.5 2.5H12.7z"/><path d="M8.5 9.7H5c-0.5 0.8-0.7 1.7-0.7 2.7 0 2.6 1.8 4.8 4.2 5.2V9.7z"/><path d="M13.4 9.7H9.9v7.9c2.4-0.4 4.2-2.5 4.2-5.2C14.1 11.4 13.9 10.5 13.4 9.7z"/><circle cx="15.7" cy="12.9" r="1"/><circle cx="15.1" cy="15.4" r="1"/><circle cx="15.3" cy="10.4" r="1"/><circle cx="2.7" cy="12.9" r="1"/><circle cx="3.3" cy="15.4" r="1"/><circle cx="3.1" cy="10.4" r="1"/></svg>
3636
</a>
3737
</div>
@@ -53,11 +53,6 @@
5353
{{ .Hash }}
5454
</div>
5555
{{ end }}
56-
{{ if .WarnMaxDirectorySize }}
57-
<div>
58-
{{ .WarnMaxDirectorySize }}
59-
</div>
60-
{{ end }}
6156
</div>
6257
{{ if .Size }}
6358
<div class="no-linebreak flex-shrink-1 ml-auto">
@@ -77,6 +72,21 @@
7772
<td></td>
7873
<td></td>
7974
</tr>
75+
{{ if gt .HTMLDirListingLimit 0 }}
76+
{{ if ge (len .Listing) .HTMLDirListingLimit }}
77+
<tr>
78+
<td>
79+
</td>
80+
<td>
81+
<p>Below listing shows only the first {{ .HTMLDirListingLimit }} items.</p>
82+
<p>To see all CIDs from this directory, adjust <code>Gateway.HTMLDirListingLimit</code>, or use the CLI:</p>
83+
<p><code>ipfs ls -s --size=false --resolve-type=false {{ .Hash }} </code></p>
84+
</td>
85+
<td></td>
86+
<td></td>
87+
</tr>
88+
{{ end }}
89+
{{ end }}
8090
{{ range .Listing }}
8191
<tr>
8292
<td class="type-icon">
@@ -95,6 +105,28 @@
95105
<td class="no-linebreak">{{ .Size }}</td>
96106
</tr>
97107
{{ end }}
108+
{{ if gt .HTMLDirListingLimit 0 }}
109+
{{ if ge (len .Listing) .HTMLDirListingLimit }}
110+
<tr>
111+
<td></td>
112+
<td>
113+
+ more
114+
</td>
115+
<td></td>
116+
<td></td>
117+
</tr>
118+
<tr>
119+
<td></td>
120+
<td>
121+
<p>Directories bigger than {{ .HTMLDirListingLimit }} items can't be rendered fully on this gateway.</p>
122+
<p>To see all CIDs from this directory, adjust <code>Gateway.HTMLDirListingLimit</code>, or use the CLI:</p>
123+
<p><code>ipfs ls -s --size=false --resolve-type=false {{ .Hash }}</code></p>
124+
</td>
125+
<td></td>
126+
<td></td>
127+
</tr>
128+
{{ end }}
129+
{{ end }}
98130
</table>
99131
</div>
100132
</div>

‎assets/dir-index-html/src/dir-index.html‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
<div class="menu-item-narrow"><a href="https://ipfs.io" target="_blank" rel="noopener noreferrer">About</a></div>
3131
<div class="menu-item-narrow"><a href="https://ipfs.io#install" target="_blank" rel="noopener noreferrer">Install</a></div>
3232
<div>
33-
<a href="https://github.com/ipfs/dir-index-html/issues/" target="_blank" rel="noopener noreferrer">
33+
<a href="https://github.com/ipfs/go-ipfs/issues/new/choose" target="_blank" rel="noopener noreferrer" title="Report a bug">
3434
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 18.4 21"><circle cx="7.5" cy="4.8" r="1"/><circle cx="11.1" cy="4.8" r="1"/><path d="M12.7 8.4c-0.5-1.5-1.9-2.5-3.5-2.5 -1.6 0-3 1-3.5 2.5H12.7z"/><path d="M8.5 9.7H5c-0.5 0.8-0.7 1.7-0.7 2.7 0 2.6 1.8 4.8 4.2 5.2V9.7z"/><path d="M13.4 9.7H9.9v7.9c2.4-0.4 4.2-2.5 4.2-5.2C14.1 11.4 13.9 10.5 13.4 9.7z"/><circle cx="15.7" cy="12.9" r="1"/><circle cx="15.1" cy="15.4" r="1"/><circle cx="15.3" cy="10.4" r="1"/><circle cx="2.7" cy="12.9" r="1"/><circle cx="3.3" cy="15.4" r="1"/><circle cx="3.1" cy="10.4" r="1"/></svg>
3535
</a>
3636
</div>
@@ -52,11 +52,6 @@
5252
{{ .Hash }}
5353
</div>
5454
{{ end }}
55-
{{ if .WarnMaxDirectorySize }}
56-
<div>
57-
{{ .WarnMaxDirectorySize }}
58-
</div>
59-
{{ end }}
6055
</div>
6156
{{ if .Size }}
6257
<div class="no-linebreak flex-shrink-1 ml-auto">
@@ -76,6 +71,21 @@
7671
<td></td>
7772
<td></td>
7873
</tr>
74+
{{ if gt .HTMLDirListingLimit 0 }}
75+
{{ if ge (len .Listing) .HTMLDirListingLimit }}
76+
<tr>
77+
<td>
78+
</td>
79+
<td>
80+
<p>Below listing shows only the first {{ .HTMLDirListingLimit }} items.</p>
81+
<p>To see all CIDs from this directory, adjust <code>Gateway.HTMLDirListingLimit</code>, or use the CLI:</p>
82+
<p><code>ipfs ls -s --size=false --resolve-type=false {{ .Hash }} </code></p>
83+
</td>
84+
<td></td>
85+
<td></td>
86+
</tr>
87+
{{ end }}
88+
{{ end }}
7989
{{ range .Listing }}
8090
<tr>
8191
<td class="type-icon">
@@ -94,6 +104,28 @@
94104
<td class="no-linebreak">{{ .Size }}</td>
95105
</tr>
96106
{{ end }}
107+
{{ if gt .HTMLDirListingLimit 0 }}
108+
{{ if ge (len .Listing) .HTMLDirListingLimit }}
109+
<tr>
110+
<td></td>
111+
<td>
112+
+ more
113+
</td>
114+
<td></td>
115+
<td></td>
116+
</tr>
117+
<tr>
118+
<td></td>
119+
<td>
120+
<p>Directories bigger than {{ .HTMLDirListingLimit }} items can't be rendered fully on this gateway.</p>
121+
<p>To see all CIDs from this directory, adjust <code>Gateway.HTMLDirListingLimit</code>, or use the CLI:</p>
122+
<p><code>ipfs ls -s --size=false --resolve-type=false {{ .Hash }}</code></p>
123+
</td>
124+
<td></td>
125+
<td></td>
126+
</tr>
127+
{{ end }}
128+
{{ end }}
97129
</table>
98130
</div>
99131
</div>

‎assets/dir-index-html/test/main.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,15 @@ const templateFile = "../dir-index.html"
1212

1313
// Copied from go-ipfs/core/corehttp/gateway_indexPage.go
1414
type listingTemplateData struct {
15-
GatewayURL string
16-
DNSLink bool
17-
Listing []directoryItem
18-
Size string
19-
Path string
20-
Breadcrumbs []breadcrumb
21-
BackLink string
22-
Hash string
15+
GatewayURL string
16+
DNSLink bool
17+
Listing []directoryItem
18+
Size string
19+
Path string
20+
Breadcrumbs []breadcrumb
21+
BackLink string
22+
Hash string
23+
HTMLDirListingLimit int
2324
}
2425

2526
type directoryItem struct {

‎config/gateway.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,12 @@ type Gateway struct {
5353
// }
5454
PathPrefixes []string
5555

56-
// MaxDirectorySize is the maximum directory size that will be listed in the
57-
// gateway. A directory being listed with a detected size above this threshold
58-
// will be stopped to avoid performance downgrade while fetching entries'
59-
// metadata. The default value of 0 means no limit will be applied.
60-
MaxDirectorySize OptionalInteger `json:",omitempty"`
56+
// HTMLDirListingLimit is the maximum number of directory items that will
57+
// be listed in HTML generated by the gateway. HTML output for directory
58+
// above this threshold will be truncated after this many items to avoid
59+
// performance downgrade while fetching entries' metadata. Setting this to
60+
// value of 0 means no limit will be applied.
61+
HTMLDirListingLimit *OptionalInteger `json:",omitempty"`
6162

6263
// FIXME: Not yet implemented
6364
APICommands []string

‎core/corehttp/gateway.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@ import (
1616
)
1717

1818
type GatewayConfig struct {
19-
Headers map[string][]string
20-
Writable bool
21-
PathPrefixes []string
22-
MaxDirectorySize int
19+
Headers map[string][]string
20+
Writable bool
21+
PathPrefixes []string
22+
HTMLDirListingLimit int
2323
}
2424

2525
// A helper function to clean up a set of headers:
@@ -90,10 +90,10 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
9090
}, headers[ACEHeadersName]...))
9191

9292
var gateway http.Handler = newGatewayHandler(GatewayConfig{
93-
Headers: headers,
94-
Writable: writable,
95-
PathPrefixes: cfg.Gateway.PathPrefixes,
96-
MaxDirectorySize: int(cfg.Gateway.MaxDirectorySize.WithDefault(0)),
93+
Headers: headers,
94+
Writable: writable,
95+
PathPrefixes: cfg.Gateway.PathPrefixes,
96+
HTMLDirListingLimit: int(cfg.Gateway.HTMLDirListingLimit.WithDefault(2500)),
9797
}, api)
9898

9999
gateway = otelhttp.NewHandler(gateway, "Gateway.Request")

‎core/corehttp/gateway_handler_unixfs.go‎

Lines changed: 2 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99

1010
files "github.com/ipfs/go-ipfs-files"
1111
"github.com/ipfs/go-ipfs/tracing"
12-
"github.com/ipfs/interface-go-ipfs-core/options"
1312
ipath "github.com/ipfs/interface-go-ipfs-core/path"
1413
"go.opentelemetry.io/otel/attribute"
1514
"go.opentelemetry.io/otel/trace"
@@ -20,13 +19,8 @@ func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter,
2019
ctx, span := tracing.Span(ctx, "Gateway", "ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
2120
defer span.End()
2221

23-
// Decouple (potential) directory context to cancel it in case it's too big.
24-
directoryContext, cancelDirContext := context.WithCancel(ctx)
25-
2622
// Handling UnixFS
27-
// FIXME: We should be using `Unixfs().Ls()` not only here but as much as
28-
// possible in this function.
29-
dr, err := i.api.Unixfs().Get(directoryContext, resolvedPath)
23+
dr, err := i.api.Unixfs().Get(ctx, resolvedPath)
3024
if err != nil {
3125
webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound)
3226
return
@@ -47,37 +41,6 @@ func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter,
4741
return
4842
}
4943

50-
// Preemptively count directory entries to stop listing directories too big.
51-
directoryTooBig := make(chan struct{})
52-
if i.config.MaxDirectorySize > 0 {
53-
dirEntryChan, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false))
54-
if err != nil {
55-
internalWebError(w, err)
56-
return
57-
}
58-
directoryEntriesNumber := 0
59-
go func() {
60-
for {
61-
select {
62-
case <-ctx.Done():
63-
return
64-
default:
65-
}
66-
select {
67-
case <-ctx.Done():
68-
return
69-
case <-dirEntryChan:
70-
directoryEntriesNumber++
71-
if directoryEntriesNumber >= i.config.MaxDirectorySize {
72-
close(directoryTooBig)
73-
cancelDirContext()
74-
return
75-
}
76-
}
77-
}
78-
}()
79-
}
80-
8144
logger.Debugw("serving unixfs directory", "path", contentPath)
82-
i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger, directoryTooBig)
45+
i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger)
8346
}

‎core/corehttp/gateway_handler_unixfs_dir.go‎

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package corehttp
22

33
import (
44
"context"
5-
"fmt"
65
"net/http"
76
"net/url"
87
gopath "path"
@@ -25,7 +24,7 @@ import (
2524
// serveDirectory returns the best representation of UnixFS directory
2625
//
2726
// It will return index.html if present, or generate directory listing otherwise.
28-
func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, begin time.Time, logger *zap.SugaredLogger, directoryTooBig <-chan struct{}) {
27+
func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, begin time.Time, logger *zap.SugaredLogger) {
2928
ctx, span := tracing.Span(ctx, "Gateway", "ServeDirectory", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
3029
defer span.End()
3130

@@ -112,7 +111,12 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
112111
// storage for directory listing
113112
var dirListing []directoryItem
114113
dirit := dir.Entries()
114+
itemCount := 0
115115
for dirit.Next() {
116+
itemCount++
117+
if itemCount > i.config.HTMLDirListingLimit {
118+
break
119+
}
116120
size := "?"
117121
if s, err := dirit.Node().Size(); err == nil {
118122
// Size may not be defined/supported. Continue anyways.
@@ -136,17 +140,6 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
136140
}
137141
dirListing = append(dirListing, di)
138142
}
139-
var warnMaxDirectorySize string
140-
if dirit.Err() != nil {
141-
select {
142-
case <-directoryTooBig:
143-
warnMaxDirectorySize = fmt.Sprintf("Directories bigger than %d items can't be rendered fully. Try using the CLI: ipfs ls -s --size=false --resolve-type=false %s",
144-
i.config.MaxDirectorySize, resolvedPath.Cid().String())
145-
default:
146-
internalWebError(w, dirit.Err())
147-
return
148-
}
149-
}
150143

151144
// construct the correct back link
152145
// https://github.com/ipfs/go-ipfs/issues/1365
@@ -194,15 +187,15 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
194187

195188
// See comment above where originalUrlPath is declared.
196189
tplData := listingTemplateData{
197-
GatewayURL: gwURL,
198-
DNSLink: dnslink,
199-
Listing: dirListing,
200-
Size: size,
201-
Path: contentPath.String(),
202-
Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
203-
BackLink: backLink,
204-
Hash: hash,
205-
WarnMaxDirectorySize: warnMaxDirectorySize,
190+
GatewayURL: gwURL,
191+
DNSLink: dnslink,
192+
Listing: dirListing,
193+
Size: size,
194+
Path: contentPath.String(),
195+
Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
196+
BackLink: backLink,
197+
Hash: hash,
198+
HTMLDirListingLimit: i.config.HTMLDirListingLimit,
206199
}
207200

208201
logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash)

‎core/corehttp/gateway_indexPage.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,15 @@ import (
1212

1313
// structs for directory listing
1414
type listingTemplateData struct {
15-
GatewayURL string
16-
DNSLink bool
17-
Listing []directoryItem
18-
Size string
19-
Path string
20-
Breadcrumbs []breadcrumb
21-
BackLink string
22-
Hash string
23-
WarnMaxDirectorySize string
15+
GatewayURL string
16+
DNSLink bool
17+
Listing []directoryItem
18+
Size string
19+
Path string
20+
Breadcrumbs []breadcrumb
21+
BackLink string
22+
Hash string
23+
HTMLDirListingLimit int
2424
}
2525

2626
type directoryItem struct {

0 commit comments

Comments
 (0)