-
-
Notifications
You must be signed in to change notification settings - Fork 3.1k
feat(gateway): JSON and CBOR response formats (IPIP-328) #9335
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Changes from 32 commits
Commits
Show all changes
39 commits
Select commit
Hold shift + click to select a range
ac8f9b1
wip: play with dag-cbor and dag-json
hacdias 3dce012
wip: add application/json and application/cbor
hacdias b557181
fix: go cases don't flow automatically :)
hacdias 4104bb7
test: add some dag-json and dag-cbor tests
hacdias 25893c5
test: improve names
hacdias e621e64
feat: allow json and cbor data types too
hacdias 44946ed
refactor: avoid encoding things that are already on their right encoding
hacdias 199ab42
fix: remove responseFormat from logging
hacdias 89eb033
refactor: simplify serveCodec to use serveRawBlock iff data encoded i…
hacdias fb50869
tests: rename current tests to indicate they're unixfs only
hacdias fc31241
refactor: do not use serveRawBlock inside serveCodec bc headers and o…
hacdias 55383cd
test: add test with pure json and cbor
hacdias a6d45c7
test: convert cbor <-> json
hacdias 1986be1
test: path traversal and dag-pb output
hacdias 9ef022e
fix: add more info about errors
hacdias cadc681
fix: add missing traversal
hacdias 6331695
Merge branch 'master' into feat/8823
hacdias 2c93672
fix: remove duplicate variable
hacdias 462c71b
Merge branch 'master' into feat/8823
hacdias 1e844c5
refactor: do not support traversal
hacdias bb98041
Update core/corehttp/gateway_handler_codec.go
hacdias 53d5878
improve PR to match spec
hacdias 4064f97
Merge branch 'master' into feat/8823
hacdias 8c6a8da
feat: little web page
hacdias b5e5ff2
feat: update doc
hacdias 8ca2a52
fix: Content-Disposition .json and .cbor
lidel b4dfa66
fix: inline disposition for JSON responses
lidel 83913c7
refactor: return 501 for unsupported pathing
lidel ff55745
docs(cbor): improved info about codec
lidel 52711d3
refactor: create template at assets/dag-index-html
lidel 7e84856
fix(dag@gw): content type and cache headers
lidel 3b89f20
Merge branch 'master' into feat/8823
hacdias 162f435
add changelog info
hacdias 12d0d7f
fix title
hacdias 2d8ba78
rm wild block
hacdias b5874e7
Merge branch 'master' into feat/8823
hacdias 0c08a76
Merge branch 'master' into feat/8823
hacdias f084f09
fix(dag-index-html): remove technical jargon
lidel 32bcd41
Merge branch 'master' into feat/8823
lidel File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| # dag-index-html | ||
|
|
||
| > HTML representation for non-UnixFS DAGs such as DAG-CBOR. | ||
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,258 @@ | ||
| package corehttp | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "fmt" | ||
| "html" | ||
| "io" | ||
| "net/http" | ||
| "strings" | ||
| "time" | ||
|
|
||
| cid "github.com/ipfs/go-cid" | ||
| ipldlegacy "github.com/ipfs/go-ipld-legacy" | ||
| ipath "github.com/ipfs/interface-go-ipfs-core/path" | ||
| "github.com/ipfs/kubo/assets" | ||
| dih "github.com/ipfs/kubo/assets/dag-index-html" | ||
| "github.com/ipfs/kubo/tracing" | ||
| "github.com/ipld/go-ipld-prime" | ||
| "github.com/ipld/go-ipld-prime/multicodec" | ||
| mc "github.com/multiformats/go-multicodec" | ||
| "go.opentelemetry.io/otel/attribute" | ||
| "go.opentelemetry.io/otel/trace" | ||
| ) | ||
|
|
||
| // codecToContentType maps the supported IPLD codecs to the HTTP Content | ||
| // Type they should have. | ||
| var codecToContentType = map[uint64]string{ | ||
| uint64(mc.Json): "application/json", | ||
| uint64(mc.Cbor): "application/cbor", | ||
| uint64(mc.DagJson): "application/vnd.ipld.dag-json", | ||
| uint64(mc.DagCbor): "application/vnd.ipld.dag-cbor", | ||
| } | ||
|
|
||
| // contentTypeToCodecs maps the HTTP Content Type to the respective | ||
| // possible codecs. If the original data is in one of those codecs, | ||
| // we stream the raw bytes. Otherwise, we encode in the last codec | ||
| // of the list. | ||
| var contentTypeToCodecs = map[string][]uint64{ | ||
| "application/json": {uint64(mc.Json), uint64(mc.DagJson)}, | ||
| "application/vnd.ipld.dag-json": {uint64(mc.DagJson)}, | ||
| "application/cbor": {uint64(mc.Cbor), uint64(mc.DagCbor)}, | ||
| "application/vnd.ipld.dag-cbor": {uint64(mc.DagCbor)}, | ||
| } | ||
|
|
||
| // contentTypeToExtension maps the HTTP Content Type to the respective file | ||
| // extension, used in Content-Disposition header when downloading the file. | ||
| var contentTypeToExtension = map[string]string{ | ||
| "application/json": ".json", | ||
| "application/vnd.ipld.dag-json": ".json", | ||
| "application/cbor": ".cbor", | ||
| "application/vnd.ipld.dag-cbor": ".cbor", | ||
| } | ||
|
|
||
| func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, requestedContentType string) { | ||
| ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) | ||
| defer span.End() | ||
|
|
||
| cidCodec := resolvedPath.Cid().Prefix().Codec | ||
| responseContentType := requestedContentType | ||
|
|
||
| // If the resolved path still has some remainder, return error for now. | ||
| // TODO: handle this when we have IPLD Patch (https://ipld.io/specs/patch/) via HTTP PUT | ||
| // TODO: (depends on https://github.com/ipfs/kubo/issues/4801 and https://github.com/ipfs/kubo/issues/4782) | ||
| if resolvedPath.Remainder() != "" { | ||
| path := strings.TrimSuffix(resolvedPath.String(), resolvedPath.Remainder()) | ||
| err := fmt.Errorf("%q of %q could not be returned: reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented: try reading %q instead", resolvedPath.Remainder(), resolvedPath.String(), path) | ||
| webError(w, "unsupported pathing", err, http.StatusNotImplemented) | ||
| return | ||
| } | ||
|
|
||
| // If no explicit content type was requested, the response will have one based on the codec from the CID | ||
| if requestedContentType == "" { | ||
| cidContentType, ok := codecToContentType[cidCodec] | ||
| if !ok { | ||
| // Should not happen unless function is called with wrong parameters. | ||
| err := fmt.Errorf("content type not found for codec: %v", cidCodec) | ||
| webError(w, "internal error", err, http.StatusInternalServerError) | ||
| return | ||
| } | ||
| responseContentType = cidContentType | ||
| } | ||
|
|
||
| // Set HTTP headers (for caching etc) | ||
| modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) | ||
| name := setCodecContentDisposition(w, r, resolvedPath, responseContentType) | ||
| w.Header().Set("Content-Type", responseContentType) | ||
| w.Header().Set("X-Content-Type-Options", "nosniff") | ||
|
|
||
| // No content type is specified by the user (via Accept, or format=). However, | ||
| // we support this format. Let's handle it. | ||
| if requestedContentType == "" { | ||
| isDAG := cidCodec == uint64(mc.DagJson) || cidCodec == uint64(mc.DagCbor) | ||
| acceptsHTML := strings.Contains(r.Header.Get("Accept"), "text/html") | ||
| download := r.URL.Query().Get("download") == "true" | ||
|
|
||
| if isDAG && acceptsHTML && !download { | ||
| i.serveCodecHTML(ctx, w, r, resolvedPath, contentPath) | ||
| } else { | ||
| i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime) | ||
| } | ||
|
|
||
| return | ||
| } | ||
|
|
||
| // Otherwise, the user has requested a specific content type. Let's first get | ||
| // the codecs that can be used with this content type. | ||
| codecs, ok := contentTypeToCodecs[requestedContentType] | ||
| if !ok { | ||
| // This is never supposed to happen unless function is called with wrong parameters. | ||
| err := fmt.Errorf("unsupported content type: %s", requestedContentType) | ||
| webError(w, err.Error(), err, http.StatusInternalServerError) | ||
| return | ||
| } | ||
|
|
||
| // If we need to convert, use the last codec (strict dag- variant) | ||
| toCodec := codecs[len(codecs)-1] | ||
|
|
||
| // If the requested content type has "dag-", ALWAYS go through the encoding | ||
| // process in order to validate the content. | ||
| if strings.Contains(requestedContentType, "dag-") { | ||
| i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime) | ||
| return | ||
| } | ||
|
|
||
| // Otherwise, check if the data is encoded with the requested content type. | ||
| // If so, we can directly stream the raw data. serveRawBlock cannot be directly | ||
| // used here as it sets different headers. | ||
| for _, codec := range codecs { | ||
| if resolvedPath.Cid().Prefix().Codec == codec { | ||
| i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime) | ||
| return | ||
| } | ||
| } | ||
|
|
||
| // Finally, if nothing of the above is true, we have to actually convert the codec. | ||
| i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime) | ||
| } | ||
|
|
||
| func (i *gatewayHandler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) { | ||
| // A HTML directory index will be presented, be sure to set the correct | ||
| // type instead of relying on autodetection (which may fail). | ||
| w.Header().Set("Content-Type", "text/html") | ||
|
|
||
| // Clear Content-Disposition -- we want HTML to be rendered inline | ||
| w.Header().Del("Content-Disposition") | ||
|
|
||
| // Generated index requires custom Etag (output may change between Kubo versions) | ||
| dagEtag := getDagIndexEtag(resolvedPath.Cid()) | ||
| w.Header().Set("Etag", dagEtag) | ||
|
|
||
| // Remove Cache-Control for now to match UnixFS dir-index-html responses | ||
| // (we don't want browser to cache HTML forever) | ||
| // TODO: if we ever change behavior for UnixFS dir listings, same changes should be applied here | ||
| w.Header().Del("Cache-Control") | ||
|
|
||
| cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec) | ||
| if err := dih.DagIndexTemplate.Execute(w, dih.DagIndexTemplateData{ | ||
| Path: contentPath.String(), | ||
| CID: resolvedPath.Cid().String(), | ||
| CodecName: cidCodec.String(), | ||
| CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), | ||
| }); err != nil { | ||
| webError(w, "failed to generate HTML listing for this DAG: try fetching raw block with ?format=raw", err, http.StatusInternalServerError) | ||
| } | ||
| } | ||
|
|
||
| func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, name string, modtime time.Time) { | ||
| blockCid := resolvedPath.Cid() | ||
| blockReader, err := i.api.Block().Get(ctx, resolvedPath) | ||
| if err != nil { | ||
| webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError) | ||
| return | ||
| } | ||
| block, err := io.ReadAll(blockReader) | ||
| if err != nil { | ||
| webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError) | ||
| return | ||
| } | ||
| content := bytes.NewReader(block) | ||
|
|
||
| // ServeContent will take care of | ||
| // If-None-Match+Etag, Content-Length and range requests | ||
| _, _, _ = ServeContent(w, r, name, modtime, content) | ||
| } | ||
|
|
||
| func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, toCodec uint64, modtime time.Time) { | ||
| obj, err := i.api.Dag().Get(ctx, resolvedPath.Cid()) | ||
| if err != nil { | ||
| webError(w, "ipfs dag get "+html.EscapeString(resolvedPath.String()), err, http.StatusInternalServerError) | ||
| return | ||
| } | ||
|
|
||
| universal, ok := obj.(ipldlegacy.UniversalNode) | ||
| if !ok { | ||
| err = fmt.Errorf("%T is not a valid IPLD node", obj) | ||
| webError(w, err.Error(), err, http.StatusInternalServerError) | ||
| return | ||
| } | ||
| finalNode := universal.(ipld.Node) | ||
|
|
||
| encoder, err := multicodec.LookupEncoder(toCodec) | ||
| if err != nil { | ||
| webError(w, err.Error(), err, http.StatusInternalServerError) | ||
| return | ||
| } | ||
|
|
||
| // Ensure IPLD node conforms to the codec specification. | ||
| var buf bytes.Buffer | ||
| err = encoder(finalNode, &buf) | ||
| if err != nil { | ||
| webError(w, err.Error(), err, http.StatusInternalServerError) | ||
| return | ||
| } | ||
|
|
||
| // Sets correct Last-Modified header. This code is borrowed from the standard | ||
| // library (net/http/server.go) as we cannot use serveFile. | ||
| if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) { | ||
| w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) | ||
| } | ||
|
|
||
| _, _ = w.Write(buf.Bytes()) | ||
| } | ||
|
|
||
| func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentType string) string { | ||
| var dispType, name string | ||
|
|
||
| ext, ok := contentTypeToExtension[contentType] | ||
| if !ok { | ||
| // Should never happen. | ||
| ext = ".bin" | ||
| } | ||
|
|
||
| if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" { | ||
| name = urlFilename | ||
| } else { | ||
| name = resolvedPath.Cid().String() + ext | ||
| } | ||
|
|
||
| // JSON should be inlined, but ?download=true should still override | ||
| if r.URL.Query().Get("download") == "true" { | ||
| dispType = "attachment" | ||
| } else { | ||
| switch ext { | ||
| case ".json": // codecs that serialize to JSON can be rendered by browsers | ||
| dispType = "inline" | ||
| default: // everything else is assumed binary / opaque bytes | ||
| dispType = "attachment" | ||
| } | ||
| } | ||
|
|
||
| setContentDispositionHeader(w, name, dispType) | ||
| return name | ||
| } | ||
|
|
||
| func getDagIndexEtag(dagCid cid.Cid) string { | ||
| return `"DagIndex-` + assets.AssetHash + `_CID-` + dagCid.String() + `"` | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.