Skip to content

Commit 4cabdfe

Browse files
authored
feat(gateway): Block and CAR response formats (#8758)
* feat: serveRawBlock implements ?format=block * feat: serveCar implements ?format=car * feat(gw): ?format= or Accept HTTP header - extracted file-like content type responses to separate .go files - Accept HTTP header with support for application/vnd.ipld.* types * fix: use .bin for raw block content-disposition .raw may be handled by something, depending on OS, and .bin seems to be universally "binary file" across all systems: https://en.wikipedia.org/wiki/List_of_filename_extensions_(A%E2%80%93E) * refactor: gateway_handler_unixfs.go - Moved UnixFS response handling to gateway_handler_unixfs*.go files. - Removed support for X-Ipfs-Gateway-Prefix (Closes #7702) * refactor: prefix cleanup and readable paths - removed dead code after X-Ipfs-Gateway-Prefix is gone (#7702) - escaped special characters in content paths returned with http.Error making them both safer and easier to reason about (e.g. when invisible whitespace Unicode is used)
1 parent 6774ef9 commit 4cabdfe

14 files changed

+992
-404
lines changed

core/corehttp/gateway_handler.go

+166-308
Large diffs are not rendered by default.
+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package corehttp
2+
3+
import (
4+
"bytes"
5+
"io/ioutil"
6+
"net/http"
7+
8+
cid "github.com/ipfs/go-cid"
9+
ipath "github.com/ipfs/interface-go-ipfs-core/path"
10+
)
11+
12+
// serveRawBlock returns bytes behind a raw block
13+
func (i *gatewayHandler) serveRawBlock(w http.ResponseWriter, r *http.Request, blockCid cid.Cid, contentPath ipath.Path) {
14+
blockReader, err := i.api.Block().Get(r.Context(), contentPath)
15+
if err != nil {
16+
webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError)
17+
return
18+
}
19+
block, err := ioutil.ReadAll(blockReader)
20+
if err != nil {
21+
webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError)
22+
return
23+
}
24+
content := bytes.NewReader(block)
25+
26+
// Set Content-Disposition
27+
name := blockCid.String() + ".bin"
28+
setContentDispositionHeader(w, name, "attachment")
29+
30+
// Set remaining headers
31+
modtime := addCacheControlHeaders(w, r, contentPath, blockCid)
32+
w.Header().Set("Content-Type", "application/vnd.ipld.raw")
33+
w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)
34+
35+
// Done: http.ServeContent will take care of
36+
// If-None-Match+Etag, Content-Length and range requests
37+
http.ServeContent(w, r, name, modtime, content)
38+
}

core/corehttp/gateway_handler_car.go

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package corehttp
2+
3+
import (
4+
"context"
5+
"net/http"
6+
7+
blocks "github.com/ipfs/go-block-format"
8+
cid "github.com/ipfs/go-cid"
9+
coreiface "github.com/ipfs/interface-go-ipfs-core"
10+
ipath "github.com/ipfs/interface-go-ipfs-core/path"
11+
gocar "github.com/ipld/go-car"
12+
selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
13+
)
14+
15+
// serveCar returns a CAR stream for specific DAG+selector
16+
func (i *gatewayHandler) serveCar(w http.ResponseWriter, r *http.Request, rootCid cid.Cid, contentPath ipath.Path) {
17+
ctx, cancel := context.WithCancel(r.Context())
18+
defer cancel()
19+
20+
// Set Content-Disposition
21+
name := rootCid.String() + ".car"
22+
setContentDispositionHeader(w, name, "attachment")
23+
24+
// Weak Etag W/ because we can't guarantee byte-for-byte identical responses
25+
// (CAR is streamed, and in theory, blocks may arrive from datastore in non-deterministic order)
26+
etag := `W/` + getEtag(r, rootCid)
27+
w.Header().Set("Etag", etag)
28+
29+
// Finish early if Etag match
30+
if r.Header.Get("If-None-Match") == etag {
31+
w.WriteHeader(http.StatusNotModified)
32+
return
33+
}
34+
35+
// Make it clear we don't support range-requests over a car stream
36+
// Partial downloads and resumes should be handled using
37+
// IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769
38+
w.Header().Set("Accept-Ranges", "none")
39+
40+
// Explicit Cache-Control to ensure fresh stream on retry.
41+
// CAR stream could be interrupted, and client should be able to resume and get full response, not the truncated one
42+
w.Header().Set("Cache-Control", "no-cache, no-transform")
43+
44+
w.Header().Set("Content-Type", "application/vnd.ipld.car; version=1")
45+
w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)
46+
47+
// Same go-car settings as dag.export command
48+
store := dagStore{dag: i.api.Dag(), ctx: ctx}
49+
50+
// TODO: support selectors passed as request param: https://github.com/ipfs/go-ipfs/issues/8769
51+
dag := gocar.Dag{Root: rootCid, Selector: selectorparse.CommonSelector_ExploreAllRecursively}
52+
car := gocar.NewSelectiveCar(ctx, store, []gocar.Dag{dag}, gocar.TraverseLinksOnlyOnce())
53+
54+
if err := car.Write(w); err != nil {
55+
// We return error as a trailer, however it is not something browsers can access
56+
// (https://github.com/mdn/browser-compat-data/issues/14703)
57+
// Due to this, we suggest client always verify that
58+
// the received CAR stream response is matching requested DAG selector
59+
w.Header().Set("X-Stream-Error", err.Error())
60+
return
61+
}
62+
}
63+
64+
type dagStore struct {
65+
dag coreiface.APIDagService
66+
ctx context.Context
67+
}
68+
69+
func (ds dagStore) Get(c cid.Cid) (blocks.Block, error) {
70+
obj, err := ds.dag.Get(ds.ctx, c)
71+
return obj, err
72+
}
+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package corehttp
2+
3+
import (
4+
"fmt"
5+
"html"
6+
"net/http"
7+
8+
files "github.com/ipfs/go-ipfs-files"
9+
ipath "github.com/ipfs/interface-go-ipfs-core/path"
10+
"go.uber.org/zap"
11+
)
12+
13+
func (i *gatewayHandler) serveUnixFs(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, logger *zap.SugaredLogger) {
14+
// Handling UnixFS
15+
dr, err := i.api.Unixfs().Get(r.Context(), resolvedPath)
16+
if err != nil {
17+
webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound)
18+
return
19+
}
20+
defer dr.Close()
21+
22+
// Handling Unixfs file
23+
if f, ok := dr.(files.File); ok {
24+
logger.Debugw("serving unixfs file", "path", contentPath)
25+
i.serveFile(w, r, contentPath, resolvedPath.Cid(), f)
26+
return
27+
}
28+
29+
// Handling Unixfs directory
30+
dir, ok := dr.(files.Directory)
31+
if !ok {
32+
internalWebError(w, fmt.Errorf("unsupported UnixFs type"))
33+
return
34+
}
35+
logger.Debugw("serving unixfs directory", "path", contentPath)
36+
i.serveDirectory(w, r, resolvedPath, contentPath, dir, logger)
37+
}
+197
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
package corehttp
2+
3+
import (
4+
"net/http"
5+
"net/url"
6+
gopath "path"
7+
"strings"
8+
9+
"github.com/dustin/go-humanize"
10+
files "github.com/ipfs/go-ipfs-files"
11+
"github.com/ipfs/go-ipfs/assets"
12+
path "github.com/ipfs/go-path"
13+
"github.com/ipfs/go-path/resolver"
14+
ipath "github.com/ipfs/interface-go-ipfs-core/path"
15+
"go.uber.org/zap"
16+
)
17+
18+
// serveDirectory returns the best representation of UnixFS directory
19+
//
20+
// It will return index.html if present, or generate directory listing otherwise.
21+
func (i *gatewayHandler) serveDirectory(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, logger *zap.SugaredLogger) {
22+
23+
// HostnameOption might have constructed an IPNS/IPFS path using the Host header.
24+
// In this case, we need the original path for constructing redirects
25+
// and links that match the requested URL.
26+
// For example, http://example.net would become /ipns/example.net, and
27+
// the redirects and links would end up as http://example.net/ipns/example.net
28+
requestURI, err := url.ParseRequestURI(r.RequestURI)
29+
if err != nil {
30+
webError(w, "failed to parse request path", err, http.StatusInternalServerError)
31+
return
32+
}
33+
originalUrlPath := requestURI.Path
34+
35+
// Check if directory has index.html, if so, serveFile
36+
idxPath := ipath.Join(resolvedPath, "index.html")
37+
idx, err := i.api.Unixfs().Get(r.Context(), idxPath)
38+
switch err.(type) {
39+
case nil:
40+
cpath := contentPath.String()
41+
dirwithoutslash := cpath[len(cpath)-1] != '/'
42+
goget := r.URL.Query().Get("go-get") == "1"
43+
if dirwithoutslash && !goget {
44+
// See comment above where originalUrlPath is declared.
45+
suffix := "/"
46+
if r.URL.RawQuery != "" {
47+
// preserve query parameters
48+
suffix = suffix + "?" + r.URL.RawQuery
49+
}
50+
51+
redirectURL := originalUrlPath + suffix
52+
logger.Debugw("serving index.html file", "to", redirectURL, "status", http.StatusFound, "path", idxPath)
53+
http.Redirect(w, r, redirectURL, http.StatusFound)
54+
return
55+
}
56+
57+
f, ok := idx.(files.File)
58+
if !ok {
59+
internalWebError(w, files.ErrNotReader)
60+
return
61+
}
62+
63+
logger.Debugw("serving index.html file", "path", idxPath)
64+
// write to request
65+
i.serveFile(w, r, idxPath, resolvedPath.Cid(), f)
66+
return
67+
case resolver.ErrNoLink:
68+
logger.Debugw("no index.html; noop", "path", idxPath)
69+
default:
70+
internalWebError(w, err)
71+
return
72+
}
73+
74+
// See statusResponseWriter.WriteHeader
75+
// and https://github.com/ipfs/go-ipfs/issues/7164
76+
// Note: this needs to occur before listingTemplate.Execute otherwise we get
77+
// superfluous response.WriteHeader call from prometheus/client_golang
78+
if w.Header().Get("Location") != "" {
79+
logger.Debugw("location moved permanently", "status", http.StatusMovedPermanently)
80+
w.WriteHeader(http.StatusMovedPermanently)
81+
return
82+
}
83+
84+
// A HTML directory index will be presented, be sure to set the correct
85+
// type instead of relying on autodetection (which may fail).
86+
w.Header().Set("Content-Type", "text/html")
87+
88+
// Generated dir index requires custom Etag (it may change between go-ipfs versions)
89+
if assets.BindataVersionHash != "" {
90+
dirEtag := `"DirIndex-` + assets.BindataVersionHash + `_CID-` + resolvedPath.Cid().String() + `"`
91+
w.Header().Set("Etag", dirEtag)
92+
if r.Header.Get("If-None-Match") == dirEtag {
93+
w.WriteHeader(http.StatusNotModified)
94+
return
95+
}
96+
}
97+
98+
if r.Method == http.MethodHead {
99+
logger.Debug("return as request's HTTP method is HEAD")
100+
return
101+
}
102+
103+
// storage for directory listing
104+
var dirListing []directoryItem
105+
dirit := dir.Entries()
106+
for dirit.Next() {
107+
size := "?"
108+
if s, err := dirit.Node().Size(); err == nil {
109+
// Size may not be defined/supported. Continue anyways.
110+
size = humanize.Bytes(uint64(s))
111+
}
112+
113+
resolved, err := i.api.ResolvePath(r.Context(), ipath.Join(resolvedPath, dirit.Name()))
114+
if err != nil {
115+
internalWebError(w, err)
116+
return
117+
}
118+
hash := resolved.Cid().String()
119+
120+
// See comment above where originalUrlPath is declared.
121+
di := directoryItem{
122+
Size: size,
123+
Name: dirit.Name(),
124+
Path: gopath.Join(originalUrlPath, dirit.Name()),
125+
Hash: hash,
126+
ShortHash: shortHash(hash),
127+
}
128+
dirListing = append(dirListing, di)
129+
}
130+
if dirit.Err() != nil {
131+
internalWebError(w, dirit.Err())
132+
return
133+
}
134+
135+
// construct the correct back link
136+
// https://github.com/ipfs/go-ipfs/issues/1365
137+
var backLink string = originalUrlPath
138+
139+
// don't go further up than /ipfs/$hash/
140+
pathSplit := path.SplitList(contentPath.String())
141+
switch {
142+
// keep backlink
143+
case len(pathSplit) == 3: // url: /ipfs/$hash
144+
145+
// keep backlink
146+
case len(pathSplit) == 4 && pathSplit[3] == "": // url: /ipfs/$hash/
147+
148+
// add the correct link depending on whether the path ends with a slash
149+
default:
150+
if strings.HasSuffix(backLink, "/") {
151+
backLink += "./.."
152+
} else {
153+
backLink += "/.."
154+
}
155+
}
156+
157+
size := "?"
158+
if s, err := dir.Size(); err == nil {
159+
// Size may not be defined/supported. Continue anyways.
160+
size = humanize.Bytes(uint64(s))
161+
}
162+
163+
hash := resolvedPath.Cid().String()
164+
165+
// Gateway root URL to be used when linking to other rootIDs.
166+
// This will be blank unless subdomain or DNSLink resolution is being used
167+
// for this request.
168+
var gwURL string
169+
170+
// Get gateway hostname and build gateway URL.
171+
if h, ok := r.Context().Value("gw-hostname").(string); ok {
172+
gwURL = "//" + h
173+
} else {
174+
gwURL = ""
175+
}
176+
177+
dnslink := hasDNSLinkOrigin(gwURL, contentPath.String())
178+
179+
// See comment above where originalUrlPath is declared.
180+
tplData := listingTemplateData{
181+
GatewayURL: gwURL,
182+
DNSLink: dnslink,
183+
Listing: dirListing,
184+
Size: size,
185+
Path: contentPath.String(),
186+
Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
187+
BackLink: backLink,
188+
Hash: hash,
189+
}
190+
191+
logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash)
192+
193+
if err := listingTemplate.Execute(w, tplData); err != nil {
194+
internalWebError(w, err)
195+
return
196+
}
197+
}

0 commit comments

Comments
 (0)