Skip to content

Commit c5c8994

Browse files
committed
docs: add example of gateway backed by CAR file
1 parent c7cc8b8 commit c5c8994

File tree

6 files changed

+1908
-0
lines changed

6 files changed

+1908
-0
lines changed
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*.car
2+
gateway
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Gateway backed by a CAR File
2+
3+
This is an example that shows how to build a Gateway backed by the contents of
4+
a CAR file. A [CAR file](https://ipld.io/specs/transport/car/) is a Content
5+
Addressable aRchive that contains blocks.
6+
7+
## Build
8+
9+
```bash
10+
> go build -o gateway
11+
```
12+
13+
## Usage
14+
15+
First of all, you will need some content stored as a CAR file. You can easily
16+
export your favorite website, or content, using:
17+
18+
```
19+
ipfs dag export <CID> > data.car
20+
```
21+
22+
Then, you can start the gateway with:
23+
24+
25+
```
26+
./gateway -c data.car -p 8040
27+
```
28+
29+
Now you can access the gateway in [localhost:8040](http://localhost:8040). It will
30+
behave like a regular IPFS Gateway, except for the fact that all contents are provided
31+
from the CAR file. Therefore, things such as IPNS resolution and fetching contents
32+
from nodes in the IPFS network won't work.
Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"io"
8+
gopath "path"
9+
10+
"github.com/ipfs/go-blockservice"
11+
"github.com/ipfs/go-cid"
12+
bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice"
13+
blockstore "github.com/ipfs/go-ipfs-blockstore"
14+
offline "github.com/ipfs/go-ipfs-exchange-offline"
15+
format "github.com/ipfs/go-ipld-format"
16+
"github.com/ipfs/go-libipfs/blocks"
17+
"github.com/ipfs/go-libipfs/files"
18+
"github.com/ipfs/go-merkledag"
19+
ipfspath "github.com/ipfs/go-path"
20+
"github.com/ipfs/go-path/resolver"
21+
"github.com/ipfs/go-unixfs"
22+
ufile "github.com/ipfs/go-unixfs/file"
23+
uio "github.com/ipfs/go-unixfs/io"
24+
"github.com/ipfs/go-unixfsnode"
25+
iface "github.com/ipfs/interface-go-ipfs-core"
26+
ifacepath "github.com/ipfs/interface-go-ipfs-core/path"
27+
carblockstore "github.com/ipld/go-car/v2/blockstore"
28+
dagpb "github.com/ipld/go-codec-dagpb"
29+
"github.com/ipld/go-ipld-prime"
30+
"github.com/ipld/go-ipld-prime/node/basicnode"
31+
"github.com/ipld/go-ipld-prime/schema"
32+
)
33+
34+
type carGateway struct {
35+
blockStore blockstore.Blockstore
36+
blockService blockservice.BlockService
37+
dagService format.DAGService
38+
resolver resolver.Resolver
39+
}
40+
41+
func newCarGateway(r io.ReaderAt) (*carGateway, error) {
42+
// Setup CAR block store, which fetches the blocks directly from a CAR file.
43+
bs, err := carblockstore.NewReadOnly(r, nil)
44+
if err != nil {
45+
return nil, err
46+
}
47+
48+
// Setup the block and DAG services, which use the CAR block store. It is also
49+
// an offline block service, so it doesn't exchange data with any other peer.
50+
blockService := blockservice.New(bs, offline.Exchange(bs))
51+
dagService := merkledag.NewDAGService(blockService)
52+
53+
// Setup the UnixFS resolver.
54+
fetcherConfig := bsfetcher.NewFetcherConfig(blockService)
55+
fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
56+
if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
57+
return tlnkNd.LinkTargetNodePrototype(), nil
58+
}
59+
return basicnode.Prototype.Any, nil
60+
})
61+
fetcher := fetcherConfig.WithReifier(unixfsnode.Reify)
62+
resolver := resolver.NewBasicResolver(fetcher)
63+
64+
return &carGateway{
65+
blockStore: bs,
66+
blockService: blockService,
67+
dagService: dagService,
68+
resolver: resolver,
69+
}, nil
70+
}
71+
72+
func (api *carGateway) GetUnixFsNode(ctx context.Context, p ifacepath.Resolved) (files.Node, error) {
73+
nd, err := api.resolveNode(ctx, p)
74+
if err != nil {
75+
return nil, err
76+
}
77+
78+
return ufile.NewUnixfsFile(ctx, api.dagService, nd)
79+
}
80+
81+
func (api *carGateway) LsUnixFsDir(ctx context.Context, p ifacepath.Resolved) (<-chan iface.DirEntry, error) {
82+
node, err := api.resolveNode(ctx, p)
83+
if err != nil {
84+
return nil, err
85+
}
86+
87+
dir, err := uio.NewDirectoryFromNode(api.dagService, node)
88+
if err != nil {
89+
return nil, err
90+
}
91+
92+
out := make(chan iface.DirEntry, uio.DefaultShardWidth)
93+
94+
go func() {
95+
defer close(out)
96+
for l := range dir.EnumLinksAsync(ctx) {
97+
select {
98+
case out <- api.processLink(ctx, l):
99+
case <-ctx.Done():
100+
return
101+
}
102+
}
103+
}()
104+
105+
return out, nil
106+
}
107+
108+
func (api *carGateway) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
109+
return api.blockService.GetBlock(ctx, c)
110+
}
111+
112+
func (api *carGateway) GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) {
113+
return nil, errors.New("not implemented")
114+
}
115+
116+
func (api *carGateway) IsCached(ctx context.Context, p ifacepath.Path) bool {
117+
rp, err := api.ResolvePath(ctx, p)
118+
if err != nil {
119+
return false
120+
}
121+
122+
_, err = api.blockService.GetBlock(ctx, rp.Cid())
123+
return err == nil
124+
}
125+
126+
func (api *carGateway) ResolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) {
127+
if _, ok := p.(ifacepath.Resolved); ok {
128+
return p.(ifacepath.Resolved), nil
129+
}
130+
131+
if err := p.IsValid(); err != nil {
132+
return nil, err
133+
}
134+
135+
if p.Namespace() != "ipfs" {
136+
return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace())
137+
}
138+
139+
ipath := ipfspath.Path(p.String())
140+
node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath)
141+
if err != nil {
142+
return nil, err
143+
}
144+
145+
root, err := cid.Parse(ipath.Segments()[1])
146+
if err != nil {
147+
return nil, err
148+
}
149+
150+
return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil
151+
}
152+
153+
func (api *carGateway) resolveNode(ctx context.Context, p ifacepath.Path) (format.Node, error) {
154+
rp, err := api.ResolvePath(ctx, p)
155+
if err != nil {
156+
return nil, err
157+
}
158+
159+
node, err := api.dagService.Get(ctx, rp.Cid())
160+
if err != nil {
161+
return nil, fmt.Errorf("get node: %w", err)
162+
}
163+
return node, nil
164+
}
165+
166+
func (api *carGateway) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry {
167+
if result.Err != nil {
168+
return iface.DirEntry{Err: result.Err}
169+
}
170+
171+
link := iface.DirEntry{
172+
Name: result.Link.Name,
173+
Cid: result.Link.Cid,
174+
}
175+
176+
switch link.Cid.Type() {
177+
case cid.Raw:
178+
link.Type = iface.TFile
179+
link.Size = result.Link.Size
180+
case cid.DagProtobuf:
181+
link.Size = result.Link.Size
182+
}
183+
184+
return link
185+
}
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
module car-file-gateway
2+
3+
go 1.19
4+
5+
require (
6+
github.com/ipfs/go-blockservice v0.5.0
7+
github.com/ipfs/go-cid v0.3.2
8+
github.com/ipfs/go-fetcher v1.6.1
9+
github.com/ipfs/go-ipfs-blockstore v1.2.0
10+
github.com/ipfs/go-ipfs-exchange-offline v0.3.0
11+
github.com/ipfs/go-ipld-format v0.4.0
12+
github.com/ipfs/go-libipfs v0.4.0
13+
github.com/ipfs/go-merkledag v0.9.0
14+
github.com/ipfs/go-path v0.3.0
15+
github.com/ipfs/go-unixfs v0.3.1
16+
github.com/ipfs/go-unixfsnode v1.5.1
17+
github.com/ipfs/interface-go-ipfs-core v0.10.0
18+
github.com/ipld/go-car/v2 v2.6.0
19+
github.com/ipld/go-codec-dagpb v1.5.0
20+
github.com/ipld/go-ipld-prime v0.19.0
21+
)
22+
23+
require (
24+
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
25+
github.com/beorn7/perks v1.0.1 // indirect
26+
github.com/cespare/xxhash v1.1.0 // indirect
27+
github.com/cespare/xxhash/v2 v2.1.2 // indirect
28+
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
29+
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
30+
github.com/dustin/go-humanize v1.0.0 // indirect
31+
github.com/gabriel-vasile/mimetype v1.4.1 // indirect
32+
github.com/go-logr/logr v1.2.3 // indirect
33+
github.com/go-logr/stdr v1.2.2 // indirect
34+
github.com/gogo/protobuf v1.3.2 // indirect
35+
github.com/golang/protobuf v1.5.2 // indirect
36+
github.com/google/uuid v1.3.0 // indirect
37+
github.com/hashicorp/golang-lru v0.5.4 // indirect
38+
github.com/ipfs/bbloom v0.0.4 // indirect
39+
github.com/ipfs/go-bitfield v1.0.0 // indirect
40+
github.com/ipfs/go-block-format v0.1.1 // indirect
41+
github.com/ipfs/go-datastore v0.6.0 // indirect
42+
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
43+
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
44+
github.com/ipfs/go-ipfs-files v0.3.0 // indirect
45+
github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
46+
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
47+
github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
48+
github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
49+
github.com/ipfs/go-ipns v0.3.0 // indirect
50+
github.com/ipfs/go-log v1.0.5 // indirect
51+
github.com/ipfs/go-log/v2 v2.5.1 // indirect
52+
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
53+
github.com/ipfs/go-verifcid v0.0.2 // indirect
54+
github.com/ipld/go-car v0.5.0 // indirect
55+
github.com/jbenet/goprocess v0.1.4 // indirect
56+
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
57+
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
58+
github.com/libp2p/go-libp2p v0.23.4 // indirect
59+
github.com/libp2p/go-openssl v0.1.0 // indirect
60+
github.com/mattn/go-isatty v0.0.17 // indirect
61+
github.com/mattn/go-pointer v0.0.1 // indirect
62+
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
63+
github.com/minio/sha256-simd v1.0.0 // indirect
64+
github.com/mr-tron/base58 v1.2.0 // indirect
65+
github.com/multiformats/go-base32 v0.1.0 // indirect
66+
github.com/multiformats/go-base36 v0.2.0 // indirect
67+
github.com/multiformats/go-multiaddr v0.8.0 // indirect
68+
github.com/multiformats/go-multibase v0.1.1 // indirect
69+
github.com/multiformats/go-multicodec v0.7.0 // indirect
70+
github.com/multiformats/go-multihash v0.2.1 // indirect
71+
github.com/multiformats/go-varint v0.0.7 // indirect
72+
github.com/opentracing/opentracing-go v1.2.0 // indirect
73+
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
74+
github.com/pkg/errors v0.9.1 // indirect
75+
github.com/polydawn/refmt v0.89.0 // indirect
76+
github.com/prometheus/client_golang v1.13.0 // indirect
77+
github.com/prometheus/client_model v0.2.0 // indirect
78+
github.com/prometheus/common v0.37.0 // indirect
79+
github.com/prometheus/procfs v0.8.0 // indirect
80+
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
81+
github.com/spaolacci/murmur3 v1.1.0 // indirect
82+
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
83+
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
84+
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
85+
go.opentelemetry.io/otel v1.12.0 // indirect
86+
go.opentelemetry.io/otel/trace v1.12.0 // indirect
87+
go.uber.org/atomic v1.10.0 // indirect
88+
go.uber.org/multierr v1.9.0 // indirect
89+
go.uber.org/zap v1.24.0 // indirect
90+
golang.org/x/crypto v0.5.0 // indirect
91+
golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2 // indirect
92+
golang.org/x/net v0.5.0 // indirect
93+
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
94+
golang.org/x/sys v0.4.0 // indirect
95+
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
96+
google.golang.org/protobuf v1.28.1 // indirect
97+
lukechampine.com/blake3 v1.1.7 // indirect
98+
)
99+
100+
replace github.com/ipfs/go-libipfs => ../../..

0 commit comments

Comments
 (0)