Skip to content

Commit 03ef327

Browse files
hacdiashacdias
committed
docs: add example of gateway backed by CAR file
1 parent c7cc8b8 commit 03ef327

File tree

10 files changed

+2104
-0
lines changed

10 files changed

+2104
-0
lines changed

‎.github/workflows/test-examples.yml

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
on: [push, pull_request]
2+
name: Go Test Examples
3+
4+
jobs:
5+
unit:
6+
defaults:
7+
run:
8+
working-directory: examples
9+
strategy:
10+
fail-fast: false
11+
matrix:
12+
os: [ "ubuntu", "windows", "macos" ]
13+
go: [ "1.18.x", "1.19.x" ]
14+
env:
15+
COVERAGES: ""
16+
runs-on: ${{ format('{0}-latest', matrix.os) }}
17+
name: ${{ matrix.os }} (go ${{ matrix.go }})
18+
steps:
19+
- uses: actions/checkout@v3
20+
with:
21+
submodules: recursive
22+
- uses: actions/setup-go@v3
23+
with:
24+
go-version: ${{ matrix.go }}
25+
- name: Go information
26+
run: |
27+
go version
28+
go env
29+
- name: Use msys2 on windows
30+
if: ${{ matrix.os == 'windows' }}
31+
shell: bash
32+
# The executable for msys2 is also called bash.cmd
33+
# https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md#shells
34+
# If we prepend its location to the PATH
35+
# subsequent 'shell: bash' steps will use msys2 instead of gitbash
36+
run: echo "C:/msys64/usr/bin" >> $GITHUB_PATH
37+
- name: Run tests
38+
uses: protocol/[email protected]
39+
with:
40+
run: go test -v -shuffle=on ./...
41+
- name: Run tests (32 bit)
42+
if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX.
43+
uses: protocol/[email protected]
44+
env:
45+
GOARCH: 386
46+
with:
47+
run: |
48+
export "PATH=${{ env.PATH_386 }}:$PATH"
49+
go test -v -shuffle=on ./...
50+
- name: Run tests with race detector
51+
if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow
52+
uses: protocol/[email protected]
53+
with:
54+
run: go test -v -race ./...

‎examples/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
gateway

‎examples/README.md

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# go-libipfs examples and tutorials
2+
3+
In this folder, you can find some examples to help you get started using go-libipfs and its associated libraries in your applications.
4+
5+
Let us know if you find any issue or if you want to contribute and add a new tutorial, feel welcome to submit a pr, thank you!
6+
7+
## Examples and Tutorials
8+
9+
- [Gateway backed by a CAR file](./gateway-car)

‎examples/gateway-car/README.md

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Gateway backed by a CAR File
2+
3+
This is an example that shows how to build a Gateway backed by the contents of
4+
a CAR file. A [CAR file](https://ipld.io/specs/transport/car/) is a Content
5+
Addressable aRchive that contains blocks.
6+
7+
## Build
8+
9+
```bash
10+
> go build -o gateway
11+
```
12+
13+
## Usage
14+
15+
First of all, you will need some content stored as a CAR file. You can easily
16+
export your favorite website, or content, using:
17+
18+
```
19+
ipfs dag export <CID> > data.car
20+
```
21+
22+
Then, you can start the gateway with:
23+
24+
25+
```
26+
./gateway -c data.car -p 8040
27+
```
28+
29+
Now you can access the gateway in [localhost:8040](http://localhost:8040). It will
30+
behave like a regular IPFS Gateway, except for the fact that all contents are provided
31+
from the CAR file. Therefore, things such as IPNS resolution and fetching contents
32+
from nodes in the IPFS network won't work.

‎examples/gateway-car/api.go

+174
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
gopath "path"
8+
9+
"github.com/ipfs/go-blockservice"
10+
"github.com/ipfs/go-cid"
11+
bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice"
12+
blockstore "github.com/ipfs/go-ipfs-blockstore"
13+
format "github.com/ipfs/go-ipld-format"
14+
"github.com/ipfs/go-libipfs/blocks"
15+
"github.com/ipfs/go-libipfs/files"
16+
"github.com/ipfs/go-merkledag"
17+
ipfspath "github.com/ipfs/go-path"
18+
"github.com/ipfs/go-path/resolver"
19+
"github.com/ipfs/go-unixfs"
20+
ufile "github.com/ipfs/go-unixfs/file"
21+
uio "github.com/ipfs/go-unixfs/io"
22+
"github.com/ipfs/go-unixfsnode"
23+
iface "github.com/ipfs/interface-go-ipfs-core"
24+
ifacepath "github.com/ipfs/interface-go-ipfs-core/path"
25+
dagpb "github.com/ipld/go-codec-dagpb"
26+
"github.com/ipld/go-ipld-prime"
27+
"github.com/ipld/go-ipld-prime/node/basicnode"
28+
"github.com/ipld/go-ipld-prime/schema"
29+
)
30+
31+
type blocksGateway struct {
32+
blockStore blockstore.Blockstore
33+
blockService blockservice.BlockService
34+
dagService format.DAGService
35+
resolver resolver.Resolver
36+
}
37+
38+
func newBlocksGateway(blockService blockservice.BlockService) (*blocksGateway, error) {
39+
// Setup the DAG services, which use the CAR block store.
40+
dagService := merkledag.NewDAGService(blockService)
41+
42+
// Setup the UnixFS resolver.
43+
fetcherConfig := bsfetcher.NewFetcherConfig(blockService)
44+
fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
45+
if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
46+
return tlnkNd.LinkTargetNodePrototype(), nil
47+
}
48+
return basicnode.Prototype.Any, nil
49+
})
50+
fetcher := fetcherConfig.WithReifier(unixfsnode.Reify)
51+
resolver := resolver.NewBasicResolver(fetcher)
52+
53+
return &blocksGateway{
54+
blockStore: blockService.Blockstore(),
55+
blockService: blockService,
56+
dagService: dagService,
57+
resolver: resolver,
58+
}, nil
59+
}
60+
61+
func (api *blocksGateway) GetUnixFsNode(ctx context.Context, p ifacepath.Resolved) (files.Node, error) {
62+
nd, err := api.resolveNode(ctx, p)
63+
if err != nil {
64+
return nil, err
65+
}
66+
67+
return ufile.NewUnixfsFile(ctx, api.dagService, nd)
68+
}
69+
70+
func (api *blocksGateway) LsUnixFsDir(ctx context.Context, p ifacepath.Resolved) (<-chan iface.DirEntry, error) {
71+
node, err := api.resolveNode(ctx, p)
72+
if err != nil {
73+
return nil, err
74+
}
75+
76+
dir, err := uio.NewDirectoryFromNode(api.dagService, node)
77+
if err != nil {
78+
return nil, err
79+
}
80+
81+
out := make(chan iface.DirEntry, uio.DefaultShardWidth)
82+
83+
go func() {
84+
defer close(out)
85+
for l := range dir.EnumLinksAsync(ctx) {
86+
select {
87+
case out <- api.processLink(ctx, l):
88+
case <-ctx.Done():
89+
return
90+
}
91+
}
92+
}()
93+
94+
return out, nil
95+
}
96+
97+
func (api *blocksGateway) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
98+
return api.blockService.GetBlock(ctx, c)
99+
}
100+
101+
func (api *blocksGateway) GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) {
102+
return nil, errors.New("not implemented")
103+
}
104+
105+
func (api *blocksGateway) IsCached(ctx context.Context, p ifacepath.Path) bool {
106+
rp, err := api.ResolvePath(ctx, p)
107+
if err != nil {
108+
return false
109+
}
110+
111+
has, _ := api.blockStore.Has(ctx, rp.Cid())
112+
return has
113+
}
114+
115+
func (api *blocksGateway) ResolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) {
116+
if _, ok := p.(ifacepath.Resolved); ok {
117+
return p.(ifacepath.Resolved), nil
118+
}
119+
120+
if err := p.IsValid(); err != nil {
121+
return nil, err
122+
}
123+
124+
if p.Namespace() != "ipfs" {
125+
return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace())
126+
}
127+
128+
ipath := ipfspath.Path(p.String())
129+
node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath)
130+
if err != nil {
131+
return nil, err
132+
}
133+
134+
root, err := cid.Parse(ipath.Segments()[1])
135+
if err != nil {
136+
return nil, err
137+
}
138+
139+
return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil
140+
}
141+
142+
func (api *blocksGateway) resolveNode(ctx context.Context, p ifacepath.Path) (format.Node, error) {
143+
rp, err := api.ResolvePath(ctx, p)
144+
if err != nil {
145+
return nil, err
146+
}
147+
148+
node, err := api.dagService.Get(ctx, rp.Cid())
149+
if err != nil {
150+
return nil, fmt.Errorf("get node: %w", err)
151+
}
152+
return node, nil
153+
}
154+
155+
func (api *blocksGateway) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry {
156+
if result.Err != nil {
157+
return iface.DirEntry{Err: result.Err}
158+
}
159+
160+
link := iface.DirEntry{
161+
Name: result.Link.Name,
162+
Cid: result.Link.Cid,
163+
}
164+
165+
switch link.Cid.Type() {
166+
case cid.Raw:
167+
link.Type = iface.TFile
168+
link.Size = result.Link.Size
169+
case cid.DagProtobuf:
170+
link.Size = result.Link.Size
171+
}
172+
173+
return link
174+
}

‎examples/gateway-car/main.go

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"io"
6+
"log"
7+
"net/http"
8+
"os"
9+
"strconv"
10+
11+
"github.com/ipfs/go-blockservice"
12+
offline "github.com/ipfs/go-ipfs-exchange-offline"
13+
"github.com/ipfs/go-libipfs/gateway"
14+
carblockstore "github.com/ipld/go-car/v2/blockstore"
15+
)
16+
17+
func main() {
18+
carFilePtr := flag.String("c", "", "path to CAR file to back this gateway from")
19+
portPtr := flag.Int("p", 8080, "port to run this gateway from")
20+
flag.Parse()
21+
22+
blockService, f, err := newBlockServiceFromCAR(*carFilePtr)
23+
if err != nil {
24+
log.Fatal(err)
25+
}
26+
defer f.Close()
27+
28+
gateway, err := newBlocksGateway(blockService)
29+
if err != nil {
30+
log.Fatal(err)
31+
}
32+
33+
handler := newHandler(gateway, *portPtr)
34+
35+
address := ":" + strconv.Itoa(*portPtr)
36+
log.Printf("Listening on %s", address)
37+
38+
if err := http.ListenAndServe(address, handler); err != nil {
39+
log.Fatal(err)
40+
}
41+
}
42+
43+
func newBlockServiceFromCAR(filepath string) (blockservice.BlockService, io.Closer, error) {
44+
r, err := os.Open(filepath)
45+
if err != nil {
46+
return nil, nil, err
47+
}
48+
49+
bs, err := carblockstore.NewReadOnly(r, nil)
50+
if err != nil {
51+
_ = r.Close()
52+
return nil, nil, err
53+
}
54+
55+
blockService := blockservice.New(bs, offline.Exchange(bs))
56+
return blockService, r, nil
57+
}
58+
59+
func newHandler(gw *blocksGateway, port int) http.Handler {
60+
headers := map[string][]string{}
61+
gateway.AddAccessControlHeaders(headers)
62+
63+
conf := gateway.Config{
64+
Headers: headers,
65+
}
66+
67+
mux := http.NewServeMux()
68+
gwHandler := gateway.NewHandler(conf, gw)
69+
mux.Handle("/ipfs/", gwHandler)
70+
mux.Handle("/ipns/", gwHandler)
71+
return mux
72+
}

0 commit comments

Comments
 (0)