Skip to content

Commit 5ae869f

Browse files
whyrusleepingwhyrusleeping
committed
implement an HAMT for unixfs directory sharding
License: MIT Signed-off-by: Jeromy <[email protected]>
1 parent 015d476 commit 5ae869f

39 files changed

+2059
-141
lines changed

‎assets/assets.go

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,17 +53,27 @@ func addAssetList(nd *core.IpfsNode, l []string) (*cid.Cid, error) {
5353
}
5454

5555
fname := filepath.Base(p)
56+
5657
c, err := cid.Decode(s)
5758
if err != nil {
5859
return nil, err
5960
}
6061

61-
if err := dirb.AddChild(nd.Context(), fname, c); err != nil {
62+
node, err := nd.DAG.Get(nd.Context(), c)
63+
if err != nil {
64+
return nil, err
65+
}
66+
67+
if err := dirb.AddChild(nd.Context(), fname, node); err != nil {
6268
return nil, fmt.Errorf("assets: could not add '%s' as a child: %s", fname, err)
6369
}
6470
}
6571

66-
dir := dirb.GetNode()
72+
dir, err := dirb.GetNode()
73+
if err != nil {
74+
return nil, err
75+
}
76+
6777
dcid, err := nd.DAG.Add(dir)
6878
if err != nil {
6979
return nil, fmt.Errorf("assets: DAG.Add(dir) failed: %s", err)

‎core/builder.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
214214
// this is kinda sketchy and could cause data loss
215215
n.Pinning = pin.NewPinner(n.Repo.Datastore(), n.DAG, internalDag)
216216
}
217-
n.Resolver = &path.Resolver{DAG: n.DAG}
217+
n.Resolver = path.NewBasicResolver(n.DAG)
218218

219219
err = n.loadFilesRoot()
220220
if err != nil {

‎core/commands/files/files.go

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
mfs "github.com/ipfs/go-ipfs/mfs"
1616
path "github.com/ipfs/go-ipfs/path"
1717
ft "github.com/ipfs/go-ipfs/unixfs"
18+
uio "github.com/ipfs/go-ipfs/unixfs/io"
1819

1920
context "context"
2021
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
@@ -253,7 +254,12 @@ func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (*dag.N
253254
return nil, err
254255
}
255256

256-
return core.Resolve(ctx, node, np)
257+
resolver := &path.Resolver{
258+
DAG: node.DAG,
259+
ResolveOnce: uio.ResolveUnixfsOnce,
260+
}
261+
262+
return core.Resolve(ctx, node.Namesys, resolver, np)
257263
default:
258264
fsn, err := mfs.Lookup(node.FilesRoot, p)
259265
if err != nil {
@@ -336,7 +342,13 @@ Examples:
336342
case *mfs.Directory:
337343
if !long {
338344
var output []mfs.NodeListing
339-
for _, name := range fsn.ListNames() {
345+
names, err := fsn.ListNames()
346+
if err != nil {
347+
res.SetError(err, cmds.ErrNormal)
348+
return
349+
}
350+
351+
for _, name := range names {
340352
output = append(output, mfs.NodeListing{
341353
Name: name,
342354
})

‎core/commands/get.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ may also specify the level of compression by specifying '-l=<1-9>'.
6363
}
6464
p := path.Path(req.Arguments()[0])
6565
ctx := req.Context()
66-
dn, err := core.Resolve(ctx, node, p)
66+
dn, err := core.Resolve(ctx, node.Namesys, node.Resolver, p)
6767
if err != nil {
6868
res.SetError(err, cmds.ErrNormal)
6969
return

‎core/commands/ls.go

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
merkledag "github.com/ipfs/go-ipfs/merkledag"
1212
path "github.com/ipfs/go-ipfs/path"
1313
unixfs "github.com/ipfs/go-ipfs/unixfs"
14+
uio "github.com/ipfs/go-ipfs/unixfs/io"
1415
unixfspb "github.com/ipfs/go-ipfs/unixfs/pb"
1516

1617
cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
@@ -74,7 +75,18 @@ The JSON output contains type information.
7475

7576
var dagnodes []*merkledag.Node
7677
for _, fpath := range paths {
77-
dagnode, err := core.Resolve(req.Context(), node, path.Path(fpath))
78+
p, err := path.ParsePath(fpath)
79+
if err != nil {
80+
res.SetError(err, cmds.ErrNormal)
81+
return
82+
}
83+
84+
r := &path.Resolver{
85+
DAG: node.DAG,
86+
ResolveOnce: uio.ResolveUnixfsOnce,
87+
}
88+
89+
dagnode, err := core.Resolve(req.Context(), node.Namesys, r, p)
7890
if err != nil {
7991
res.SetError(err, cmds.ErrNormal)
8092
return

‎core/commands/object/diff.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,13 @@ Example:
7373

7474
ctx := req.Context()
7575

76-
obj_a, err := core.Resolve(ctx, node, pa)
76+
obj_a, err := core.Resolve(ctx, node.Namesys, node.Resolver, pa)
7777
if err != nil {
7878
res.SetError(err, cmds.ErrNormal)
7979
return
8080
}
8181

82-
obj_b, err := core.Resolve(ctx, node, pb)
82+
obj_b, err := core.Resolve(ctx, node.Namesys, node.Resolver, pb)
8383
if err != nil {
8484
res.SetError(err, cmds.ErrNormal)
8585
return

‎core/commands/object/object.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ is the raw data of the object.
9393
return
9494
}
9595

96-
node, err := core.Resolve(req.Context(), n, fpath)
96+
node, err := core.Resolve(req.Context(), n.Namesys, n.Resolver, fpath)
9797
if err != nil {
9898
res.SetError(err, cmds.ErrNormal)
9999
return
@@ -132,7 +132,7 @@ multihash.
132132
}
133133

134134
fpath := path.Path(req.Arguments()[0])
135-
node, err := core.Resolve(req.Context(), n, fpath)
135+
node, err := core.Resolve(req.Context(), n.Namesys, n.Resolver, fpath)
136136
if err != nil {
137137
res.SetError(err, cmds.ErrNormal)
138138
return
@@ -195,7 +195,7 @@ This command outputs data in the following encodings:
195195

196196
fpath := path.Path(req.Arguments()[0])
197197

198-
object, err := core.Resolve(req.Context(), n, fpath)
198+
object, err := core.Resolve(req.Context(), n.Namesys, n.Resolver, fpath)
199199
if err != nil {
200200
res.SetError(err, cmds.ErrNormal)
201201
return
@@ -262,7 +262,7 @@ var ObjectStatCmd = &cmds.Command{
262262

263263
fpath := path.Path(req.Arguments()[0])
264264

265-
object, err := core.Resolve(req.Context(), n, fpath)
265+
object, err := core.Resolve(req.Context(), n.Namesys, n.Resolver, fpath)
266266
if err != nil {
267267
res.SetError(err, cmds.ErrNormal)
268268
return

‎core/commands/object/patch.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ the limit will not be respected by the network.
7373
return
7474
}
7575

76-
rootnd, err := core.Resolve(req.Context(), nd, root)
76+
rootnd, err := core.Resolve(req.Context(), nd.Namesys, nd.Resolver, root)
7777
if err != nil {
7878
res.SetError(err, cmds.ErrNormal)
7979
return
@@ -135,7 +135,7 @@ Example:
135135
return
136136
}
137137

138-
root, err := core.Resolve(req.Context(), nd, rp)
138+
root, err := core.Resolve(req.Context(), nd.Namesys, nd.Resolver, rp)
139139
if err != nil {
140140
res.SetError(err, cmds.ErrNormal)
141141
return
@@ -193,7 +193,7 @@ Removes a link by the given name from root.
193193
return
194194
}
195195

196-
root, err := core.Resolve(req.Context(), nd, rootp)
196+
root, err := core.Resolve(req.Context(), nd.Namesys, nd.Resolver, rootp)
197197
if err != nil {
198198
res.SetError(err, cmds.ErrNormal)
199199
return
@@ -262,7 +262,7 @@ to a file containing 'bar', and returns the hash of the new object.
262262
return
263263
}
264264

265-
root, err := core.Resolve(req.Context(), nd, rootp)
265+
root, err := core.Resolve(req.Context(), nd.Namesys, nd.Resolver, rootp)
266266
if err != nil {
267267
res.SetError(err, cmds.ErrNormal)
268268
return
@@ -288,7 +288,7 @@ to a file containing 'bar', and returns the hash of the new object.
288288

289289
e := dagutils.NewDagEditor(root, nd.DAG)
290290

291-
childnd, err := core.Resolve(req.Context(), nd, childp)
291+
childnd, err := core.Resolve(req.Context(), nd.Namesys, nd.Resolver, childp)
292292
if err != nil {
293293
res.SetError(err, cmds.ErrNormal)
294294
return

‎core/commands/pin.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,7 @@ func pinLsKeys(args []string, typeStr string, ctx context.Context, n *core.IpfsN
277277
return nil, err
278278
}
279279

280-
dagNode, err := core.Resolve(ctx, n, pth)
280+
dagNode, err := core.Resolve(ctx, n.Namesys, n.Resolver, pth)
281281
if err != nil {
282282
return nil, err
283283
}

‎core/commands/publish.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ func publish(ctx context.Context, n *core.IpfsNode, k crypto.PrivKey, ref path.P
135135

136136
if opts.verifyExists {
137137
// verify the path exists
138-
_, err := core.Resolve(ctx, n, ref)
138+
_, err := core.Resolve(ctx, n.Namesys, n.Resolver, ref)
139139
if err != nil {
140140
return nil, err
141141
}

‎core/commands/refs.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,8 +197,13 @@ var refsMarshallerMap = cmds.MarshalerMap{
197197

198198
func objectsForPaths(ctx context.Context, n *core.IpfsNode, paths []string) ([]*dag.Node, error) {
199199
objects := make([]*dag.Node, len(paths))
200-
for i, p := range paths {
201-
o, err := core.Resolve(ctx, n, path.Path(p))
200+
for i, sp := range paths {
201+
p, err := path.ParsePath(sp)
202+
if err != nil {
203+
return nil, err
204+
}
205+
206+
o, err := core.Resolve(ctx, n.Namesys, n.Resolver, p)
202207
if err != nil {
203208
return nil, err
204209
}

‎core/commands/resolve.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ Resolve the value of an IPFS DAG path:
9999
return
100100
}
101101

102-
node, err := core.Resolve(req.Context(), n, p)
102+
node, err := core.Resolve(req.Context(), n.Namesys, n.Resolver, p)
103103
if err != nil {
104104
res.SetError(err, cmds.ErrNormal)
105105
return

‎core/commands/tar.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ var tarCatCmd = &cmds.Command{
9494
return
9595
}
9696

97-
root, err := core.Resolve(req.Context(), nd, p)
97+
root, err := core.Resolve(req.Context(), nd.Namesys, nd.Resolver, p)
9898
if err != nil {
9999
res.SetError(err, cmds.ErrNormal)
100100
return

‎core/commands/unixfs/ls.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
merkledag "github.com/ipfs/go-ipfs/merkledag"
1313
path "github.com/ipfs/go-ipfs/path"
1414
unixfs "github.com/ipfs/go-ipfs/unixfs"
15+
uio "github.com/ipfs/go-ipfs/unixfs/io"
1516
unixfspb "github.com/ipfs/go-ipfs/unixfs/pb"
1617
)
1718

@@ -87,7 +88,13 @@ possible, please use 'ipfs ls' instead.
8788

8889
for _, fpath := range paths {
8990
ctx := req.Context()
90-
merkleNode, err := core.Resolve(ctx, node, path.Path(fpath))
91+
92+
resolver := &path.Resolver{
93+
DAG: node.DAG,
94+
ResolveOnce: uio.ResolveUnixfsOnce,
95+
}
96+
97+
merkleNode, err := core.Resolve(ctx, node.Namesys, resolver, path.Path(fpath))
9198
if err != nil {
9299
res.SetError(err, cmds.ErrNormal)
93100
return

‎core/core.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ import (
6060
pin "github.com/ipfs/go-ipfs/pin"
6161
repo "github.com/ipfs/go-ipfs/repo"
6262
config "github.com/ipfs/go-ipfs/repo/config"
63-
uio "github.com/ipfs/go-ipfs/unixfs/io"
63+
ft "github.com/ipfs/go-ipfs/unixfs"
6464
u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
6565
)
6666

@@ -504,7 +504,7 @@ func (n *IpfsNode) loadFilesRoot() error {
504504

505505
switch {
506506
case err == ds.ErrNotFound || val == nil:
507-
nd = uio.NewEmptyDirectory()
507+
nd = ft.EmptyDirNode()
508508
_, err := n.DAG.Add(nd)
509509
if err != nil {
510510
return fmt.Errorf("failure writing to dagstore: %s", err)

‎core/corehttp/gateway_handler.go

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
dag "github.com/ipfs/go-ipfs/merkledag"
1717
dagutils "github.com/ipfs/go-ipfs/merkledag/utils"
1818
path "github.com/ipfs/go-ipfs/path"
19+
ft "github.com/ipfs/go-ipfs/unixfs"
1920
uio "github.com/ipfs/go-ipfs/unixfs/io"
2021

2122
"context"
@@ -152,7 +153,13 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
152153
ipnsHostname = true
153154
}
154155

155-
nd, err := core.Resolve(ctx, i.node, path.Path(urlPath))
156+
p, err := path.ParsePath(urlPath)
157+
if err != nil {
158+
webError(w, "Invalid Path Error", err, http.StatusBadRequest)
159+
return
160+
}
161+
162+
nd, err := core.Resolve(ctx, i.node.Namesys, i.node.Resolver, p)
156163
// If node is in offline mode the error code and message should be different
157164
if err == core.ErrNoNamesys && !i.node.OnlineMode() {
158165
w.WriteHeader(http.StatusServiceUnavailable)
@@ -233,8 +240,14 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
233240
return
234241
}
235242

243+
p, err := path.ParsePath(urlPath + "/index.html")
244+
if err != nil {
245+
internalWebError(w, err)
246+
return
247+
}
248+
236249
// return index page instead.
237-
nd, err := core.Resolve(ctx, i.node, path.Path(urlPath+"/index.html"))
250+
nd, err := core.Resolve(ctx, i.node.Namesys, i.node.Resolver, p)
238251
if err != nil {
239252
internalWebError(w, err)
240253
return
@@ -342,7 +355,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
342355

343356
var newnode *dag.Node
344357
if rsegs[len(rsegs)-1] == "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" {
345-
newnode = uio.NewEmptyDirectory()
358+
newnode = ft.EmptyDirNode()
346359
} else {
347360
putNode, err := i.newDagFromReader(r.Body)
348361
if err != nil {
@@ -358,7 +371,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
358371
}
359372

360373
var newcid *cid.Cid
361-
rnode, err := core.Resolve(ctx, i.node, rootPath)
374+
rnode, err := core.Resolve(ctx, i.node.Namesys, i.node.Resolver, rootPath)
362375
switch ev := err.(type) {
363376
case path.ErrNoLink:
364377
// ev.Node < node where resolve failed
@@ -377,7 +390,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
377390
}
378391

379392
e := dagutils.NewDagEditor(rnode, i.node.DAG)
380-
err = e.InsertNodeAtPath(ctx, newPath, newnode, uio.NewEmptyDirectory)
393+
err = e.InsertNodeAtPath(ctx, newPath, newnode, ft.EmptyDirNode)
381394
if err != nil {
382395
webError(w, "putHandler: InsertNodeAtPath failed", err, http.StatusInternalServerError)
383396
return

‎core/corerepo/pinning.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,12 @@ import (
2727
func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]*cid.Cid, error) {
2828
dagnodes := make([]*merkledag.Node, 0)
2929
for _, fpath := range paths {
30-
dagnode, err := core.Resolve(ctx, n, path.Path(fpath))
30+
p, err := path.ParsePath(fpath)
31+
if err != nil {
32+
return nil, err
33+
}
34+
35+
dagnode, err := core.Resolve(ctx, n.Namesys, n.Resolver, p)
3136
if err != nil {
3237
return nil, fmt.Errorf("pin: %s", err)
3338
}

‎core/coreunix/add.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,12 @@ func (adder *Adder) outputDirs(path string, fsn mfs.FSNode) error {
220220
case *mfs.File:
221221
return nil
222222
case *mfs.Directory:
223-
for _, name := range fsn.ListNames() {
223+
names, err := fsn.ListNames()
224+
if err != nil {
225+
return err
226+
}
227+
228+
for _, name := range names {
224229
child, err := fsn.Child(name)
225230
if err != nil {
226231
return err

0 commit comments

Comments
 (0)