Merge pull request #3042 from ipfs/feat/hamt-sharding

Implement unixfs sharding

Author: whyrusleeping

assets/assets.go

@@ -59,17 +59,27 @@ func addAssetList(nd *core.IpfsNode, l []string) (*cid.Cid, error) {
 		}
 
 		fname := filepath.Base(p)
+
 		c, err := cid.Decode(s)
 		if err != nil {
 			return nil, err
 		}
 
-		if err := dirb.AddChild(nd.Context(), fname, c); err != nil {
+		node, err := nd.DAG.Get(nd.Context(), c)
+		if err != nil {
+			return nil, err
+		}
+
+		if err := dirb.AddChild(nd.Context(), fname, node); err != nil {
 			return nil, fmt.Errorf("assets: could not add '%s' as a child: %s", fname, err)
 		}
 	}
 
-	dir := dirb.GetNode()
+	dir, err := dirb.GetNode()
+	if err != nil {
+		return nil, err
+	}
+
 	dcid, err := nd.DAG.Add(dir)
 	if err != nil {
 		return nil, fmt.Errorf("assets: DAG.Add(dir) failed: %s", err)

commands/request.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"bufio"
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -10,7 +11,6 @@ import (
 	"strconv"
 	"time"
 
-	context "context"
 	"github.com/ipfs/go-ipfs/commands/files"
 	"github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/repo/config"

core/builder.go

@@ -18,6 +18,7 @@ import (
 	pin "github.com/ipfs/go-ipfs/pin"
 	repo "github.com/ipfs/go-ipfs/repo"
 	cfg "github.com/ipfs/go-ipfs/repo/config"
+	uio "github.com/ipfs/go-ipfs/unixfs/io"
 
 	ci "gx/ipfs/QmPGxZ1DP2w45WcogpW1h43BvseXbfke9N91qotpoQcUeS/go-libp2p-crypto"
 	ds "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore"
@@ -175,6 +176,9 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
 		return err
 	}
 
+	// TEMP: setting global sharding switch here
+	uio.UseHAMTSharding = conf.Experimental.ShardingEnabled
+
 	opts.HasBloomFilterSize = conf.Datastore.BloomFilterSize
 	if !cfg.Permament {
 		opts.HasBloomFilterSize = 0

core/commands/files/files.go

@@ -265,16 +265,7 @@ func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (node.N
 			ResolveOnce: uio.ResolveUnixfsOnce,
 		}
 
-		nd, err := core.Resolve(ctx, node.Namesys, resolver, np)
-		if err != nil {
-			return nil, err
-		}
-		pbnd, ok := nd.(*dag.ProtoNode)
-		if !ok {
-			return nil, dag.ErrNotProtobuf
-		}
-
-		return pbnd, nil
+		return core.Resolve(ctx, node.Namesys, resolver, np)
 	default:
 		fsn, err := mfs.Lookup(node.FilesRoot, p)
 		if err != nil {
@@ -357,14 +348,20 @@ Examples:
 		case *mfs.Directory:
 			if !long {
 				var output []mfs.NodeListing
-				for _, name := range fsn.ListNames() {
+				names, err := fsn.ListNames(req.Context())
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
+				}
+
+				for _, name := range names {
 					output = append(output, mfs.NodeListing{
 						Name: name,
 					})
 				}
 				res.SetOutput(&FilesLsOutput{output})
 			} else {
-				listing, err := fsn.List()
+				listing, err := fsn.List(req.Context())
 				if err != nil {
 					res.SetError(err, cmds.ErrNormal)
 					return

core/commands/ls.go

@@ -105,11 +105,24 @@ The JSON output contains type information.
 
 		output := make([]LsObject, len(req.Arguments()))
 		for i, dagnode := range dagnodes {
+			dir, err := uio.NewDirectoryFromNode(nd.DAG, dagnode)
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+
+			links, err := dir.Links(req.Context())
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+
 			output[i] = LsObject{
 				Hash:  paths[i],
-				Links: make([]LsLink, len(dagnode.Links())),
+				Links: make([]LsLink, len(links)),
 			}
-			for j, link := range dagnode.Links() {
+
+			for j, link := range links {
 				t := unixfspb.Data_DataType(-1)
 
 				linkNode, err := link.GetNode(req.Context(), dserv)

core/coreunix/add.go

@@ -21,6 +21,7 @@ import (
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	mfs "github.com/ipfs/go-ipfs/mfs"
 	"github.com/ipfs/go-ipfs/pin"
+	posinfo "github.com/ipfs/go-ipfs/thirdparty/posinfo"
 	unixfs "github.com/ipfs/go-ipfs/unixfs"
 
 	ds "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore"
@@ -189,15 +190,18 @@ func (adder *Adder) PinRoot() error {
 func (adder *Adder) Finalize() (node.Node, error) {
 	root := adder.mr.GetValue()
 
-	// cant just call adder.RootNode() here as we need the name for printing
-	rootNode, err := root.GetNode()
+	err := root.Flush()
 	if err != nil {
 		return nil, err
 	}
 
 	var name string
 	if !adder.Wrap {
-		name = rootNode.Links()[0].Name
+		children, err := root.(*mfs.Directory).ListNames(adder.ctx)
+		if err != nil {
+			return nil, err
+		}
+		name = children[0]
 
 		dir, ok := adder.mr.GetValue().(*mfs.Directory)
 		if !ok {
@@ -228,7 +232,12 @@ func (adder *Adder) outputDirs(path string, fsn mfs.FSNode) error {
 	case *mfs.File:
 		return nil
 	case *mfs.Directory:
-		for _, name := range fsn.ListNames() {
+		names, err := fsn.ListNames(adder.ctx)
+		if err != nil {
+			return err
+		}
+
+		for _, name := range names {
 			child, err := fsn.Child(name)
 			if err != nil {
 				return err
@@ -344,6 +353,10 @@ func (adder *Adder) addNode(node node.Node, path string) error {
 		path = node.Cid().String()
 	}
 
+	if pi, ok := node.(*posinfo.FilestoreNode); ok {
+		node = pi.Node
+	}
+
 	dir := gopath.Dir(path)
 	if dir != "." {
 		if err := mfs.Mkdir(adder.mr, dir, true, false); err != nil {

fuse/ipns/ipns_unix.go

@@ -16,7 +16,6 @@ import (
 	namesys "github.com/ipfs/go-ipfs/namesys"
 	path "github.com/ipfs/go-ipfs/path"
 	ft "github.com/ipfs/go-ipfs/unixfs"
-	uio "github.com/ipfs/go-ipfs/unixfs/io"
 
 	ci "gx/ipfs/QmPGxZ1DP2w45WcogpW1h43BvseXbfke9N91qotpoQcUeS/go-libp2p-crypto"
 	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
@@ -100,7 +99,7 @@ func loadRoot(ctx context.Context, rt *keyRoot, ipfs *core.IpfsNode, name string
 	switch err {
 	case nil:
 	case namesys.ErrResolveFailed:
-		node = uio.NewEmptyDirectory()
+		node = ft.EmptyDirNode()
 	default:
 		log.Errorf("looking up %s: %s", p, err)
 		return nil, err
@@ -325,7 +324,7 @@ func (s *Directory) Lookup(ctx context.Context, name string) (fs.Node, error) {
 // ReadDirAll reads the link structure as directory entries
 func (dir *Directory) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 	var entries []fuse.Dirent
-	listing, err := dir.dir.List()
+	listing, err := dir.dir.List(ctx)
 	if err != nil {
 		return nil, err
 	}