Skip to content

Commit 6912f47

Browse files
whyrusleepingwhyrusleeping
authored
Merge pull request #2960 from ipfs/feat/mdag-caching
cache encoded data when reading dag nodes from disk
2 parents f7676f4 + bf23516 commit 6912f47

36 files changed

+144
-122
lines changed

‎core/commands/add.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
core "github.com/ipfs/go-ipfs/core"
1313
dagtest "github.com/ipfs/go-ipfs/merkledag/test"
1414
mfs "github.com/ipfs/go-ipfs/mfs"
15+
ft "github.com/ipfs/go-ipfs/unixfs"
1516
u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
1617
)
1718

@@ -160,7 +161,7 @@ You can now refer to the added file in a gateway, like so:
160161

161162
if hash {
162163
md := dagtest.Mock()
163-
mr, err := mfs.NewRoot(req.Context(), md, coreunix.NewDirNode(), nil)
164+
mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
164165
if err != nil {
165166
res.SetError(err, cmds.ErrNormal)
166167
return

‎core/commands/files/files.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
163163
return nil, err
164164
}
165165

166-
d, err := ft.FromBytes(nd.Data)
166+
d, err := ft.FromBytes(nd.Data())
167167
if err != nil {
168168
return nil, err
169169
}
@@ -871,7 +871,7 @@ func getFileHandle(r *mfs.Root, path string, create bool) (*mfs.File, error) {
871871
return nil, fmt.Errorf("%s was not a directory", dirname)
872872
}
873873

874-
nd := &dag.Node{Data: ft.FilePBData(nil, 0)}
874+
nd := dag.NodeWithData(ft.FilePBData(nil, 0))
875875
err = pdir.AddChild(fname, nd)
876876
if err != nil {
877877
return nil, err

‎core/commands/ls.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ format:
110110
}
111111
}
112112
if linkNode != nil {
113-
d, err := unixfs.FromBytes(linkNode.Data)
113+
d, err := unixfs.FromBytes(linkNode.Data())
114114
if err != nil {
115115
res.SetError(err, cmds.ErrNormal)
116116
return

‎core/commands/object/object.go

+6-7
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ is the raw data of the object.
9393
res.SetError(err, cmds.ErrNormal)
9494
return
9595
}
96-
res.SetOutput(bytes.NewReader(node.Data))
96+
res.SetOutput(bytes.NewReader(node.Data()))
9797
},
9898
}
9999

@@ -198,7 +198,7 @@ This command outputs data in the following encodings:
198198

199199
node := &Node{
200200
Links: make([]Link, len(object.Links)),
201-
Data: string(object.Data),
201+
Data: string(object.Data()),
202202
}
203203

204204
for i, link := range object.Links {
@@ -438,9 +438,7 @@ Available templates:
438438
func nodeFromTemplate(template string) (*dag.Node, error) {
439439
switch template {
440440
case "unixfs-dir":
441-
nd := new(dag.Node)
442-
nd.Data = ft.FolderPBData()
443-
return nd, nil
441+
return ft.EmptyDirNode(), nil
444442
default:
445443
return nil, fmt.Errorf("template '%s' not found", template)
446444
}
@@ -566,9 +564,10 @@ func deserializeNode(node *Node, dataFieldEncoding string) (*dag.Node, error) {
566564
dagnode := new(dag.Node)
567565
switch dataFieldEncoding {
568566
case "text":
569-
dagnode.Data = []byte(node.Data)
567+
dagnode.SetData([]byte(node.Data))
570568
case "base64":
571-
dagnode.Data, _ = base64.StdEncoding.DecodeString(node.Data)
569+
data, _ := base64.StdEncoding.DecodeString(node.Data)
570+
dagnode.SetData(data)
572571
default:
573572
return nil, fmt.Errorf("Unkown data field encoding")
574573
}

‎core/commands/object/patch.go

+3-5
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ the limit will not be respected by the network.
9191
return
9292
}
9393

94-
rootnd.Data = append(rootnd.Data, data...)
94+
rootnd.SetData(append(rootnd.Data(), data...))
9595

9696
newkey, err := nd.DAG.Add(rootnd)
9797
if err != nil {
@@ -153,7 +153,7 @@ Example:
153153
return
154154
}
155155

156-
root.Data = data
156+
root.SetData(data)
157157

158158
newkey, err := nd.DAG.Add(root)
159159
if err != nil {
@@ -287,9 +287,7 @@ to a file containing 'bar', and returns the hash of the new object.
287287

288288
var createfunc func() *dag.Node
289289
if create {
290-
createfunc = func() *dag.Node {
291-
return &dag.Node{Data: ft.FolderPBData()}
292-
}
290+
createfunc = ft.EmptyDirNode
293291
}
294292

295293
e := dagutils.NewDagEditor(root, nd.DAG)

‎core/commands/unixfs/ls.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ Example:
101101
continue
102102
}
103103

104-
unixFSNode, err := unixfs.FromBytes(merkleNode.Data)
104+
unixFSNode, err := unixfs.FromBytes(merkleNode.Data())
105105
if err != nil {
106106
res.SetError(err, cmds.ErrNormal)
107107
return
@@ -128,7 +128,7 @@ Example:
128128
res.SetError(err, cmds.ErrNormal)
129129
return
130130
}
131-
d, err := unixfs.FromBytes(linkNode.Data)
131+
d, err := unixfs.FromBytes(linkNode.Data())
132132
if err != nil {
133133
res.SetError(err, cmds.ErrNormal)
134134
return

‎core/corehttp/gateway_handler.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -391,7 +391,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
391391

392392
case nil:
393393
// object set-data case
394-
rnode.Data = newnode.Data
394+
rnode.SetData(newnode.Data())
395395

396396
newkey, err = i.node.DAG.Add(rnode)
397397
if err != nil {

‎core/coreunix/add.go

+2-9
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@ import (
2929

3030
var log = logging.Logger("coreunix")
3131

32-
var folderData = unixfs.FolderPBData()
33-
3432
// how many bytes of progress to wait before sending a progress update message
3533
const progressReaderIncrement = 1024 * 256
3634

@@ -67,7 +65,7 @@ type AddedObject struct {
6765
}
6866

6967
func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) {
70-
mr, err := mfs.NewRoot(ctx, ds, NewDirNode(), nil)
68+
mr, err := mfs.NewRoot(ctx, ds, unixfs.EmptyDirNode(), nil)
7169
if err != nil {
7270
return nil, err
7371
}
@@ -392,7 +390,7 @@ func (adder *Adder) addFile(file files.File) error {
392390
return err
393391
}
394392

395-
dagnode := &dag.Node{Data: sdata}
393+
dagnode := dag.NodeWithData(sdata)
396394
_, err = adder.dagService.Add(dagnode)
397395
if err != nil {
398396
return err
@@ -488,11 +486,6 @@ func NewMemoryDagService() dag.DAGService {
488486
return dag.NewDAGService(bsrv)
489487
}
490488

491-
// TODO: generalize this to more than unix-fs nodes.
492-
func NewDirNode() *dag.Node {
493-
return &dag.Node{Data: unixfs.FolderPBData()}
494-
}
495-
496489
// from core/commands/object.go
497490
func getOutput(dagnode *dag.Node) (*Object, error) {
498491
key, err := dagnode.Key()

‎core/coreunix/metadata.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ func AddMetadataTo(n *core.IpfsNode, skey string, m *ft.Metadata) (string, error
2121
return "", err
2222
}
2323

24-
mdnode.Data = mdata
24+
mdnode.SetData(mdata)
2525
if err := mdnode.AddNodeLinkClean("file", nd); err != nil {
2626
return "", err
2727
}
@@ -42,5 +42,5 @@ func Metadata(n *core.IpfsNode, skey string) (*ft.Metadata, error) {
4242
return nil, err
4343
}
4444

45-
return ft.MetadataFromBytes(nd.Data)
45+
return ft.MetadataFromBytes(nd.Data())
4646
}

‎fuse/ipns/common.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
55

66
"github.com/ipfs/go-ipfs/core"
7-
mdag "github.com/ipfs/go-ipfs/merkledag"
87
nsys "github.com/ipfs/go-ipfs/namesys"
98
path "github.com/ipfs/go-ipfs/path"
109
ft "github.com/ipfs/go-ipfs/unixfs"
@@ -14,7 +13,7 @@ import (
1413
// InitializeKeyspace sets the ipns record for the given key to
1514
// point to an empty directory.
1615
func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
17-
emptyDir := &mdag.Node{Data: ft.FolderPBData()}
16+
emptyDir := ft.EmptyDirNode()
1817
nodek, err := n.DAG.Add(emptyDir)
1918
if err != nil {
2019
return err

‎fuse/ipns/ipns_unix.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,7 @@ func (fi *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
481481

482482
func (dir *Directory) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
483483
// New 'empty' file
484-
nd := &dag.Node{Data: ft.FilePBData(nil, 0)}
484+
nd := dag.NodeWithData(ft.FilePBData(nil, 0))
485485
err := dir.dir.AddChild(req.Name, nd)
486486
if err != nil {
487487
return nil, nil, err

‎fuse/readonly/readonly_unix.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ type Node struct {
8484

8585
func (s *Node) loadData() error {
8686
s.cached = new(ftpb.Data)
87-
return proto.Unmarshal(s.Nd.Data, s.cached)
87+
return proto.Unmarshal(s.Nd.Data(), s.cached)
8888
}
8989

9090
// Attr returns the attributes of a given node.

‎importer/helpers/helpers.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func NewUnixfsBlock() *UnixfsNode {
5959

6060
// NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node
6161
func NewUnixfsNodeFromDag(nd *dag.Node) (*UnixfsNode, error) {
62-
mb, err := ft.FSNodeFromBytes(nd.Data)
62+
mb, err := ft.FSNodeFromBytes(nd.Data())
6363
if err != nil {
6464
return nil, err
6565
}
@@ -126,6 +126,6 @@ func (n *UnixfsNode) GetDagNode() (*dag.Node, error) {
126126
if err != nil {
127127
return nil, err
128128
}
129-
n.node.Data = data
129+
n.node.SetData(data)
130130
return n.node, nil
131131
}

‎importer/trickle/trickle_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ func TestAppendSingleBytesToEmpty(t *testing.T) {
524524
data := []byte("AB")
525525

526526
nd := new(merkledag.Node)
527-
nd.Data = ft.FilePBData(nil, 0)
527+
nd.SetData(ft.FilePBData(nil, 0))
528528

529529
dbp := &h.DagBuilderParams{
530530
Dagserv: ds,
@@ -562,7 +562,7 @@ func TestAppendSingleBytesToEmpty(t *testing.T) {
562562
}
563563

564564
func printDag(nd *merkledag.Node, ds merkledag.DAGService, indent int) {
565-
pbd, err := ft.FromBytes(nd.Data)
565+
pbd, err := ft.FromBytes(nd.Data())
566566
if err != nil {
567567
panic(err)
568568
}

‎importer/trickle/trickledag.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGServi
241241
return errors.New("expected direct block")
242242
}
243243

244-
pbn, err := ft.FromBytes(nd.Data)
244+
pbn, err := ft.FromBytes(nd.Data())
245245
if err != nil {
246246
return err
247247
}
@@ -253,7 +253,7 @@ func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGServi
253253
}
254254

255255
// Verify this is a branch node
256-
pbn, err := ft.FromBytes(nd.Data)
256+
pbn, err := ft.FromBytes(nd.Data())
257257
if err != nil {
258258
return err
259259
}

‎merkledag/coding.go

+8-3
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ func (n *Node) unmarshal(encoded []byte) error {
3333
}
3434
sort.Stable(LinkSlice(n.Links)) // keep links sorted
3535

36-
n.Data = pbn.GetData()
36+
n.data = pbn.GetData()
37+
n.encoded = encoded
3738
return nil
3839
}
3940

@@ -62,8 +63,8 @@ func (n *Node) getPBNode() *pb.PBNode {
6263
pbn.Links[i].Hash = []byte(l.Hash)
6364
}
6465

65-
if len(n.Data) > 0 {
66-
pbn.Data = n.Data
66+
if len(n.data) > 0 {
67+
pbn.Data = n.data
6768
}
6869
return pbn
6970
}
@@ -73,11 +74,15 @@ func (n *Node) getPBNode() *pb.PBNode {
7374
func (n *Node) EncodeProtobuf(force bool) ([]byte, error) {
7475
sort.Stable(LinkSlice(n.Links)) // keep links sorted
7576
if n.encoded == nil || force {
77+
n.cached = nil
7678
var err error
7779
n.encoded, err = n.Marshal()
7880
if err != nil {
7981
return nil, err
8082
}
83+
}
84+
85+
if n.cached == nil {
8186
n.cached = u.Hash(n.encoded)
8287
}
8388

‎merkledag/merkledag.go

+4
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,9 @@ func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
9393
}
9494
return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
9595
}
96+
97+
res.cached = k.ToMultihash()
98+
9699
return res, nil
97100
}
98101

@@ -147,6 +150,7 @@ func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeO
147150
out <- &NodeOption{Err: err}
148151
return
149152
}
153+
nd.cached = b.Key().ToMultihash()
150154

151155
// buffered, no need to select
152156
out <- &NodeOption{Node: nd}

‎merkledag/merkledag_test.go

+9-9
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@ func getDagservAndPinner(t *testing.T) dagservAndPinner {
4646

4747
func TestNode(t *testing.T) {
4848

49-
n1 := &Node{Data: []byte("beep")}
50-
n2 := &Node{Data: []byte("boop")}
51-
n3 := &Node{Data: []byte("beep boop")}
49+
n1 := NodeWithData([]byte("beep"))
50+
n2 := NodeWithData([]byte("boop"))
51+
n3 := NodeWithData([]byte("beep boop"))
5252
if err := n3.AddNodeLink("beep-link", n1); err != nil {
5353
t.Error(err)
5454
}
@@ -58,7 +58,7 @@ func TestNode(t *testing.T) {
5858

5959
printn := func(name string, n *Node) {
6060
fmt.Println(">", name)
61-
fmt.Println("data:", string(n.Data))
61+
fmt.Println("data:", string(n.Data()))
6262

6363
fmt.Println("links:")
6464
for _, l := range n.Links {
@@ -118,8 +118,8 @@ func SubtestNodeStat(t *testing.T, n *Node) {
118118
expected := NodeStat{
119119
NumLinks: len(n.Links),
120120
BlockSize: len(enc),
121-
LinksSize: len(enc) - len(n.Data), // includes framing.
122-
DataSize: len(n.Data),
121+
LinksSize: len(enc) - len(n.Data()), // includes framing.
122+
DataSize: len(n.Data()),
123123
CumulativeSize: int(cumSize),
124124
Hash: k.B58String(),
125125
}
@@ -255,7 +255,7 @@ func TestEmptyKey(t *testing.T) {
255255

256256
func TestCantGet(t *testing.T) {
257257
dsp := getDagservAndPinner(t)
258-
a := &Node{Data: []byte("A")}
258+
a := NodeWithData([]byte("A"))
259259

260260
k, err := a.Key()
261261
if err != nil {
@@ -339,7 +339,7 @@ func TestFetchFailure(t *testing.T) {
339339

340340
top := new(Node)
341341
for i := 0; i < 10; i++ {
342-
nd := &Node{Data: []byte{byte('a' + i)}}
342+
nd := NodeWithData([]byte{byte('a' + i)})
343343
_, err := ds.Add(nd)
344344
if err != nil {
345345
t.Fatal(err)
@@ -352,7 +352,7 @@ func TestFetchFailure(t *testing.T) {
352352
}
353353

354354
for i := 0; i < 10; i++ {
355-
nd := &Node{Data: []byte{'f', 'a' + byte(i)}}
355+
nd := NodeWithData([]byte{'f', 'a' + byte(i)})
356356
_, err := ds_bad.Add(nd)
357357
if err != nil {
358358
t.Fatal(err)

0 commit comments

Comments
 (0)