Merge pull request ipfs#4888 from ipfs/fix/dedup-keys

dedup keys in GetMany

Author: whyrusleeping

merkledag.go

@@ -201,7 +201,20 @@ func (n *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *ipld.
 	return getNodesFromBG(ctx, n.Blocks, keys)
 }
 
+func dedupKeys(keys []*cid.Cid) []*cid.Cid {
+	set := cid.NewSet()
+	for _, c := range keys {
+		set.Add(c)
+	}
+	if set.Len() == len(keys) {
+		return keys
+	}
+	return set.Keys()
+}
+
 func getNodesFromBG(ctx context.Context, bs bserv.BlockGetter, keys []*cid.Cid) <-chan *ipld.NodeOption {
+	keys = dedupKeys(keys)
+
 	out := make(chan *ipld.NodeOption, len(keys))
 	blocks := bs.GetBlocks(ctx, keys)
 	var count int

merkledag_test.go

@@ -550,6 +550,36 @@ func TestCidRawDoesnNeedData(t *testing.T) {
 	}
 }
 
+func TestGetManyDuplicate(t *testing.T) {
+	ctx := context.Background()
+
+	srv := NewDAGService(dstest.Bserv())
+
+	nd := NodeWithData([]byte("foo"))
+	if err := srv.Add(ctx, nd); err != nil {
+		t.Fatal(err)
+	}
+	nds := srv.GetMany(ctx, []*cid.Cid{nd.Cid(), nd.Cid(), nd.Cid()})
+	out, ok := <-nds
+	if !ok {
+		t.Fatal("expecting node foo")
+	}
+	if out.Err != nil {
+		t.Fatal(out.Err)
+	}
+	if !out.Node.Cid().Equals(nd.Cid()) {
+		t.Fatal("got wrong node")
+	}
+	out, ok = <-nds
+	if ok {
+		if out.Err != nil {
+			t.Fatal(out.Err)
+		} else {
+			t.Fatal("expecting no more nodes")
+		}
+	}
+}
+
 func TestEnumerateAsyncFailsNotFound(t *testing.T) {
 	ctx := context.Background()