Merge commit 'cdd5285f16af665e5fd5d3592f53b2134281e76a' into HEAD

* commit 'cdd5285f16af665e5fd5d3592f53b2134281e76a':
  add cmd: clean up default logic of --progress option
  add cmd: use .Default(true) for pin option.
  Revert "Merge pull request ipfs#2657 from ipfs/feature/add-defaults-to-add"
  bitswap: add wantlist fullness to protobuf messages
  Enable parallel builds on CircleCI
  Fix PHONY name in Makefile
  Run coveralls if COVERALLS_TOKEN is set
  Switch unixfs.Metadata.MimeType to optional
  Fix bad formatting introduced by e855047
  blockstore.AllKeyChan: fix/cleanup error handling
  blockstore.AllKeyChan: avoid channels by using the new NextSync method
  ulimit: handle freebsd ulimit code separately from the rest of the unixes
  Add test for flags.
  bitswap: increase wantlist resend delay to one minute
  ds-help: avoid unnecessary allocs when posssible and make use of RawKey
  fix formatting on error call
  "block rm": make channel large enough to avoid blocking

# Conflicts:
#	Makefile
#	core/commands/add.go

Author: liliuhai

.gitignore

@@ -5,6 +5,7 @@
 *.swp
 .ipfsconfig
 *.out
+*.coverprofile
 *.test
 *.orig
 *~

Makefile

@@ -3,10 +3,19 @@ IPFS_MIN_GO_VERSION = 1.7
 IPFS_MIN_GX_VERSION = 0.6
 IPFS_MIN_GX_GO_VERSION = 1.1
 
-ifeq ($(TEST_NO_FUSE),1)
-  go_test=IPFS_REUSEPORT=false go test -tags nofuse
+
+export IPFS_REUSEPORT=false
+
+ifneq ($(COVERALLS_TOKEN), )
+  covertools_rule = covertools
+  GOT = overalls -project=github.com/ipfs/go-ipfs -covermode atomic -ignore=.git,Godeps,thirdparty,test -- $(GOTFLAGS)
 else
-  go_test=IPFS_REUSEPORT=false go test
+  covertools_rule = $()
+  GOT = go test $(GOTFLAGS) ./...
+endif
+
+ifeq ($(TEST_NO_FUSE),1)
+  GOTFLAGS += -tags nofuse
 endif
 
 ifeq ($(OS),Windows_NT)
@@ -48,9 +57,14 @@ gx_check: ${gx_bin} ${gx-go_bin}
 path_check:
 	@bin/check_go_path $(realpath $(shell pwd)) $(realpath $(addsuffix /src/github.com/ipfs/go-ipfs,$(subst $(GOPATH_DELIMITER), ,$(GOPATH))))
 
-deps: go_check gx_check path_check
+deps: go_check gx_check path_check $(covertools_rule)
 	${gx_bin} --verbose install --global
 
+covertools:
+	go get -u github.com/mattn/goveralls
+	go get -u golang.org/x/tools/cmd/cover
+	go get -u github.com/Kubuxu/overalls
+
 # saves/vendors third-party dependencies to Godeps/_workspace
 # -r flag rewrites import paths to use the vendored path
 # ./... performs operation on all packages in tree
@@ -67,7 +81,7 @@ clean:
 uninstall:
 	$(MAKE) -C cmd/ipfs uninstall
 
-PHONY += all help godep gx_check
+PHONY += all help godep gx_check covertools
 PHONY += go_check deps vendor ipfs_lib install build nofuse clean uninstall
 
 ##############################################################
@@ -85,14 +99,14 @@ test_3node:
 test_go_fmt:
 	bin/test-go-fmt
 
-test_go_short:
-	$(go_test) -test.short ./...
 
-test_go_expensive:
-	$(go_test) ./...
-
-test_go_race:
-	$(go_test) ./... -race
+test_go_short: GOTFLAGS += -test.short
+test_go_race: GOTFLAGS += -race
+test_go_expensive test_go_short test_go_race:
+	$(GOT)
+ifneq ($(COVERALLS_TOKEN), )
+	goveralls -coverprofile=overalls.coverprofile -service $(SERVICE)
+endif
 
 test_sharness_short:
 	$(MAKE) -j1 -C test/sharness/

blocks/blockstore/blockstore.go

@@ -181,44 +181,27 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error)
 		return nil, err
 	}
 
-	// this function is here to compartmentalize
-	get := func() (*cid.Cid, bool) {
-		select {
-		case <-ctx.Done():
-			return nil, false
-		case e, more := <-res.Next():
-			if !more {
-				return nil, false
-			}
-			if e.Error != nil {
-				log.Debug("blockstore.AllKeysChan got err:", e.Error)
-				return nil, false
-			}
-
-			// need to convert to key.Key using key.KeyFromDsKey.
-			c, err := dshelp.DsKeyStringToCid(e.Key)
-			if err != nil {
-				log.Warningf("error parsing key from DsKey: ", err)
-				return nil, true
-			}
-
-			return c, true
-		}
-	}
-
 	output := make(chan *cid.Cid, dsq.KeysOnlyBufSize)
 	go func() {
 		defer func() {
-			res.Process().Close() // ensure exit (signals early exit, too)
+			res.Close() // ensure exit (signals early exit, too)
 			close(output)
 		}()
 
 		for {
-			k, ok := get()
+			e, ok := res.NextSync()
 			if !ok {
 				return
 			}
-			if k == nil {
+			if e.Error != nil {
+				log.Errorf("blockstore.AllKeysChan got err:", e.Error)
+				return
+			}
+
+			// need to convert to key.Key using key.KeyFromDsKey.
+			k, err := dshelp.DsKeyToCid(ds.RawKey(e.Key))
+			if err != nil {
+				log.Warningf("error parsing key from DsKey: ", err)
 				continue
 			}
 

blocks/blockstore/util/remove.go

@@ -27,7 +27,10 @@ type RmBlocksOpts struct {
 	Force  bool
 }
 
-func RmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid, opts RmBlocksOpts) error {
+func RmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, cids []*cid.Cid, opts RmBlocksOpts) (<-chan interface{}, error) {
+	// make the channel large enough to hold any result to avoid
+	// blocking while holding the GCLock
+	out := make(chan interface{}, len(cids))
 	go func() {
 		defer close(out)
 
@@ -47,7 +50,7 @@ func RmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, out chan<- interface{}, c
 			}
 		}
 	}()
-	return nil
+	return out, nil
 }
 
 func FilterPinned(pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid) []*cid.Cid {

circle.yml

@@ -7,6 +7,8 @@ machine:
     CIRCLE: 1
     IMPORT_PATH: "github.com/ipfs/go-ipfs"
     GOPATH: "$HOME/.go_workspace"
+    GOBIN: "$GOPATH/bin"
+    SERVICE: "circle-ci"
 
   post:
     - sudo rm -rf /usr/local/go
@@ -35,5 +37,7 @@ test:
   override:
     - make test_go_expensive:
         pwd: "../.go_workspace/src/$IMPORT_PATH"
+        parallel: true
     - make test_sharness_expensive:
         pwd: "../.go_workspace/src/$IMPORT_PATH"
+        parallel: true

cmd/ipfs/ulimit.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"os"
+	"strconv"
+)
+
+var ipfsFileDescNum = uint64(1024)
+
+func init() {
+	if val := os.Getenv("IPFS_FD_MAX"); val != "" {
+		n, err := strconv.Atoi(val)
+		if err != nil {
+			log.Errorf("bad value for IPFS_FD_MAX: %s", err)
+		} else {
+			ipfsFileDescNum = uint64(n)
+		}
+	}
+}

cmd/ipfs/ulimit_freebsd.go

@@ -0,0 +1,44 @@
+// +build freebsd
+
+package main
+
+import (
+	"fmt"
+	"syscall"
+)
+
+func init() {
+	fileDescriptorCheck = checkAndSetUlimit
+}
+
+func checkAndSetUlimit() error {
+	var rLimit syscall.Rlimit
+	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+	if err != nil {
+		return fmt.Errorf("error getting rlimit: %s", err)
+	}
+
+	ipfsFileDescNum := int64(ipfsFileDescNum)
+
+	var setting bool
+	if rLimit.Cur < ipfsFileDescNum {
+		if rLimit.Max < ipfsFileDescNum {
+			log.Error("adjusting max")
+			rLimit.Max = ipfsFileDescNum
+		}
+		fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum)
+		rLimit.Cur = ipfsFileDescNum
+		setting = true
+	}
+
+	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+	if err != nil {
+		return fmt.Errorf("error setting ulimit: %s", err)
+	}
+
+	if setting {
+		fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum)
+	}
+
+	return nil
+}

cmd/ipfs/ulimit_unix.go

@@ -1,25 +1,13 @@
-// +build darwin freebsd linux netbsd openbsd
+// +build darwin linux netbsd openbsd
 
 package main
 
 import (
 	"fmt"
-	"os"
-	"strconv"
 	"syscall"
 )
 
-var ipfsFileDescNum = uint64(1024)
-
 func init() {
-	if val := os.Getenv("IPFS_FD_MAX"); val != "" {
-		n, err := strconv.Atoi(val)
-		if err != nil {
-			log.Errorf("bad value for IPFS_FD_MAX: %s", err)
-		} else {
-			ipfsFileDescNum = uint64(n)
-		}
-	}
 	fileDescriptorCheck = checkAndSetUlimit
 }
 

core/commands/add.go

@@ -43,11 +43,9 @@ const (
 
 var AddCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
-		Tagline: "Add a file to ipfs.",
+		Tagline: "Add a file or directory to ipfs.",
 		ShortDescription: `
-Adds contents of <path> to ipfs. Use -r to add directories.
-Note that directories are added recursively, to form the ipfs
-MerkleDAG.
+Adds contents of <path> to ipfs. Use -r to add directories (recursively).
 `,
 		LongDescription: `
 Adds contents of <path> to ipfs. Use -r to add directories.
@@ -76,23 +74,27 @@ You can now refer to the added file in a gateway, like so:
 	},
 	Options: []cmds.Option{
 		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
-		cmds.BoolOption(quietOptionName, "q", "Write minimal output.").Default(false),
-		cmds.BoolOption(silentOptionName, "Write no output.").Default(false),
+		cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
+		cmds.BoolOption(silentOptionName, "Write no output."),
 		cmds.BoolOption(progressOptionName, "p", "Stream progress data."),
-		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation.").Default(false),
-		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk.").Default(false),
-		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object.").Default(false),
-		cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add.").Default(false),
+		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
+		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
+		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
+		cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."),
 		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
 		cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true),
 		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
 		cmds.BoolOption(islibOptionName, "Is for lib").Default(false),
 	},
 	PreRun: func(req cmds.Request) error {
-		if quiet, _, _ := req.Option(quietOptionName).Bool(); quiet {
+		quiet, _, _ := req.Option(quietOptionName).Bool()
+		silent, _, _ := req.Option(silentOptionName).Bool()
+
+		if quiet || silent {
 			return nil
 		}
 
+		// ipfs cli progress bar defaults to true unless quiet or silent is used
 		_, found, _ := req.Option(progressOptionName).Bool()
 		if !found {
 			req.SetOption(progressOptionName, true)
@@ -280,22 +282,12 @@ You can now refer to the added file in a gateway, like so:
 			return
 		}
 
-		silent, _, err := req.Option(silentOptionName).Bool()
-		if err != nil {
-			res.SetError(u.ErrCast(), cmds.ErrNormal)
-			return
-		}
-
 		islib, _, err := req.Option(islibOptionName).Bool()
 		if err != nil {
 			res.SetError(u.ErrCast(), cmds.ErrNormal)
 			return
 		}
 
-		if !quiet && !silent {
-			progress = true
-		}
-
 		var bar *pb.ProgressBar
 		if progress {
 			bar = pb.New64(0).SetUnits(pb.U_BYTES)

core/commands/block.go

@@ -254,16 +254,15 @@ It takes a list of base58 encoded multihashs to remove.
 
 			cids = append(cids, c)
 		}
-		outChan := make(chan interface{})
-		err = util.RmBlocks(n.Blockstore, n.Pinning, outChan, cids, util.RmBlocksOpts{
+		ch, err := util.RmBlocks(n.Blockstore, n.Pinning, cids, util.RmBlocksOpts{
 			Quiet: quiet,
 			Force: force,
 		})
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}
-		res.SetOutput((<-chan interface{})(outChan))
+		res.SetOutput(ch)
 	},
 	PostRun: func(req cmds.Request, res cmds.Response) {
 		if res.Error() != nil {

exchange/bitswap/bitswap.go

@@ -57,7 +57,7 @@ func init() {
 	}
 }
 
-var rebroadcastDelay = delay.Fixed(time.Second * 10)
+var rebroadcastDelay = delay.Fixed(time.Minute)
 
 // New initializes a BitSwap instance that communicates over the provided
 // BitSwapNetwork. This function registers the returned instance as the network

exchange/bitswap/message/message.go

@@ -189,6 +189,7 @@ func (m *impl) ToProtoV0() *pb.Message {
 			Cancel:   proto.Bool(e.Cancel),
 		})
 	}
+	pbm.Wantlist.Full = proto.Bool(m.full)
 	for _, b := range m.Blocks() {
 		pbm.Blocks = append(pbm.Blocks, b.RawData())
 	}
@@ -205,6 +206,7 @@ func (m *impl) ToProtoV1() *pb.Message {
 			Cancel:   proto.Bool(e.Cancel),
 		})
 	}
+	pbm.Wantlist.Full = proto.Bool(m.full)
 	for _, b := range m.Blocks() {
 		blk := &pb.Message_Block{
 			Data:   b.RawData(),

exchange/bitswap/message/message_test.go

@@ -118,6 +118,10 @@ func TestToNetFromNetPreservesWantList(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	if !copied.Full() {
+		t.Fatal("fullness attribute got dropped on marshal")
+	}
+
 	keys := make(map[string]bool)
 	for _, k := range copied.Wantlist() {
 		keys[k.Cid.KeyString()] = true

exchange/bitswap/wantmanager.go

@@ -227,7 +227,7 @@ func (mq *msgQueue) doWork(ctx context.Context) {
 
 		err = mq.openSender(ctx)
 		if err != nil {
-			log.Error("couldnt open sender again after SendMsg(%s) failed: %s", mq.p, err)
+			log.Errorf("couldnt open sender again after SendMsg(%s) failed: %s", mq.p, err)
 			// TODO(why): what do we do now?
 			// I think the *right* answer is to probably put the message we're
 			// trying to send back, and then return to waiting for new work or