Submodules init and update

_examples/clone/main.go

@@ -17,8 +17,9 @@ func main() {
 	Info("git clone %s %s", url, directory)
 
 	r, err := git.PlainClone(directory, false, &git.CloneOptions{
-		URL:   url,
-		Depth: 1,
+		URL:                 url,
+		RecursiveSubmodules: true,
+		Depth:               1,
 	})
 
 	CheckIfError(err)

common.go

@@ -1,23 +1,6 @@
 package git
 
-import (
-	"strings"
-
-	"srcd.works/go-git.v4/config"
-	"srcd.works/go-git.v4/plumbing/storer"
-)
-
-// Storer is a generic storage of objects, references and any information
-// related to a particular repository. The package srcd.works/go-git.v4/storage
-// contains two implementation a filesystem base implementation (such as `.git`)
-// and a memory implementations being ephemeral
-type Storer interface {
-	storer.EncodedObjectStorer
-	storer.ReferenceStorer
-	storer.ShallowStorer
-	storer.IndexStorer
-	config.ConfigStorer
-}
+import "strings"
 
 // countLines returns the number of lines in a string à la git, this is
 // The newline character is assumed to be '\n'.  The empty string

doc.go

@@ -7,4 +7,4 @@
 // It is highly extensible, we have been following the open/close principle in
 // its design to facilitate extensions, mainly focusing the efforts on the
 // persistence of the objects.
-package git
+package git // import "srcd.works/go-git.v4"

options.go

@@ -32,6 +32,10 @@ type CloneOptions struct {
 	SingleBranch bool
 	// Limit fetching to the specified number of commits
 	Depth int
+	// RecursiveSubmodules after the clone is created, initialize all submodules
+	// within, using their default settings. This option is ignored if the
+	// cloned repository does not have a worktree
+	RecursiveSubmodules bool
 	// Progress is where the human readable information sent by the server is
 	// stored, if nil nothing is stored and the capability (if supported)
 	// no-progress, is sent to the server to avoid send this information

plumbing/object/file.go

@@ -81,7 +81,7 @@ func (iter *FileIter) Next() (*File, error) {
 			return nil, err
 		}
 
-		if entry.Mode.IsDir() {
+		if entry.Mode.IsDir() || entry.Mode == SubmoduleMode {
 			continue
 		}
 

plumbing/object/file_test.go

@@ -247,3 +247,29 @@ func (s *FileSuite) TestFileIter(c *C) {
 
 	c.Assert(count, Equals, 1)
 }
+
+func (s *FileSuite) TestFileIterSubmodule(c *C) {
+	st, err := filesystem.NewStorage(fixtures.ByTag("submodule").One().DotGit())
+	c.Assert(err, IsNil)
+
+	hash := plumbing.NewHash("a692ec699bff9117c1ed91752afbb7d9d272ebef")
+	commit, err := GetCommit(st, hash)
+	c.Assert(err, IsNil)
+
+	tree, err := commit.Tree()
+	c.Assert(err, IsNil)
+
+	expected := []string{
+		".gitmodules",
+	}
+
+	var count int
+	i := tree.Files()
+	i.ForEach(func(f *File) error {
+		c.Assert(f.Name, Equals, expected[count])
+		count++
+		return nil
+	})
+
+	c.Assert(count, Equals, 1)
+}

plumbing/object/tree.go

@@ -375,11 +375,6 @@ func (w *TreeWalker) Next() (name string, entry TreeEntry, err error) {
 			return
 		}
 
-		if entry.Mode == SubmoduleMode {
-			err = nil
-			continue
-		}
-
 		if entry.Mode.IsDir() {
 			obj, err = GetTree(w.s, entry.Hash)
 		}

plumbing/object/tree_test.go

@@ -4,7 +4,10 @@ import (
 	"io"
 	"os"
 
+	fixtures "github.com/src-d/go-git-fixtures"
+
 	"srcd.works/go-git.v4/plumbing"
+	"srcd.works/go-git.v4/storage/filesystem"
 
 	. "gopkg.in/check.v1"
 	"srcd.works/go-git.v4/plumbing/storer"
@@ -262,6 +265,41 @@ func (s *TreeSuite) TestTreeWalkerNextNonRecursive(c *C) {
 	c.Assert(count, Equals, 8)
 }
 
+func (s *TreeSuite) TestTreeWalkerNextSubmodule(c *C) {
+	st, err := filesystem.NewStorage(fixtures.ByTag("submodule").One().DotGit())
+	c.Assert(err, IsNil)
+
+	hash := plumbing.NewHash("a692ec699bff9117c1ed91752afbb7d9d272ebef")
+	commit, err := GetCommit(st, hash)
+	c.Assert(err, IsNil)
+
+	tree, err := commit.Tree()
+	c.Assert(err, IsNil)
+
+	expected := []string{
+		".gitmodules",
+		"basic",
+		"itself",
+	}
+
+	var count int
+	walker := NewTreeWalker(tree, true)
+	for {
+		name, entry, err := walker.Next()
+		if err == io.EOF {
+			break
+		}
+
+		c.Assert(err, IsNil)
+		c.Assert(entry, NotNil)
+		c.Assert(name, Equals, expected[count])
+
+		count++
+	}
+
+	c.Assert(count, Equals, 3)
+}
+
 var treeWalkerExpects = []struct {
 	Path, Mode, Name, Hash, Tree string
 }{{

remote.go

@@ -16,6 +16,7 @@ import (
 	"srcd.works/go-git.v4/plumbing/storer"
 	"srcd.works/go-git.v4/plumbing/transport"
 	"srcd.works/go-git.v4/plumbing/transport/client"
+	"srcd.works/go-git.v4/storage"
 	"srcd.works/go-git.v4/storage/memory"
 	"srcd.works/go-git.v4/utils/ioutil"
 )
@@ -25,10 +26,10 @@ var NoErrAlreadyUpToDate = errors.New("already up-to-date")
 // Remote represents a connection to a remote repository
 type Remote struct {
 	c *config.RemoteConfig
-	s Storer
+	s storage.Storer
 }
 
-func newRemote(s Storer, c *config.RemoteConfig) *Remote {
+func newRemote(s storage.Storer, c *config.RemoteConfig) *Remote {
 	return &Remote{s: s, c: c}
 }
 
@@ -321,7 +322,9 @@ func getHaves(localRefs storer.ReferenceStorer) ([]plumbing.Hash, error) {
 	return haves, nil
 }
 
-func getWants(spec []config.RefSpec, localStorer Storer, remoteRefs storer.ReferenceStorer) ([]plumbing.Hash, error) {
+func getWants(
+	spec []config.RefSpec, localStorer storage.Storer, remoteRefs storer.ReferenceStorer,
+) ([]plumbing.Hash, error) {
 	wantTags := true
 	for _, s := range spec {
 		if !s.IsWildcard() {

remote_test.go

@@ -11,6 +11,7 @@ import (
 	"srcd.works/go-git.v4/config"
 	"srcd.works/go-git.v4/plumbing"
 	"srcd.works/go-git.v4/plumbing/storer"
+	"srcd.works/go-git.v4/storage"
 	"srcd.works/go-git.v4/storage/filesystem"
 	"srcd.works/go-git.v4/storage/memory"
 
@@ -126,7 +127,7 @@ func (s *RemoteSuite) TestFetchWithProgress(c *C) {
 }
 
 type mockPackfileWriter struct {
-	Storer
+	storage.Storer
 	PackfileWriterCalled bool
 }
 

repository.go

@@ -10,6 +10,7 @@ import (
 	"srcd.works/go-git.v4/plumbing"
 	"srcd.works/go-git.v4/plumbing/object"
 	"srcd.works/go-git.v4/plumbing/storer"
+	"srcd.works/go-git.v4/storage"
 	"srcd.works/go-git.v4/storage/filesystem"
 
 	"srcd.works/go-billy.v1"
@@ -29,7 +30,7 @@ var (
 
 // Repository represents a git repository
 type Repository struct {
-	Storer Storer
+	Storer storage.Storer
 
 	r  map[string]*Remote
 	wt billy.Filesystem
@@ -38,7 +39,7 @@ type Repository struct {
 // Init creates an empty git repository, based on the given Storer and worktree.
 // The worktree Filesystem is optional, if nil a bare repository is created. If
 // the given storer is not empty ErrRepositoryAlreadyExists is returned
-func Init(s Storer, worktree billy.Filesystem) (*Repository, error) {
+func Init(s storage.Storer, worktree billy.Filesystem) (*Repository, error) {
 	r := newRepository(s, worktree)
 	_, err := r.Reference(plumbing.HEAD, false)
 	switch err {
@@ -66,7 +67,7 @@ func Init(s Storer, worktree billy.Filesystem) (*Repository, error) {
 // The worktree can be nil when the repository being opened is bare, if the
 // repository is a normal one (not bare) and worktree is nil the err
 // ErrWorktreeNotProvided is returned
-func Open(s Storer, worktree billy.Filesystem) (*Repository, error) {
+func Open(s storage.Storer, worktree billy.Filesystem) (*Repository, error) {
 	_, err := s.Reference(plumbing.HEAD)
 	if err == plumbing.ErrReferenceNotFound {
 		return nil, ErrRepositoryNotExists
@@ -91,7 +92,7 @@ func Open(s Storer, worktree billy.Filesystem) (*Repository, error) {
 // Clone a repository into the given Storer and worktree Filesystem with the
 // given options, if worktree is nil a bare repository is created. If the given
 // storer is not empty ErrRepositoryAlreadyExists is returned
-func Clone(s Storer, worktree billy.Filesystem, o *CloneOptions) (*Repository, error) {
+func Clone(s storage.Storer, worktree billy.Filesystem, o *CloneOptions) (*Repository, error) {
 	r, err := Init(s, worktree)
 	if err != nil {
 		return nil, err
@@ -159,7 +160,7 @@ func PlainClone(path string, isBare bool, o *CloneOptions) (*Repository, error)
 	return r, r.clone(o)
 }
 
-func newRepository(s Storer, worktree billy.Filesystem) *Repository {
+func newRepository(s storage.Storer, worktree billy.Filesystem) *Repository {
 	return &Repository{
 		Storer: s,
 		wt:     worktree,
@@ -247,12 +248,6 @@ func (r *Repository) clone(o *CloneOptions) error {
 		return err
 	}
 
-	// marks the repository as bare in the config, until we have Worktree, all
-	// the repository are bare
-	if err := r.setIsBare(true); err != nil {
-		return err
-	}
-
 	c := &config.RemoteConfig{
 		Name: o.RemoteName,
 		URL:  o.URL,
@@ -270,11 +265,13 @@ func (r *Repository) clone(o *CloneOptions) error {
 		Progress: o.Progress,
 	})
 	if err != nil {
+
 		return err
 	}
 
 	head, err := storer.ResolveReference(remoteRefs, o.ReferenceName)
 	if err != nil {
+
 		return err
 	}
 
@@ -283,12 +280,33 @@ func (r *Repository) clone(o *CloneOptions) error {
 	}
 
 	if err := r.updateWorktree(); err != nil {
+		fmt.Println("q", err)
 		return err
 	}
 
+	if o.RecursiveSubmodules && r.wt != nil {
+		if err := r.initSubmodules(); err != nil {
+			return err
+		}
+	}
+
 	return r.updateRemoteConfig(remote, o, c, head)
 }
 
+func (r *Repository) initSubmodules() error {
+	w, err := r.Worktree()
+	if err != nil {
+		return err
+	}
+
+	s, err := w.Submodules()
+	if err != nil {
+		return err
+	}
+
+	return s.Init()
+}
+
 func (r *Repository) cloneRefSpec(o *CloneOptions,
 	c *config.RemoteConfig) []config.RefSpec {
 

repository_test.go

@@ -516,26 +516,21 @@ func (s *RepositorySuite) TestPullProgress(c *C) {
 }
 
 func (s *RepositorySuite) TestPullAdd(c *C) {
-	path := fixtures.Basic().One().Worktree().Base()
+	path := fixtures.Basic().ByTag("worktree").One().Worktree().Base()
 
-	r, _ := Init(memory.NewStorage(), nil)
-	err := r.clone(&CloneOptions{
+	r, err := Clone(memory.NewStorage(), nil, &CloneOptions{
 		URL: fmt.Sprintf("file://%s", filepath.Join(path, ".git")),
 	})
 
 	c.Assert(err, IsNil)
 
 	storage := r.Storer.(*memory.Storage)
-	c.Assert(storage.Objects, HasLen, 31)
+	c.Assert(storage.Objects, HasLen, 28)
 
 	branch, err := r.Reference("refs/heads/master", false)
 	c.Assert(err, IsNil)
 	c.Assert(branch.Hash().String(), Equals, "6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
 
-	branch, err = r.Reference("refs/remotes/origin/branch", false)
-	c.Assert(err, IsNil)
-	c.Assert(branch.Hash().String(), Equals, "e8d3ffab552895c19b9fcf7aa264d277cde33881")
-
 	ExecuteOnPath(c, path,
 		"touch foo",
 		"git add foo",
@@ -546,16 +541,11 @@ func (s *RepositorySuite) TestPullAdd(c *C) {
 	c.Assert(err, IsNil)
 
 	// the commit command has introduced a new commit, tree and blob
-	c.Assert(storage.Objects, HasLen, 34)
+	c.Assert(storage.Objects, HasLen, 31)
 
 	branch, err = r.Reference("refs/heads/master", false)
 	c.Assert(err, IsNil)
 	c.Assert(branch.Hash().String(), Not(Equals), "6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
-
-	// the commit command, was in the local branch, so the remote should be read ok
-	branch, err = r.Reference("refs/remotes/origin/branch", false)
-	c.Assert(err, IsNil)
-	c.Assert(branch.Hash().String(), Equals, "e8d3ffab552895c19b9fcf7aa264d277cde33881")
 }
 
 func (s *RepositorySuite) TestPushToEmptyRepository(c *C) {

storage/filesystem/internal/dotgit/dotgit.go

@@ -20,10 +20,10 @@ const (
 	configPath     = "config"
 	indexPath      = "index"
 	shallowPath    = "shallow"
-
-	objectsPath = "objects"
-	packPath    = "pack"
-	refsPath    = "refs"
+	modulePath     = "module"
+	objectsPath    = "objects"
+	packPath       = "pack"
+	refsPath       = "refs"
 
 	packExt = ".pack"
 	idxExt  = ".idx"
@@ -390,6 +390,11 @@ func (d *DotGit) readReferenceFile(refsPath, refFile string) (ref *plumbing.Refe
 	return plumbing.NewReferenceFromStrings(refFile, line), nil
 }
 
+// Module return a billy.Filesystem poiting to the module folder
+func (d *DotGit) Module(name string) billy.Filesystem {
+	return d.fs.Dir(d.fs.Join(modulePath, name))
+}
+
 func isHex(s string) bool {
 	for _, b := range []byte(s) {
 		if isNum(b) {