gitbase: use only one cache for all reposiories

Also fix a bug getting the cache size from cli.

Signed-off-by: Javi Fontan <[email protected]>

Author: jfontan

CHANGELOG.md

@@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- Fix bug that created multiple object cache with incorrect size ([#898](https://github.com/src-d/gitbase/pull/898))
+
 ## [0.22.0-beta1] - 2019-06-20
 
 ### Added

cmd/gitbase/command/server.go

@@ -220,10 +220,10 @@ func (c *Server) buildDatabase() error {
 		)
 	}
 
-	c.rootLibrary = libraries.New(libraries.Options{})
-	c.pool = gitbase.NewRepositoryPool(c.CacheSize*cache.MiByte, c.rootLibrary)
+	c.sharedCache = cache.NewObjectLRU(c.CacheSize * cache.MiByte)
 
-	c.sharedCache = cache.NewObjectLRU(512 * cache.MiByte)
+	c.rootLibrary = libraries.New(libraries.Options{})
+	c.pool = gitbase.NewRepositoryPool(c.sharedCache, c.rootLibrary)
 
 	if err := c.addDirectories(); err != nil {
 		return err

cmd/gitbase/command/server_test.go

@@ -3,7 +3,11 @@ package command
 import (
 	"testing"
 
+	"io/ioutil"
+	"os"
+
 	"github.com/stretchr/testify/require"
+	"gopkg.in/src-d/go-git.v4/plumbing"
 )
 
 func TestDirectories(t *testing.T) {
@@ -147,3 +151,48 @@ func TestDirectories(t *testing.T) {
 		})
 	}
 }
+
+func TestCache(t *testing.T) {
+	require := require.New(t)
+
+	tmpDir, err := ioutil.TempDir("", "gitbase")
+	require.NoError(err)
+	func() {
+		require.NoError(os.RemoveAll(tmpDir))
+	}()
+
+	server := &Server{
+		CacheSize:   512,
+		Format:      "siva",
+		Bucket:      0,
+		LogLevel:    "debug",
+		Directories: []string{"../../../_testdata"},
+		IndexDir:    tmpDir,
+	}
+
+	err = server.buildDatabase()
+	require.NoError(err)
+
+	cache := server.sharedCache
+	pool := server.pool
+	hash := plumbing.NewHash("dbfab055c70379219cbcf422f05316fdf4e1aed3")
+
+	_, ok := cache.Get(hash)
+	require.False(ok)
+
+	repo, err := pool.GetRepo("015da2f4-6d89-7ec8-5ac9-a38329ea875b")
+	require.NoError(err)
+
+	_, ok = repo.Cache().Get(hash)
+	require.False(ok)
+	require.Equal(cache, repo.Cache())
+
+	_, err = repo.CommitObject(hash)
+	require.NoError(err)
+
+	_, ok = cache.Get(hash)
+	require.True(ok)
+
+	_, ok = repo.Cache().Get(hash)
+	require.True(ok)
+}

common_test.go

@@ -49,7 +49,7 @@ func buildSession(
 	lib, err := newMultiLibrary()
 	require.NoError(err)
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 	for _, fixture := range repos {
 		path := fixture.Worktree().Root()
 		ok, err := IsGitRepo(path)
@@ -173,7 +173,7 @@ func setupSivaCloseRepos(t *testing.T, dir string) (*sql.Context, *closedLibrary
 	require.NoError(err)
 
 	closedLib := &closedLibrary{multiLibrary: lib}
-	pool := NewRepositoryPool(cache.DefaultMaxSize, closedLib)
+	pool := NewRepositoryPool(cache.NewObjectLRUDefault(), closedLib)
 
 	cwd, err := os.Getwd()
 	require.NoError(err)
@@ -298,7 +298,7 @@ func newMultiPool() (*multiLibrary, *RepositoryPool, error) {
 		return nil, nil, err
 	}
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 
 	return lib, pool, err
 }

database_test.go

@@ -15,7 +15,7 @@ func TestDatabase_Tables(t *testing.T) {
 	require := require.New(t)
 
 	lib := libraries.New(libraries.Options{})
-	db := getDB(t, testDBName, NewRepositoryPool(0, lib))
+	db := getDB(t, testDBName, NewRepositoryPool(nil, lib))
 
 	tables := db.Tables()
 	var tableNames []string
@@ -46,7 +46,7 @@ func TestDatabase_Name(t *testing.T) {
 	require := require.New(t)
 
 	lib := libraries.New(libraries.Options{})
-	db := getDB(t, testDBName, NewRepositoryPool(0, lib))
+	db := getDB(t, testDBName, NewRepositoryPool(nil, lib))
 	require.Equal(testDBName, db.Name())
 }
 

integration_test.go

@@ -51,7 +51,7 @@ func TestIntegration(t *testing.T) {
 	require.NoError(t, err)
 	lib.AddLocation(loc)
 
-	pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := gitbase.NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 	engine := newBaseEngine(pool)
 
 	testCases := []struct {
@@ -639,7 +639,7 @@ func TestMissingHeadRefs(t *testing.T) {
 	})
 	require.NoError(err)
 
-	pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := gitbase.NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 	engine := newBaseEngine(pool)
 
 	session := gitbase.NewSession(pool)
@@ -1013,7 +1013,7 @@ func setup(t testing.TB) (*sqle.Engine, *gitbase.RepositoryPool, func()) {
 	require.NoError(t, err)
 	lib.AddLocation(loc)
 
-	pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := gitbase.NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 
 	cleanup := func() {
 		require.NoError(t, fixtures.Clean())

internal/function/commit_stats_test.go

@@ -108,7 +108,7 @@ func setupPool(t *testing.T) (*gitbase.RepositoryPool, func()) {
 	require.NoError(t, err)
 	lib.AddLocation(loc)
 
-	pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := gitbase.NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 
 	return pool, cleanup
 }

internal/function/uast_test.go

@@ -423,11 +423,6 @@ func bblfshFixtures(
 func setup(t *testing.T) (*sql.Context, func()) {
 	t.Helper()
 
-	// pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize)
-	// for _, f := range fixtures.ByTag("worktree") {
-	// 	pool.AddGit(f.Worktree().Root())
-	// }
-
 	// create library directory and move repo inside
 
 	path := fixtures.ByTag("worktree").One().Worktree().Root()
@@ -445,7 +440,7 @@ func setup(t *testing.T) (*sql.Context, func()) {
 	require.NoError(t, err)
 	lib.AddLocation(loc)
 
-	pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := gitbase.NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 
 	session := gitbase.NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))

internal/rule/squashjoins_test.go

@@ -22,7 +22,7 @@ func TestAnalyzeSquashJoinsExchange(t *testing.T) {
 
 	catalog := sql.NewCatalog()
 	catalog.AddDatabase(
-		gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0, lib)),
+		gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(nil, lib)),
 	)
 	a := analyzer.NewBuilder(catalog).
 		WithParallelism(2).
@@ -61,7 +61,7 @@ func TestAnalyzeSquashNaturalJoins(t *testing.T) {
 
 	catalog := sql.NewCatalog()
 	catalog.AddDatabase(
-		gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0, lib)),
+		gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(nil, lib)),
 	)
 	a := analyzer.NewBuilder(catalog).
 		WithParallelism(2).
@@ -2289,7 +2289,7 @@ func TestIsJoinLeafSquashable(t *testing.T) {
 
 func TestOrderedTableNames(t *testing.T) {
 	lib := libraries.New(libraries.Options{})
-	tables := gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(0, lib)).Tables()
+	tables := gitbase.NewDatabase("foo", gitbase.NewRepositoryPool(nil, lib)).Tables()
 
 	input := []sql.Table{
 		tables[gitbase.BlobsTableName],
@@ -2831,5 +2831,5 @@ func (l dummyLookup) Indexes() []string {
 
 func gitbaseTables() map[string]sql.Table {
 	lib := libraries.New(libraries.Options{})
-	return gitbase.NewDatabase("", gitbase.NewRepositoryPool(0, lib)).Tables()
+	return gitbase.NewDatabase("", gitbase.NewRepositoryPool(nil, lib)).Tables()
 }

repositories_test.go

@@ -22,7 +22,7 @@ func TestRepositoriesTable(t *testing.T) {
 	lib, err := newMultiLibrary()
 	require.NoError(err)
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 
 	path := fixtures.Basic().ByTag("worktree").One().Worktree().Root()
 	for _, id := range repoIDs {

repository_pool.go

@@ -74,13 +74,13 @@ type RepositoryPool struct {
 	library borges.Library
 }
 
-// NewRepositoryPool initializes a new RepositoryPool with LRU cache.
+// NewRepositoryPool holds a repository library and a shared object cache.
 func NewRepositoryPool(
-	maxCacheSize cache.FileSize,
+	c cache.Object,
 	lib borges.Library,
 ) *RepositoryPool {
 	return &RepositoryPool{
-		cache:   cache.NewObjectLRU(maxCacheSize),
+		cache:   c,
 		library: lib,
 	}
 }

repository_pool_test.go

@@ -20,7 +20,7 @@ func TestRepositoryPoolBasic(t *testing.T) {
 	lib, err := newMultiLibrary()
 	require.NoError(err)
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 
 	iter, err := pool.RepoIter()
 	require.NoError(err)
@@ -66,7 +66,7 @@ func TestRepositoryPoolGit(t *testing.T) {
 	lib, err := newMultiLibrary()
 	require.NoError(err)
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 
 	require.NoError(lib.AddPlain(path, path, nil))
 
@@ -107,7 +107,7 @@ func TestRepositoryPoolIterator(t *testing.T) {
 	lib, err := newMultiLibrary()
 	require.NoError(err)
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 	err = lib.AddPlain("0", path, nil)
 	require.NoError(err)
 	err = lib.AddPlain("1", path, nil)
@@ -145,7 +145,7 @@ func TestRepositoryPoolSiva(t *testing.T) {
 	cwd, err := os.Getwd()
 	require.NoError(err)
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 	path := filepath.Join(cwd, "_testdata")
 
 	require.NoError(