Skip to content

Commit 001dbf1

Browse files
zeripathzeripath
authored
Defer Last Commit Info (#16467)
One of the biggest reasons for slow repository browsing is that we wait until last commit information has been generated for all files in the repository. This PR proposes deferring this generation to a new POST endpoint that does the look up outside of the main page request. Signed-off-by: Andrew Thornton <[email protected]>
1 parent 88fa9f3 commit 001dbf1

13 files changed

+321
-151
lines changed

‎modules/git/commit_info_gogit.go

+35-26
Original file line numberDiff line numberDiff line change
@@ -44,20 +44,17 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
4444
return nil, nil, err
4545
}
4646
if len(unHitPaths) > 0 {
47-
revs2, err := GetLastCommitForPaths(ctx, c, treePath, unHitPaths)
47+
revs2, err := GetLastCommitForPaths(ctx, cache, c, treePath, unHitPaths)
4848
if err != nil {
4949
return nil, nil, err
5050
}
5151

5252
for k, v := range revs2 {
53-
if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
54-
return nil, nil, err
55-
}
5653
revs[k] = v
5754
}
5855
}
5956
} else {
60-
revs, err = GetLastCommitForPaths(ctx, c, treePath, entryPaths)
57+
revs, err = GetLastCommitForPaths(ctx, nil, c, treePath, entryPaths)
6158
}
6259
if err != nil {
6360
return nil, nil, err
@@ -70,25 +67,29 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
7067
commitsInfo[i] = CommitInfo{
7168
Entry: entry,
7269
}
70+
71+
// Check if we have found a commit for this entry in time
7372
if rev, ok := revs[entry.Name()]; ok {
7473
entryCommit := convertCommit(rev)
7574
commitsInfo[i].Commit = entryCommit
76-
if entry.IsSubModule() {
77-
subModuleURL := ""
78-
var fullPath string
79-
if len(treePath) > 0 {
80-
fullPath = treePath + "/" + entry.Name()
81-
} else {
82-
fullPath = entry.Name()
83-
}
84-
if subModule, err := commit.GetSubModule(fullPath); err != nil {
85-
return nil, nil, err
86-
} else if subModule != nil {
87-
subModuleURL = subModule.URL
88-
}
89-
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
90-
commitsInfo[i].SubModuleFile = subModuleFile
75+
}
76+
77+
// If the entry if a submodule add a submodule file for this
78+
if entry.IsSubModule() {
79+
subModuleURL := ""
80+
var fullPath string
81+
if len(treePath) > 0 {
82+
fullPath = treePath + "/" + entry.Name()
83+
} else {
84+
fullPath = entry.Name()
9185
}
86+
if subModule, err := commit.GetSubModule(fullPath); err != nil {
87+
return nil, nil, err
88+
} else if subModule != nil {
89+
subModuleURL = subModule.URL
90+
}
91+
subModuleFile := NewSubModuleFile(commitsInfo[i].Commit, subModuleURL, entry.ID.String())
92+
commitsInfo[i].SubModuleFile = subModuleFile
9293
}
9394
}
9495

@@ -175,7 +176,9 @@ func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cac
175176
}
176177

177178
// GetLastCommitForPaths returns last commit information
178-
func GetLastCommitForPaths(ctx context.Context, c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
179+
func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
180+
refSha := c.ID().String()
181+
179182
// We do a tree traversal with nodes sorted by commit time
180183
heap := binaryheap.NewWith(func(a, b interface{}) int {
181184
if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
@@ -192,10 +195,13 @@ func GetLastCommitForPaths(ctx context.Context, c cgobject.CommitNode, treePath
192195

193196
// Start search from the root commit and with full set of paths
194197
heap.Push(&commitAndPaths{c, paths, initialHashes})
195-
198+
heaploop:
196199
for {
197200
select {
198201
case <-ctx.Done():
202+
if ctx.Err() == context.DeadlineExceeded {
203+
break heaploop
204+
}
199205
return nil, ctx.Err()
200206
default:
201207
}
@@ -233,14 +239,14 @@ func GetLastCommitForPaths(ctx context.Context, c cgobject.CommitNode, treePath
233239
}
234240

235241
var remainingPaths []string
236-
for i, path := range current.paths {
242+
for i, pth := range current.paths {
237243
// The results could already contain some newer change for the same path,
238244
// so don't override that and bail out on the file early.
239-
if resultNodes[path] == nil {
245+
if resultNodes[pth] == nil {
240246
if pathUnchanged[i] {
241247
// The path existed with the same hash in at least one parent so it could
242248
// not have been changed in this commit directly.
243-
remainingPaths = append(remainingPaths, path)
249+
remainingPaths = append(remainingPaths, pth)
244250
} else {
245251
// There are few possible cases how can we get here:
246252
// - The path didn't exist in any parent, so it must have been created by
@@ -250,7 +256,10 @@ func GetLastCommitForPaths(ctx context.Context, c cgobject.CommitNode, treePath
250256
// - We are looking at a merge commit and the hash of the file doesn't
251257
// match any of the hashes being merged. This is more common for directories,
252258
// but it can also happen if a file is changed through conflict resolution.
253-
resultNodes[path] = current.commit
259+
resultNodes[pth] = current.commit
260+
if err := cache.Put(refSha, path.Join(treePath, pth), current.commit.ID().String()); err != nil {
261+
return nil, err
262+
}
254263
}
255264
}
256265
}

‎modules/git/commit_info_nogogit.go

+24-23
Original file line numberDiff line numberDiff line change
@@ -37,21 +37,18 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
3737
}
3838
if len(unHitPaths) > 0 {
3939
sort.Strings(unHitPaths)
40-
commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths)
40+
commits, err := GetLastCommitForPaths(ctx, cache, commit, treePath, unHitPaths)
4141
if err != nil {
4242
return nil, nil, err
4343
}
4444

4545
for pth, found := range commits {
46-
if err := cache.Put(commit.ID.String(), path.Join(treePath, pth), found.ID.String()); err != nil {
47-
return nil, nil, err
48-
}
4946
revs[pth] = found
5047
}
5148
}
5249
} else {
5350
sort.Strings(entryPaths)
54-
revs, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths)
51+
revs, err = GetLastCommitForPaths(ctx, nil, commit, treePath, entryPaths)
5552
}
5653
if err != nil {
5754
return nil, nil, err
@@ -62,27 +59,31 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
6259
commitsInfo[i] = CommitInfo{
6360
Entry: entry,
6461
}
62+
63+
// Check if we have found a commit for this entry in time
6564
if entryCommit, ok := revs[entry.Name()]; ok {
6665
commitsInfo[i].Commit = entryCommit
67-
if entry.IsSubModule() {
68-
subModuleURL := ""
69-
var fullPath string
70-
if len(treePath) > 0 {
71-
fullPath = treePath + "/" + entry.Name()
72-
} else {
73-
fullPath = entry.Name()
74-
}
75-
if subModule, err := commit.GetSubModule(fullPath); err != nil {
76-
return nil, nil, err
77-
} else if subModule != nil {
78-
subModuleURL = subModule.URL
79-
}
80-
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
81-
commitsInfo[i].SubModuleFile = subModuleFile
82-
}
8366
} else {
8467
log.Debug("missing commit for %s", entry.Name())
8568
}
69+
70+
// If the entry if a submodule add a submodule file for this
71+
if entry.IsSubModule() {
72+
subModuleURL := ""
73+
var fullPath string
74+
if len(treePath) > 0 {
75+
fullPath = treePath + "/" + entry.Name()
76+
} else {
77+
fullPath = entry.Name()
78+
}
79+
if subModule, err := commit.GetSubModule(fullPath); err != nil {
80+
return nil, nil, err
81+
} else if subModule != nil {
82+
subModuleURL = subModule.URL
83+
}
84+
subModuleFile := NewSubModuleFile(commitsInfo[i].Commit, subModuleURL, entry.ID.String())
85+
commitsInfo[i].SubModuleFile = subModuleFile
86+
}
8687
}
8788

8889
// Retrieve the commit for the treePath itself (see above). We basically
@@ -121,9 +122,9 @@ func getLastCommitForPathsByCache(ctx context.Context, commitID, treePath string
121122
}
122123

123124
// GetLastCommitForPaths returns last commit information
124-
func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) {
125+
func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) {
125126
// We read backwards from the commit to obtain all of the commits
126-
revs, err := WalkGitLog(ctx, commit.repo, commit, treePath, paths...)
127+
revs, err := WalkGitLog(ctx, cache, commit.repo, commit, treePath, paths...)
127128
if err != nil {
128129
return nil, err
129130
}

‎modules/git/last_commit_cache.go

+3
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string {
2626

2727
// Put put the last commit id with commit and entry path
2828
func (c *LastCommitCache) Put(ref, entryPath, commitID string) error {
29+
if c == nil || c.cache == nil {
30+
return nil
31+
}
2932
log.Debug("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
3033
return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl())
3134
}

‎modules/git/last_commit_cache_gogit.go

+2-6
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ package git
99

1010
import (
1111
"context"
12-
"path"
1312

1413
"code.gitea.io/gitea/modules/log"
1514

@@ -93,15 +92,12 @@ func (c *LastCommitCache) recursiveCache(ctx context.Context, index cgobject.Com
9392
entryMap[entry.Name()] = entry
9493
}
9594

96-
commits, err := GetLastCommitForPaths(ctx, index, treePath, entryPaths)
95+
commits, err := GetLastCommitForPaths(ctx, c, index, treePath, entryPaths)
9796
if err != nil {
9897
return err
9998
}
10099

101-
for entry, cm := range commits {
102-
if err := c.Put(index.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil {
103-
return err
104-
}
100+
for entry := range commits {
105101
if entryMap[entry].IsDir() {
106102
subTree, err := tree.SubTree(entry)
107103
if err != nil {

‎modules/git/last_commit_cache_nogogit.go

+5-11
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ package git
1010
import (
1111
"bufio"
1212
"context"
13-
"path"
1413

1514
"code.gitea.io/gitea/modules/log"
1615
)
@@ -80,28 +79,23 @@ func (c *LastCommitCache) recursiveCache(ctx context.Context, commit *Commit, tr
8079
}
8180

8281
entryPaths := make([]string, len(entries))
83-
entryMap := make(map[string]*TreeEntry)
8482
for i, entry := range entries {
8583
entryPaths[i] = entry.Name()
86-
entryMap[entry.Name()] = entry
8784
}
8885

89-
commits, err := GetLastCommitForPaths(ctx, commit, treePath, entryPaths)
86+
_, err = WalkGitLog(ctx, c, commit.repo, commit, treePath, entryPaths...)
9087
if err != nil {
9188
return err
9289
}
9390

94-
for entry, entryCommit := range commits {
95-
if err := c.Put(commit.ID.String(), path.Join(treePath, entry), entryCommit.ID.String()); err != nil {
96-
return err
97-
}
91+
for _, treeEntry := range entries {
9892
// entryMap won't contain "" therefore skip this.
99-
if treeEntry := entryMap[entry]; treeEntry != nil && treeEntry.IsDir() {
100-
subTree, err := tree.SubTree(entry)
93+
if treeEntry.IsDir() {
94+
subTree, err := tree.SubTree(treeEntry.Name())
10195
if err != nil {
10296
return err
10397
}
104-
if err := c.recursiveCache(ctx, commit, subTree, entry, level-1); err != nil {
98+
if err := c.recursiveCache(ctx, commit, subTree, treeEntry.Name(), level-1); err != nil {
10599
return err
106100
}
107101
}

‎modules/git/log_name_status.go

+12-1
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,9 @@ func (g *LogNameStatusRepoParser) Close() {
275275
}
276276

277277
// WalkGitLog walks the git log --name-status for the head commit in the provided treepath and files
278-
func WalkGitLog(ctx context.Context, repo *Repository, head *Commit, treepath string, paths ...string) (map[string]string, error) {
278+
func WalkGitLog(ctx context.Context, cache *LastCommitCache, repo *Repository, head *Commit, treepath string, paths ...string) (map[string]string, error) {
279+
headRef := head.ID.String()
280+
279281
tree, err := head.SubTree(treepath)
280282
if err != nil {
281283
return nil, err
@@ -339,6 +341,9 @@ heaploop:
339341
for {
340342
select {
341343
case <-ctx.Done():
344+
if ctx.Err() == context.DeadlineExceeded {
345+
break heaploop
346+
}
342347
g.Close()
343348
return nil, ctx.Err()
344349
default:
@@ -360,10 +365,16 @@ heaploop:
360365
changed[i] = false
361366
if results[i] == "" {
362367
results[i] = current.CommitID
368+
if err := cache.Put(headRef, path.Join(treepath, paths[i]), current.CommitID); err != nil {
369+
return nil, err
370+
}
363371
delete(path2idx, paths[i])
364372
remaining--
365373
if results[0] == "" {
366374
results[0] = current.CommitID
375+
if err := cache.Put(headRef, treepath, current.CommitID); err != nil {
376+
return nil, err
377+
}
367378
delete(path2idx, "")
368379
remaining--
369380
}

‎modules/git/notes_gogit.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
)
1818

1919
// GetNote retrieves the git-notes data for a given commit.
20+
// FIXME: Add LastCommitCache support
2021
func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) error {
2122
log.Trace("Searching for git note corresponding to the commit %q in the repository %q", commitID, repo.Path)
2223
notes, err := repo.GetCommit(NotesRef)
@@ -75,7 +76,7 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note)
7576
return err
7677
}
7778

78-
lastCommits, err := GetLastCommitForPaths(ctx, commitNode, "", []string{path})
79+
lastCommits, err := GetLastCommitForPaths(ctx, nil, commitNode, "", []string{path})
7980
if err != nil {
8081
log.Error("Unable to get the commit for the path %q. Error: %v", path, err)
8182
return err

‎modules/git/notes_nogogit.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
)
1717

1818
// GetNote retrieves the git-notes data for a given commit.
19+
// FIXME: Add LastCommitCache support
1920
func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) error {
2021
log.Trace("Searching for git note corresponding to the commit %q in the repository %q", commitID, repo.Path)
2122
notes, err := repo.GetCommit(NotesRef)
@@ -75,7 +76,7 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note)
7576
path = path[idx+1:]
7677
}
7778

78-
lastCommits, err := GetLastCommitForPaths(ctx, notes, treePath, []string{path})
79+
lastCommits, err := GetLastCommitForPaths(ctx, nil, notes, treePath, []string{path})
7980
if err != nil {
8081
log.Error("Unable to get the commit for the path %q. Error: %v", treePath, err)
8182
return err

0 commit comments

Comments
 (0)