Skip to content

Commit a741466

Browse files
zeripathzeripath
committed
Defer Last Commit Info
One of the biggest reasons for slow repository browsing is that we wait until last commit information has been generated for all files in the repository. This PR proposes deferring this generation to a new POST endpoint that does the look up outside of the main page request. Signed-off-by: Andrew Thornton <[email protected]>
1 parent b08e14b commit a741466

File tree

13 files changed

+296
-151
lines changed

13 files changed

+296
-151
lines changed

‎modules/git/commit_info_gogit.go

Lines changed: 35 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -43,20 +43,17 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
4343
return nil, nil, err
4444
}
4545
if len(unHitPaths) > 0 {
46-
revs2, err := GetLastCommitForPaths(ctx, c, treePath, unHitPaths)
46+
revs2, err := GetLastCommitForPaths(ctx, cache, c, treePath, unHitPaths)
4747
if err != nil {
4848
return nil, nil, err
4949
}
5050

5151
for k, v := range revs2 {
52-
if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
53-
return nil, nil, err
54-
}
5552
revs[k] = v
5653
}
5754
}
5855
} else {
59-
revs, err = GetLastCommitForPaths(ctx, c, treePath, entryPaths)
56+
revs, err = GetLastCommitForPaths(ctx, nil, c, treePath, entryPaths)
6057
}
6158
if err != nil {
6259
return nil, nil, err
@@ -69,25 +66,29 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
6966
commitsInfo[i] = CommitInfo{
7067
Entry: entry,
7168
}
69+
70+
// Check if we have found a commit for this entry in time
7271
if rev, ok := revs[entry.Name()]; ok {
7372
entryCommit := convertCommit(rev)
7473
commitsInfo[i].Commit = entryCommit
75-
if entry.IsSubModule() {
76-
subModuleURL := ""
77-
var fullPath string
78-
if len(treePath) > 0 {
79-
fullPath = treePath + "/" + entry.Name()
80-
} else {
81-
fullPath = entry.Name()
82-
}
83-
if subModule, err := commit.GetSubModule(fullPath); err != nil {
84-
return nil, nil, err
85-
} else if subModule != nil {
86-
subModuleURL = subModule.URL
87-
}
88-
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
89-
commitsInfo[i].SubModuleFile = subModuleFile
74+
}
75+
76+
// If the entry if a submodule add a submodule file for this
77+
if entry.IsSubModule() {
78+
subModuleURL := ""
79+
var fullPath string
80+
if len(treePath) > 0 {
81+
fullPath = treePath + "/" + entry.Name()
82+
} else {
83+
fullPath = entry.Name()
9084
}
85+
if subModule, err := commit.GetSubModule(fullPath); err != nil {
86+
return nil, nil, err
87+
} else if subModule != nil {
88+
subModuleURL = subModule.URL
89+
}
90+
subModuleFile := NewSubModuleFile(commitsInfo[i].Commit, subModuleURL, entry.ID.String())
91+
commitsInfo[i].SubModuleFile = subModuleFile
9192
}
9293
}
9394

@@ -174,7 +175,9 @@ func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cac
174175
}
175176

176177
// GetLastCommitForPaths returns last commit information
177-
func GetLastCommitForPaths(ctx context.Context, c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
178+
func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
179+
refSha := c.ID().String()
180+
178181
// We do a tree traversal with nodes sorted by commit time
179182
heap := binaryheap.NewWith(func(a, b interface{}) int {
180183
if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
@@ -191,10 +194,13 @@ func GetLastCommitForPaths(ctx context.Context, c cgobject.CommitNode, treePath
191194

192195
// Start search from the root commit and with full set of paths
193196
heap.Push(&commitAndPaths{c, paths, initialHashes})
194-
197+
heaploop:
195198
for {
196199
select {
197200
case <-ctx.Done():
201+
if ctx.Err() == context.DeadlineExceeded {
202+
break heaploop
203+
}
198204
return nil, ctx.Err()
199205
default:
200206
}
@@ -232,14 +238,14 @@ func GetLastCommitForPaths(ctx context.Context, c cgobject.CommitNode, treePath
232238
}
233239

234240
var remainingPaths []string
235-
for i, path := range current.paths {
241+
for i, pth := range current.paths {
236242
// The results could already contain some newer change for the same path,
237243
// so don't override that and bail out on the file early.
238-
if resultNodes[path] == nil {
244+
if resultNodes[pth] == nil {
239245
if pathUnchanged[i] {
240246
// The path existed with the same hash in at least one parent so it could
241247
// not have been changed in this commit directly.
242-
remainingPaths = append(remainingPaths, path)
248+
remainingPaths = append(remainingPaths, pth)
243249
} else {
244250
// There are few possible cases how can we get here:
245251
// - The path didn't exist in any parent, so it must have been created by
@@ -249,7 +255,10 @@ func GetLastCommitForPaths(ctx context.Context, c cgobject.CommitNode, treePath
249255
// - We are looking at a merge commit and the hash of the file doesn't
250256
// match any of the hashes being merged. This is more common for directories,
251257
// but it can also happen if a file is changed through conflict resolution.
252-
resultNodes[path] = current.commit
258+
resultNodes[pth] = current.commit
259+
if err := cache.Put(refSha, path.Join(treePath, pth), current.commit.ID().String()); err != nil {
260+
return nil, err
261+
}
253262
}
254263
}
255264
}

‎modules/git/commit_info_nogogit.go

Lines changed: 24 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -36,21 +36,18 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
3636
}
3737
if len(unHitPaths) > 0 {
3838
sort.Strings(unHitPaths)
39-
commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths)
39+
commits, err := GetLastCommitForPaths(ctx, cache, commit, treePath, unHitPaths)
4040
if err != nil {
4141
return nil, nil, err
4242
}
4343

4444
for pth, found := range commits {
45-
if err := cache.Put(commit.ID.String(), path.Join(treePath, pth), found.ID.String()); err != nil {
46-
return nil, nil, err
47-
}
4845
revs[pth] = found
4946
}
5047
}
5148
} else {
5249
sort.Strings(entryPaths)
53-
revs, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths)
50+
revs, err = GetLastCommitForPaths(ctx, nil, commit, treePath, entryPaths)
5451
}
5552
if err != nil {
5653
return nil, nil, err
@@ -61,27 +58,31 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
6158
commitsInfo[i] = CommitInfo{
6259
Entry: entry,
6360
}
61+
62+
// Check if we have found a commit for this entry in time
6463
if entryCommit, ok := revs[entry.Name()]; ok {
6564
commitsInfo[i].Commit = entryCommit
66-
if entry.IsSubModule() {
67-
subModuleURL := ""
68-
var fullPath string
69-
if len(treePath) > 0 {
70-
fullPath = treePath + "/" + entry.Name()
71-
} else {
72-
fullPath = entry.Name()
73-
}
74-
if subModule, err := commit.GetSubModule(fullPath); err != nil {
75-
return nil, nil, err
76-
} else if subModule != nil {
77-
subModuleURL = subModule.URL
78-
}
79-
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
80-
commitsInfo[i].SubModuleFile = subModuleFile
81-
}
8265
} else {
8366
log.Debug("missing commit for %s", entry.Name())
8467
}
68+
69+
// If the entry if a submodule add a submodule file for this
70+
if entry.IsSubModule() {
71+
subModuleURL := ""
72+
var fullPath string
73+
if len(treePath) > 0 {
74+
fullPath = treePath + "/" + entry.Name()
75+
} else {
76+
fullPath = entry.Name()
77+
}
78+
if subModule, err := commit.GetSubModule(fullPath); err != nil {
79+
return nil, nil, err
80+
} else if subModule != nil {
81+
subModuleURL = subModule.URL
82+
}
83+
subModuleFile := NewSubModuleFile(commitsInfo[i].Commit, subModuleURL, entry.ID.String())
84+
commitsInfo[i].SubModuleFile = subModuleFile
85+
}
8586
}
8687

8788
// Retrieve the commit for the treePath itself (see above). We basically
@@ -120,9 +121,9 @@ func getLastCommitForPathsByCache(ctx context.Context, commitID, treePath string
120121
}
121122

122123
// GetLastCommitForPaths returns last commit information
123-
func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) {
124+
func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) {
124125
// We read backwards from the commit to obtain all of the commits
125-
revs, err := WalkGitLog(ctx, commit.repo, commit, treePath, paths...)
126+
revs, err := WalkGitLog(ctx, cache, commit.repo, commit, treePath, paths...)
126127
if err != nil {
127128
return nil, err
128129
}

‎modules/git/last_commit_cache.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string {
2626

2727
// Put put the last commit id with commit and entry path
2828
func (c *LastCommitCache) Put(ref, entryPath, commitID string) error {
29+
if c == nil || c.cache == nil {
30+
return nil
31+
}
2932
log.Debug("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
3033
return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl())
3134
}

‎modules/git/last_commit_cache_gogit.go

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ package git
88

99
import (
1010
"context"
11-
"path"
1211

1312
"code.gitea.io/gitea/modules/log"
1413

@@ -92,15 +91,12 @@ func (c *LastCommitCache) recursiveCache(ctx context.Context, index cgobject.Com
9291
entryMap[entry.Name()] = entry
9392
}
9493

95-
commits, err := GetLastCommitForPaths(ctx, index, treePath, entryPaths)
94+
commits, err := GetLastCommitForPaths(ctx, c, index, treePath, entryPaths)
9695
if err != nil {
9796
return err
9897
}
9998

100-
for entry, cm := range commits {
101-
if err := c.Put(index.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil {
102-
return err
103-
}
99+
for entry := range commits {
104100
if entryMap[entry].IsDir() {
105101
subTree, err := tree.SubTree(entry)
106102
if err != nil {

‎modules/git/last_commit_cache_nogogit.go

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ package git
99
import (
1010
"bufio"
1111
"context"
12-
"path"
1312

1413
"code.gitea.io/gitea/modules/log"
1514
)
@@ -79,28 +78,23 @@ func (c *LastCommitCache) recursiveCache(ctx context.Context, commit *Commit, tr
7978
}
8079

8180
entryPaths := make([]string, len(entries))
82-
entryMap := make(map[string]*TreeEntry)
8381
for i, entry := range entries {
8482
entryPaths[i] = entry.Name()
85-
entryMap[entry.Name()] = entry
8683
}
8784

88-
commits, err := GetLastCommitForPaths(ctx, commit, treePath, entryPaths)
85+
_, err = WalkGitLog(ctx, c, commit.repo, commit, treePath, entryPaths...)
8986
if err != nil {
9087
return err
9188
}
9289

93-
for entry, entryCommit := range commits {
94-
if err := c.Put(commit.ID.String(), path.Join(treePath, entry), entryCommit.ID.String()); err != nil {
95-
return err
96-
}
90+
for _, treeEntry := range entries {
9791
// entryMap won't contain "" therefore skip this.
98-
if treeEntry := entryMap[entry]; treeEntry != nil && treeEntry.IsDir() {
99-
subTree, err := tree.SubTree(entry)
92+
if treeEntry.IsDir() {
93+
subTree, err := tree.SubTree(treeEntry.Name())
10094
if err != nil {
10195
return err
10296
}
103-
if err := c.recursiveCache(ctx, commit, subTree, entry, level-1); err != nil {
97+
if err := c.recursiveCache(ctx, commit, subTree, treeEntry.Name(), level-1); err != nil {
10498
return err
10599
}
106100
}

‎modules/git/log_name_status.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,9 @@ func (g *LogNameStatusRepoParser) Close() {
267267
}
268268

269269
// WalkGitLog walks the git log --name-status for the head commit in the provided treepath and files
270-
func WalkGitLog(ctx context.Context, repo *Repository, head *Commit, treepath string, paths ...string) (map[string]string, error) {
270+
func WalkGitLog(ctx context.Context, cache *LastCommitCache, repo *Repository, head *Commit, treepath string, paths ...string) (map[string]string, error) {
271+
headRef := head.ID.String()
272+
271273
tree, err := head.SubTree(treepath)
272274
if err != nil {
273275
return nil, err
@@ -328,6 +330,9 @@ heaploop:
328330
for {
329331
select {
330332
case <-ctx.Done():
333+
if ctx.Err() == context.DeadlineExceeded {
334+
break heaploop
335+
}
331336
return nil, ctx.Err()
332337
default:
333338
}
@@ -348,10 +353,16 @@ heaploop:
348353
changed[i] = false
349354
if results[i] == "" {
350355
results[i] = current.CommitID
356+
if err := cache.Put(headRef, path.Join(treepath, paths[i]), current.CommitID); err != nil {
357+
return nil, err
358+
}
351359
delete(path2idx, paths[i])
352360
remaining--
353361
if results[0] == "" {
354362
results[0] = current.CommitID
363+
if err := cache.Put(headRef, treepath, current.CommitID); err != nil {
364+
return nil, err
365+
}
355366
delete(path2idx, "")
356367
remaining--
357368
}

‎modules/git/notes_gogit.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
)
1515

1616
// GetNote retrieves the git-notes data for a given commit.
17+
// FIXME: Add LastCommitCache support
1718
func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) error {
1819
notes, err := repo.GetCommit(NotesRef)
1920
if err != nil {
@@ -63,7 +64,7 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note)
6364
return err
6465
}
6566

66-
lastCommits, err := GetLastCommitForPaths(ctx, commitNode, "", []string{path})
67+
lastCommits, err := GetLastCommitForPaths(ctx, nil, commitNode, "", []string{path})
6768
if err != nil {
6869
return err
6970
}

‎modules/git/notes_nogogit.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
)
1414

1515
// GetNote retrieves the git-notes data for a given commit.
16+
// FIXME: Add LastCommitCache support
1617
func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) error {
1718
notes, err := repo.GetCommit(NotesRef)
1819
if err != nil {
@@ -64,7 +65,7 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note)
6465
path = path[idx+1:]
6566
}
6667

67-
lastCommits, err := GetLastCommitForPaths(ctx, notes, treePath, []string{path})
68+
lastCommits, err := GetLastCommitForPaths(ctx, nil, notes, treePath, []string{path})
6869
if err != nil {
6970
return err
7071
}

0 commit comments

Comments
 (0)