Cache last commit to accelerate the repository directory page visit

lunny · lunny · commit 30e7727724f3 · 2020-01-31T13:34:33.000+08:00
diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample
@@ -632,6 +632,24 @@ SENDMAIL_PATH = sendmail
 SENDMAIL_ARGS =
 
 [cache]
+; if the cache enabled
+ENABLED = true
+; Either "memory", "redis", or "memcache", default is "memory"
+ADAPTER = memory
+; For "memory" only, GC interval in seconds, default is 60
+INTERVAL = 60
+; For "redis" and "memcache", connection host address
+; redis: network=tcp,addr=:6379,password=macaron,db=0,pool_size=100,idle_timeout=180
+; memcache: `127.0.0.1:11211`
+HOST =
+; Time to keep items in cache if not used, default is 16 hours.
+; Setting it to 0 disables caching
+ITEM_TTL = 16h
+
+; Last commit cache
+[cache.last_commit]
+; if the cache enabled
+ENABLED = true
 ; Either "memory", "redis", or "memcache", default is "memory"
 ADAPTER = memory
 ; For "memory" only, GC interval in seconds, default is 60
@@ -643,6 +661,8 @@ HOST =
 ; Time to keep items in cache if not used, default is 16 hours.
 ; Setting it to 0 disables caching
 ITEM_TTL = 16h
+; Only enable the cache when repository's commits count great than
+COMMITS_COUNT = 1000
 
 [session]
 ; Either "memory", "file", or "redis", default is "memory"
diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
@@ -383,13 +383,25 @@ relation to port exhaustion.
 
 ## Cache (`cache`)
 
+- `ENABLED`: **true**: Enable the cache.
 - `ADAPTER`: **memory**: Cache engine adapter, either `memory`, `redis`, or `memcache`.
 - `INTERVAL`: **60**: Garbage Collection interval (sec), for memory cache only.
 - `HOST`: **\<empty\>**: Connection string for `redis` and `memcache`.
    - Redis: `network=tcp,addr=127.0.0.1:6379,password=macaron,db=0,pool_size=100,idle_timeout=180`
    - Memcache: `127.0.0.1:9090;127.0.0.1:9091`
 - `ITEM_TTL`: **16h**: Time to keep items in cache if not used, Setting it to 0 disables caching.
 
+## Cache - LastCommitCache settings (`cache.last_commit`)
+
+- `ENABLED`: **true**: Enable the cache.
+- `ADAPTER`: **memory**: Cache engine adapter, either `memory`, `redis`, or `memcache`.
+- `INTERVAL`: **60**: Garbage Collection interval (sec), for memory cache only.
+- `HOST`: **\<empty\>**: Connection string for `redis` and `memcache`.
+   - Redis: `network=tcp,addr=127.0.0.1:6379,password=macaron,db=0,pool_size=100,idle_timeout=180`
+   - Memcache: `127.0.0.1:9090;127.0.0.1:9091`
+- `ITEM_TTL`: **16h**: Time to keep items in cache if not used, Setting it to 0 disables caching.
+- `COMMITS_COUNT`: **1000**: Only enable the cache when repository's commits count great than.
+
 ## Session (`session`)
 
 - `PROVIDER`: **memory**: Session engine provider \[memory, file, redis, mysql, couchbase, memcache, nodb, postgres\].
diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
@@ -148,13 +148,25 @@ menu:
 
 ## Cache (`cache`)
 
+- `ENABLED`: **true**: 是否启用。
 - `ADAPTER`: **memory**: 缓存引擎，可以为 `memory`, `redis` 或 `memcache`。
 - `INTERVAL`: **60**: 只对内存缓存有效，GC间隔，单位秒。
 - `HOST`: **\<empty\>**: 针对redis和memcache有效，主机地址和端口。
     - Redis: `network=tcp,addr=127.0.0.1:6379,password=macaron,db=0,pool_size=100,idle_timeout=180`
     - Memache: `127.0.0.1:9090;127.0.0.1:9091`
 - `ITEM_TTL`: **16h**: 缓存项目失效时间，设置为 0 则禁用缓存。
 
+## Cache - LastCommitCache settings (`cache.last_commit`)
+
+- `ENABLED`: **true**: 是否启用。
+- `ADAPTER`: **memory**: 缓存引擎，可以为 `memory`, `redis` 或 `memcache`。
+- `INTERVAL`: **60**: 只对内存缓存有效，GC间隔，单位秒。
+- `HOST`: **\<empty\>**: 针对redis和memcache有效，主机地址和端口。
+    - Redis: `network=tcp,addr=127.0.0.1:6379,password=macaron,db=0,pool_size=100,idle_timeout=180`
+    - Memache: `127.0.0.1:9090;127.0.0.1:9091`
+- `ITEM_TTL`: **86400h**: 缓存项目失效时间，设置为 0 则禁用缓存。
+- `COMMITS_COUNT`: **1000**: 仅当仓库的提交数大于时才启用缓存。
+
 ## Session (`session`)
 
 - `PROVIDER`: Session 内容存储方式，可选 `memory`, `file`, `redis` 或 `mysql`。
diff --git a/modules/cache/cache.go b/modules/cache/cache.go
@@ -16,20 +16,35 @@ import (
 	_ "gitea.com/macaron/cache/redis"
 )
 
-var conn mc.Cache
+var (
+	conn            mc.Cache
+	lastCommitCache mc.Cache
+)
+
+func newCache(cacheConfig setting.Cache) (mc.Cache, error) {
+	return mc.NewCacher(cacheConfig.Adapter, mc.Options{
+		Adapter:       cacheConfig.Adapter,
+		AdapterConfig: cacheConfig.Conn,
+		Interval:      cacheConfig.Interval,
+	})
+}
 
 // NewContext start cache service
 func NewContext() error {
-	if setting.CacheService == nil || conn != nil {
-		return nil
+	var err error
+
+	if conn == nil && setting.CacheService.Enabled {
+		if conn, err = newCache(setting.CacheService.Cache); err != nil {
+			return err
+		}
+	}
+
+	if lastCommitCache == nil && setting.CacheService.LastCommit.Enabled {
+		if lastCommitCache, err = newCache(setting.CacheService.LastCommit.Cache); err != nil {
+			return err
+		}
 	}
 
-	var err error
-	conn, err = mc.NewCacher(setting.CacheService.Adapter, mc.Options{
-		Adapter:       setting.CacheService.Adapter,
-		AdapterConfig: setting.CacheService.Conn,
-		Interval:      setting.CacheService.Interval,
-	})
 	return err
 }
 
diff --git a/modules/cache/last_commit.go b/modules/cache/last_commit.go
@@ -0,0 +1,62 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+
+	mc "gitea.com/macaron/cache"
+	"gopkg.in/src-d/go-git.v4/plumbing/object"
+)
+
+// LastCommitCache represents a cache to store last commit
+type LastCommitCache struct {
+	repoPath    string
+	ttl         int64
+	repo        *git.Repository
+	commitCache map[string]*object.Commit
+	mc.Cache
+}
+
+// NewLastCommitCache creates a new last commit cache for repo
+func NewLastCommitCache(repoPath string, gitRepo *git.Repository, ttl int64) *LastCommitCache {
+	return &LastCommitCache{
+		repoPath:    repoPath,
+		repo:        gitRepo,
+		commitCache: make(map[string]*object.Commit),
+		ttl:         ttl,
+		Cache:       lastCommitCache,
+	}
+}
+
+func (c LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) {
+	v := c.Cache.Get(fmt.Sprintf("%s:%s:%s", c.repoPath, ref, entryPath))
+	if vs, ok := v.(string); ok {
+		log.Trace("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
+		if commit, ok := c.commitCache[vs]; ok {
+			log.Trace("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
+			return commit, nil
+		}
+		id, err := c.repo.ConvertToSHA1(vs)
+		if err != nil {
+			return nil, err
+		}
+		commit, err := c.repo.GoGitRepo().CommitObject(id)
+		if err != nil {
+			return nil, err
+		}
+		c.commitCache[vs] = commit
+		return commit, nil
+	}
+	return nil, nil
+}
+
+func (c LastCommitCache) Put(ref, entryPath, commitID string) error {
+	log.Trace("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
+	return c.Cache.Put(fmt.Sprintf("%s:%s:%s", c.repoPath, ref, entryPath), commitID, c.ttl)
+}
diff --git a/modules/git/cache.go b/modules/git/cache.go
@@ -4,8 +4,10 @@
 
 package git
 
+import "gopkg.in/src-d/go-git.v4/plumbing/object"
+
 // LastCommitCache cache
 type LastCommitCache interface {
-	Get(repoPath, ref, entryPath string) (*Commit, error)
-	Put(repoPath, ref, entryPath string, commit *Commit) error
+	Get(ref, entryPath string) (*object.Commit, error)
+	Put(ref, entryPath, commitID string) error
 }
diff --git a/modules/git/commit_info.go b/modules/git/commit_info.go
@@ -5,6 +5,8 @@
 package git
 
 import (
+	"path"
+
 	"github.com/emirpasic/gods/trees/binaryheap"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
@@ -30,7 +32,29 @@ func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCom
 		return nil, nil, err
 	}
 
-	revs, err := getLastCommitForPaths(c, treePath, entryPaths)
+	var revs map[string]*object.Commit
+	if cache != nil {
+		var unHitPaths []string
+		revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
+		if err != nil {
+			return nil, nil, err
+		}
+		if len(unHitPaths) > 0 {
+			revs2, err := getLastCommitForPaths(c, treePath, unHitPaths)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			for k, v := range revs2 {
+				if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
+					return nil, nil, err
+				}
+				revs[k] = v
+			}
+		}
+	} else {
+		revs, err = getLastCommitForPaths(c, treePath, entryPaths)
+	}
 	if err != nil {
 		return nil, nil, err
 	}
@@ -127,6 +151,25 @@ func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[
 	return hashes, nil
 }
 
+func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache LastCommitCache) (map[string]*object.Commit, []string, error) {
+	var unHitEntryPaths []string
+	var results = make(map[string]*object.Commit)
+	for _, p := range paths {
+		lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
+		if err != nil {
+			return nil, nil, err
+		}
+		if lastCommit != nil {
+			results[p] = lastCommit
+			continue
+		}
+
+		unHitEntryPaths = append(unHitEntryPaths, p)
+	}
+
+	return results, unHitEntryPaths, nil
+}
+
 func getLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
 	// We do a tree traversal with nodes sorted by commit time
 	heap := binaryheap.NewWith(func(a, b interface{}) int {
diff --git a/modules/setting/cache.go b/modules/setting/cache.go
@@ -13,31 +13,81 @@ import (
 
 // Cache represents cache settings
 type Cache struct {
+	Enabled  bool
 	Adapter  string
 	Interval int
 	Conn     string
-	TTL      time.Duration
+	TTL      time.Duration `ini:"ITEM_TTL"`
 }
 
 var (
 	// CacheService the global cache
-	CacheService *Cache
+	CacheService = struct {
+		Cache
+
+		LastCommit struct {
+			Cache
+			CommitsCount int64
+		} `ini:"cache.last_commit"`
+	}{
+		Cache: Cache{
+			Enabled:  true,
+			Adapter:  "memory",
+			Interval: 60,
+			TTL:      16 * time.Hour,
+		},
+		LastCommit: struct {
+			Cache
+			CommitsCount int64
+		}{
+			Cache: Cache{
+				Enabled:  true,
+				Adapter:  "memory",
+				Interval: 60,
+				TTL:      86400 * time.Hour,
+			},
+			CommitsCount: 1000,
+		},
+	}
 )
 
 func newCacheService() {
 	sec := Cfg.Section("cache")
-	CacheService = &Cache{
-		Adapter: sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"}),
+	if err := sec.MapTo(&CacheService); err != nil {
+		log.Fatal("Failed to map Cache settings: %v", err)
 	}
+
+	CacheService.Adapter = sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"})
 	switch CacheService.Adapter {
 	case "memory":
-		CacheService.Interval = sec.Key("INTERVAL").MustInt(60)
 	case "redis", "memcache":
 		CacheService.Conn = strings.Trim(sec.Key("HOST").String(), "\" ")
+	case "": // disable cache
+		CacheService.Enabled = false
 	default:
 		log.Fatal("Unknown cache adapter: %s", CacheService.Adapter)
 	}
-	CacheService.TTL = sec.Key("ITEM_TTL").MustDuration(16 * time.Hour)
 
-	log.Info("Cache Service Enabled")
+	if CacheService.Enabled {
+		log.Info("Cache Service Enabled")
+	}
+
+	sec = Cfg.Section("cache.last_commit")
+
+	CacheService.LastCommit.Adapter = sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"})
+	switch CacheService.LastCommit.Adapter {
+	case "memory":
+	case "redis", "memcache":
+		CacheService.LastCommit.Conn = strings.Trim(sec.Key("HOST").String(), "\" ")
+	case "": // disable cache
+		CacheService.LastCommit.Enabled = false
+	default:
+		log.Fatal("Unknown cache.last_commit adapter: %s", CacheService.LastCommit.Adapter)
+	}
+
+	CacheService.LastCommit.CommitsCount = sec.Key("COMMITS_COUNT").MustInt64(1000)
+
+	if CacheService.LastCommit.Enabled {
+		log.Info("Last Commit Cache Service Enabled")
+	}
 }
diff --git a/routers/repo/view.go b/routers/repo/view.go
@@ -17,6 +17,7 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/git"
@@ -49,8 +50,13 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 	}
 	entries.CustomSort(base.NaturalSortLess)
 
+	var c git.LastCommitCache
+	if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount {
+		c = cache.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()))
+	}
+
 	var latestCommit *git.Commit
-	ctx.Data["Files"], latestCommit, err = entries.GetCommitsInfo(ctx.Repo.Commit, ctx.Repo.TreePath, nil)
+	ctx.Data["Files"], latestCommit, err = entries.GetCommitsInfo(ctx.Repo.Commit, ctx.Repo.TreePath, c)
 	if err != nil {
 		ctx.ServerError("GetCommitsInfo", err)
 		return
