Skip to content

Commit ad2642a

Browse files
lafrikslafriks
authored
Language statistics bar for repositories (#8037)
* Implementation for calculating language statistics Impement saving code language statistics to database Implement rendering langauge stats Add primary laguage to show in repository list Implement repository stats indexer queue Add indexer test Refactor to use queue module * Do not timeout for queues
1 parent 37892be commit ad2642a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+182948
-55
lines changed

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ require (
8585
github.com/sergi/go-diff v1.0.0
8686
github.com/shurcooL/httpfs v0.0.0-20190527155220-6a4d4a70508b // indirect
8787
github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd
88+
github.com/src-d/enry/v2 v2.1.0
8889
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 // indirect
8990
github.com/stretchr/testify v1.4.0
9091
github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 // indirect

‎go.sum

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,8 +508,13 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
508508
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
509509
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
510510
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
511+
github.com/src-d/enry v1.7.3 h1:jG2fmEaQaURh0qqU/sn82BRzVa6d4EVHJIw6gc98bak=
512+
github.com/src-d/enry/v2 v2.1.0 h1:z1L8t+B8bh3mmjPkJrgOTnVRpFGmTPJsplHX9wAn6BI=
513+
github.com/src-d/enry/v2 v2.1.0/go.mod h1:qQeCMRwzMF3ckeGr+h0tJLdxXnq+NVZsIDMELj0t028=
511514
github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
512515
github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
516+
github.com/src-d/go-oniguruma v1.1.0 h1:EG+Nm5n2JqWUaCjtM0NtutPxU7ZN5Tp50GWrrV8bTww=
517+
github.com/src-d/go-oniguruma v1.1.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM=
513518
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 h1:JNEGSiWg6D3lcBCMCBqN3ELniXujt+0QNHLhNnO0w3s=
514519
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2/go.mod h1:mjqs7N0Q6m5HpR7QfXVBZXZWSqTjQLeTujjA/xUp2uw=
515520
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -530,6 +535,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV
530535
github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
531536
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
532537
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
538+
github.com/toqueteos/trie v1.0.0 h1:8i6pXxNUXNRAqP246iibb7w/pSFquNTQ+uNfriG7vlk=
539+
github.com/toqueteos/trie v1.0.0/go.mod h1:Ywk48QhEqhU1+DwhMkJ2x7eeGxDHiGkAdc9+0DYcbsM=
533540
github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
534541
github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM=
535542
github.com/tstranex/u2f v1.0.0 h1:HhJkSzDDlVSVIVt7pDJwCHQj67k7A5EeBgPmeD+pVsQ=
@@ -747,6 +754,8 @@ gopkg.in/testfixtures.v2 v2.5.0 h1:N08B7l2GzFQenyYbzqthDnKAA+cmb17iAZhhFxr7JHw=
747754
gopkg.in/testfixtures.v2 v2.5.0/go.mod h1:vyAq+MYCgNpR29qitQdLZhdbLFf4mR/2MFJRFoQZZ2M=
748755
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
749756
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
757+
gopkg.in/toqueteos/substring.v1 v1.0.2 h1:urLqCeMm6x/eTuQa1oZerNw8N1KNOIp5hD5kGL7lFsE=
758+
gopkg.in/toqueteos/substring.v1 v1.0.2/go.mod h1:Eb2Z1UYehlVK8LYW2WBVR2rwbujsz3aX8XDrM1vbNew=
750759
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
751760
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
752761
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=

‎integrations/testlogger.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import (
1313
"strings"
1414
"sync"
1515
"testing"
16-
"time"
1716

1817
"code.gitea.io/gitea/modules/log"
1918
"code.gitea.io/gitea/modules/queue"
@@ -101,7 +100,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() {
101100
}
102101
writerCloser.setT(&t)
103102
return func() {
104-
if err := queue.GetManager().FlushAll(context.Background(), 20*time.Second); err != nil {
103+
if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil {
105104
t.Errorf("Flushing queues failed with error %v", err)
106105
}
107106
_ = writerCloser.Close()

‎models/migrations/migrations.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,8 @@ var migrations = []Migration{
186186
NewMigration("Add some columns on review for migration", addReviewMigrateInfo),
187187
// v126 -> v127
188188
NewMigration("Fix topic repository count", fixTopicRepositoryCount),
189+
// v127 -> v128
190+
NewMigration("add repository code language statistics", addLanguageStats),
189191
}
190192

191193
// Migrate database to current version

‎models/migrations/v127.go

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Copyright 2020 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package migrations
6+
7+
import (
8+
"fmt"
9+
10+
"code.gitea.io/gitea/modules/timeutil"
11+
12+
"xorm.io/xorm"
13+
)
14+
15+
func addLanguageStats(x *xorm.Engine) error {
16+
// LanguageStat see models/repo_language_stats.go
17+
type LanguageStat struct {
18+
ID int64 `xorm:"pk autoincr"`
19+
RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
20+
CommitID string
21+
IsPrimary bool
22+
Language string `xorm:"VARCHAR(30) UNIQUE(s) INDEX NOT NULL"`
23+
Percentage float32 `xorm:"NUMERIC(5,2) NOT NULL DEFAULT 0"`
24+
Color string `xorm:"-"`
25+
CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
26+
}
27+
28+
type RepoIndexerType int
29+
30+
// RepoIndexerStatus see models/repo_stats_indexer.go
31+
type RepoIndexerStatus struct {
32+
ID int64 `xorm:"pk autoincr"`
33+
RepoID int64 `xorm:"INDEX(s)"`
34+
CommitSha string `xorm:"VARCHAR(40)"`
35+
IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"`
36+
}
37+
38+
if err := x.Sync2(new(LanguageStat)); err != nil {
39+
return fmt.Errorf("Sync2: %v", err)
40+
}
41+
if err := x.Sync2(new(RepoIndexerStatus)); err != nil {
42+
return fmt.Errorf("Sync2: %v", err)
43+
}
44+
return nil
45+
}

‎models/models.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ func init() {
116116
new(OAuth2AuthorizationCode),
117117
new(OAuth2Grant),
118118
new(Task),
119+
new(LanguageStat),
119120
)
120121

121122
gonicNames := []string{"SSL", "UID"}

‎models/repo.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -175,8 +175,9 @@ type Repository struct {
175175
*Mirror `xorm:"-"`
176176
Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"`
177177

178-
RenderingMetas map[string]string `xorm:"-"`
179-
Units []*RepoUnit `xorm:"-"`
178+
RenderingMetas map[string]string `xorm:"-"`
179+
Units []*RepoUnit `xorm:"-"`
180+
PrimaryLanguage *LanguageStat `xorm:"-"`
180181

181182
IsFork bool `xorm:"INDEX NOT NULL DEFAULT false"`
182183
ForkID int64 `xorm:"INDEX"`
@@ -185,7 +186,8 @@ type Repository struct {
185186
TemplateID int64 `xorm:"INDEX"`
186187
TemplateRepo *Repository `xorm:"-"`
187188
Size int64 `xorm:"NOT NULL DEFAULT 0"`
188-
IndexerStatus *RepoIndexerStatus `xorm:"-"`
189+
CodeIndexerStatus *RepoIndexerStatus `xorm:"-"`
190+
StatsIndexerStatus *RepoIndexerStatus `xorm:"-"`
189191
IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"`
190192
CloseIssuesViaCommitInAnyBranch bool `xorm:"NOT NULL DEFAULT false"`
191193
Topics []string `xorm:"TEXT JSON"`
@@ -1504,6 +1506,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
15041506
&Notification{RepoID: repoID},
15051507
&CommitStatus{RepoID: repoID},
15061508
&RepoIndexerStatus{RepoID: repoID},
1509+
&LanguageStat{RepoID: repoID},
15071510
&Comment{RefRepoID: repoID},
15081511
&Task{RepoID: repoID},
15091512
); err != nil {

‎models/repo_indexer.go

Lines changed: 58 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,21 +10,32 @@ import (
1010
"xorm.io/builder"
1111
)
1212

13+
// RepoIndexerType specifies the repository indexer type
14+
type RepoIndexerType int
15+
16+
const (
17+
// RepoIndexerTypeCode code indexer
18+
RepoIndexerTypeCode RepoIndexerType = iota // 0
19+
// RepoIndexerTypeStats repository stats indexer
20+
RepoIndexerTypeStats // 1
21+
)
22+
1323
// RepoIndexerStatus status of a repo's entry in the repo indexer
1424
// For now, implicitly refers to default branch
1525
type RepoIndexerStatus struct {
16-
ID int64 `xorm:"pk autoincr"`
17-
RepoID int64 `xorm:"INDEX"`
18-
CommitSha string `xorm:"VARCHAR(40)"`
26+
ID int64 `xorm:"pk autoincr"`
27+
RepoID int64 `xorm:"INDEX(s)"`
28+
CommitSha string `xorm:"VARCHAR(40)"`
29+
IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"`
1930
}
2031

2132
// GetUnindexedRepos returns repos which do not have an indexer status
22-
func GetUnindexedRepos(maxRepoID int64, page, pageSize int) ([]int64, error) {
33+
func GetUnindexedRepos(indexerType RepoIndexerType, maxRepoID int64, page, pageSize int) ([]int64, error) {
2334
ids := make([]int64, 0, 50)
2435
cond := builder.Cond(builder.IsNull{
2536
"repo_indexer_status.id",
2637
})
27-
sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id")
38+
sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id AND repo_indexer_status.indexer_type = ?", indexerType)
2839
if maxRepoID > 0 {
2940
cond = builder.And(cond, builder.Lte{
3041
"repository.id": maxRepoID,
@@ -43,40 +54,64 @@ func GetUnindexedRepos(maxRepoID int64, page, pageSize int) ([]int64, error) {
4354
return ids, err
4455
}
4556

46-
// GetIndexerStatus loads repo codes indxer status
47-
func (repo *Repository) GetIndexerStatus() error {
48-
if repo.IndexerStatus != nil {
49-
return nil
57+
// getIndexerStatus loads repo codes indxer status
58+
func (repo *Repository) getIndexerStatus(e Engine, indexerType RepoIndexerType) (*RepoIndexerStatus, error) {
59+
switch indexerType {
60+
case RepoIndexerTypeCode:
61+
if repo.CodeIndexerStatus != nil {
62+
return repo.CodeIndexerStatus, nil
63+
}
64+
case RepoIndexerTypeStats:
65+
if repo.StatsIndexerStatus != nil {
66+
return repo.StatsIndexerStatus, nil
67+
}
5068
}
51-
status := &RepoIndexerStatus{RepoID: repo.ID}
52-
has, err := x.Get(status)
69+
status := &RepoIndexerStatus{RepoID: repo.ID, IndexerType: indexerType}
70+
has, err := e.Get(status)
5371
if err != nil {
54-
return err
72+
return nil, err
5573
} else if !has {
5674
status.CommitSha = ""
5775
}
58-
repo.IndexerStatus = status
59-
return nil
76+
switch indexerType {
77+
case RepoIndexerTypeCode:
78+
repo.CodeIndexerStatus = status
79+
case RepoIndexerTypeStats:
80+
repo.StatsIndexerStatus = status
81+
}
82+
return status, nil
6083
}
6184

62-
// UpdateIndexerStatus updates indexer status
63-
func (repo *Repository) UpdateIndexerStatus(sha string) error {
64-
if err := repo.GetIndexerStatus(); err != nil {
85+
// GetIndexerStatus loads repo codes indxer status
86+
func (repo *Repository) GetIndexerStatus(indexerType RepoIndexerType) (*RepoIndexerStatus, error) {
87+
return repo.getIndexerStatus(x, indexerType)
88+
}
89+
90+
// updateIndexerStatus updates indexer status
91+
func (repo *Repository) updateIndexerStatus(e Engine, indexerType RepoIndexerType, sha string) error {
92+
status, err := repo.getIndexerStatus(e, indexerType)
93+
if err != nil {
6594
return fmt.Errorf("UpdateIndexerStatus: Unable to getIndexerStatus for repo: %s Error: %v", repo.FullName(), err)
6695
}
67-
if len(repo.IndexerStatus.CommitSha) == 0 {
68-
repo.IndexerStatus.CommitSha = sha
69-
_, err := x.Insert(repo.IndexerStatus)
96+
97+
if len(status.CommitSha) == 0 {
98+
status.CommitSha = sha
99+
_, err := e.Insert(status)
70100
if err != nil {
71101
return fmt.Errorf("UpdateIndexerStatus: Unable to insert repoIndexerStatus for repo: %s Sha: %s Error: %v", repo.FullName(), sha, err)
72102
}
73103
return nil
74104
}
75-
repo.IndexerStatus.CommitSha = sha
76-
_, err := x.ID(repo.IndexerStatus.ID).Cols("commit_sha").
77-
Update(repo.IndexerStatus)
105+
status.CommitSha = sha
106+
_, err = e.ID(status.ID).Cols("commit_sha").
107+
Update(status)
78108
if err != nil {
79109
return fmt.Errorf("UpdateIndexerStatus: Unable to update repoIndexerStatus for repo: %s Sha: %s Error: %v", repo.FullName(), sha, err)
80110
}
81111
return nil
82112
}
113+
114+
// UpdateIndexerStatus updates indexer status
115+
func (repo *Repository) UpdateIndexerStatus(indexerType RepoIndexerType, sha string) error {
116+
return repo.updateIndexerStatus(x, indexerType, sha)
117+
}

0 commit comments

Comments
 (0)