Skip to content

Commit 462284e

Browse files
lunnylunny
authored
Use batch insert on migrating repository to make the process faster (#7050)
* Use batch insert on migrating repository to make the process faster * fix lint * fix tests * fix comments
1 parent e463bda commit 462284e

File tree

7 files changed

+339
-238
lines changed

7 files changed

+339
-238
lines changed

‎models/migrate.go

Lines changed: 80 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -6,38 +6,58 @@ package models
66

77
import "github.com/go-xorm/xorm"
88

9-
// InsertIssue insert one issue to database
10-
func InsertIssue(issue *Issue, labelIDs []int64) error {
9+
// InsertMilestones creates milestones of repository.
10+
func InsertMilestones(ms ...*Milestone) (err error) {
11+
if len(ms) == 0 {
12+
return nil
13+
}
14+
1115
sess := x.NewSession()
12-
if err := sess.Begin(); err != nil {
16+
defer sess.Close()
17+
if err = sess.Begin(); err != nil {
1318
return err
1419
}
1520

16-
if err := insertIssue(sess, issue, labelIDs); err != nil {
21+
// to return the id, so we should not use batch insert
22+
for _, m := range ms {
23+
if _, err = sess.NoAutoTime().Insert(m); err != nil {
24+
return err
25+
}
26+
}
27+
28+
if _, err = sess.Exec("UPDATE `repository` SET num_milestones = num_milestones + ? WHERE id = ?", len(ms), ms[0].RepoID); err != nil {
1729
return err
1830
}
1931
return sess.Commit()
2032
}
2133

22-
func insertIssue(sess *xorm.Session, issue *Issue, labelIDs []int64) error {
23-
if issue.MilestoneID > 0 {
24-
sess.Incr("num_issues")
25-
if issue.IsClosed {
26-
sess.Incr("num_closed_issues")
27-
}
28-
if _, err := sess.ID(issue.MilestoneID).NoAutoTime().Update(new(Milestone)); err != nil {
34+
// InsertIssues insert issues to database
35+
func InsertIssues(issues ...*Issue) error {
36+
sess := x.NewSession()
37+
if err := sess.Begin(); err != nil {
38+
return err
39+
}
40+
41+
for _, issue := range issues {
42+
if err := insertIssue(sess, issue); err != nil {
2943
return err
3044
}
3145
}
46+
return sess.Commit()
47+
}
48+
49+
func insertIssue(sess *xorm.Session, issue *Issue) error {
3250
if _, err := sess.NoAutoTime().Insert(issue); err != nil {
3351
return err
3452
}
35-
var issueLabels = make([]IssueLabel, 0, len(labelIDs))
36-
for _, labelID := range labelIDs {
53+
var issueLabels = make([]IssueLabel, 0, len(issue.Labels))
54+
var labelIDs = make([]int64, 0, len(issue.Labels))
55+
for _, label := range issue.Labels {
3756
issueLabels = append(issueLabels, IssueLabel{
3857
IssueID: issue.ID,
39-
LabelID: labelID,
58+
LabelID: label.ID,
4059
})
60+
labelIDs = append(labelIDs, label.ID)
4161
}
4262
if _, err := sess.Insert(issueLabels); err != nil {
4363
return err
@@ -61,84 +81,93 @@ func insertIssue(sess *xorm.Session, issue *Issue, labelIDs []int64) error {
6181
if issue.IsClosed {
6282
sess.Incr("num_closed_issues")
6383
}
64-
if _, err := sess.In("id", labelIDs).Update(new(Label)); err != nil {
84+
if _, err := sess.In("id", labelIDs).NoAutoTime().Update(new(Label)); err != nil {
6585
return err
6686
}
6787

6888
if issue.MilestoneID > 0 {
69-
if _, err := sess.ID(issue.MilestoneID).SetExpr("completeness", "num_closed_issues * 100 / num_issues").Update(new(Milestone)); err != nil {
89+
sess.Incr("num_issues")
90+
if issue.IsClosed {
91+
sess.Incr("num_closed_issues")
92+
}
93+
94+
if _, err := sess.ID(issue.MilestoneID).
95+
SetExpr("completeness", "num_closed_issues * 100 / num_issues").
96+
NoAutoTime().
97+
Update(new(Milestone)); err != nil {
7098
return err
7199
}
72100
}
73101

74102
return nil
75103
}
76104

77-
// InsertComment inserted a comment
78-
func InsertComment(comment *Comment) error {
105+
// InsertIssueComments inserts many comments of issues.
106+
func InsertIssueComments(comments []*Comment) error {
107+
if len(comments) == 0 {
108+
return nil
109+
}
110+
111+
var issueIDs = make(map[int64]bool)
112+
for _, comment := range comments {
113+
issueIDs[comment.IssueID] = true
114+
}
115+
79116
sess := x.NewSession()
80117
defer sess.Close()
81118
if err := sess.Begin(); err != nil {
82119
return err
83120
}
84-
if _, err := sess.NoAutoTime().Insert(comment); err != nil {
121+
if _, err := sess.NoAutoTime().Insert(comments); err != nil {
85122
return err
86123
}
87-
if _, err := sess.ID(comment.IssueID).Incr("num_comments").Update(new(Issue)); err != nil {
88-
return err
124+
for issueID := range issueIDs {
125+
if _, err := sess.Exec("UPDATE issue set num_comments = (SELECT count(*) FROM comment WHERE issue_id = ?) WHERE id = ?", issueID, issueID); err != nil {
126+
return err
127+
}
89128
}
90129
return sess.Commit()
91130
}
92131

93-
// InsertPullRequest inserted a pull request
94-
func InsertPullRequest(pr *PullRequest, labelIDs []int64) error {
132+
// InsertPullRequests inserted pull requests
133+
func InsertPullRequests(prs ...*PullRequest) error {
95134
sess := x.NewSession()
96135
defer sess.Close()
97136
if err := sess.Begin(); err != nil {
98137
return err
99138
}
100-
if err := insertIssue(sess, pr.Issue, labelIDs); err != nil {
101-
return err
102-
}
103-
pr.IssueID = pr.Issue.ID
104-
if _, err := sess.NoAutoTime().Insert(pr); err != nil {
105-
return err
139+
for _, pr := range prs {
140+
if err := insertIssue(sess, pr.Issue); err != nil {
141+
return err
142+
}
143+
pr.IssueID = pr.Issue.ID
144+
if _, err := sess.NoAutoTime().Insert(pr); err != nil {
145+
return err
146+
}
106147
}
148+
107149
return sess.Commit()
108150
}
109151

110-
// MigrateRelease migrates release
111-
func MigrateRelease(rel *Release) error {
152+
// InsertReleases migrates release
153+
func InsertReleases(rels ...*Release) error {
112154
sess := x.NewSession()
113155
if err := sess.Begin(); err != nil {
114156
return err
115157
}
116158

117-
var oriRel = Release{
118-
RepoID: rel.RepoID,
119-
TagName: rel.TagName,
120-
}
121-
exist, err := sess.Get(&oriRel)
122-
if err != nil {
123-
return err
124-
}
125-
if !exist {
159+
for _, rel := range rels {
126160
if _, err := sess.NoAutoTime().Insert(rel); err != nil {
127161
return err
128162
}
129-
} else {
130-
rel.ID = oriRel.ID
131-
if _, err := sess.ID(rel.ID).Cols("target, title, note, is_tag, num_commits").Update(rel); err != nil {
132-
return err
133-
}
134-
}
135163

136-
for i := 0; i < len(rel.Attachments); i++ {
137-
rel.Attachments[i].ReleaseID = rel.ID
138-
}
164+
for i := 0; i < len(rel.Attachments); i++ {
165+
rel.Attachments[i].ReleaseID = rel.ID
166+
}
139167

140-
if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil {
141-
return err
168+
if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil {
169+
return err
170+
}
142171
}
143172

144173
return sess.Commit()

‎modules/migrations/base/comment.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import "time"
99

1010
// Comment is a standard comment information
1111
type Comment struct {
12+
IssueIndex int64
1213
PosterName string
1314
PosterEmail string
1415
Created time.Time

‎modules/migrations/base/uploader.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@
55

66
package base
77

8-
// Uploader uploads all the informations
8+
// Uploader uploads all the informations of one repository
99
type Uploader interface {
1010
CreateRepo(repo *Repository, includeUncyclo bool) error
11-
CreateMilestone(milestone *Milestone) error
12-
CreateRelease(release *Release) error
13-
CreateLabel(label *Label) error
14-
CreateIssue(issue *Issue) error
15-
CreateComment(issueNumber int64, comment *Comment) error
16-
CreatePullRequest(pr *PullRequest) error
11+
CreateMilestones(milestones ...*Milestone) error
12+
CreateReleases(releases ...*Release) error
13+
CreateLabels(labels ...*Label) error
14+
CreateIssues(issues ...*Issue) error
15+
CreateComments(comments ...*Comment) error
16+
CreatePullRequests(prs ...*PullRequest) error
1717
Rollback() error
1818
}

0 commit comments

Comments
 (0)