Sync releases table with tags on push and for mirrors (#2459)

* Sync releases table with tags on push and for mirrors

* Code style fixes

* Fix api to return only releases

* Optimize release creation and update
Minimize posibility of race conditions

* Fix release lower tag name updating

* handle tag reference update by addionally comparing commit id

Author: lafriks

models/migrations/migrations.go

@@ -132,6 +132,8 @@ var migrations = []Migration{
 	NewMigration("migrate protected branch struct", migrateProtectedBranchStruct),
 	// v41 -> v42
 	NewMigration("add default value to user prohibit_login", addDefaultValueToUserProhibitLogin),
+	// v42 -> v43
+	NewMigration("add tags to releases and sync existing repositories", releaseAddColumnIsTagAndSyncTags),
 }
 
 // Migrate database to current version

models/migrations/v42.go

@@ -0,0 +1,57 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"code.gitea.io/git"
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/go-xorm/xorm"
+)
+
+// ReleaseV39 describes the added field for Release
+type ReleaseV39 struct {
+	IsTag bool `xorm:"NOT NULL DEFAULT false"`
+}
+
+// TableName will be invoked by XORM to customrize the table name
+func (*ReleaseV39) TableName() string {
+	return "release"
+}
+
+func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error {
+	if err := x.Sync2(new(ReleaseV39)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+
+	// For the sake of SQLite3, we can't use x.Iterate here.
+	offset := 0
+	pageSize := 20
+	for {
+		repos := make([]*models.Repository, 0, pageSize)
+		if err := x.Table("repository").Asc("id").Limit(pageSize, offset).Find(&repos); err != nil {
+			return fmt.Errorf("select repos [offset: %d]: %v", offset, err)
+		}
+		for _, repo := range repos {
+			gitRepo, err := git.OpenRepository(repo.RepoPath())
+			if err != nil {
+				log.Warn("OpenRepository: %v", err)
+				continue
+			}
+
+			if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
+				log.Warn("SyncReleasesWithTags: %v", err)
+			}
+		}
+		if len(repos) < pageSize {
+			break
+		}
+		offset += pageSize
+	}
+	return nil
+}

models/release.go

@@ -34,7 +34,8 @@ type Release struct {
 	NumCommitsBehind int64  `xorm:"-"`
 	Note             string `xorm:"TEXT"`
 	IsDraft          bool   `xorm:"NOT NULL DEFAULT false"`
-	IsPrerelease     bool
+	IsPrerelease     bool   `xorm:"NOT NULL DEFAULT false"`
+	IsTag            bool   `xorm:"NOT NULL DEFAULT false"`
 
 	Attachments []*Attachment `xorm:"-"`
 
@@ -139,17 +140,18 @@ func createTag(gitRepo *git.Repository, rel *Release) error {
 				}
 				return err
 			}
-		} else {
-			commit, err := gitRepo.GetTagCommit(rel.TagName)
-			if err != nil {
-				return fmt.Errorf("GetTagCommit: %v", err)
-			}
+			rel.LowerTagName = strings.ToLower(rel.TagName)
+		}
+		commit, err := gitRepo.GetTagCommit(rel.TagName)
+		if err != nil {
+			return fmt.Errorf("GetTagCommit: %v", err)
+		}
 
-			rel.Sha1 = commit.ID.String()
-			rel.NumCommits, err = commit.CommitsCount()
-			if err != nil {
-				return fmt.Errorf("CommitsCount: %v", err)
-			}
+		rel.Sha1 = commit.ID.String()
+		rel.CreatedUnix = commit.Author.When.Unix()
+		rel.NumCommits, err = commit.CommitsCount()
+		if err != nil {
+			return fmt.Errorf("CommitsCount: %v", err)
 		}
 	}
 	return nil
@@ -236,6 +238,7 @@ func GetReleaseByID(id int64) (*Release, error) {
 // FindReleasesOptions describes the conditions to Find releases
 type FindReleasesOptions struct {
 	IncludeDrafts bool
+	IncludeTags   bool
 	TagNames      []string
 }
 
@@ -246,6 +249,9 @@ func (opts *FindReleasesOptions) toConds(repoID int64) builder.Cond {
 	if !opts.IncludeDrafts {
 		cond = cond.And(builder.Eq{"is_draft": false})
 	}
+	if !opts.IncludeTags {
+		cond = cond.And(builder.Eq{"is_tag": false})
+	}
 	if len(opts.TagNames) > 0 {
 		cond = cond.And(builder.In("tag_name", opts.TagNames))
 	}
@@ -361,6 +367,8 @@ func UpdateRelease(gitRepo *git.Repository, rel *Release, attachmentUUIDs []stri
 	if err = createTag(gitRepo, rel); err != nil {
 		return err
 	}
+	rel.LowerTagName = strings.ToLower(rel.TagName)
+
 	_, err = x.Id(rel.ID).AllCols().Update(rel)
 	if err != nil {
 		return err
@@ -397,11 +405,64 @@ func DeleteReleaseByID(id int64, u *User, delTag bool) error {
 		if err != nil && !strings.Contains(stderr, "not found") {
 			return fmt.Errorf("git tag -d: %v - %s", err, stderr)
 		}
-	}
 
-	if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil {
-		return fmt.Errorf("Delete: %v", err)
+		if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil {
+			return fmt.Errorf("Delete: %v", err)
+		}
+	} else {
+		rel.IsTag = true
+		rel.IsDraft = false
+		rel.IsPrerelease = false
+		rel.Title = ""
+		rel.Note = ""
+
+		if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
+			return fmt.Errorf("Update: %v", err)
+		}
 	}
 
 	return nil
 }
+
+// SyncReleasesWithTags synchronizes release table with repository tags
+func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error {
+	existingRelTags := make(map[string]struct{})
+	opts := FindReleasesOptions{IncludeDrafts: true, IncludeTags: true}
+	for page := 1; ; page++ {
+		rels, err := GetReleasesByRepoID(repo.ID, opts, page, 100)
+		if err != nil {
+			return fmt.Errorf("GetReleasesByRepoID: %v", err)
+		}
+		if len(rels) == 0 {
+			break
+		}
+		for _, rel := range rels {
+			if rel.IsDraft {
+				continue
+			}
+			commitID, err := gitRepo.GetTagCommitID(rel.TagName)
+			if err != nil {
+				return fmt.Errorf("GetTagCommitID: %v", err)
+			}
+			if !gitRepo.IsTagExist(rel.TagName) || commitID != rel.Sha1 {
+				if err := pushUpdateDeleteTag(repo, gitRepo, rel.TagName); err != nil {
+					return fmt.Errorf("pushUpdateDeleteTag: %v", err)
+				}
+			} else {
+				existingRelTags[strings.ToLower(rel.TagName)] = struct{}{}
+			}
+		}
+	}
+	tags, err := gitRepo.GetTags()
+	if err != nil {
+		return fmt.Errorf("GetTags: %v", err)
+	}
+	for _, tagName := range tags {
+		if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok {
+			if err := pushUpdateAddTag(repo, gitRepo, tagName); err != nil {
+				return fmt.Errorf("pushUpdateAddTag: %v", err)
+			}
+		}
+	}
+	return nil
+}

models/repo.go

@@ -940,6 +940,10 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
 		if headBranch != nil {
 			repo.DefaultBranch = headBranch.Name
 		}
+
+		if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
+			log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
+		}
 	}
 
 	if err = repo.UpdateSize(); err != nil {

models/repo_mirror.go

@@ -13,6 +13,7 @@ import (
 	"github.com/go-xorm/xorm"
 	"gopkg.in/ini.v1"
 
+	"code.gitea.io/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/setting"
@@ -156,6 +157,15 @@ func (m *Mirror) runSync() bool {
 		return false
 	}
 
+	gitRepo, err := git.OpenRepository(repoPath)
+	if err != nil {
+		log.Error(4, "OpenRepository: %v", err)
+		return false
+	}
+	if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
+		log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
+	}
+
 	if err := m.Repo.UpdateSize(); err != nil {
 		log.Error(4, "Failed to update size for mirror repository: %v", err)
 	}

models/update.go

@@ -81,6 +81,93 @@ func PushUpdate(branch string, opt PushUpdateOptions) error {
 	return nil
 }
 
+func pushUpdateDeleteTag(repo *Repository, gitRepo *git.Repository, tagName string) error {
+	rel, err := GetRelease(repo.ID, tagName)
+	if err != nil {
+		if IsErrReleaseNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("GetRelease: %v", err)
+	}
+	if rel.IsTag {
+		if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil {
+			return fmt.Errorf("Delete: %v", err)
+		}
+	} else {
+		rel.IsDraft = true
+		rel.NumCommits = 0
+		rel.Sha1 = ""
+		if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
+			return fmt.Errorf("Update: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func pushUpdateAddTag(repo *Repository, gitRepo *git.Repository, tagName string) error {
+	rel, err := GetRelease(repo.ID, tagName)
+	if err != nil && !IsErrReleaseNotExist(err) {
+		return fmt.Errorf("GetRelease: %v", err)
+	}
+
+	tag, err := gitRepo.GetTag(tagName)
+	if err != nil {
+		return fmt.Errorf("GetTag: %v", err)
+	}
+	commit, err := tag.Commit()
+	if err != nil {
+		return fmt.Errorf("Commit: %v", err)
+	}
+	tagCreatedUnix := commit.Author.When.Unix()
+
+	author, err := GetUserByEmail(commit.Author.Email)
+	if err != nil && !IsErrUserNotExist(err) {
+		return fmt.Errorf("GetUserByEmail: %v", err)
+	}
+
+	commitsCount, err := commit.CommitsCount()
+	if err != nil {
+		return fmt.Errorf("CommitsCount: %v", err)
+	}
+
+	if rel == nil {
+		rel = &Release{
+			RepoID:       repo.ID,
+			Title:        "",
+			TagName:      tagName,
+			LowerTagName: strings.ToLower(tagName),
+			Target:       "",
+			Sha1:         commit.ID.String(),
+			NumCommits:   commitsCount,
+			Note:         "",
+			IsDraft:      false,
+			IsPrerelease: false,
+			IsTag:        true,
+			CreatedUnix:  tagCreatedUnix,
+		}
+		if author != nil {
+			rel.PublisherID = author.ID
+		}
+
+		if _, err = x.InsertOne(rel); err != nil {
+			return fmt.Errorf("InsertOne: %v", err)
+		}
+	} else {
+		rel.Sha1 = commit.ID.String()
+		rel.CreatedUnix = tagCreatedUnix
+		rel.NumCommits = commitsCount
+		rel.IsDraft = false
+		if rel.IsTag && author != nil {
+			rel.PublisherID = author.ID
+		}
+		if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
+			return fmt.Errorf("Update: %v", err)
+		}
+	}
+	return nil
+}
+
 func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
 	isNewRef := opts.OldCommitID == git.EmptySHA
 	isDelRef := opts.NewCommitID == git.EmptySHA
@@ -106,23 +193,31 @@ func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
 		return nil, fmt.Errorf("GetRepositoryByName: %v", err)
 	}
 
+	gitRepo, err := git.OpenRepository(repoPath)
+	if err != nil {
+		return nil, fmt.Errorf("OpenRepository: %v", err)
+	}
+
 	if isDelRef {
+		// Tag has been deleted
+		if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
+			err = pushUpdateDeleteTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
+			if err != nil {
+				return nil, fmt.Errorf("pushUpdateDeleteTag: %v", err)
+			}
+		}
 		log.GitLogger.Info("Reference '%s' has been deleted from '%s/%s' by %s",
 			opts.RefFullName, opts.RepoUserName, opts.RepoName, opts.PusherName)
 		return repo, nil
 	}
 
-	gitRepo, err := git.OpenRepository(repoPath)
-	if err != nil {
-		return nil, fmt.Errorf("OpenRepository: %v", err)
-	}
-
 	if err = repo.UpdateSize(); err != nil {
 		log.Error(4, "Failed to update size for repository: %v", err)
 	}
 
 	// Push tags.
 	if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
+		pushUpdateAddTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
 		if err := CommitRepoAction(CommitRepoActionOptions{
 			PusherName:  opts.PusherName,
 			RepoOwnerID: owner.ID,

modules/context/repo.go

@@ -357,6 +357,7 @@ func RepoAssignment() macaron.Handler {
 
 		count, err := models.GetReleaseCountByRepoID(ctx.Repo.Repository.ID, models.FindReleasesOptions{
 			IncludeDrafts: false,
+			IncludeTags:   true,
 		})
 		if err != nil {
 			ctx.Handle(500, "GetReleaseCountByRepoID", err)