Git migration UX (#12619)

* Initial work

Signed-off-by: jolheiser <[email protected]>

* Implementation

Signed-off-by: jolheiser <[email protected]>

* Fix gitlab and token cloning

Signed-off-by: jolheiser <[email protected]>

* Imports and JS

Signed-off-by: jolheiser <[email protected]>

* Fix test

Signed-off-by: jolheiser <[email protected]>

* Linting

Signed-off-by: jolheiser <[email protected]>

* Generate swagger

Signed-off-by: jolheiser <[email protected]>

* Move mirror toggle and rename options

Signed-off-by: jolheiser <[email protected]>

Co-authored-by: Lauris BH <[email protected]>
Co-authored-by: Lunny Xiao <[email protected]>

Author: 3 people

modules/auth/repo_form.go

@@ -56,8 +56,10 @@ func (f *CreateRepoForm) Validate(ctx *macaron.Context, errs binding.Errors) bin
 type MigrateRepoForm struct {
 	// required: true
 	CloneAddr    string `json:"clone_addr" binding:"Required"`
+	Service      int    `json:"service"`
 	AuthUsername string `json:"auth_username"`
 	AuthPassword string `json:"auth_password"`
+	AuthToken    string `json:"auth_token"`
 	// required: true
 	UID int64 `json:"uid" binding:"Required"`
 	// required: true

modules/migrations/base/downloader.go

@@ -7,13 +7,20 @@ package base
 
 import (
 	"context"
+	"io"
 	"time"
 
 	"code.gitea.io/gitea/modules/structs"
 )
 
+// AssetDownloader downloads an asset (attachment) for a release
+type AssetDownloader interface {
+	GetAsset(tag string, id int64) (io.ReadCloser, error)
+}
+
 // Downloader downloads the site repo informations
 type Downloader interface {
+	AssetDownloader
 	SetContext(context.Context)
 	GetRepoInfo() (*Repository, error)
 	GetTopics() ([]string, error)
@@ -28,7 +35,6 @@ type Downloader interface {
 
 // DownloaderFactory defines an interface to match a downloader implementation and create a downloader
 type DownloaderFactory interface {
-	Match(opts MigrateOptions) (bool, error)
 	New(opts MigrateOptions) (Downloader, error)
 	GitServiceType() structs.GitServiceType
 }

modules/migrations/base/release.go

@@ -8,7 +8,7 @@ import "time"
 
 // ReleaseAsset represents a release asset
 type ReleaseAsset struct {
-	URL           string
+	ID            int64
 	Name          string
 	ContentType   *string
 	Size          *int

modules/migrations/base/uploader.go

@@ -11,7 +11,7 @@ type Uploader interface {
 	CreateRepo(repo *Repository, opts MigrateOptions) error
 	CreateTopics(topic ...string) error
 	CreateMilestones(milestones ...*Milestone) error
-	CreateReleases(releases ...*Release) error
+	CreateReleases(downloader Downloader, releases ...*Release) error
 	SyncTags() error
 	CreateLabels(labels ...*Label) error
 	CreateIssues(issues ...*Issue) error

modules/migrations/git.go

@@ -6,6 +6,7 @@ package migrations
 
 import (
 	"context"
+	"io"
 
 	"code.gitea.io/gitea/modules/migrations/base"
 )
@@ -64,6 +65,11 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
 	return nil, ErrNotSupported
 }
 
+// GetAsset returns an asset
+func (g *PlainGitDownloader) GetAsset(_ string, _ int64) (io.ReadCloser, error) {
+	return nil, ErrNotSupported
+}
+
 // GetIssues returns issues according page and perPage
 func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
 	return nil, false, ErrNotSupported

modules/migrations/gitea.go

@@ -93,12 +93,15 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
 	}
 
 	var remoteAddr = repo.CloneURL
-	if len(opts.AuthUsername) > 0 {
+	if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
 		u, err := url.Parse(repo.CloneURL)
 		if err != nil {
 			return err
 		}
 		u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
+		if len(opts.AuthToken) > 0 {
+			u.User = url.UserPassword("oauth2", opts.AuthToken)
+		}
 		remoteAddr = u.String()
 	}
 
@@ -210,7 +213,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
 }
 
 // CreateReleases creates releases
-func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
+func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases ...*base.Release) error {
 	var rels = make([]*models.Release, 0, len(releases))
 	for _, release := range releases {
 		var rel = models.Release{
@@ -269,13 +272,11 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 
 			// download attachment
 			err = func() error {
-				resp, err := http.Get(asset.URL)
+				rc, err := downloader.GetAsset(rel.TagName, asset.ID)
 				if err != nil {
 					return err
 				}
-				defer resp.Body.Close()
-
-				_, err = storage.Attachments.Save(attach.RelativePath(), resp.Body)
+				_, err = storage.Attachments.Save(attach.RelativePath(), rc)
 				return err
 			}()
 			if err != nil {

modules/migrations/gitea_test.go

@@ -26,7 +26,7 @@ func TestGiteaUploadRepo(t *testing.T) {
 	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
 
 	var (
-		downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder")
+		downloader = NewGithubDownloaderV3("", "", "", "go-xorm", "builder")
 		repoName   = "builder-" + time.Now().Format("2006-01-02-15-04-05")
 		uploader   = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
 	)

modules/migrations/github.go

@@ -6,8 +6,11 @@
 package migrations
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
@@ -37,16 +40,6 @@ func init() {
 type GithubDownloaderV3Factory struct {
 }
 
-// Match returns ture if the migration remote URL matched this downloader factory
-func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error) {
-	u, err := url.Parse(opts.CloneAddr)
-	if err != nil {
-		return false, err
-	}
-
-	return strings.EqualFold(u.Host, "github.com") && opts.AuthUsername != "", nil
-}
-
 // New returns a Downloader related to this factory according MigrateOptions
 func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Downloader, error) {
 	u, err := url.Parse(opts.CloneAddr)
@@ -60,7 +53,7 @@ func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Download
 
 	log.Trace("Create github downloader: %s/%s", oldOwner, oldName)
 
-	return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
+	return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
 }
 
 // GitServiceType returns the type of git service
@@ -81,7 +74,7 @@ type GithubDownloaderV3 struct {
 }
 
 // NewGithubDownloaderV3 creates a github Downloader via github v3 API
-func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *GithubDownloaderV3 {
+func NewGithubDownloaderV3(userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
 	var downloader = GithubDownloaderV3{
 		userName:  userName,
 		password:  password,
@@ -90,23 +83,19 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith
 		repoName:  repoName,
 	}
 
-	var client *http.Client
-	if userName != "" {
-		if password == "" {
-			ts := oauth2.StaticTokenSource(
-				&oauth2.Token{AccessToken: userName},
-			)
-			client = oauth2.NewClient(downloader.ctx, ts)
-		} else {
-			client = &http.Client{
-				Transport: &http.Transport{
-					Proxy: func(req *http.Request) (*url.URL, error) {
-						req.SetBasicAuth(userName, password)
-						return nil, nil
-					},
-				},
-			}
-		}
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy: func(req *http.Request) (*url.URL, error) {
+				req.SetBasicAuth(userName, password)
+				return nil, nil
+			},
+		},
+	}
+	if token != "" {
+		ts := oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: token},
+		)
+		client = oauth2.NewClient(downloader.ctx, ts)
 	}
 	downloader.client = github.NewClient(client)
 	return &downloader
@@ -290,10 +279,8 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
 	}
 
 	for _, asset := range rel.Assets {
-		u, _ := url.Parse(*asset.BrowserDownloadURL)
-		u.User = url.UserPassword(g.userName, g.password)
 		r.Assets = append(r.Assets, base.ReleaseAsset{
-			URL:           u.String(),
+			ID:            *asset.ID,
 			Name:          *asset.Name,
 			ContentType:   asset.ContentType,
 			Size:          asset.Size,
@@ -331,6 +318,18 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
 	return releases, nil
 }
 
+// GetAsset returns an asset
+func (g *GithubDownloaderV3) GetAsset(_ string, id int64) (io.ReadCloser, error) {
+	asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, id, http.DefaultClient)
+	if err != nil {
+		return nil, err
+	}
+	if asset == nil {
+		return ioutil.NopCloser(bytes.NewBufferString(redir)), nil
+	}
+	return asset, nil
+}
+
 // GetIssues returns issues according start and limit
 func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
 	opt := &github.IssueListByRepoOptions{

modules/migrations/github_test.go

@@ -64,7 +64,7 @@ func assertLabelEqual(t *testing.T, name, color, description string, label *base
 
 func TestGitHubDownloadRepo(t *testing.T) {
 	GithubLimitRateRemaining = 3 //Wait at 3 remaining since we could have 3 CI in //
-	downloader := NewGithubDownloaderV3(os.Getenv("GITHUB_READ_TOKEN"), "", "go-gitea", "test_repo")
+	downloader := NewGithubDownloaderV3("", "", os.Getenv("GITHUB_READ_TOKEN"), "go-gitea", "test_repo")
 	err := downloader.RefreshRate()
 	assert.NoError(t, err)
 

modules/migrations/gitlab.go

@@ -8,6 +8,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
 	"net/url"
 	"strings"
 	"time"
@@ -32,21 +34,6 @@ func init() {
 type GitlabDownloaderFactory struct {
 }
 
-// Match returns true if the migration remote URL matched this downloader factory
-func (f *GitlabDownloaderFactory) Match(opts base.MigrateOptions) (bool, error) {
-	var matched bool
-
-	u, err := url.Parse(opts.CloneAddr)
-	if err != nil {
-		return false, err
-	}
-	if strings.EqualFold(u.Host, "gitlab.com") && opts.AuthUsername != "" {
-		matched = true
-	}
-
-	return matched, nil
-}
-
 // New returns a Downloader related to this factory according MigrateOptions
 func (f *GitlabDownloaderFactory) New(opts base.MigrateOptions) (base.Downloader, error) {
 	u, err := url.Parse(opts.CloneAddr)
@@ -56,10 +43,11 @@ func (f *GitlabDownloaderFactory) New(opts base.MigrateOptions) (base.Downloader
 
 	baseURL := u.Scheme + "://" + u.Host
 	repoNameSpace := strings.TrimPrefix(u.Path, "/")
+	repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git")
 
 	log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
 
-	return NewGitlabDownloader(baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword), nil
+	return NewGitlabDownloader(baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword, opts.AuthToken), nil
 }
 
 // GitServiceType returns the type of git service
@@ -85,15 +73,13 @@ type GitlabDownloader struct {
 // NewGitlabDownloader creates a gitlab Downloader via gitlab API
 //   Use either a username/password, personal token entered into the username field, or anonymous/public access
 //   Note: Public access only allows very basic access
-func NewGitlabDownloader(baseURL, repoPath, username, password string) *GitlabDownloader {
+func NewGitlabDownloader(baseURL, repoPath, username, password, token string) *GitlabDownloader {
 	var gitlabClient *gitlab.Client
 	var err error
-	if username != "" {
-		if password == "" {
-			gitlabClient, err = gitlab.NewClient(username, gitlab.WithBaseURL(baseURL))
-		} else {
-			gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
-		}
+	if token != "" {
+		gitlabClient, err = gitlab.NewClient(token, gitlab.WithBaseURL(baseURL))
+	} else {
+		gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
 	}
 
 	if err != nil {
@@ -271,7 +257,7 @@ func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
 }
 
 func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release {
-
+	var zero int
 	r := &base.Release{
 		TagName:         rel.TagName,
 		TargetCommitish: rel.Commit.ID,
@@ -284,9 +270,11 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
 
 	for k, asset := range rel.Assets.Links {
 		r.Assets = append(r.Assets, base.ReleaseAsset{
-			URL:         asset.URL,
-			Name:        asset.Name,
-			ContentType: &rel.Assets.Sources[k].Format,
+			ID:            int64(asset.ID),
+			Name:          asset.Name,
+			ContentType:   &rel.Assets.Sources[k].Format,
+			Size:          &zero,
+			DownloadCount: &zero,
 		})
 	}
 	return r
@@ -315,6 +303,21 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
 	return releases, nil
 }
 
+// GetAsset returns an asset
+func (g *GitlabDownloader) GetAsset(tag string, id int64) (io.ReadCloser, error) {
+	link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, tag, int(id))
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.Get(link.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	// resp.Body is closed by the uploader
+	return resp.Body, nil
+}
+
 // GetIssues returns issues according start and limit
 //   Note: issue label description and colors are not supported by the go-gitlab library at this time
 //   TODO: figure out how to transfer issue reactions

modules/migrations/gitlab_test.go

@@ -27,7 +27,7 @@ func TestGitlabDownloadRepo(t *testing.T) {
 		t.Skipf("Can't access test repo, skipping %s", t.Name())
 	}
 
-	downloader := NewGitlabDownloader("https://gitlab.com", "gitea/test_repo", gitlabPersonalAccessToken, "")
+	downloader := NewGitlabDownloader("https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken)
 	if downloader == nil {
 		t.Fatal("NewGitlabDownloader is nil")
 	}