add black list and white list support for migrating repositories

Author: lunny

custom/conf/app.ini.sample

@@ -803,3 +803,11 @@ IS_INPUT_FILE = false
 ENABLED = false
 ; If you want to add authorization, specify a token here
 TOKEN =
+
+[migration]
+; Whitelist for migrating, default is blank. Blank means everything will be allowed.
+; Multiple domains could be separated by commas.
+WHITELISTED_DOMAINS =
+; Blacklist for migrating, default is blank. Multiple domains could be separated by commas.
+; When WHITELISTED_DOMAINS is not blank, this option will be ignored.
+BLACKLISTED_DOMAINS =

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -511,9 +511,15 @@ Two special environment variables are passed to the render command:
 - `GITEA_PREFIX_RAW`, which contains the current URL prefix in the `raw` path tree. To be used as prefix for image paths.
 
 ## Time (`time`)
+
 - `FORMAT`: Time format to diplay on UI. i.e. RFC1123 or 2006-01-02 15:04:05
 - `DEFAULT_UI_LOCATION`: Default location of time on the UI, so that we can display correct user's time on UI. i.e. Shanghai/Asia
 
+## Migraions (`migration`)
+
+- `WHITELISTED_DOMAINS`: ****: Domains whitelist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
+- `BLACKLISTED_DOMAINS`: ****: Domains blacklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `WHITELISTED_DOMAINS` is not blank, this option will be ignored. 
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -240,9 +240,15 @@ IS_INPUT_FILE = false
 - IS_INPUT_FILE: 输入方式是最后一个参数为文件路径还是从标准输入读取。
 
 ## Time (`time`)
+
 - `FORMAT`: 显示在界面上的时间格式。比如： RFC1123 或者 2006-01-02 15:04:05
 - `DEFAULT_UI_LOCATION`: 默认显示在界面上的时区，默认为本地时区。比如： Asia/Shanghai
 
+## Migraions (`migration`)
+
+- `WHITELISTED_DOMAINS`: ****: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。
+- `BLACKLISTED_DOMAINS`: ****: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `WHITELISTED_DOMAINS` 不为空，此选项将会被忽略。
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。

modules/matchlist/matchlist.go

@@ -0,0 +1,43 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package matchlist
+
+import (
+	"github.com/gobwas/glob"
+)
+
+// Matchlist represents a black or white list
+type Matchlist struct {
+	rules []string
+	ruleGlobs []glob.Glob
+}
+
+// NewMatchlist creates a new black or white list
+func NewMatchlist(rules ...string) (*Matchlist, error) {
+	list := Matchlist{
+		rules: rules,
+		ruleGlobs: make([]glob.Glob, 0, len(rules)),
+	}
+
+	for _, rule := range list.rules {
+		rg, err := glob.Compile(rule)
+		if err != nil {
+			return nil, err
+		}
+		list.ruleGlobs = append(list.ruleGlobs, rg)
+	}
+
+	return &list, nil
+}
+
+// Match will matches
+func (b *Matchlist) Match(u string) bool {
+	for _, r := range b.ruleGlobs {
+		if r.Match(u) {
+			return true
+		}
+	}
+	return false
+}

modules/migrations/migrate.go

@@ -6,9 +6,15 @@
 package migrations
 
 import (
+	"fmt"
+	"net/url"
+	"strings"
+
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/matchlist"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/setting"
 )
 
 // MigrateOptions is equal to base.MigrateOptions
@@ -23,8 +29,34 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) {
 	factories = append(factories, factory)
 }
 
+func isMigrateURLAllowed(remoteURL string) (bool, error) {
+	u, err := url.Parse(remoteURL)
+	if err != nil {
+		return false, err
+	}
+
+	if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") {
+		if len(setting.Migration.WhitelistedDomains) > 0 {
+			if !whitelist.Match(u.Host) {
+				return false, fmt.Errorf("Migrate from %v is not allowed", u.Host)
+			}
+		} else {
+			if blacklist.Match(u.Host) {
+				return false, fmt.Errorf("Migrate from %v is not allowed", u.Host)
+			}
+		}
+	}
+
+	return true, nil
+}
+
 // MigrateRepository migrate repository according MigrateOptions
 func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
+	allowed, err := isMigrateURLAllowed(opts.RemoteURL)
+	if !allowed {
+		return nil, err
+	}
+	
 	var (
 		downloader base.Downloader
 		uploader   = NewGiteaLocalUploader(doer, ownerName, opts.Name)
@@ -250,3 +282,23 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	return nil
 }
+
+var (
+	whitelist *matchlist.Matchlist
+	blacklist *matchlist.Matchlist
+)
+
+// Init migrations service
+func Init() error {
+	var err error
+	whitelist, err = matchlist.NewMatchlist(setting.Migration.WhitelistedDomains...)
+	if err != nil {
+		return fmt.Errorf("Init migration whitelist domains failed: %v", err)
+	}
+
+	blacklist, err = matchlist.NewMatchlist(setting.Migration.BlacklistedDomains...)
+	if err != nil {
+		return fmt.Errorf("Init migration blacklist domains failed: %v", err)
+	}
+	return nil
+}

modules/migrations/migrate_test.go

@@ -0,0 +1,38 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMigrateWhiteBlacklist(t *testing.T) {
+	setting.Migration.WhitelistedDomains = []string{"github.com"}
+	assert.NoError(t, Init())
+
+	allowed, err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git")
+	assert.False(t, allowed)
+	assert.Error(t, err)
+
+	allowed, err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git")
+	assert.True(t, allowed)
+	assert.NoError(t, err)
+
+	setting.Migration.WhitelistedDomains = []string{}
+	setting.Migration.BlacklistedDomains = []string{"github.com"}
+	assert.NoError(t, Init())
+
+	allowed, err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git")
+	assert.True(t, allowed)
+	assert.NoError(t, err)
+
+	allowed, err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git")
+	assert.False(t, allowed)
+	assert.Error(t, err)
+}

modules/setting/migrate.go

@@ -0,0 +1,25 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"fmt"
+)
+
+var Migration = struct {
+	WhitelistedDomains []string
+	BlacklistedDomains []string
+} {
+	WhitelistedDomains: []string{},
+	BlacklistedDomains: []string{},
+}
+
+// InitMigrationConfig represents load migration configurations
+func InitMigrationConfig() error {
+	if err := Cfg.Section("migration").MapTo(&Migration); err != nil {
+		return fmt.Errorf("Failed to map Migration settings: %v", err)
+	}
+	return nil
+}

routers/init.go

@@ -19,6 +19,7 @@ import (
 	"code.gitea.io/gitea/modules/mailer"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/external"
+	repo_migrations "code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/ssh"
 
@@ -101,6 +102,10 @@ func GlobalInit() {
 		models.InitSyncMirrors()
 		models.InitDeliverHooks()
 		models.InitTestPullRequests()
+
+		if err := repo_migrations.Init(); err != nil {
+			log.Fatal("Failed to initialize migrations: %v", err)
+		}
 	}
 	if setting.EnableSQLite3 {
 		log.Info("SQLite3 Supported")

vendor/modules.txt

@@ -524,6 +524,7 @@ golang.org/x/tools/go/internal/gcimporter
 golang.org/x/tools/internal/fastwalk
 golang.org/x/tools/internal/module
 # google.golang.org/appengine v1.6.2
+# google.golang.org/appengine v1.6.1
 google.golang.org/appengine/cloudsql
 google.golang.org/appengine/urlfetch
 google.golang.org/appengine/internal