Merge branch 'main' into feature-jwt-asymmetric

Author: techknowlogick

MAINTAINERS

@@ -42,3 +42,4 @@ Norwin Roosen <[email protected]> (@noerw)
 Kyle Dumont <[email protected]> (@kdumontnu)
 Patrick Schratz <[email protected]> (@pat-s)
 Janis Estelmann <[email protected]> (@KN4CK3R)
+Steven Kriegler <[email protected]> (@justusbunsi)

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -270,6 +270,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 - `SSH_AUTHORIZED_PRINCIPALS_ALLOW`: **off** or **username, email**: \[off, username, email, anything\]: Specify the principals values that users are allowed to use as principal. When set to `anything` no checks are done on the principal string. When set to `off` authorized principal are not allowed to be set.
 - `SSH_CREATE_AUTHORIZED_PRINCIPALS_FILE`: **false/true**: Gitea will create a authorized_principals file by default when it is not using the internal ssh server and `SSH_AUTHORIZED_PRINCIPALS_ALLOW` is not `off`.
 - `SSH_AUTHORIZED_PRINCIPALS_BACKUP`: **false/true**: Enable SSH Authorized Principals Backup when rewriting all keys, default is true if `SSH_AUTHORIZED_PRINCIPALS_ALLOW` is not `off`.
+- `SSH_AUTHORIZED_KEYS_COMMAND_TEMPLATE`: **{{.AppPath}} --config={{.CustomConf}} serv key-{{.Key.ID}}**: Set the template for the command to passed on authorized keys. Possible keys are: AppPath, AppWorkPath, CustomConf, CustomPath, Key - where Key is a `models.PublicKey` and the others are strings which are shellquoted.
 - `SSH_SERVER_CIPHERS`: **aes128-ctr, aes192-ctr, aes256-ctr, [email protected], arcfour256, arcfour128**: For the built-in SSH server, choose the ciphers to support for SSH connections, for system SSH this setting has no effect.
 - `SSH_SERVER_KEY_EXCHANGES`: **diffie-hellman-group1-sha1, diffie-hellman-group14-sha1, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, [email protected]**: For the built-in SSH server, choose the key exchange algorithms to support for SSH connections, for system SSH this setting has no effect.
 - `SSH_SERVER_MACS`: **[email protected], hmac-sha2-256, hmac-sha1, hmac-sha1-96**: For the built-in SSH server, choose the MACs to support for SSH connections, for system SSH this setting has no effect

docs/content/doc/advanced/repo-mirror.en-us.md

@@ -0,0 +1,88 @@
+---
+date: "2021-05-13T00:00:00-00:00"
+title: "Repository Mirror"
+slug: "repo-mirror"
+weight: 45
+toc: false
+draft: false
+menu:
+  sidebar:
+    parent: "advanced"
+    name: "Repository Mirror"
+    weight: 45
+    identifier: "repo-mirror"
+---
+
+# Repository Mirror
+
+Repository mirroring allows for the mirroring of repositories to and from external sources. You can use it to mirror branches, tags, and commits between repositories.
+
+**Table of Contents**
+
+{{< toc >}}
+
+## Use cases
+
+The following are some possible use cases for repository mirroring:
+
+- You migrated to Gitea but still need to keep your project in another source. In that case, you can simply set it up to mirror to Gitea (pull) and all the essential history of commits, tags, and branches are available in your Gitea instance.
+- You have old projects in another source that you don’t use actively anymore, but don’t want to remove for archiving purposes. In that case, you can create a push mirror so that your active Gitea repository can push its changes to the old location.
+
+## Pulling from a remote repository
+
+For an existing remote repository, you can set up pull mirroring as follows:
+
+1. Select **New Migration** in the **Create...** menu on the top right.
+2. Select the remote repository service.
+3. Enter a repository URL.
+4. If the repository needs authentication fill in your authentication information.
+5. Check the box **This repository will be a mirror**.
+5. Select **Migrate repository** to save the configuration.
+
+The repository now gets mirrored periodically from the remote repository. You can force a sync by selecting **Synchronize Now** in the repository settings.
+
+## Pushing to a remote repository
+
+For an existing repository, you can set up push mirroring as follows:
+
+1. In your repository, go to **Settings** > **Repository**, and then the **Mirror Settings** section.
+2. Enter a repository URL.
+3. If the repository needs authentication expand the **Authorization** section and fill in your authentication information.
+4. Select **Add Push Mirror** to save the configuration.
+
+The repository now gets mirrored periodically to the remote repository. You can force a sync by selecting **Synchronize Now**. In case of an error a message displayed to help you resolve it.
+
+:exclamation::exclamation: **NOTE:** This will force push to the remote repository. This will overwrite any changes in the remote repository! :exclamation::exclamation:
+
+### Setting up a push mirror from Gitea to GitHub
+
+To set up a mirror from Gitea to GitHub, you need to follow these steps:
+
+1. Create a [GitHub personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with the *public_repo* box checked.
+2. Fill in the **Git Remote Repository URL**: `https://github.com/<your_github_group>/<your_github_project>.git`.
+3. Fill in the **Authorization** fields with your GitHub username and the personal access token.
+4. Select **Add Push Mirror** to save the configuration.
+
+The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button.
+
+### Setting up a push mirror from Gitea to GitLab
+
+To set up a mirror from Gitea to GitLab, you need to follow these steps:
+
+1. Create a [GitLab personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) with *write_repository* scope.
+2. Fill in the **Git Remote Repository URL**: `https://<destination host>/<your_gitlab_group_or_name>/<your_gitlab_project>.git`.
+3. Fill in the **Authorization** fields with `oauth2` as **Username** and your GitLab personal access token as **Password**.
+4. Select **Add Push Mirror** to save the configuration.
+
+The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button.
+
+### Setting up a push mirror from Gitea to Bitbucket
+
+To set up a mirror from Gitea to Bitbucket, you need to follow these steps:
+
+1. Create a [Bitbucket app password](https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/) with the *Repository Write* box checked.
+2. Fill in the **Git Remote Repository URL**: `https://bitbucket.org/<your_bitbucket_group_or_name>/<your_bitbucket_project>.git`.
+3. Fill in the **Authorization** fields with your Bitbucket username and the app password as **Password**.
+4. Select **Add Push Mirror** to save the configuration.
+
+The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button.

services/mirror/mirror_test.go renamed to integrations/mirror_pull_test.go

@@ -2,28 +2,24 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
-package mirror
+package integrations
 
 import (
 	"context"
-	"path/filepath"
 	"testing"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/git"
 	migration "code.gitea.io/gitea/modules/migrations/base"
 	"code.gitea.io/gitea/modules/repository"
+	mirror_service "code.gitea.io/gitea/services/mirror"
 	release_service "code.gitea.io/gitea/services/release"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestMain(m *testing.M) {
-	models.MainTest(m, filepath.Join("..", ".."))
-}
-
-func TestRelease_MirrorDelete(t *testing.T) {
-	assert.NoError(t, models.PrepareTestDatabase())
+func TestMirrorPull(t *testing.T) {
+	defer prepareTestEnv(t)()
 
 	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
 	repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
@@ -76,7 +72,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
 	err = mirror.GetMirror()
 	assert.NoError(t, err)
 
-	_, ok := runSync(ctx, mirror.Mirror)
+	ok := mirror_service.SyncPullMirror(ctx, mirror.ID)
 	assert.True(t, ok)
 
 	count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions)
@@ -87,7 +83,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true))
 
-	_, ok = runSync(ctx, mirror.Mirror)
+	ok = mirror_service.SyncPullMirror(ctx, mirror.ID)
 	assert.True(t, ok)
 
 	count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions)

integrations/mirror_push_test.go

@@ -0,0 +1,86 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/repository"
+	"code.gitea.io/gitea/modules/setting"
+	mirror_service "code.gitea.io/gitea/services/mirror"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMirrorPush(t *testing.T) {
+	onGiteaRun(t, testMirrorPush)
+}
+
+func testMirrorPush(t *testing.T, u *url.URL) {
+	defer prepareTestEnv(t)()
+
+	setting.Migrations.AllowLocalNetworks = true
+
+	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
+	srcRepo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
+
+	mirrorRepo, err := repository.CreateRepository(user, user, models.CreateRepoOptions{
+		Name: "test-push-mirror",
+	})
+	assert.NoError(t, err)
+
+	ctx := NewAPITestContext(t, user.LowerName, srcRepo.Name)
+
+	doCreatePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword)(t)
+
+	mirrors, err := models.GetPushMirrorsByRepoID(srcRepo.ID)
+	assert.NoError(t, err)
+	assert.Len(t, mirrors, 1)
+
+	ok := mirror_service.SyncPushMirror(context.Background(), mirrors[0].ID)
+	assert.True(t, ok)
+
+	srcGitRepo, err := git.OpenRepository(srcRepo.RepoPath())
+	assert.NoError(t, err)
+	defer srcGitRepo.Close()
+
+	srcCommit, err := srcGitRepo.GetBranchCommit("master")
+	assert.NoError(t, err)
+
+	mirrorGitRepo, err := git.OpenRepository(mirrorRepo.RepoPath())
+	assert.NoError(t, err)
+	defer mirrorGitRepo.Close()
+
+	mirrorCommit, err := mirrorGitRepo.GetBranchCommit("master")
+	assert.NoError(t, err)
+
+	assert.Equal(t, srcCommit.ID, mirrorCommit.ID)
+}
+
+func doCreatePushMirror(ctx APITestContext, address, username, password string) func(t *testing.T) {
+	return func(t *testing.T) {
+		csrf := GetCSRF(t, ctx.Session, fmt.Sprintf("/%s/%s/settings", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame)))
+
+		req := NewRequestWithValues(t, "POST", fmt.Sprintf("/%s/%s/settings", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame)), map[string]string{
+			"_csrf":                csrf,
+			"action":               "push-mirror-add",
+			"push_mirror_address":  address,
+			"push_mirror_username": username,
+			"push_mirror_password": password,
+			"push_mirror_interval": "0",
+		})
+		ctx.Session.MakeRequest(t, req, http.StatusFound)
+
+		flashCookie := ctx.Session.GetCookie("macaron_flash")
+		assert.NotNil(t, flashCookie)
+		assert.Contains(t, flashCookie.Value, "success")
+	}
+}

models/issue.go

@@ -1127,9 +1127,18 @@ func sortIssuesSession(sess *xorm.Session, sortType string, priorityRepoID int64
 		sess.Desc("issue.priority")
 	case "nearduedate":
 		// 253370764800 is 01/01/9999 @ 12:00am (UTC)
-		sess.OrderBy("CASE WHEN issue.deadline_unix = 0 THEN 253370764800 ELSE issue.deadline_unix END ASC")
+		sess.Join("LEFT", "milestone", "issue.milestone_id = milestone.id").
+			OrderBy("CASE " +
+				"WHEN issue.deadline_unix = 0 AND (milestone.deadline_unix = 0 OR milestone.deadline_unix IS NULL) THEN 253370764800 " +
+				"WHEN milestone.deadline_unix = 0 OR milestone.deadline_unix IS NULL THEN issue.deadline_unix " +
+				"WHEN milestone.deadline_unix < issue.deadline_unix OR issue.deadline_unix = 0 THEN milestone.deadline_unix " +
+				"ELSE issue.deadline_unix END ASC")
 	case "farduedate":
-		sess.Desc("issue.deadline_unix")
+		sess.Join("LEFT", "milestone", "issue.milestone_id = milestone.id").
+			OrderBy("CASE " +
+				"WHEN milestone.deadline_unix IS NULL THEN issue.deadline_unix " +
+				"WHEN milestone.deadline_unix < issue.deadline_unix OR issue.deadline_unix = 0 THEN milestone.deadline_unix " +
+				"ELSE issue.deadline_unix END DESC")
 	case "priorityrepo":
 		sess.OrderBy("CASE WHEN issue.repo_id = " + strconv.FormatInt(priorityRepoID, 10) + " THEN 1 ELSE 2 END, issue.created_unix DESC")
 	default:

models/issue_comment.go

@@ -762,6 +762,8 @@ func updateCommentInfos(e *xorm.Session, opts *CreateCommentOptions, comment *Co
 			}
 		}
 		fallthrough
+	case CommentTypeReview:
+		fallthrough
 	case CommentTypeComment:
 		if _, err = e.Exec("UPDATE `issue` SET num_comments=num_comments+1 WHERE id=?", opts.Issue.ID); err != nil {
 			return err

models/migrations/migrations.go

@@ -315,6 +315,8 @@ var migrations = []Migration{
 	NewMigration("Always save primary email on email address table", addPrimaryEmail2EmailAddress),
 	// v182 -> v183
 	NewMigration("Add issue resource index table", addIssueResourceIndexTable),
+	// v183 -> v184
+	NewMigration("Create PushMirror table", createPushMirrorTable),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v180.go

@@ -5,9 +5,6 @@
 package migrations
 
 import (
-	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/migrations/base"
-	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 
 	jsoniter "github.com/json-iterator/go"
@@ -16,20 +13,38 @@ import (
 )
 
 func deleteMigrationCredentials(x *xorm.Engine) (err error) {
+	// Task represents a task
+	type Task struct {
+		ID             int64
+		DoerID         int64 `xorm:"index"` // operator
+		OwnerID        int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero
+		RepoID         int64 `xorm:"index"`
+		Type           int
+		Status         int `xorm:"index"`
+		StartTime      int64
+		EndTime        int64
+		PayloadContent string `xorm:"TEXT"`
+		Errors         string `xorm:"TEXT"` // if task failed, saved the error reason
+		Created        int64  `xorm:"created"`
+	}
+
+	const TaskTypeMigrateRepo = 0
+	const TaskStatusStopped = 2
+
 	const batchSize = 100
 
 	// only match migration tasks, that are not pending or running
 	cond := builder.Eq{
-		"type": structs.TaskTypeMigrateRepo,
+		"type": TaskTypeMigrateRepo,
 	}.And(builder.Gte{
-		"status": structs.TaskStatusStopped,
+		"status": TaskStatusStopped,
 	})
 
 	sess := x.NewSession()
 	defer sess.Close()
 
 	for start := 0; ; start += batchSize {
-		tasks := make([]*models.Task, 0, batchSize)
+		tasks := make([]*Task, 0, batchSize)
 		if err = sess.Limit(batchSize, start).Where(cond, 0).Find(&tasks); err != nil {
 			return
 		}
@@ -55,7 +70,41 @@ func deleteMigrationCredentials(x *xorm.Engine) (err error) {
 }
 
 func removeCredentials(payload string) (string, error) {
-	var opts base.MigrateOptions
+	// MigrateOptions defines the way a repository gets migrated
+	// this is for internal usage by migrations module and func who interact with it
+	type MigrateOptions struct {
+		// required: true
+		CloneAddr             string `json:"clone_addr" binding:"Required"`
+		CloneAddrEncrypted    string `json:"clone_addr_encrypted,omitempty"`
+		AuthUsername          string `json:"auth_username"`
+		AuthPassword          string `json:"-"`
+		AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"`
+		AuthToken             string `json:"-"`
+		AuthTokenEncrypted    string `json:"auth_token_encrypted,omitempty"`
+		// required: true
+		UID int `json:"uid" binding:"Required"`
+		// required: true
+		RepoName        string `json:"repo_name" binding:"Required"`
+		Mirror          bool   `json:"mirror"`
+		LFS             bool   `json:"lfs"`
+		LFSEndpoint     string `json:"lfs_endpoint"`
+		Private         bool   `json:"private"`
+		Description     string `json:"description"`
+		OriginalURL     string
+		GitServiceType  int
+		Uncyclo            bool
+		Issues          bool
+		Milestones      bool
+		Labels          bool
+		Releases        bool
+		Comments        bool
+		PullRequests    bool
+		ReleaseAssets   bool
+		MigrateToRepoID int64
+		MirrorInterval  string `json:"mirror_interval"`
+	}
+
+	var opts MigrateOptions
 	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	err := json.Unmarshal([]byte(payload), &opts)
 	if err != nil {
@@ -64,7 +113,7 @@ func removeCredentials(payload string) (string, error) {
 
 	opts.AuthPassword = ""
 	opts.AuthToken = ""
-	opts.CloneAddr = util.SanitizeURLCredentials(opts.CloneAddr, true)
+	opts.CloneAddr = util.NewStringURLSanitizer(opts.CloneAddr, true).Replace(opts.CloneAddr)
 
 	confBytes, err := json.Marshal(opts)
 	if err != nil {