Merge branch 'master' into feature/Pinned_repo

Author: a1012112796

docs/content/doc/help/faq.en-us.md

@@ -47,7 +47,8 @@ Also see [Support Options]({{< relref "doc/help/seek-help.en-us.md" >}})
 * [How can I enable password reset](#how-can-i-enable-password-reset)
 * [How can a user's password be changed](#how-can-a-user-s-password-be-changed)
 * [Why is my markdown broken](#why-is-my-markdown-broken)
-
+* [Errors during upgrade on MySQL: Error 1118: Row size too large.](#upgrade-errors-with-mysql)
+* [Why are emoji broken on MySQL](#why-are-emoji-broken-on-mysql)
 
 ## Difference between 1.x and 1.x.x downloads
 Version 1.7.x will be used for this example.  
@@ -308,3 +309,30 @@ There is no setting for password resets. It is enabled when a [mail service]({{<
 In Gitea version `1.11` we moved to [goldmark](https://github.com/yuin/goldmark) for markdown rendering, which is [CommonMark](https://commonmark.org/) compliant.  
 If you have markdown that worked as you expected prior to version `1.11` and after upgrading it's not working anymore, please look through the CommonMark spec to see whether the problem is due to a bug or non-compliant syntax.  
 If it is the latter, _usually_ there is a compliant alternative listed in the spec.
+
+## Upgrade errors with MySQL
+
+If you are receiving errors on upgrade of Gitea using MySQL that read:
+
+> `ORM engine initialization failed: migrate: do migrate: Error: 1118: Row size too large...`
+
+Please run `gitea convert` or run `ALTER TABLE table_name ROW_FORMAT=dynamic;` for each table in the database.
+
+The underlying problem is that the space allocated for indices by the default row format
+is too small. Gitea requires that the `ROWFORMAT` for its tables is `DYNAMIC`.
+
+If you are receiving an error line containing `Error 1071: Specified key was too long; max key length is 1000 bytes...`
+then you are attempting to run Gitea on tables which use the ISAM engine. While this may have worked by chance in previous versions of Gitea, it has never been officially supported and
+you must use InnoDB. You should run `ALTER TABLE table_name ENGINE=InnoDB;` for each table in the database.
+
+## Why Are Emoji Broken On MySQL
+
+Unfortunately MySQL's `utf8` charset does not completely allow all possible UTF-8 characters, in particular Emoji.
+They created a new charset and collation called `utf8mb4` that allows for emoji to be stored but tables which use
+the `utf8` charset, and connections which use the `utf8` charset will not use this.
+
+Please run `gitea convert`, or run `ALTER DATABASE database_name CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;`
+for the database_name and run `ALTER TABLE table_name CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;`
+for each table in the database.
+
+You will also need to change the app.ini database charset to `CHARSET=utf8mb4`.

integrations/repo_migrate_test.go

@@ -5,15 +5,17 @@
 package integrations
 
 import (
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
+	"code.gitea.io/gitea/modules/structs"
 	"github.com/stretchr/testify/assert"
 )
 
 func testRepoMigrate(t testing.TB, session *TestSession, cloneAddr, repoName string) *httptest.ResponseRecorder {
-	req := NewRequest(t, "GET", "/repo/migrate")
+	req := NewRequest(t, "GET", fmt.Sprintf("/repo/migrate?service_type=%d", structs.PlainGitService)) // render plain git migration page
 	resp := session.MakeRequest(t, req, http.StatusOK)
 	htmlDoc := NewHTMLParser(t, resp.Body)
 
@@ -28,8 +30,8 @@ func testRepoMigrate(t testing.TB, session *TestSession, cloneAddr, repoName str
 		"clone_addr": cloneAddr,
 		"uid":        uid,
 		"repo_name":  repoName,
-	},
-	)
+		"service":    fmt.Sprintf("%d", structs.PlainGitService),
+	})
 	resp = session.MakeRequest(t, req, http.StatusFound)
 
 	return resp

models/user.go

@@ -1419,11 +1419,21 @@ func getUserEmailsByNames(e Engine, names []string) []string {
 }
 
 // GetMaileableUsersByIDs gets users from ids, but only if they can receive mails
-func GetMaileableUsersByIDs(ids []int64) ([]*User, error) {
+func GetMaileableUsersByIDs(ids []int64, isMention bool) ([]*User, error) {
 	if len(ids) == 0 {
 		return nil, nil
 	}
 	ous := make([]*User, 0, len(ids))
+
+	if isMention {
+		return ous, x.In("id", ids).
+			Where("`type` = ?", UserTypeIndividual).
+			And("`prohibit_login` = ?", false).
+			And("`is_active` = ?", true).
+			And("`email_notifications_preference` IN ( ?, ?)", EmailNotificationsEnabled, EmailNotificationsOnMention).
+			Find(&ous)
+	}
+
 	return ous, x.In("id", ids).
 		Where("`type` = ?", UserTypeIndividual).
 		And("`prohibit_login` = ?", false).

models/user_test.go

@@ -389,3 +389,20 @@ func TestGetUserIDsByNames(t *testing.T) {
 	assert.Error(t, err)
 	assert.Equal(t, []int64(nil), IDs)
 }
+
+func TestGetMaileableUsersByIDs(t *testing.T) {
+	results, err := GetMaileableUsersByIDs([]int64{1, 4}, false)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(results))
+	if len(results) > 1 {
+		assert.Equal(t, results[0].ID, 1)
+	}
+
+	results, err = GetMaileableUsersByIDs([]int64{1, 4}, true)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(results))
+	if len(results) > 2 {
+		assert.Equal(t, results[0].ID, 1)
+		assert.Equal(t, results[1].ID, 4)
+	}
+}

modules/migrations/gitlab.go

@@ -47,7 +47,7 @@ func (f *GitlabDownloaderFactory) New(ctx context.Context, opts base.MigrateOpti
 
 	log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
 
-	return NewGitlabDownloader(ctx, baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword, opts.AuthToken), nil
+	return NewGitlabDownloader(ctx, baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword, opts.AuthToken)
 }
 
 // GitServiceType returns the type of git service
@@ -73,7 +73,7 @@ type GitlabDownloader struct {
 // NewGitlabDownloader creates a gitlab Downloader via gitlab API
 //   Use either a username/password, personal token entered into the username field, or anonymous/public access
 //   Note: Public access only allows very basic access
-func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) *GitlabDownloader {
+func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
 	var gitlabClient *gitlab.Client
 	var err error
 	if token != "" {
@@ -84,27 +84,27 @@ func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, passw
 
 	if err != nil {
 		log.Trace("Error logging into gitlab: %v", err)
-		return nil
+		return nil, err
 	}
 
 	// Grab and store project/repo ID here, due to issues using the URL escaped path
 	gr, _, err := gitlabClient.Projects.GetProject(repoPath, nil, nil, gitlab.WithContext(ctx))
 	if err != nil {
 		log.Trace("Error retrieving project: %v", err)
-		return nil
+		return nil, err
 	}
 
 	if gr == nil {
 		log.Trace("Error getting project, project is nil")
-		return nil
+		return nil, errors.New("Error getting project, project is nil")
 	}
 
 	return &GitlabDownloader{
 		ctx:      ctx,
 		client:   gitlabClient,
 		repoID:   gr.ID,
 		repoName: gr.Name,
-	}
+	}, nil
 }
 
 // SetContext set context
@@ -114,10 +114,6 @@ func (g *GitlabDownloader) SetContext(ctx context.Context) {
 
 // GetRepoInfo returns a repository information
 func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) {
-	if g == nil {
-		return nil, errors.New("error: GitlabDownloader is nil")
-	}
-
 	gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))
 	if err != nil {
 		return nil, err
@@ -154,10 +150,6 @@ func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) {
 
 // GetTopics return gitlab topics
 func (g *GitlabDownloader) GetTopics() ([]string, error) {
-	if g == nil {
-		return nil, errors.New("error: GitlabDownloader is nil")
-	}
-
 	gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))
 	if err != nil {
 		return nil, err
@@ -167,9 +159,6 @@ func (g *GitlabDownloader) GetTopics() ([]string, error) {
 
 // GetMilestones returns milestones
 func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
-	if g == nil {
-		return nil, errors.New("error: GitlabDownloader is nil")
-	}
 	var perPage = 100
 	var state = "all"
 	var milestones = make([]*base.Milestone, 0, perPage)
@@ -226,11 +215,21 @@ func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
 	return milestones, nil
 }
 
+func (g *GitlabDownloader) normalizeColor(val string) string {
+	val = strings.TrimLeft(val, "#")
+	val = strings.ToLower(val)
+	if len(val) == 3 {
+		c := []rune(val)
+		val = fmt.Sprintf("%c%c%c%c%c%c", c[0], c[0], c[1], c[1], c[2], c[2])
+	}
+	if len(val) != 6 {
+		return ""
+	}
+	return val
+}
+
 // GetLabels returns labels
 func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
-	if g == nil {
-		return nil, errors.New("error: GitlabDownloader is nil")
-	}
 	var perPage = 100
 	var labels = make([]*base.Label, 0, perPage)
 	for i := 1; ; i++ {
@@ -244,7 +243,7 @@ func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
 		for _, label := range ls {
 			baseLabel := &base.Label{
 				Name:        label.Name,
-				Color:       strings.TrimLeft(label.Color, "#)"),
+				Color:       g.normalizeColor(label.Color),
 				Description: label.Description,
 			}
 			labels = append(labels, baseLabel)
@@ -466,7 +465,6 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro
 
 // GetPullRequests returns pull requests according page and perPage
 func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) {
-
 	opt := &gitlab.ListProjectMergeRequestsOptions{
 		ListOptions: gitlab.ListOptions{
 			PerPage: perPage,
@@ -576,7 +574,6 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque
 
 // GetReviews returns pull requests review
 func (g *GitlabDownloader) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
-
 	state, _, err := g.client.MergeRequestApprovals.GetApprovalState(g.repoID, int(pullRequestNumber), gitlab.WithContext(g.ctx))
 	if err != nil {
 		return nil, err

modules/migrations/gitlab_test.go

@@ -6,6 +6,7 @@ package migrations
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"os"
 	"testing"
@@ -28,9 +29,9 @@ func TestGitlabDownloadRepo(t *testing.T) {
 		t.Skipf("Can't access test repo, skipping %s", t.Name())
 	}
 
-	downloader := NewGitlabDownloader(context.Background(), "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken)
-	if downloader == nil {
-		t.Fatal("NewGitlabDownloader is nil")
+	downloader, err := NewGitlabDownloader(context.Background(), "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken)
+	if err != nil {
+		t.Fatal(fmt.Sprintf("NewGitlabDownloader is nil: %v", err))
 	}
 	repo, err := downloader.GetRepoInfo()
 	assert.NoError(t, err)

modules/structs/repo.go

@@ -5,6 +5,7 @@
 package structs
 
 import (
+	"strings"
 	"time"
 )
 
@@ -205,17 +206,7 @@ const (
 // Name represents the service type's name
 // WARNNING: the name have to be equal to that on goth's library
 func (gt GitServiceType) Name() string {
-	switch gt {
-	case GithubService:
-		return "github"
-	case GiteaService:
-		return "gitea"
-	case GitlabService:
-		return "gitlab"
-	case GogsService:
-		return "gogs"
-	}
-	return ""
+	return strings.ToLower(gt.Title())
 }
 
 // Title represents the service type's proper title

modules/templates/helper.go

@@ -694,7 +694,7 @@ func ActionContent2Commits(act Actioner) *repository.PushCommits {
 // DiffTypeToStr returns diff type name
 func DiffTypeToStr(diffType int) string {
 	diffTypes := map[int]string{
-		1: "add", 2: "modify", 3: "del", 4: "rename",
+		1: "add", 2: "modify", 3: "del", 4: "rename", 5: "copy",
 	}
 	return diffTypes[diffType]
 }

options/locale/locale_cs-CZ.ini

@@ -726,7 +726,6 @@ migrate.permission_denied=Není dovoleno importovat místní repozitáře.
 migrate.invalid_local_path=Místní cesta je neplatná, buď neexistuje nebo není adresářem.
 migrate.failed=Přenesení selhalo: %v
 migrate.lfs_mirror_unsupported=Zrcadlení LFS objektů není podporováno - použijte místo toho „git lfs fetch --all“ a „git lfs push --all“.
-migrate.migrate_items_options=Pro migraci položek ze služby, která je podporuje, je potřeba autentizace.
 migrated_from=Migrováno z <a href="%[1]s">%[2]s</a>
 migrated_from_fake=Migrováno z %[1]s
 migrate.migrating=Probíhá migrace z <b>%s</b> ...

options/locale/locale_de-DE.ini

@@ -726,7 +726,6 @@ migrate.permission_denied=Du hast keine Berechtigung zum Importieren lokaler Rep
 migrate.invalid_local_path=Der lokale Pfad ist ungültig, existiert nicht oder ist kein Ordner.
 migrate.failed=Fehler bei der Migration: %v
 migrate.lfs_mirror_unsupported=Spiegeln von LFS-Objekten wird nicht unterstützt - nutze stattdessen 'git lfs fetch --all' und 'git lfs push --all'.
-migrate.migrate_items_options=Authentifizierung wird benötigt, um Elemente aus einem Dienst zu migrieren, der sie unterstützt.
 migrated_from=Migriert von <a href="%[1]s">%[2]s</a>
 migrated_from_fake=Migriert von %[1]s
 migrate.migrating=Migriere von <b>%s</b> ...

options/locale/locale_en-US.ini

@@ -730,6 +730,7 @@ migrate_items_milestones = Milestones
 migrate_items_labels = Labels
 migrate_items_issues = Issues
 migrate_items_pullrequests = Pull Requests
+migrate_items_merge_requests = Merge Requests
 migrate_items_releases = Releases
 migrate_repo = Migrate Repository
 migrate.clone_address = Migrate / Clone From URL
@@ -739,11 +740,15 @@ migrate.permission_denied = You are not allowed to import local repositories.
 migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory."
 migrate.failed = Migration failed: %v
 migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'git lfs fetch --all' and 'git lfs push --all' instead.
-migrate.migrate_items_options = Authentication is needed to migrate items from a service that supports them.
+migrate.migrate_items_options = Access Token is required to migrate additional items
 migrated_from = Migrated from <a href="%[1]s">%[2]s</a>
 migrated_from_fake = Migrated From %[1]s
+migrate.migrate = Migrate From %s
 migrate.migrating = Migrating from <b>%s</b> ...
 migrate.migrating_failed = Migrating from <b>%s</b> failed.
+migrate.github.description = Migrating data from Github.com or Github Enterprise.
+migrate.git.description = Migrating or Mirroring git data from Git services
+migrate.gitlab.description = Migrating data from GitLab.com or Self-Hosted gitlab server.
 
 mirror_from = mirror of
 forked_from = forked from

options/locale/locale_es-ES.ini

@@ -299,6 +299,8 @@ authorize_title=¿Autorizar a "%s" a acceder a su cuenta?
 authorization_failed=Autorización fallida
 authorization_failed_desc=La autorización ha fallado porque hemos detectado una solicitud no válida. Por favor, póngase en contacto con el mantenedor de la aplicación que ha intentado autorizar.
 sspi_auth_failed=Fallo en la autenticación SSPI
+password_pwned=La contraseña que eligió está en una <a target="_blank" rel="noopener noreferrer" href="https://haveibeenpwned.com/Passwords">lista de contraseñas robadas</a> previamente expuestas en violaciones de datos públicos. Por favor, inténtalo de nuevo con una contraseña diferente.
+password_pwned_err=No se pudo completar la solicitud a HaveIBeenPwned
 
 [mail]
 activate_account=Por favor, active su cuenta
@@ -717,6 +719,7 @@ migrate_items_milestones=Hitos
 migrate_items_labels=Etiquetas
 migrate_items_issues=Incidencias
 migrate_items_pullrequests=Pull Requests
+migrate_items_merge_requests=Merge Requests
 migrate_items_releases=Lanzamientos
 migrate_repo=Migrar Repositorio
 migrate.clone_address=Migrar / Clonar desde URL
@@ -726,11 +729,15 @@ migrate.permission_denied=No te está permitido importar repositorios locales.
 migrate.invalid_local_path=La ruta local es inválida. No existe o no es un directorio.
 migrate.failed=Migración fallida: %v
 migrate.lfs_mirror_unsupported=La replicación de objetos LFS no está soportada - use 'git lfs fetch --all' y 'git lfs push --all' en su lugar.
-migrate.migrate_items_options=Se requiere autenticación para migrar elementos de un servicio que los soporte.
+migrate.migrate_items_options=Un token de acceso es necesario para migrar elementos adicionales
 migrated_from=Migrado desde <a href="%[1]s">%[2]s</a>
 migrated_from_fake=Migrado desde %[1]s
+migrate.migrate=Migrar desde %s
 migrate.migrating=Migrando desde <b>%s</b>...
 migrate.migrating_failed=La migración desde <b>%s</b> ha fallado.
+migrate.github.description=Migrar datos de Github.com o Github Enterprise.
+migrate.git.description=Migrar o replicar de datos de git desde los servicios de Git
+migrate.gitlab.description=Migrar datos de GitLab.com o servidor gitlab autoalojado.
 
 mirror_from=réplica de
 forked_from=forkeado de
@@ -759,6 +766,7 @@ code=Código
 code.desc=Acceder código fuente, archivos, commits, y ramas.
 branch=Rama
 tree=Árbol
+clear_ref=`Borrar referencia actual`
 filter_branch_and_tag=Filtrar por rama o etiqueta
 branches=Ramas
 tags=Etiquetas

options/locale/locale_ja-JP.ini

@@ -726,7 +726,6 @@ migrate.permission_denied=ローカルリポジトリをインポートする権
 migrate.invalid_local_path=ローカルパスが無効です。 存在しないかディレクトリではありません。
 migrate.failed=移行に失敗しました: %v
 migrate.lfs_mirror_unsupported=LFSオブジェクトのミラーはサポートされていません。 代わりに 'git lfs fetch --all' と 'git lfs push --all' を使ってください。
-migrate.migrate_items_options=サービスがサポートしている項目を移行するには、認証が必要です。
 migrated_from=<a href="%[1]s">%[2]s</a>から移行
 migrated_from_fake=%[1]sから移行
 migrate.migrating=<b>%s</b> から移行しています ...