Remove diacritics

Author: kdumontnu

docs/content/administration/config-cheat-sheet.en-us.md

@@ -598,7 +598,7 @@ And the following unique queues:
   - userid - use the userid / sub attribute
   - nickname - use the nickname attribute
   - email - use the username part of the email attribute
-  - email-normalized - use the username part of the email attribute,with single-quotes removed and any other non-supported username characters replaced with a `-` character
+  - email-normalized - use the username part of the email attribute with diacritics and single-quotes removed, and any other non-supported username characters replaced with a `-` character
 - `UPDATE_AVATAR`: **false**: Update avatar if available from oauth2 provider. Update will be performed on each login.
 - `ACCOUNT_LINKING`: **login**: How to handle if an account / email already exists:
   - disabled - show an error

models/user/user.go

@@ -13,6 +13,7 @@ import (
 	"regexp"
 	"strings"
 	"time"
+	"unicode"
 
 	_ "image/jpeg" // Needed for jpeg support
 
@@ -29,6 +30,9 @@ import (
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/validation"
+	"golang.org/x/text/runes"
+	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/norm"
 
 	"xorm.io/builder"
 )
@@ -516,13 +520,19 @@ func GetUserSalt() (string, error) {
 	return hex.EncodeToString(rBytes), nil
 }
 
-var validUsernameRE = regexp.MustCompile(`[^\w-.]`)
+var invalidUsernameCharsRE = regexp.MustCompile(`[^\w-.]`)
+var removeDiacriticsTransform = transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
+var removeChars = strings.NewReplacer("'", "")
 
-// normalizeUserName returns a string with single-quotes removed,
-// and any other non-supported username characters replaced with
+// normalizeUserName returns a string with single-quotes and diacritics
+// removed, and any other non-supported username characters replaced with
 // a `-` character
-func NormalizeUserName(s string) string {
-	return validUsernameRE.ReplaceAllString(strings.ReplaceAll(s, "'", ""), "-")
+func NormalizeUserName(s string) (string, error) {
+	strDiacriticsRemoved, n, err := transform.String(removeDiacriticsTransform, s)
+	if err != nil {
+		return "", fmt.Errorf("Failed to normalize character `%v` in provided username `%v`", s[n], s)
+	}
+	return invalidUsernameCharsRE.ReplaceAllString(removeChars.Replace(strDiacriticsRemoved), "-"), nil
 }
 
 var (

models/user/user_test.go

@@ -552,17 +552,18 @@ func Test_NormalizeUserFromEmail(t *testing.T) {
 		IsNormalizedValid bool
 	}{
 		{"test", "test", true},
-		{"Sinéad.O'Connor", "Sin-ad.OConnor", true},
-		{"Æsir", "-sir", false},
+		{"Sinéad.O'Connor", "Sinead.OConnor", true},
+		{"Æsir", "-sir", false}, // Currently unsupported
 		// \u00e9\u0065\u0301
-		{"éé", "-e-", false},
+		{"éé", "ee", true},
 		{"Awareness Hub", "Awareness-Hub", true},
 		{"double__underscore", "double__underscore", false}, // We should consider squashing double non-alpha characters
 		{".bad.", ".bad.", false},
 		{"new😀user", "new-user", true},
 	}
 	for _, testCase := range testCases {
-		normalizedName := user_model.NormalizeUserName(testCase.Input)
+		normalizedName, err := user_model.NormalizeUserName(testCase.Input)
+		assert.NoError(t, err)
 		assert.EqualValues(t, testCase.Expected, normalizedName)
 		if testCase.IsNormalizedValid {
 			assert.NoError(t, user_model.IsUsableUsername(normalizedName))

routers/web/auth/auth.go

@@ -368,16 +368,16 @@ func handleSignInFull(ctx *context.Context, u *user_model.User, remember, obeyRe
 	return setting.AppSubURL + "/"
 }
 
-func getUserName(gothUser *goth.User) string {
+func getUserName(gothUser *goth.User) (string, error) {
 	switch setting.OAuth2Client.Username {
 	case setting.OAuth2UsernameEmail:
-		return strings.Split(gothUser.Email, "@")[0]
+		return strings.Split(gothUser.Email, "@")[0], nil
 	case setting.OAuth2UsernameEmailNormalized:
 		return user_model.NormalizeUserName(strings.Split(gothUser.Email, "@")[0])
 	case setting.OAuth2UsernameNickname:
-		return gothUser.NickName
+		return gothUser.NickName, nil
 	default: // OAuth2UsernameUserid
-		return gothUser.UserID
+		return gothUser.UserID, nil
 	}
 }
 

routers/web/auth/linkaccount.go

@@ -55,7 +55,11 @@ func LinkAccount(ctx *context.Context) {
 	}
 
 	gu, _ := gothUser.(goth.User)
-	uname := getUserName(&gu)
+	uname, err := getUserName(&gu)
+	if err != nil {
+		ctx.ServerError("UserSignIn", err)
+		return
+	}
 	email := gu.Email
 	ctx.Data["user_name"] = uname
 	ctx.Data["email"] = email

routers/web/auth/oauth.go

@@ -970,8 +970,13 @@ func SignInOAuthCallback(ctx *context.Context) {
 				ctx.ServerError("CreateUser", err)
 				return
 			}
+			uname, err := getUserName(&gothUser)
+			if err != nil {
+				ctx.ServerError("UserSignIn", err)
+				return
+			}
 			u = &user_model.User{
-				Name:        getUserName(&gothUser),
+				Name:        uname,
 				FullName:    gothUser.Name,
 				Email:       gothUser.Email,
 				LoginType:   auth.OAuth2,