Add support of utf8mb4 for mysql (#6992)

Author: lunny

custom/conf/app.ini.sample

@@ -260,6 +260,9 @@ PASSWD =
 ; For Postgres, either "disable" (default), "require", or "verify-full"
 ; For MySQL, either "false" (default), "true", or "skip-verify"
 SSL_MODE = disable
+; For MySQL only, either "utf8" or "utf8mb4", default is "utf8".
+; NOTICE: for "utf8mb4" you must use MySQL InnoDB > 5.6. Gitea is unable to check this.
+CHARSET = utf8
 ; For "sqlite3" and "tidb", use an absolute path when you start gitea as service
 PATH = data/gitea.db
 ; For "sqlite3" only. Query timeout

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -160,6 +160,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `USER`: **root**: Database username.
 - `PASSWD`: **\<empty\>**: Database user password. Use \`your password\` for quoting if you use special characters in the password.
 - `SSL_MODE`: **disable**: For PostgreSQL and MySQL only.
+- `CHARSET`: **utf8**: For MySQL only, either "utf8" or "utf8mb4", default is "utf8". NOTICE: for "utf8mb4" you must use MySQL InnoDB > 5.6. Gitea is unable to check this.
 - `PATH`: **data/gitea.db**: For SQLite3 only, the database file path.
 - `LOG_SQL`: **true**: Log the executed SQL.
 - `DB_RETRIES`: **10**: How many ORM init / DB connect attempts allowed.

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -78,7 +78,8 @@ menu:
 - `NAME`: 数据库名称。
 - `USER`: 数据库用户名。
 - `PASSWD`: 数据库用户密码。
-- `SSL_MODE`: PostgreSQL数据库是否启用SSL模式。
+- `SSL_MODE`: MySQL 或 PostgreSQL数据库是否启用SSL模式。
+- `CHARSET`: **utf8**: 仅当数据库为 MySQL 时有效, 可以为 "utf8" 或 "utf8mb4"。注意：如果使用 "utf8mb4"，你的 MySQL InnoDB 版本必须在 5.6 以上。
 - `PATH`: Tidb 或者 SQLite3 数据文件存放路径。
 - `LOG_SQL`: **true**: 显示生成的SQL，默认为真。
 

models/models.go

@@ -59,8 +59,8 @@ var (
 
 	// DbCfg holds the database settings
 	DbCfg struct {
-		Type, Host, Name, User, Passwd, Path, SSLMode string
-		Timeout                                       int
+		Type, Host, Name, User, Passwd, Path, SSLMode, Charset string
+		Timeout                                                int
 	}
 
 	// EnableSQLite3 use SQLite3
@@ -160,6 +160,7 @@ func LoadConfigs() {
 		DbCfg.Passwd = sec.Key("PASSWD").String()
 	}
 	DbCfg.SSLMode = sec.Key("SSL_MODE").MustString("disable")
+	DbCfg.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"})
 	DbCfg.Path = sec.Key("PATH").MustString(filepath.Join(setting.AppDataPath, "gitea.db"))
 	DbCfg.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
 }
@@ -222,8 +223,8 @@ func getEngine() (*xorm.Engine, error) {
 		if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL
 			tls = "false"
 		}
-		connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=utf8&parseTime=true&tls=%s",
-			DbCfg.User, DbCfg.Passwd, connType, DbCfg.Host, DbCfg.Name, Param, tls)
+		connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s",
+			DbCfg.User, DbCfg.Passwd, connType, DbCfg.Host, DbCfg.Name, Param, DbCfg.Charset, tls)
 	case "postgres":
 		connStr = getPostgreSQLConnectionString(DbCfg.Host, DbCfg.User, DbCfg.Passwd, DbCfg.Name, Param, DbCfg.SSLMode)
 	case "mssql":

modules/auth/user_form.go

@@ -23,6 +23,7 @@ type InstallForm struct {
 	DbPasswd string
 	DbName   string
 	SSLMode  string
+	Charset  string `binding:"Required;In(utf8,utf8mb4)"`
 	DbPath   string
 
 	AppName      string `binding:"Required" locale:"install.app_name"`

options/locale/locale_en-US.ini

@@ -86,8 +86,9 @@ host = Host
 user = Username
 password = Password
 db_name = Database Name
-db_helper = Note to MySQL users: please use the InnoDB storage engine and the 'utf8_general_ci' character set.
+db_helper = Note to MySQL users: please use the InnoDB storage engine and if you use "utf8mb4", your InnoDB version must be greater than 5.6 .
 ssl_mode = SSL
+charset = Charset
 path = Path
 sqlite_helper = File path for the SQLite3 database.<br>Enter an absolute path if you run Gitea as a service.
 err_empty_db_path = The SQLite3 database path cannot be empty.

public/js/index.js

@@ -587,15 +587,14 @@ function initInstall() {
         var tidbDefault = 'data/gitea_tidb';
 
         var dbType = $(this).val();
-        if (dbType === "SQLite3" || dbType === "TiDB") {
+        if (dbType === "SQLite3") {
             $('#sql_settings').hide();
             $('#pgsql_settings').hide();
+            $('#mysql_settings').hide();
             $('#sqlite_settings').show();
 
             if (dbType === "SQLite3" && $('#db_path').val() == tidbDefault) {
                 $('#db_path').val(sqliteDefault);
-            } else if (dbType === "TiDB" && $('#db_path').val() == sqliteDefault) {
-                $('#db_path').val(tidbDefault);
             }
             return;
         }
@@ -610,6 +609,7 @@ function initInstall() {
         $('#sql_settings').show();
 
         $('#pgsql_settings').toggle(dbType === "PostgreSQL");
+        $('#mysql_settings').toggle(dbType === "MySQL");
         $.each(dbDefaults, function(_type, defaultHost) {
             if ($('#db_host').val() == defaultHost) {
                 $('#db_host').val(dbDefaults[dbType]);

routers/install.go

@@ -150,6 +150,7 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) {
 	models.DbCfg.Passwd = form.DbPasswd
 	models.DbCfg.Name = form.DbName
 	models.DbCfg.SSLMode = form.SSLMode
+	models.DbCfg.Charset = form.Charset
 	models.DbCfg.Path = form.DbPath
 
 	if (models.DbCfg.Type == "sqlite3") &&

templates/install.tmpl

@@ -28,7 +28,7 @@
 						</div>
 					</div>
 
-					<div id="sql_settings" class="{{if or (eq .CurDbOption "SQLite3") (eq .CurDbOption "TiDB")}}hide{{end}}">
+					<div id="sql_settings" class="{{if or (eq .CurDbOption "SQLite3")}}hide{{end}}">
 						<div class="inline required field {{if .Err_DbSetting}}error{{end}}">
 							<label for="db_host">{{.i18n.Tr "install.host"}}</label>
 							<input id="db_host" name="db_host" value="{{.db_host}}">
@@ -64,6 +64,21 @@
 						</div>
 					</div>
 
+					<div id="mysql_settings" class="{{if not (eq .CurDbOption "MySQL")}}hide{{end}}">
+						<div class="inline required field">
+							<label>{{.i18n.Tr "install.charset"}}</label>
+							<div class="ui selection database type dropdown">
+								<input type="hidden" name="charset" value="{{if .charset}}{{.charset}}{{else}}utf8{{end}}">
+								<div class="default text">utf8</div>
+								<i class="dropdown icon"></i>
+								<div class="menu">
+									<div class="item" data-value="utf8">utf8</div>
+									<div class="item" data-value="utf8mb4">utf8mb4</div>
+								</div>
+							</div>
+						</div>
+					</div>
+
 					<div id="sqlite_settings" class="{{if not (or (eq .CurDbOption "SQLite3") (eq .CurDbOption "TiDB"))}}hide{{end}}">
 						<div class="inline required field {{if or .Err_DbPath .Err_DbSetting}}error{{end}}">
 							<label for="db_path">{{.i18n.Tr "install.path"}}</label>