utf8mb4 encoding (#7959)

* use utf8mb4 character set for connections to mysql

* use utf8mb4 character set for tables, shorten varchar fields used in unique indexes

* migration type to update table character set

* update table character sets

* set charset for temp_user.status

* gofmt

pkg/services/sqlstore/migrator/dialect.go

@@ -29,6 +29,7 @@ type Dialect interface {
 
 	TableCheckSql(tableName string) (string, []interface{})
 	RenameTable(oldName string, newName string) string
+	UpdateTableSql(tableName string, columns []*Column) string
 }
 
 func NewDialect(name string) Dialect {
@@ -102,7 +103,7 @@ func (b *BaseDialect) CreateTableSql(table *Table) string {
 
 	sql = sql[:len(sql)-2] + ")"
 	if b.dialect.SupportEngine() {
-		sql += " ENGINE=InnoDB DEFAULT CHARSET UTF8 "
+		sql += " ENGINE=InnoDB DEFAULT CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci"
 	}
 
 	sql += ";"
@@ -160,3 +161,7 @@ func (db *BaseDialect) DropIndexSql(tableName string, index *Index) string {
 	name = index.XName(tableName)
 	return fmt.Sprintf("DROP INDEX %v ON %s", quote(name), quote(tableName))
 }
+
+func (db *BaseDialect) UpdateTableSql(tableName string, columns []*Column) string {
+	return "-- NOT REQUIRED"
+}

pkg/services/sqlstore/migrator/migrations.go

@@ -200,3 +200,17 @@ func (m *CopyTableDataMigration) IfTableExists(tableName string) *CopyTableDataM
 func (m *CopyTableDataMigration) Sql(d Dialect) string {
 	return d.CopyTableData(m.sourceTable, m.targetTable, m.sourceCols, m.targetCols)
 }
+
+type TableCharsetMigration struct {
+	MigrationBase
+	tableName string
+	columns   []*Column
+}
+
+func NewTableCharsetMigration(tableName string, columns []*Column) *TableCharsetMigration {
+	return &TableCharsetMigration{tableName: tableName, columns: columns}
+}
+
+func (m *TableCharsetMigration) Sql(d Dialect) string {
+	return d.UpdateTableSql(m.tableName, m.columns)
+}

pkg/services/sqlstore/migrator/mysql_dialect.go

@@ -1,6 +1,9 @@
 package migrator
 
-import "strconv"
+import (
+	"strconv"
+	"strings"
+)
 
 type Mysql struct {
 	BaseDialect
@@ -76,6 +79,12 @@ func (db *Mysql) SqlType(c *Column) string {
 	} else if hasLen1 {
 		res += "(" + strconv.Itoa(c.Length) + ")"
 	}
+
+	switch c.Type {
+	case DB_Char, DB_Varchar, DB_NVarchar, DB_TinyText, DB_Text, DB_MediumText, DB_LongText:
+		res += " CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
+	}
+
 	return res
 }
 
@@ -84,3 +93,15 @@ func (db *Mysql) TableCheckSql(tableName string) (string, []interface{}) {
 	sql := "SELECT `TABLE_NAME` from `INFORMATION_SCHEMA`.`TABLES` WHERE `TABLE_SCHEMA`=? and `TABLE_NAME`=?"
 	return sql, args
 }
+
+func (db *Mysql) UpdateTableSql(tableName string, columns []*Column) string {
+	var statements = []string{}
+
+	statements = append(statements, "DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+
+	for _, col := range columns {
+		statements = append(statements, "MODIFY "+col.StringNoPk(db))
+	}
+
+	return "ALTER TABLE " + db.Quote(tableName) + " " + strings.Join(statements, ", ") + ";"
+}

pkg/services/sqlstore/migrator/postgres_dialect.go

@@ -3,6 +3,7 @@ package migrator
 import (
 	"fmt"
 	"strconv"
+	"strings"
 )
 
 type Postgres struct {
@@ -112,3 +113,13 @@ func (db *Postgres) DropIndexSql(tableName string, index *Index) string {
 	idxName := index.XName(tableName)
 	return fmt.Sprintf("DROP INDEX %v", quote(idxName))
 }
+
+func (db *Postgres) UpdateTableSql(tableName string, columns []*Column) string {
+	var statements = []string{}
+
+	for _, col := range columns {
+		statements = append(statements, "ALTER "+db.QuoteStr()+col.Name+db.QuoteStr()+" TYPE "+db.SqlType(col))
+	}
+
+	return "ALTER TABLE " + db.Quote(tableName) + " " + strings.Join(statements, ", ") + ";"
+}