utf8mb4 encoding (#7959)

* use utf8mb4 character set for connections to mysql

* use utf8mb4 character set for tables, shorten varchar fields used in unique indexes

* migration type to update table character set

* update table character sets

* set charset for temp_user.status

* gofmt

pkg/services/sqlstore/migrations/alert_mig.go

@@ -50,7 +50,7 @@ func addAlertMigrations(mg *Migrator) {
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "org_id", Type: DB_BigInt, Nullable: false},
-			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
 			{Name: "settings", Type: DB_Text, Nullable: false},
 			{Name: "created", Type: DB_DateTime, Nullable: false},
@@ -67,4 +67,19 @@ func addAlertMigrations(mg *Migrator) {
 	}))
 	mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0]))
 
+	mg.AddMigration("Update alert table charset", NewTableCharsetMigration("alert", []*Column{
+		{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
+		{Name: "message", Type: DB_Text, Nullable: false},
+		{Name: "state", Type: DB_NVarchar, Length: 255, Nullable: false},
+		{Name: "settings", Type: DB_Text, Nullable: false},
+		{Name: "severity", Type: DB_Text, Nullable: false},
+		{Name: "execution_error", Type: DB_Text, Nullable: false},
+		{Name: "eval_data", Type: DB_Text, Nullable: true},
+	}))
+
+	mg.AddMigration("Update alert_notification table charset", NewTableCharsetMigration("alert_notification", []*Column{
+		{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
+		{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
+		{Name: "settings", Type: DB_Text, Nullable: false},
+	}))
 }

pkg/services/sqlstore/migrations/annotation_mig.go

@@ -44,4 +44,14 @@ func addAnnotationMig(mg *Migrator) {
 	mg.AddMigration("add index annotation 2 v3", NewAddIndexMigration(table, table.Indices[2]))
 	mg.AddMigration("add index annotation 3 v3", NewAddIndexMigration(table, table.Indices[3]))
 	mg.AddMigration("add index annotation 4 v3", NewAddIndexMigration(table, table.Indices[4]))
+
+	mg.AddMigration("Update annotation table charset", NewTableCharsetMigration("annotation", []*Column{
+		{Name: "type", Type: DB_NVarchar, Length: 25, Nullable: false},
+		{Name: "title", Type: DB_Text, Nullable: false},
+		{Name: "text", Type: DB_Text, Nullable: false},
+		{Name: "metric", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "prev_state", Type: DB_NVarchar, Length: 25, Nullable: false},
+		{Name: "new_state", Type: DB_NVarchar, Length: 25, Nullable: false},
+		{Name: "data", Type: DB_Text, Nullable: false},
+	}))
 }

pkg/services/sqlstore/migrations/apikey_mig.go

@@ -8,7 +8,7 @@ func addApiKeyMigrations(mg *Migrator) {
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "account_id", Type: DB_BigInt, Nullable: false},
-			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "key", Type: DB_Varchar, Length: 64, Nullable: false},
 			{Name: "role", Type: DB_NVarchar, Length: 255, Nullable: false},
 			{Name: "created", Type: DB_DateTime, Nullable: false},
@@ -41,8 +41,8 @@ func addApiKeyMigrations(mg *Migrator) {
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "org_id", Type: DB_BigInt, Nullable: false},
-			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "key", Type: DB_Varchar, Length: 255, Nullable: false},
+			{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
+			{Name: "key", Type: DB_Varchar, Length: 190, Nullable: false},
 			{Name: "role", Type: DB_NVarchar, Length: 255, Nullable: false},
 			{Name: "created", Type: DB_DateTime, Nullable: false},
 			{Name: "updated", Type: DB_DateTime, Nullable: false},
@@ -72,4 +72,10 @@ func addApiKeyMigrations(mg *Migrator) {
 	}))
 
 	mg.AddMigration("Drop old table api_key_v1", NewDropTableMigration("api_key_v1"))
+
+	mg.AddMigration("Update api_key table charset", NewTableCharsetMigration("api_key", []*Column{
+		{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
+		{Name: "key", Type: DB_Varchar, Length: 190, Nullable: false},
+		{Name: "role", Type: DB_NVarchar, Length: 255, Nullable: false},
+	}))
 }

pkg/services/sqlstore/migrations/dashboard_mig.go

@@ -8,7 +8,7 @@ func addDashboardMigration(mg *Migrator) {
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "version", Type: DB_Int, Nullable: false},
-			{Name: "slug", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "slug", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false},
 			{Name: "data", Type: DB_Text, Nullable: false},
 			{Name: "account_id", Type: DB_BigInt, Nullable: false},
@@ -56,7 +56,7 @@ func addDashboardMigration(mg *Migrator) {
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "version", Type: DB_Int, Nullable: false},
-			{Name: "slug", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "slug", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false},
 			{Name: "data", Type: DB_Text, Nullable: false},
 			{Name: "org_id", Type: DB_BigInt, Nullable: false},
@@ -125,4 +125,15 @@ func addDashboardMigration(mg *Migrator) {
 	mg.AddMigration("Add index for dashboard_id in dashboard_tag", NewAddIndexMigration(dashboardTagV1, &Index{
 		Cols: []string{"dashboard_id"}, Type: IndexType,
 	}))
+
+	mg.AddMigration("Update dashboard table charset", NewTableCharsetMigration("dashboard", []*Column{
+		{Name: "slug", Type: DB_NVarchar, Length: 190, Nullable: false},
+		{Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false},
+		{Name: "plugin_id", Type: DB_NVarchar, Nullable: true, Length: 255},
+		{Name: "data", Type: DB_MediumText, Nullable: false},
+	}))
+
+	mg.AddMigration("Update dashboard_tag table charset", NewTableCharsetMigration("dashboard_tag", []*Column{
+		{Name: "term", Type: DB_NVarchar, Length: 50, Nullable: false},
+	}))
 }

pkg/services/sqlstore/migrations/dashboard_snapshot_mig.go

@@ -8,7 +8,7 @@ func addDashboardSnapshotMigrations(mg *Migrator) {
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "key", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "key", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "dashboard", Type: DB_Text, Nullable: false},
 			{Name: "expires", Type: DB_DateTime, Nullable: false},
 			{Name: "created", Type: DB_DateTime, Nullable: false},
@@ -28,8 +28,8 @@ func addDashboardSnapshotMigrations(mg *Migrator) {
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "key", Type: DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "delete_key", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "key", Type: DB_NVarchar, Length: 190, Nullable: false},
+			{Name: "delete_key", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "org_id", Type: DB_BigInt, Nullable: false},
 			{Name: "user_id", Type: DB_BigInt, Nullable: false},
 			{Name: "external", Type: DB_Bool, Nullable: false},
@@ -54,4 +54,12 @@ func addDashboardSnapshotMigrations(mg *Migrator) {
 		Sqlite("SELECT 0 WHERE 0;").
 		Postgres("SELECT 0;").
 		Mysql("ALTER TABLE dashboard_snapshot MODIFY dashboard MEDIUMTEXT;"))
+
+	mg.AddMigration("Update dashboard_snapshot table charset", NewTableCharsetMigration("dashboard_snapshot", []*Column{
+		{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
+		{Name: "key", Type: DB_NVarchar, Length: 190, Nullable: false},
+		{Name: "delete_key", Type: DB_NVarchar, Length: 190, Nullable: false},
+		{Name: "external_url", Type: DB_NVarchar, Length: 255, Nullable: false},
+		{Name: "dashboard", Type: DB_MediumText, Nullable: false},
+	}))
 }

pkg/services/sqlstore/migrations/datasource_mig.go

@@ -10,7 +10,7 @@ func addDataSourceMigration(mg *Migrator) {
 			{Name: "account_id", Type: DB_BigInt, Nullable: false},
 			{Name: "version", Type: DB_Int, Nullable: false},
 			{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "access", Type: DB_NVarchar, Length: 255, Nullable: false},
 			{Name: "url", Type: DB_NVarchar, Length: 255, Nullable: false},
 			{Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true},
@@ -49,7 +49,7 @@ func addDataSourceMigration(mg *Migrator) {
 			{Name: "org_id", Type: DB_BigInt, Nullable: false},
 			{Name: "version", Type: DB_Int, Nullable: false},
 			{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "access", Type: DB_NVarchar, Length: 255, Nullable: false},
 			{Name: "url", Type: DB_NVarchar, Length: 255, Nullable: false},
 			{Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true},
@@ -106,4 +106,18 @@ func addDataSourceMigration(mg *Migrator) {
 	mg.AddMigration("Add secure json data column", NewAddColumnMigration(tableV2, &Column{
 		Name: "secure_json_data", Type: DB_Text, Nullable: true,
 	}))
+
+	mg.AddMigration("Update data_source table charset", NewTableCharsetMigration(tableV2.Name, []*Column{
+		{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
+		{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
+		{Name: "access", Type: DB_NVarchar, Length: 255, Nullable: false},
+		{Name: "url", Type: DB_NVarchar, Length: 255, Nullable: false},
+		{Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "user", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "database", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "basic_auth_user", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "basic_auth_password", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "json_data", Type: DB_Text, Nullable: true},
+		{Name: "secure_json_data", Type: DB_Text, Nullable: true},
+	}))
 }

pkg/services/sqlstore/migrations/org_mig.go

@@ -8,7 +8,7 @@ func addOrgMigrations(mg *Migrator) {
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "version", Type: DB_Int, Nullable: false},
-			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "address1", Type: DB_NVarchar, Length: 255, Nullable: true},
 			{Name: "address2", Type: DB_NVarchar, Length: 255, Nullable: true},
 			{Name: "city", Type: DB_NVarchar, Length: 255, Nullable: true},
@@ -68,4 +68,19 @@ func addOrgMigrations(mg *Migrator) {
 
 	mg.AddMigration("Drop old table account", NewDropTableMigration("account"))
 	mg.AddMigration("Drop old table account_user", NewDropTableMigration("account_user"))
+
+	mg.AddMigration("Update org table charset", NewTableCharsetMigration("org", []*Column{
+		{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
+		{Name: "address1", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "address2", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "city", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "state", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "zip_code", Type: DB_NVarchar, Length: 50, Nullable: true},
+		{Name: "country", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "billing_email", Type: DB_NVarchar, Length: 255, Nullable: true},
+	}))
+
+	mg.AddMigration("Update org_user table charset", NewTableCharsetMigration("org_user", []*Column{
+		{Name: "role", Type: DB_NVarchar, Length: 20},
+	}))
 }

pkg/services/sqlstore/migrations/playlist_mig.go

@@ -32,4 +32,15 @@ func addPlaylistMigrations(mg *Migrator) {
 	}
 
 	mg.AddMigration("create playlist item table v2", NewAddTableMigration(playlistItemV2))
+
+	mg.AddMigration("Update playlist table charset", NewTableCharsetMigration("playlist", []*Column{
+		{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
+		{Name: "interval", Type: DB_NVarchar, Length: 255, Nullable: false},
+	}))
+
+	mg.AddMigration("Update playlist_item table charset", NewTableCharsetMigration("playlist_item", []*Column{
+		{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
+		{Name: "value", Type: DB_Text, Nullable: false},
+		{Name: "title", Type: DB_Text, Nullable: false},
+	}))
 }

pkg/services/sqlstore/migrations/plugin_setting.go

@@ -9,7 +9,7 @@ func addAppSettingsMigration(mg *Migrator) {
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "org_id", Type: DB_BigInt, Nullable: true},
-			{Name: "plugin_id", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "plugin_id", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "enabled", Type: DB_Bool, Nullable: false},
 			{Name: "pinned", Type: DB_Bool, Nullable: false},
 			{Name: "json_data", Type: DB_Text, Nullable: true},
@@ -32,4 +32,10 @@ func addAppSettingsMigration(mg *Migrator) {
 		Name: "plugin_version", Type: DB_NVarchar, Nullable: true, Length: 50,
 	}))
 
+	mg.AddMigration("Update plugin_setting table charset", NewTableCharsetMigration("plugin_setting", []*Column{
+		{Name: "plugin_id", Type: DB_NVarchar, Length: 190, Nullable: false},
+		{Name: "json_data", Type: DB_Text, Nullable: true},
+		{Name: "secure_json_data", Type: DB_Text, Nullable: true},
+		{Name: "plugin_version", Type: DB_NVarchar, Nullable: true, Length: 50},
+	}))
 }

pkg/services/sqlstore/migrations/preferences_mig.go

@@ -29,4 +29,9 @@ func addPreferencesMigrations(mg *Migrator) {
 
 	// create table
 	mg.AddMigration("create preferences table v3", NewAddTableMigration(preferencesV2))
+
+	mg.AddMigration("Update preferences table charset", NewTableCharsetMigration("preferences", []*Column{
+		{Name: "timezone", Type: DB_NVarchar, Length: 50, Nullable: false},
+		{Name: "theme", Type: DB_NVarchar, Length: 20, Nullable: false},
+	}))
 }

pkg/services/sqlstore/migrations/quota_mig.go

@@ -12,7 +12,7 @@ func addQuotaMigration(mg *Migrator) {
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "org_id", Type: DB_BigInt, Nullable: true},
 			{Name: "user_id", Type: DB_BigInt, Nullable: true},
-			{Name: "target", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "target", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "limit", Type: DB_BigInt, Nullable: false},
 			{Name: "created", Type: DB_DateTime, Nullable: false},
 			{Name: "updated", Type: DB_DateTime, Nullable: false},
@@ -25,4 +25,8 @@ func addQuotaMigration(mg *Migrator) {
 
 	//-------  indexes ------------------
 	addTableIndicesMigrations(mg, "v1", quotaV1)
+
+	mg.AddMigration("Update quota table charset", NewTableCharsetMigration("quota", []*Column{
+		{Name: "target", Type: DB_NVarchar, Length: 190, Nullable: false},
+	}))
 }

pkg/services/sqlstore/migrations/temp_user.go

@@ -35,4 +35,13 @@ func addTempUserMigrations(mg *Migrator) {
 	// create table
 	mg.AddMigration("create temp user table v1-7", NewAddTableMigration(tempUserV1))
 	addTableIndicesMigrations(mg, "v1-7", tempUserV1)
+
+	mg.AddMigration("Update temp_user table charset", NewTableCharsetMigration("temp_user", []*Column{
+		{Name: "email", Type: DB_NVarchar, Length: 255},
+		{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "role", Type: DB_NVarchar, Length: 20, Nullable: true},
+		{Name: "code", Type: DB_NVarchar, Length: 255},
+		{Name: "status", Type: DB_Varchar, Length: 20},
+		{Name: "remote_addr", Type: DB_Varchar, Length: 255, Nullable: true},
+	}))
 }

pkg/services/sqlstore/migrations/user_mig.go

@@ -8,8 +8,8 @@ func addUserMigrations(mg *Migrator) {
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "version", Type: DB_Int, Nullable: false},
-			{Name: "login", Type: DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "email", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "login", Type: DB_NVarchar, Length: 190, Nullable: false},
+			{Name: "email", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: true},
 			{Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true},
 			{Name: "salt", Type: DB_NVarchar, Length: 50, Nullable: true},
@@ -47,8 +47,8 @@ func addUserMigrations(mg *Migrator) {
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "version", Type: DB_Int, Nullable: false},
-			{Name: "login", Type: DB_NVarchar, Length: 255, Nullable: false},
-			{Name: "email", Type: DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "login", Type: DB_NVarchar, Length: 190, Nullable: false},
+			{Name: "email", Type: DB_NVarchar, Length: 190, Nullable: false},
 			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: true},
 			{Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true},
 			{Name: "salt", Type: DB_NVarchar, Length: 50, Nullable: true},
@@ -92,4 +92,15 @@ func addUserMigrations(mg *Migrator) {
 	mg.AddMigration("Add column help_flags1 to user table", NewAddColumnMigration(userV2, &Column{
 		Name: "help_flags1", Type: DB_BigInt, Nullable: false, Default: "0",
 	}))
+
+	mg.AddMigration("Update user table charset", NewTableCharsetMigration("user", []*Column{
+		{Name: "login", Type: DB_NVarchar, Length: 190, Nullable: false},
+		{Name: "email", Type: DB_NVarchar, Length: 190, Nullable: false},
+		{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "password", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "salt", Type: DB_NVarchar, Length: 50, Nullable: true},
+		{Name: "rands", Type: DB_NVarchar, Length: 50, Nullable: true},
+		{Name: "company", Type: DB_NVarchar, Length: 255, Nullable: true},
+		{Name: "theme", Type: DB_NVarchar, Length: 255, Nullable: true},
+	}))
 }