public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Spanner support enhancements (#101634)

Browse Source 
* Adds ability to run integration tests against spanner (by using GRAFANA_TEST_DB=spanner env variable. SPANNER_DB variable then specifies database to use: spannertest, emulator or string like /projects/<project>/instances/<instance>/databases/<db>)
* Adds feature to migration dialects to create database from a snapshot, instead of running individual migrations.
* Adds first version of Spanner snapshot, prepared from "OSS" migrations.
* Uses generated bit-reversed-positive values instead of auto_increment. (As an experiment)
This commit is contained in:
Peter Štibraný
2025-03-06 16:11:20 +01:00
committed by GitHub
parent 43f56c5ca1
commit 9e690bd620
18 changed files with 1477 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/sqlstore/migrations/dialect_migration_test.go
+120
View File
@@ -0,0 +1,120 @@
//go:build enterprise || pro
package migrations
import (
"encoding/json"
"fmt"
"os"
"strconv"
"testing"
"github.com/stretchr/testify/require"
"gopkg.in/ini.v1"
"xorm.io/core"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/setting"
"xorm.io/xorm"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/services/sqlstore/sqlutil"
)
func setupTestDB(t *testing.T) (*migrator.Migrator, *xorm.Engine) {
t.Helper()
dbType := sqlutil.GetTestDBType()
testDB, err := sqlutil.GetTestDB(dbType)
require.NoError(t, err)
t.Cleanup(testDB.Cleanup)
x, err := xorm.NewEngine(testDB.DriverName, testDB.ConnStr)
require.NoError(t, err)
t.Cleanup(func() {
if err := x.Close(); err != nil {
fmt.Printf("failed to close xorm engine: %v", err)
}
})
err = migrator.NewDialect(x.DriverName()).CleanDB(x)
require.NoError(t, err)
mg := migrator.NewMigrator(x, &setting.Cfg{
Logger: log.New("users.test"),
Raw: ini.Empty(),
})
migrations := &OSSMigrations{}
migrations.AddMigration(mg)
err = mg.Start(false, 0)
require.NoError(t, err)
return mg, x
}
// This "test" migrates database from scratch, and then generates Spanner DDL statements for re-creating the same database.
func TestMigrateToSpannerDialect(t *testing.T) {
mg, eng := setupTestDB(t)
tables, err := eng.DBMetas()
require.NoError(t, err)
var statements []string
spannerDialect := migrator.NewSpannerDialect()
for _, table := range tables {
t := &migrator.Table{
Name: table.Name,
Columns: nil,
PrimaryKeys: table.PrimaryKeys,
Indices: nil,
}
for _, c := range table.Columns() {
col := &migrator.Column{
Name: c.Name,
Type: c.SQLType.Name,
Length: c.Length,
Length2: c.Length2,
Nullable: c.Nullable,
IsPrimaryKey: c.IsPrimaryKey,
IsAutoIncrement: c.IsAutoIncrement,
IsLatin: false,
Default: c.Default,
}
if (col.Type == core.Bool || col.Type == core.TinyInt) && c.Default != "" {
b, err := strconv.ParseBool(c.Default)
if err == nil {
// Format bool values as true/false.
col.Default = strconv.FormatBool(b)
}
}
t.Columns = append(t.Columns, col)
}
for _, ix := range table.Indexes {
nix := &migrator.Index{
Name: ix.Name,
Type: ix.Type,
Cols: ix.Cols,
}
t.Indices = append(t.Indices, nix)
}
statements = append(statements, spannerDialect.CreateTableSQL(t))
for _, nix := range t.Indices {
if nix.Name != "PRIMARY_KEY" {
statements = append(statements, spannerDialect.CreateIndexSQL(table.Name, nix))
}
}
}
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
require.NoError(t, enc.Encode(statements))
fmt.Println()
require.NoError(t, enc.Encode(mg.GetMigrationIDs(true)))
}
pkg/services/sqlstore/migrations/migrations_test.go
+5 -3
View File
@@ -13,6 +13,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/ini.v1"
"xorm.io/xorm"
. "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
@@ -76,7 +77,8 @@ func TestIntegrationMigrationLock(t *testing.T) {
}
dbType := sqlutil.GetTestDBType()
if dbType == SQLite {
// skip for SQLite and Spanner since there is no database locking (only migrator locking)
if dbType == SQLite || dbType == Spanner {
t.Skip()
}
@@ -233,8 +235,8 @@ func TestMigratorLocking(t *testing.T) {
func TestDatabaseLocking(t *testing.T) {
dbType := sqlutil.GetTestDBType()
// skip for SQLite since there is no database locking (only migrator locking)
if dbType == SQLite {
// skip for SQLite and Spanner since there is no database locking (only migrator locking)
if dbType == SQLite || dbType == Spanner {
t.Skip()
}
pkg/services/sqlstore/migrations/usermig/service_account_multiple_org_login_migrator.go
+11 -11
View File
@@ -38,12 +38,12 @@ func (p *ServiceAccountsSameLoginCrossOrgs) Exec(sess *xorm.Session, mg *migrato
case migrator.Postgres:
_, err = p.sess.Exec(`
UPDATE "user"
SET login = 'sa-' || org_id::text || '-' ||
CASE
WHEN login LIKE 'sa-%' THEN SUBSTRING(login FROM 4)
ELSE login
END
WHERE login IS NOT NULL
SET login = 'sa-' || org_id::text || '-' ||
CASE
WHEN login LIKE 'sa-%' THEN SUBSTRING(login FROM 4)
ELSE login
END
WHERE login IS NOT NULL
AND is_service_account = true
AND login NOT LIKE 'sa-' || org_id::text || '-%';
`)
@@ -56,7 +56,7 @@ func (p *ServiceAccountsSameLoginCrossOrgs) Exec(sess *xorm.Session, mg *migrato
ELSE login
END
)
WHERE login IS NOT NULL
WHERE login IS NOT NULL
AND is_service_account = 1
AND login NOT LIKE CONCAT('sa-', org_id, '-%');
`)
@@ -68,7 +68,7 @@ func (p *ServiceAccountsSameLoginCrossOrgs) Exec(sess *xorm.Session, mg *migrato
WHEN SUBSTR(login, 1, 3) = 'sa-' THEN SUBSTR(login, 4)
ELSE login
END
WHERE login IS NOT NULL
WHERE login IS NOT NULL
AND is_service_account = 1
AND login NOT LIKE 'sa-' || CAST(org_id AS TEXT) || '-%';
`)
@@ -96,7 +96,7 @@ func (p *ServiceAccountsDeduplicateOrgInLogin) Exec(sess *xorm.Session, mg *migr
_, err = sess.Exec(`
UPDATE "user" AS u
SET login = 'sa-' || org_id::text || SUBSTRING(login FROM LENGTH('sa-' || org_id::text || '-' || org_id::text)+1)
WHERE login IS NOT NULL
WHERE login IS NOT NULL
AND is_service_account = true
AND login LIKE 'sa-' || org_id::text || '-' || org_id::text || '-%'
AND NOT EXISTS (
@@ -123,8 +123,8 @@ func (p *ServiceAccountsDeduplicateOrgInLogin) Exec(sess *xorm.Session, mg *migr
AND u.is_service_account = 1
AND u.login LIKE 'sa-'||CAST(u.org_id AS TEXT)||'-'||CAST(u.org_id AS TEXT)||'-%'
AND NOT EXISTS (
SELECT 1
FROM ` + dialect.Quote("user") + `AS u2
SELECT 1
FROM ` + dialect.Quote("user") + `AS u2
WHERE u2.login = 'sa-' || CAST(u.org_id AS TEXT) || SUBSTRING(u.login, LENGTH('sa-'||CAST(u.org_id AS TEXT)||'-'||CAST(u.org_id AS TEXT))+1)
);;
`)