public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove support for Google Spanner database. (#105846)

Browse Source 
* Remove support for Google Spanner database.
This commit is contained in:
Peter Štibraný
2025-05-23 11:35:59 +02:00
committed by GitHub
parent 9769871a88
commit c4d3eb1cd0
50 changed files with 26 additions and 1265 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/sqlstore/database_config.go
-2
View File
@@ -203,8 +203,6 @@ func (dbCfg *DatabaseConfig) buildConnectionString(cfg *setting.Cfg, features fe
}
cnnstr += buildExtraConnectionString('&', dbCfg.UrlQueryParams)
case migrator.Spanner:
cnnstr = dbCfg.Name
default:
return fmt.Errorf("unknown database type: %s", dbCfg.Type)
}
pkg/services/sqlstore/migrations/migrations_test.go
+4 -4
View File
@@ -77,8 +77,8 @@ func TestIntegrationMigrationLock(t *testing.T) {
}
dbType := sqlutil.GetTestDBType()
// skip for SQLite and Spanner since there is no database locking (only migrator locking)
if dbType == SQLite || dbType == Spanner {
// skip for SQLite since there is no database locking (only migrator locking)
if dbType == SQLite {
t.Skip()
}
@@ -235,8 +235,8 @@ func TestMigratorLocking(t *testing.T) {
func TestDatabaseLocking(t *testing.T) {
dbType := sqlutil.GetTestDBType()
// skip for SQLite and Spanner since there is no database locking (only migrator locking)
if dbType == SQLite || dbType == Spanner {
// skip for SQLite since there is no database locking (only migrator locking)
if dbType == SQLite {
t.Skip()
}
pkg/services/sqlstore/migrations/star_mig.go
-9
View File
@@ -82,15 +82,6 @@ func RunStarMigrations(sess *xorm.Session, driverName string) error {
star.org_id = dashboard.org_id,
star.updated = NOW()
WHERE star.dashboard_uid IS NULL OR star.org_id IS NULL;`
case Spanner:
sql = `UPDATE star
SET
dashboard_uid = (SELECT uid FROM dashboard WHERE dashboard.id = star.dashboard_id),
org_id = (SELECT org_id FROM dashboard WHERE dashboard.id = star.dashboard_id),
updated = CURRENT_TIMESTAMP()
WHERE
(dashboard_uid IS NULL OR org_id IS NULL)
AND EXISTS (SELECT 1 FROM dashboard WHERE dashboard.id = star.dashboard_id)`
}
if _, err := sess.Exec(sql); err != nil {
pkg/services/sqlstore/migrations/user_mig.go
+1 -2
View File
@@ -154,8 +154,7 @@ func addUserMigrations(mg *Migrator) {
mg.AddMigration("Make sure users uid are set", NewRawSQLMigration("").
SQLite("UPDATE user SET uid=printf('u%09d',id) WHERE uid is NULL OR uid = '';").
Postgres("UPDATE `user` SET uid='u' || lpad('' || id::text,9,'0') WHERE uid is NULL OR uid = '';").
Mysql("UPDATE user SET uid=concat('u',lpad(id,9,'0')) WHERE uid is NULL OR uid = '';").
Spanner("UPDATE user SET uid=concat('u',lpad(CAST(id AS STRING),9,'0')) WHERE uid IS NULL OR uid = '';"))
Mysql("UPDATE user SET uid=concat('u',lpad(id,9,'0')) WHERE uid is NULL OR uid = '';"))
mg.AddMigration("Add unique index user_uid", NewAddIndexMigration(userV2, &Index{
Cols: []string{"uid"}, Type: UniqueIndex,
pkg/services/sqlstore/migrations/usermig/service_account_multiple_org_login_migrator.go
-26
View File
@@ -72,19 +72,6 @@ func (p *ServiceAccountsSameLoginCrossOrgs) Exec(sess *xorm.Session, mg *migrato
AND is_service_account = 1
AND login NOT LIKE 'sa-' || CAST(org_id AS TEXT) || '-%';
`)
case migrator.Spanner:
_, err = p.sess.Exec(`
UPDATE user
SET login = CONCAT('sa-', CAST(org_id AS STRING), '-',
CASE
WHEN login LIKE 'sa-%' THEN SUBSTRING(login, 4)
ELSE login
END
)
WHERE login IS NOT NULL
AND is_service_account
AND login NOT LIKE CONCAT('sa-', CAST(org_id AS STRING), '-%')
`)
default:
return fmt.Errorf("dialect not supported: %s", p.dialect)
@@ -142,19 +129,6 @@ func (p *ServiceAccountsDeduplicateOrgInLogin) Exec(sess *xorm.Session, mg *migr
WHERE u2.login = 'sa-' || CAST(u.org_id AS TEXT) || SUBSTRING(u.login, LENGTH('sa-'||CAST(u.org_id AS TEXT)||'-'||CAST(u.org_id AS TEXT))+1)
);;
`)
case migrator.Spanner:
_, err = sess.Exec(`
UPDATE ` + dialect.Quote("user") + ` AS u
SET login = 'sa-' || CAST(u.org_id AS STRING) || SUBSTRING(u.login, LENGTH('sa-'||CAST(u.org_id AS STRING)||'-'||CAST(u.org_id AS STRING))+1)
WHERE u.login IS NOT NULL
AND u.is_service_account
AND u.login LIKE 'sa-'||CAST(u.org_id AS STRING)||'-'||CAST(u.org_id AS STRING)||'-%'
AND NOT EXISTS (
SELECT 1
FROM ` + dialect.Quote("user") + `AS u2
WHERE u2.login = 'sa-' || CAST(u.org_id AS STRING) || SUBSTRING(u.login, LENGTH('sa-'||CAST(u.org_id AS STRING)||'-'||CAST(u.org_id AS STRING))+1)
);;
`)
default:
return fmt.Errorf("dialect not supported: %s", dialect)
}
pkg/services/sqlstore/migrator/migrations.go
-4
View File
@@ -85,10 +85,6 @@ func (m *RawSQLMigration) Mssql(sql string) *RawSQLMigration {
return m.Set(MSSQL, sql)
}
func (m *RawSQLMigration) Spanner(sql string) *RawSQLMigration {
return m.Set(Spanner, sql)
}
type AddColumnMigration struct {
MigrationBase
tableName string
pkg/services/sqlstore/migrator/migrator.go
-5
View File
@@ -417,11 +417,6 @@ func (mg *Migrator) InTransaction(callback dbTransactionFunc) error {
sess := mg.DBEngine.NewSession()
defer sess.Close()
// XXX: Spanner cannot execute DDL statements in transactions
if mg.Dialect.DriverName() == Spanner {
return callback(sess)
}
if err := sess.Begin(); err != nil {
return err
}
pkg/services/sqlstore/migrator/spanner_dialect.go
-355
View File
@@ -1,355 +0,0 @@
//go:build enterprise || pro
package migrator
import (
"context"
_ "embed"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"cloud.google.com/go/spanner"
database "cloud.google.com/go/spanner/admin/database/apiv1"
"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
"github.com/googleapis/gax-go/v2"
spannerdriver "github.com/googleapis/go-sql-spanner"
"github.com/grafana/grafana/pkg/util/xorm"
"google.golang.org/grpc/codes"
"github.com/grafana/dskit/concurrency"
utilspanner "github.com/grafana/grafana/pkg/util/spanner"
"github.com/grafana/grafana/pkg/util/xorm/core"
)
type SpannerDialect struct {
BaseDialect
d core.Dialect
}
func init() {
supportedDialects[Spanner] = NewSpannerDialect
}
func NewSpannerDialect() Dialect {
d := SpannerDialect{d: core.QueryDialect(Spanner)}
d.dialect = &d
d.driverName = Spanner
return &d
}
func (s *SpannerDialect) AutoIncrStr() string { return s.d.AutoIncrStr() }
func (s *SpannerDialect) Quote(name string) string { return s.d.Quote(name) }
func (s *SpannerDialect) SupportEngine() bool { return s.d.SupportEngine() }
func (s *SpannerDialect) LikeOperator(column string, wildcardBefore bool, pattern string, wildcardAfter bool) (string, string) {
param := strings.ToLower(pattern)
if wildcardBefore {
param = "%" + param
}
if wildcardAfter {
param = param + "%"
}
return fmt.Sprintf("LOWER(%s) LIKE ?", column), param
}
func (s *SpannerDialect) IndexCheckSQL(tableName, indexName string) (string, []any) {
return s.d.IndexCheckSql(tableName, indexName)
}
func (s *SpannerDialect) SQLType(col *Column) string {
c := core.NewColumn(col.Name, "", core.SQLType{Name: col.Type}, col.Length, col.Length2, col.Nullable)
return s.d.SqlType(c)
}
func (s *SpannerDialect) BatchSize() int { return 1000 }
func (s *SpannerDialect) BooleanValue(b bool) any {
return b
}
func (s *SpannerDialect) BooleanStr(b bool) string {
if b {
return "true"
}
return "false"
}
func (s *SpannerDialect) ErrorMessage(err error) string {
return spanner.ErrDesc(spanner.ToSpannerError(err))
}
func (s *SpannerDialect) IsDeadlock(err error) bool {
return spanner.ErrCode(spanner.ToSpannerError(err)) == codes.Aborted
}
func (s *SpannerDialect) IsUniqueConstraintViolation(err error) bool {
return spanner.ErrCode(spanner.ToSpannerError(err)) == codes.AlreadyExists
}
func (s *SpannerDialect) CreateTableSQL(table *Table) string {
t := core.NewEmptyTable()
t.Name = table.Name
t.PrimaryKeys = table.PrimaryKeys
for _, c := range table.Columns {
col := core.NewColumn(c.Name, c.Name, core.SQLType{Name: c.Type}, c.Length, c.Length2, c.Nullable)
col.IsAutoIncrement = c.IsAutoIncrement
col.Default = c.Default
t.AddColumn(col)
}
if len(t.PrimaryKeys) == 0 {
for _, ix := range table.Indices {
if ix.Name == "PRIMARY_KEY" {
t.PrimaryKeys = append(t.PrimaryKeys, ix.Cols...)
}
}
}
return s.d.CreateTableSql(t, t.Name, "", "")
}
func (s *SpannerDialect) CreateIndexSQL(tableName string, index *Index) string {
idx := core.NewIndex(index.Name, index.Type)
idx.Cols = index.Cols
return s.d.CreateIndexSql(tableName, idx)
}
func (s *SpannerDialect) UpsertMultipleSQL(tableName string, keyCols, updateCols []string, count int) (string, error) {
return "", errors.New("not supported")
}
func (s *SpannerDialect) DropIndexSQL(tableName string, index *Index) string {
return fmt.Sprintf("DROP INDEX %v", s.Quote(index.XName(tableName)))
}
func (s *SpannerDialect) DropTable(tableName string) string {
return fmt.Sprintf("DROP TABLE %s", s.Quote(tableName))
}
func (s *SpannerDialect) ColStringNoPk(col *Column) string {
sql := s.dialect.Quote(col.Name) + " "
sql += s.dialect.SQLType(col) + " "
if s.dialect.ShowCreateNull() && !col.Nullable {
sql += "NOT NULL "
}
if col.Default != "" {
// Default value must be in parentheses.
sql += "DEFAULT (" + s.dialect.Default(col) + ") "
}
return sql
}
func (s *SpannerDialect) TruncateDBTables(engine *xorm.Engine) error {
// Get tables names only, no columns or indexes.
tables, err := engine.Dialect().GetTables()
if err != nil {
return err
}
sess := engine.NewSession()
defer sess.Close()
var statements []string
for _, table := range tables {
switch table.Name {
case "":
continue
case "autoincrement_sequences":
// Don't delete sequence number for migration_log.id column.
statements = append(statements, fmt.Sprintf("DELETE FROM %v WHERE name <> 'migration_log:id'", s.Quote(table.Name)))
case "migration_log":
continue
case "dashboard_acl":
// keep default dashboard permissions
statements = append(statements, fmt.Sprintf("DELETE FROM %v WHERE dashboard_id != -1 AND org_id != -1;", s.Quote(table.Name)))
default:
statements = append(statements, fmt.Sprintf("DELETE FROM %v WHERE TRUE;", s.Quote(table.Name)))
}
}
// Run statements concurrently.
return concurrency.ForEachJob(context.Background(), len(statements), 10, func(ctx context.Context, idx int) error {
_, err := sess.Exec(statements[idx])
return err
})
}
// CleanDB drops all existing tables and their indexes.
func (s *SpannerDialect) CleanDB(engine *xorm.Engine) error {
tables, err := engine.DBMetas()
if err != nil {
return err
}
// Collect all DROP statements.
changeStreams, err := s.findChangeStreams(engine)
if err != nil {
return err
}
statements := make([]string, 0, len(tables)+len(changeStreams))
for _, cs := range changeStreams {
statements = append(statements, fmt.Sprintf("DROP CHANGE STREAM `%s`", cs))
}
for _, table := range tables {
// Indexes must be dropped first, otherwise dropping tables fails.
for _, index := range table.Indexes {
if !index.IsRegular {
// Don't drop primary key.
continue
}
sql := fmt.Sprintf("DROP INDEX %s", s.Quote(index.XName(table.Name)))
statements = append(statements, sql)
}
sql := fmt.Sprintf("DROP TABLE %s", s.Quote(table.Name))
statements = append(statements, sql)
}
if len(statements) == 0 {
return nil
}
return s.executeDDLStatements(context.Background(), engine, statements)
}
//go:embed snapshot/spanner-ddl.json
var snapshotDDL string
//go:embed snapshot/spanner-log.json
var snapshotMigrations string
func (s *SpannerDialect) CreateDatabaseFromSnapshot(ctx context.Context, engine *xorm.Engine, tableName string) error {
var statements, migrationIDs []string
err := json.Unmarshal([]byte(snapshotDDL), &statements)
if err != nil {
return err
}
err = json.Unmarshal([]byte(snapshotMigrations), &migrationIDs)
if err != nil {
return err
}
err = s.executeDDLStatements(ctx, engine, statements)
if err != nil {
return err
}
return s.recordMigrationsToMigrationLog(engine, migrationIDs, tableName)
}
func (s *SpannerDialect) recordMigrationsToMigrationLog(engine *xorm.Engine, migrationIDs []string, tableName string) error {
now := time.Now()
makeRecord := func(id string) MigrationLog {
return MigrationLog{
MigrationID: id,
SQL: "",
Success: true,
Timestamp: now,
}
}
sess := engine.NewSession()
defer sess.Close()
// Insert records in batches to avoid many roundtrips to database.
// Inserting all records at once fails due to "Number of parameters in query exceeds the maximum
// allowed limit of 950." error, so we use smaller batches.
const batchSize = 100
if err := sess.Begin(); err != nil {
return err
}
records := make([]MigrationLog, 0, len(migrationIDs))
for _, mid := range migrationIDs {
records = append(records, makeRecord(mid))
if len(records) >= batchSize {
if _, err := sess.Table(tableName).InsertMulti(records); err != nil {
err2 := sess.Rollback()
return errors.Join(fmt.Errorf("failed to insert migration logs: %w", err), err2)
}
records = records[:0]
}
}
// Insert remaining records.
if len(records) > 0 {
if _, err := sess.Table(tableName).InsertMulti(records); err != nil {
err2 := sess.Rollback()
return errors.Join(fmt.Errorf("failed to insert migration logs: %w", err), err2)
}
}
if err := sess.Commit(); err != nil {
return err
}
return nil
}
// Spanner can be very slow at executing single DDL statements (it can take up to a minute), but when
// many DDL statements are batched together, Spanner is *much* faster (total time to execute all statements
// is often in tens of seconds). We can't execute batch of DDL statements using sql wrapper, we use "database admin client"
// from Spanner library instead.
func (s *SpannerDialect) executeDDLStatements(ctx context.Context, engine *xorm.Engine, statements []string) error {
// Datasource name contains string used for sql.Open.
dsn := engine.Dialect().DataSourceName()
cfg, err := spannerdriver.ExtractConnectorConfig(dsn)
if err != nil {
return err
}
opts := utilspanner.ConnectorConfigToClientOptions(cfg)
databaseAdminClient, err := database.NewDatabaseAdminClient(ctx, opts...)
if err != nil {
return fmt.Errorf("failed to create database admin client: %v", err)
}
//nolint:errcheck // If the databaseAdminClient.Close fails, we simply don't care.
defer databaseAdminClient.Close()
databaseName := fmt.Sprintf("projects/%s/instances/%s/databases/%s", cfg.Project, cfg.Instance, cfg.Database)
op, err := databaseAdminClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{
Database: databaseName,
Statements: statements,
}, gax.WithTimeout(0)) /* disable default timeout */
if err != nil {
return fmt.Errorf("failed to start database DDL update: %v", err)
}
err = op.Wait(ctx, gax.WithTimeout(0)) /* disable default timeout */
if err != nil {
return fmt.Errorf("failed to apply database DDL update: %v", err)
}
return nil
}
func (s *SpannerDialect) UnionDistinct() string {
return "UNION DISTINCT"
}
func (s *SpannerDialect) findChangeStreams(engine *xorm.Engine) ([]string, error) {
var result []string
query := `SELECT c.CHANGE_STREAM_NAME
FROM INFORMATION_SCHEMA.CHANGE_STREAMS AS C
WHERE C.CHANGE_STREAM_CATALOG=''
AND C.CHANGE_STREAM_SCHEMA=''`
rows, err := engine.DB().Query(query)
if err != nil {
return nil, err
}
//nolint:errcheck // If the rows.Close fails, we simply don't care.
defer rows.Close()
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return nil, err
}
result = append(result, name)
}
return result, nil
}
pkg/services/sqlstore/migrator/types.go
-1
View File
@@ -12,7 +12,6 @@ const (
SQLite = "sqlite3"
MySQL = "mysql"
MSSQL = "mssql"
Spanner = "spanner"
)
type Migration interface {
pkg/services/sqlstore/permissions/dashboard_test.go
-13
View File
@@ -444,9 +444,6 @@ func TestIntegration_DashboardNestedPermissionFilter(t *testing.T) {
expectedResult: []string{"parent"},
},
}
if db.IsTestDBSpanner() {
t.Skip("skipping integration test")
}
var orgID int64 = 1
@@ -554,9 +551,6 @@ func TestIntegration_DashboardNestedPermissionFilter_WithSelfContainedPermission
expectedResult: []string{"parent"},
},
}
if db.IsTestDBSpanner() {
t.Skip("skipping integration test")
}
var orgID int64 = 1
@@ -665,10 +659,6 @@ func TestIntegration_DashboardNestedPermissionFilter_WithActionSets(t *testing.T
},
}
if db.IsTestDBSpanner() {
t.Skip("skipping integration test")
}
var orgID int64 = 1
for _, tc := range testCases {
@@ -757,9 +747,6 @@ func setupTest(t *testing.T, numFolders, numDashboards int, permissions []access
// Insert dashboards in batches
batchSize := 500
if db.IsTestDBSpanner() {
batchSize = 30 // spanner has a limit of 950 parameters per query
}
for i := 0; i < len(dashes); i += batchSize {
end := i + batchSize
if end > len(dashes) {
pkg/services/sqlstore/session.go
-5
View File
@@ -146,11 +146,6 @@ func (sess *DBSession) WithReturningID(driverName string, query string, args []a
return id, err
}
} else {
if driverName == migrator.Spanner {
// Only works with INSERT statements.
query = fmt.Sprintf("%s THEN RETURN id", query)
}
sqlOrArgs := append([]any{query}, args...)
res, err := sess.Exec(sqlOrArgs...)
if err != nil {
pkg/services/sqlstore/session/session.go
-5
View File
@@ -110,11 +110,6 @@ func execWithReturningId(ctx context.Context, driverName string, query string, s
}
return id, nil
} else {
if driverName == "spanner" {
// LastInsertId requires THEN RETURN directive.
query = fmt.Sprintf("%s THEN RETURN id", query)
}
res, err := sess.Exec(ctx, query, args...)
if err != nil {
return id, err
pkg/services/sqlstore/session_test.go
+1 -5
View File
@@ -8,8 +8,6 @@ import (
"github.com/mattn/go-sqlite3"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
grpcstatus "google.golang.org/grpc/status"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
)
@@ -141,12 +139,10 @@ func getRetryErrors(t *testing.T, store *SQLStore) []error {
switch store.GetDialect().DriverName() {
case migrator.SQLite:
retryErrors = []error{sqlite3.Error{Code: sqlite3.ErrBusy}, sqlite3.Error{Code: sqlite3.ErrLocked}}
case migrator.Spanner:
retryErrors = []error{grpcstatus.Error(codes.Aborted, "aborted transaction")}
}
if len(retryErrors) == 0 {
t.Skip("This test only works with sqlite or spanner")
t.Skip("This test only works with sqlite")
}
return retryErrors
}
pkg/services/sqlstore/sqlstore.go
-16
View File
@@ -373,11 +373,6 @@ func (ss *SQLStore) RecursiveQueriesAreSupported() (bool, error) {
return *ss.recursiveQueriesAreSupported, nil
}
recursiveQueriesAreSupported := func() (bool, error) {
if ss.GetDBType() == migrator.Spanner {
// no need to try...
return false, nil
}
var result []int
if err := ss.WithDbSession(context.Background(), func(sess *DBSession) error {
recQry := `WITH RECURSIVE cte (n) AS
@@ -414,7 +409,6 @@ var testSQLStoreSetup = false
var testSQLStore *SQLStore
var testSQLStoreMutex sync.Mutex
var testSQLStoreCleanup []func()
var testSQLStoreSkipTestsOnBackend string // When not empty and matches DB type, test is skipped.
// InitTestDBOpt contains options for InitTestDB.
type InitTestDBOpt struct {
@@ -459,12 +453,6 @@ func SetupTestDB() {
testSQLStoreSetup = true
}
func SkipTestsOnSpanner() {
testSQLStoreMutex.Lock()
defer testSQLStoreMutex.Unlock()
testSQLStoreSkipTestsOnBackend = "spanner"
}
func CleanupTestDB() {
testSQLStoreMutex.Lock()
defer testSQLStoreMutex.Unlock()
@@ -549,10 +537,6 @@ func TestMain(m *testing.M) {
if testSQLStore == nil {
dbType := sqlutil.GetTestDBType()
if testSQLStoreSkipTestsOnBackend != "" && testSQLStoreSkipTestsOnBackend == dbType {
t.Skipf("test skipped when using DB type %s", testSQLStoreSkipTestsOnBackend)
}
// set test db config
cfg := setting.NewCfg()
// nolint:staticcheck
pkg/services/sqlstore/sqlstore_testinfra.go
+1 -127
View File
@@ -9,18 +9,9 @@ import (
"encoding/hex"
"fmt"
"os"
"strconv"
"testing"
"time"
database "cloud.google.com/go/spanner/admin/database/apiv1"
"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
"cloud.google.com/go/spanner/spannertest"
spannerdriver "github.com/googleapis/go-sql-spanner"
"google.golang.org/api/option"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/registry"
@@ -269,9 +260,6 @@ func createTemporaryDatabase(tb TestingTB) (*testDB, error) {
// SQLite doesn't have a concept of a database server, so we always create a new file with no connections required.
return newSQLite3DB(tb)
}
if dbType == "spanner" {
return newSpannerDB(tb)
}
// On the remaining databases, we first connect to the configured credentials, create a new database, then return this new database's info as a connection string.
// We use databases rather than schemas as MySQL has no concept of schemas, so this aligns them more closely.
@@ -326,121 +314,7 @@ func createTemporaryDatabase(tb TestingTB) (*testDB, error) {
func generateDatabaseName() string {
// The database name has to be unique amongst all tests. It is highly unlikely we will have a collision here.
// The database name has to be <= 64 chars long on MySQL, and <= 31 chars on Postgres.
// Database ID length on Spanner must be between 2 and 30 characters. (https://cloud.google.com/spanner/quotas#database-limits)
return "grafana_test_" + randomLowerHex(17)
}
func newSpannerDB(tb TestingTB) (*testDB, error) {
// See https://github.com/googleapis/go-sql-spanner/blob/main/driver.go#L56-L81 for connection string options.
spannerDB := env("SPANNER_DB", "emulator")
if spannerDB == "spannertest" {
// Start new in-memory spannertest instance. This is mostly useless for our tests
// (spannertest doesn't support many things that we use), but added for completion.
// Each spannertest instance is a separate db.
srv, err := spannertest.NewServer("localhost:0")
if err != nil {
return nil, err
}
tb.Cleanup(srv.Close)
return &testDB{
Driver: "spanner",
Conn: fmt.Sprintf("%s/projects/grafanatest/instances/grafanatest/databases/grafanatest;usePlainText=true", srv.Addr),
}, nil
}
conn := spannerDB
if spannerDB == "emulator" {
host := env("SPANNER_EMULATOR_HOST", "localhost:9010")
conn = fmt.Sprintf("%s/projects/grafanatest/instances/grafanatest/databases/grafanatest;usePlainText=true", host)
}
cfg, err := spannerdriver.ExtractConnectorConfig(conn)
if err != nil {
return nil, err
}
clientOptions := spannerConnectorConfigToClientOptions(cfg)
dbname := generateDatabaseName()
fullDbName := fmt.Sprintf("projects/%s/instances/%s/databases/%s", cfg.Project, cfg.Instance, dbname)
dbCreated := false
databaseAdminClient, err := database.NewDatabaseAdminClient(tb.Context(), clientOptions...)
if err != nil {
return nil, fmt.Errorf("failed to create database admin client: %v", err)
}
tb.Cleanup(func() {
if dbCreated {
// Drop database in the cleanup.
// Can't use tb.Context() here, since that is canceled before calling Cleanup functions.
err := databaseAdminClient.DropDatabase(context.Background(), &databasepb.DropDatabaseRequest{
Database: fullDbName,
})
if err != nil {
tb.Logf("Failed to drop Spanner database %s due to error %v", fullDbName, err)
} else {
tb.Logf("Dropped temporary Spanner database %s", fullDbName)
}
}
_ = databaseAdminClient.Close()
})
op, err := databaseAdminClient.CreateDatabase(tb.Context(), &databasepb.CreateDatabaseRequest{
Parent: fmt.Sprintf("projects/%s/instances/%s", cfg.Project, cfg.Instance),
CreateStatement: fmt.Sprintf("CREATE DATABASE `%s`", dbname),
DatabaseDialect: databasepb.DatabaseDialect_GOOGLE_STANDARD_SQL,
})
if err != nil {
return nil, fmt.Errorf("failed to create database: %v", err)
}
_, err = op.Wait(tb.Context())
if err != nil {
return nil, fmt.Errorf("failed to create database: %v", err)
}
tb.Logf("Created temporary Spanner database %s", fullDbName)
dbCreated = true
// Rebuild connection string, but change database to ID of just-created database.
// Example: `localhost:9010/projects/test-project/instances/test-instance/databases/test-database;usePlainText=true;disableRouteToLeader=true;enableEndToEndTracing=true`
connString := ""
if cfg.Host != "" {
connString = fmt.Sprintf("%s/", cfg.Host)
}
// Use new DB name instead of cfg.Database.
connString = connString + fmt.Sprintf("projects/%s/instances/%s/databases/%s", cfg.Project, cfg.Instance, dbname)
for k, v := range cfg.Params {
connString = connString + fmt.Sprintf(";%s=%s", k, v)
}
return &testDB{
Driver: "spanner",
Conn: connString,
}, nil
}
// This is same code as xorm.SpannerConnectorConfigToClientOptions, but we cannot use that because it's under "enterprise" build tag.
func spannerConnectorConfigToClientOptions(connectorConfig spannerdriver.ConnectorConfig) []option.ClientOption {
var opts []option.ClientOption
if connectorConfig.Host != "" {
opts = append(opts, option.WithEndpoint(connectorConfig.Host))
}
if strval, ok := connectorConfig.Params["credentials"]; ok {
opts = append(opts, option.WithCredentialsFile(strval))
}
if strval, ok := connectorConfig.Params["credentialsjson"]; ok {
opts = append(opts, option.WithCredentialsJSON([]byte(strval)))
}
if strval, ok := connectorConfig.Params["useplaintext"]; ok {
if val, err := strconv.ParseBool(strval); err == nil && val {
opts = append(opts,
option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())),
option.WithoutAuthentication())
}
}
return opts
return "grafana_test_" + randomLowerHex(18)
}
func env(name, fallback string) string {
pkg/services/sqlstore/sqlutil/sqlutil.go
-50
View File
@@ -6,8 +6,6 @@ import (
"io/fs"
"os"
"path/filepath"
"cloud.google.com/go/spanner/spannertest"
)
// ITestDB is an interface of arguments for testing db
@@ -45,8 +43,6 @@ func GetTestDB(dbType string) (*TestDB, error) {
return postgresTestDB()
case "sqlite3":
return sqLite3TestDB()
case "spanner":
return spannerTestDB()
}
return nil, fmt.Errorf("unknown test db type: %s", dbType)
@@ -156,49 +152,3 @@ func postgresTestDB() (*TestDB, error) {
Cleanup: func() {},
}, nil
}
func spannerTestDB() (*TestDB, error) {
// See https://github.com/googleapis/go-sql-spanner/blob/main/driver.go#L56-L81 for connection string options.
spannerDB := os.Getenv("SPANNER_DB")
if spannerDB == "" {
spannerDB = "emulator"
}
if spannerDB == "spannertest" {
// Start in-memory spannertest instance.
srv, err := spannertest.NewServer("localhost:0")
if err != nil {
return nil, err
}
return &TestDB{
DriverName: "spanner",
ConnStr: fmt.Sprintf("%s/projects/grafanatest/instances/grafanatest/databases/grafanatest;usePlainText=true", srv.Addr),
Cleanup: srv.Close,
}, nil
}
if spannerDB == "emulator" {
host := os.Getenv("SPANNER_EMULATOR_HOST")
if host == "" {
host = "localhost:9010"
}
// To create instance and database manually, run:
//
// $ curl "localhost:9020/v1/projects/grafanatest/instances" --data '{"instanceId": "'grafanatest'"}'
// $ curl "localhost:9020/v1/projects/grafanatest/instances/grafanatest/databases" --data '{"createStatement": "CREATE DATABASE `grafanatest`"}'
return &TestDB{
DriverName: "spanner",
ConnStr: fmt.Sprintf("%s/projects/grafanatest/instances/grafanatest/databases/grafanatest;usePlainText=true;inMemSequenceGenerator=true", host),
Cleanup: func() {},
}, nil
}
return &TestDB{
DriverName: "spanner",
ConnStr: spannerDB,
Cleanup: func() {},
}, nil
}
pkg/services/sqlstore/transactions.go
+1 -1
View File
@@ -61,7 +61,7 @@ func (ss *SQLStore) inTransactionWithRetryCtx(ctx context.Context, engine *xorm.
return err
}
// special handling of database locked errors for sqlite and spanner, then we can retry 5 times
// special handling of database locked errors for sqlite, then we can retry 5 times
if r, ok := engine.Dialect().(xorm.DialectWithRetryableErrors); ok {
if retry < ss.dbCfg.TransactionRetries && r.RetryOnError(err) {
if rollErr := sess.Rollback(); rollErr != nil {