From 4927376f32048ebe926d314d16458759c74940fa Mon Sep 17 00:00:00 2001 From: Georges Chaudy Date: Wed, 19 Mar 2025 13:56:54 +0100 Subject: [PATCH] unistore: use the same connection string as grafana (#102387) --- pkg/storage/unified/README.md | 31 ---------- .../unified/sql/db/dbimpl/db_engine.go | 28 +++++++++ .../unified/sql/db/dbimpl/db_engine_test.go | 59 +++++++++++++++++++ pkg/storage/unified/sql/db/dbimpl/dbimpl.go | 28 +++------ .../dbimpl/regression_incident_2144_test.go | 4 +- 5 files changed, 97 insertions(+), 53 deletions(-) diff --git a/pkg/storage/unified/README.md b/pkg/storage/unified/README.md index d19d679b77c..f08da73b33a 100644 --- a/pkg/storage/unified/README.md +++ b/pkg/storage/unified/README.md @@ -203,37 +203,6 @@ then run: kubectl --kubeconfig=./grafana.kubeconfig create -f folder-generate.yaml ``` -### Use a separate database - -By default Unified Storage uses the Grafana database. To run against a separate database, update `custom.ini` by adding the following section to it: - -``` -[resource_api] -db_type = mysql -db_host = localhost:3306 -db_name = grafana -db_user = -db_pass = -``` - -MySQL and Postgres are both supported. The `` and `` values can be found in the following devenv docker compose files: [MySQL](https://github.com/grafana/grafana/blob/main/devenv/docker/blocks/mysql/docker-compose.yaml#L6-L7) and [Postgres](https://github.com/grafana/grafana/blob/main/devenv/docker/blocks/postgres/docker-compose.yaml#L4-L5). - -Then, run -```sh -make devenv sources= -``` -where source is either `mysql` or `postgres`. - -Finally, run the Grafana backend with - -```sh -bra run -``` -or -```sh -make run -``` - ### Run as a GRPC service #### Start GRPC storage-server diff --git a/pkg/storage/unified/sql/db/dbimpl/db_engine.go b/pkg/storage/unified/sql/db/dbimpl/db_engine.go index 529cd163eb4..b4300899128 100644 --- a/pkg/storage/unified/sql/db/dbimpl/db_engine.go +++ b/pkg/storage/unified/sql/db/dbimpl/db_engine.go @@ -11,6 +11,8 @@ import ( "xorm.io/xorm" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/unified/sql/db" ) @@ -18,6 +20,31 @@ import ( // driver. const tlsConfigName = "db_engine_tls" +func getEngine(cfg *setting.Cfg) (*xorm.Engine, error) { + dbSection := cfg.SectionWithEnvOverrides("database") + dbType := dbSection.Key("type").String() + if dbType == "" { + return nil, fmt.Errorf("no database type specified") + } + + switch dbType { + case dbTypeMySQL, dbTypePostgres, dbTypeSQLite: + config, err := sqlstore.NewDatabaseConfig(cfg, nil) + if err != nil { + return nil, nil + } + + engine, err := xorm.NewEngine(dbType, config.ConnectionString) + if err != nil { + return nil, fmt.Errorf("open database: %w", err) + } + return engine, nil + default: + return nil, fmt.Errorf("unsupported database type: %s", dbType) + } +} + +// Deprecated: use getEngine instead func getEngineMySQL(getter confGetter) (*xorm.Engine, error) { config := mysql.NewConfig() config.User = getter.String("user") @@ -108,6 +135,7 @@ func configureTLS(getter confGetter, config *mysql.Config) error { return nil } +// Deprecated: use getEngine instead func getEnginePostgres(getter confGetter) (*xorm.Engine, error) { dsnKV := map[string]string{ "user": getter.String("user"), diff --git a/pkg/storage/unified/sql/db/dbimpl/db_engine_test.go b/pkg/storage/unified/sql/db/dbimpl/db_engine_test.go index 8b39100ffb3..d85260b161a 100644 --- a/pkg/storage/unified/sql/db/dbimpl/db_engine_test.go +++ b/pkg/storage/unified/sql/db/dbimpl/db_engine_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/require" ) @@ -29,6 +30,64 @@ func newValidMySQLGetter(withKeyPrefix bool) confGetter { }, prefix) } +func TestGetEngine(t *testing.T) { + t.Parallel() + + t.Run("MySQL engine", func(t *testing.T) { + t.Parallel() + cfg := setting.NewCfg() + dbSection := cfg.SectionWithEnvOverrides("database") + dbSection.Key("type").SetValue(dbTypeMySQL) + dbSection.Key("host").SetValue("/var/run/mysql.socket") + dbSection.Key("name").SetValue("grafana") + dbSection.Key("user").SetValue("user") + dbSection.Key("password").SetValue("password") + + engine, err := getEngine(cfg) + require.NoError(t, err) + require.NotNil(t, engine) + }) + + t.Run("Postgres engine", func(t *testing.T) { + t.Parallel() + cfg := setting.NewCfg() + dbSection := cfg.SectionWithEnvOverrides("database") + dbSection.Key("type").SetValue(dbTypePostgres) + dbSection.Key("host").SetValue("localhost") + dbSection.Key("name").SetValue("grafana") + dbSection.Key("user").SetValue("user") + dbSection.Key("password").SetValue("password") + + engine, err := getEngine(cfg) + require.NoError(t, err) + require.NotNil(t, engine) + }) + + t.Run("SQLite engine", func(t *testing.T) { + t.Parallel() + cfg := setting.NewCfg() + dbSection := cfg.SectionWithEnvOverrides("database") + dbSection.Key("type").SetValue(dbTypeSQLite) + dbSection.Key("path").SetValue(":memory:") + + engine, err := getEngine(cfg) + require.NoError(t, err) + require.NotNil(t, engine) + }) + + t.Run("Unknown database type", func(t *testing.T) { + t.Parallel() + cfg := setting.NewCfg() + dbSection := cfg.SectionWithEnvOverrides("database") + dbSection.Key("type").SetValue("unknown") + + engine, err := getEngine(cfg) + require.Error(t, err) + require.Nil(t, engine) + require.Contains(t, err.Error(), "unsupported database type") + }) +} + func TestGetEngineMySQLFromConfig(t *testing.T) { t.Parallel() diff --git a/pkg/storage/unified/sql/db/dbimpl/dbimpl.go b/pkg/storage/unified/sql/db/dbimpl/dbimpl.go index 13e6b7fbe42..98fb6f59af0 100644 --- a/pkg/storage/unified/sql/db/dbimpl/dbimpl.go +++ b/pkg/storage/unified/sql/db/dbimpl/dbimpl.go @@ -23,6 +23,7 @@ import ( const ( dbTypeMySQL = "mysql" dbTypePostgres = "postgres" + dbTypeSQLite = "sqlite3" ) const grafanaDBInstrumentQueriesKey = "instrument_queries" @@ -88,8 +89,7 @@ func newResourceDBProvider(grafanaDB infraDB.DB, cfg *setting.Cfg, tracer trace. dbType := getter.String("type") grafanaDBType := fallbackGetter.String("type") switch { - // First try with the config in the "resource_api" section, which is - // specific to Unified Storage + // Deprecated: First try with the config in the "resource_api" section, which is specific to Unified Storage case dbType == dbTypePostgres: p.registerMetrics = true p.engine, err = getEnginePostgres(getter) @@ -100,37 +100,23 @@ func newResourceDBProvider(grafanaDB infraDB.DB, cfg *setting.Cfg, tracer trace. p.engine, err = getEngineMySQL(getter) return p, err - // TODO: add support for SQLite - case dbType != "": return p, fmt.Errorf("invalid db type specified: %s", dbType) - // If we have an empty Resource API db config, try with the core Grafana - // database config - - case grafanaDBType == dbTypePostgres: + // If we have an empty Resource API db config, try with the core Grafana database config + case grafanaDBType != "": p.registerMetrics = true - p.engine, err = getEnginePostgres(fallbackGetter) + p.engine, err = getEngine(cfg) return p, err - - case grafanaDBType == dbTypeMySQL: - p.registerMetrics = true - p.engine, err = getEngineMySQL(fallbackGetter) - return p, err - - // TODO: add support for SQLite - case grafanaDB != nil: - // try to use the grafana db connection - + // try to use the grafana db connection (should only happen in tests) if fallbackGetter.Bool(grafanaDBInstrumentQueriesKey) { return nil, errGrafanaDBInstrumentedNotSupported } p.engine = grafanaDB.GetEngine() return p, nil - default: - return p, fmt.Errorf("no db connection provided") + return p, fmt.Errorf("no database type specified") } } diff --git a/pkg/storage/unified/sql/db/dbimpl/regression_incident_2144_test.go b/pkg/storage/unified/sql/db/dbimpl/regression_incident_2144_test.go index 38b2f8dc89e..dfbcd93aea7 100644 --- a/pkg/storage/unified/sql/db/dbimpl/regression_incident_2144_test.go +++ b/pkg/storage/unified/sql/db/dbimpl/regression_incident_2144_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util/testutil" ) @@ -104,7 +105,8 @@ func TestReproIncident2144UsingGrafanaDB(t *testing.T) { t.Run("Resource API provides a reasonable error for this case", func(t *testing.T) { t.Parallel() - cfg := newCfgFromIniMap(t, cfgMap) + cfg := setting.NewCfg() + cfg.SectionWithEnvOverrides("database").Key(grafanaDBInstrumentQueriesKey).SetValue("true") resourceDB, err := ProvideResourceDB(grafanaDB, cfg, nil) require.Nil(t, resourceDB) require.Error(t, err)