From 724517dc40fbbf38abb87c1735bfce6347eb36ac Mon Sep 17 00:00:00 2001 From: Mikel Vuka Date: Mon, 8 Apr 2024 11:45:39 +0200 Subject: [PATCH] Provisioning: datasources auto deletion (#83034) --- .../administration/provisioning/index.md | 12 +++++ pkg/services/datasources/datasources.go | 3 ++ .../fakes/fake_datasource_service.go | 10 ++++ pkg/services/datasources/models.go | 6 +++ .../datasources/service/datasource.go | 4 ++ pkg/services/datasources/service/store.go | 14 ++++++ .../datasources/service/store_test.go | 36 ++++++++++++++ .../datasources/config_reader_test.go | 47 ++++++++++++++++++- .../provisioning/datasources/datasources.go | 23 +++++++++ .../after-auto-deletion/provision1.yaml | 6 +++ .../after-auto-deletion/provision3.yaml | 7 +++ .../before-auto-deletion/provision1.yaml | 10 ++++ .../before-auto-deletion/provision2.yaml | 5 ++ .../before-auto-deletion/provision3.yaml | 11 +++++ .../provisioning/datasources/types.go | 9 +++- .../sqlstore/migrations/datasource_mig.go | 4 ++ 16 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision1.yaml create mode 100644 pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision3.yaml create mode 100644 pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision1.yaml create mode 100644 pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision2.yaml create mode 100644 pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision3.yaml diff --git a/docs/sources/administration/provisioning/index.md b/docs/sources/administration/provisioning/index.md index 0b6d76bc92c..c76903a7461 100644 --- a/docs/sources/administration/provisioning/index.md +++ b/docs/sources/administration/provisioning/index.md @@ -81,6 +81,14 @@ If the data source already exists, Grafana reconfigures it to match the provisio The configuration file can also list data sources to automatically delete, called `deleteDatasources`. Grafana deletes the data sources listed in `deleteDatasources` _before_ adding or updating those in the `datasources` list. +You can configure Grafana to automatically delete provisioned data sources when they're removed from the provisioning file. +To do so, add `prune: true` to the root of your provisioning file. +With this configuration, Grafana also removes the provisioned data sources if you remove the provisioning file entirely. + +{{% admonition type="note" %}} +The `prune` parameter is available in Grafana v11.1 and higher. +{{% /admonition %}} + ### Running multiple Grafana instances If you run multiple instances of Grafana, add a version number to each data source in the configuration and increase it when you update the configuration. @@ -100,6 +108,10 @@ deleteDatasources: - name: Graphite orgId: 1 +# Mark provisioned data sources for deletion if they are no longer in a provisioning file. +# It takes no effect if data sources are already listed in the deleteDatasources section. +prune: true + # List of data sources to insert/update depending on what's # available in the database. datasources: diff --git a/pkg/services/datasources/datasources.go b/pkg/services/datasources/datasources.go index 40c2328ef04..67bdf3a595d 100644 --- a/pkg/services/datasources/datasources.go +++ b/pkg/services/datasources/datasources.go @@ -21,6 +21,9 @@ type DataSourceService interface { // GetAllDataSources gets all datasources. GetAllDataSources(ctx context.Context, query *GetAllDataSourcesQuery) (res []*DataSource, err error) + // GetPrunableProvisionedDataSources gets all provisioned data sources that can be pruned. + GetPrunableProvisionedDataSources(ctx context.Context) (res []*DataSource, err error) + // GetDataSourcesByType gets datasources by type. GetDataSourcesByType(ctx context.Context, query *GetDataSourcesByTypeQuery) ([]*DataSource, error) diff --git a/pkg/services/datasources/fakes/fake_datasource_service.go b/pkg/services/datasources/fakes/fake_datasource_service.go index b49e17b1deb..f117f7b4af8 100644 --- a/pkg/services/datasources/fakes/fake_datasource_service.go +++ b/pkg/services/datasources/fakes/fake_datasource_service.go @@ -45,6 +45,16 @@ func (s *FakeDataSourceService) GetAllDataSources(ctx context.Context, query *da return s.DataSources, nil } +func (s *FakeDataSourceService) GetPrunableProvisionedDataSources(ctx context.Context) (res []*datasources.DataSource, err error) { + var dataSources []*datasources.DataSource + for _, dataSource := range s.DataSources { + if dataSource.IsPrunable { + dataSources = append(dataSources, dataSource) + } + } + return dataSources, nil +} + func (s *FakeDataSourceService) GetDataSourcesByType(ctx context.Context, query *datasources.GetDataSourcesByTypeQuery) ([]*datasources.DataSource, error) { var dataSources []*datasources.DataSource for _, datasource := range s.DataSources { diff --git a/pkg/services/datasources/models.go b/pkg/services/datasources/models.go index 68568868480..3c12d5ff758 100644 --- a/pkg/services/datasources/models.go +++ b/pkg/services/datasources/models.go @@ -62,6 +62,8 @@ type DataSource struct { SecureJsonData map[string][]byte `json:"secureJsonData"` ReadOnly bool `json:"readOnly"` UID string `json:"uid" xorm:"uid"` + // swagger:ignore + IsPrunable bool `xorm:"is_prunable"` Created time.Time `json:"created,omitempty"` Updated time.Time `json:"updated,omitempty"` @@ -161,6 +163,8 @@ type AddDataSourceCommand struct { JsonData *simplejson.Json `json:"jsonData"` SecureJsonData map[string]string `json:"secureJsonData"` UID string `json:"uid"` + // swagger:ignore + IsPrunable bool OrgID int64 `json:"-"` UserID int64 `json:"-"` @@ -185,6 +189,8 @@ type UpdateDataSourceCommand struct { SecureJsonData map[string]string `json:"secureJsonData"` Version int `json:"version"` UID string `json:"uid"` + // swagger:ignore + IsPrunable bool OrgID int64 `json:"-"` ID int64 `json:"-"` diff --git a/pkg/services/datasources/service/datasource.go b/pkg/services/datasources/service/datasource.go index d8757d97dc3..37c493e3b1a 100644 --- a/pkg/services/datasources/service/datasource.go +++ b/pkg/services/datasources/service/datasource.go @@ -174,6 +174,10 @@ func (s *Service) GetAllDataSources(ctx context.Context, query *datasources.GetA return s.SQLStore.GetAllDataSources(ctx, query) } +func (s *Service) GetPrunableProvisionedDataSources(ctx context.Context) (res []*datasources.DataSource, err error) { + return s.SQLStore.GetPrunableProvisionedDataSources(ctx) +} + func (s *Service) GetDataSourcesByType(ctx context.Context, query *datasources.GetDataSourcesByTypeQuery) ([]*datasources.DataSource, error) { if query.AliasIDs == nil { // Populate alias IDs from plugin store diff --git a/pkg/services/datasources/service/store.go b/pkg/services/datasources/service/store.go index 22f90a9303d..70c0c21f289 100644 --- a/pkg/services/datasources/service/store.go +++ b/pkg/services/datasources/service/store.go @@ -30,6 +30,7 @@ type Store interface { AddDataSource(context.Context, *datasources.AddDataSourceCommand) (*datasources.DataSource, error) UpdateDataSource(context.Context, *datasources.UpdateDataSourceCommand) (*datasources.DataSource, error) GetAllDataSources(ctx context.Context, query *datasources.GetAllDataSourcesQuery) (res []*datasources.DataSource, err error) + GetPrunableProvisionedDataSources(ctx context.Context) (res []*datasources.DataSource, err error) Count(context.Context, *quota.ScopeParameters) (*quota.Map, error) } @@ -124,6 +125,16 @@ func (ss *SqlStore) GetDataSourcesByType(ctx context.Context, query *datasources }) } +// GetPrunableProvisionedDataSources returns all data sources that can be pruned +func (ss *SqlStore) GetPrunableProvisionedDataSources(ctx context.Context) ([]*datasources.DataSource, error) { + prunableQuery := "is_prunable = ?" + + dataSources := make([]*datasources.DataSource, 0) + return dataSources, ss.db.WithDbSession(ctx, func(sess *db.Session) error { + return sess.Where(prunableQuery, ss.db.GetDialect().BooleanStr(true)).Asc("id").Find(&dataSources) + }) +} + // DeleteDataSource removes a datasource by org_id as well as either uid (preferred), id, or name // and is added to the bus. It also removes permissions related to the datasource. func (ss *SqlStore) DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error { @@ -261,6 +272,7 @@ func (ss *SqlStore) AddDataSource(ctx context.Context, cmd *datasources.AddDataS Version: 1, ReadOnly: cmd.ReadOnly, UID: cmd.UID, + IsPrunable: cmd.IsPrunable, } if _, err := sess.Insert(ds); err != nil { @@ -328,12 +340,14 @@ func (ss *SqlStore) UpdateDataSource(ctx context.Context, cmd *datasources.Updat ReadOnly: cmd.ReadOnly, Version: cmd.Version + 1, UID: cmd.UID, + IsPrunable: cmd.IsPrunable, } sess.UseBool("is_default") sess.UseBool("basic_auth") sess.UseBool("with_credentials") sess.UseBool("read_only") + sess.UseBool("is_prunable") // Make sure database field is zeroed out if empty. We want to migrate away from this field. sess.MustCols("database") // Make sure password are zeroed out if empty. We do this as we want to migrate passwords from diff --git a/pkg/services/datasources/service/store_test.go b/pkg/services/datasources/service/store_test.go index 942136ac2e0..70352aef450 100644 --- a/pkg/services/datasources/service/store_test.go +++ b/pkg/services/datasources/service/store_test.go @@ -489,5 +489,41 @@ func TestIntegrationDataAccess(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(dataSources)) }) + + t.Run("Get prunable data sources", func(t *testing.T) { + db := db.InitTestDB(t) + ss := SqlStore{db: db} + + _, errPrunable := ss.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ + OrgID: 10, + Name: "ElasticsearchPrunable", + Type: "other", + Access: datasources.DS_ACCESS_DIRECT, + URL: "http://test", + Database: "site", + ReadOnly: true, + IsPrunable: true, + }) + require.NoError(t, errPrunable) + + _, errNotPrunable := ss.AddDataSource(context.Background(), &datasources.AddDataSourceCommand{ + OrgID: 10, + Name: "ElasticsearchNotPrunable", + Type: "other", + Access: datasources.DS_ACCESS_DIRECT, + URL: "http://test", + Database: "site", + ReadOnly: true, + }) + require.NoError(t, errNotPrunable) + + dataSources, err := ss.GetPrunableProvisionedDataSources(context.Background()) + + require.NoError(t, err) + require.Equal(t, 1, len(dataSources)) + + dataSource := dataSources[0] + require.Equal(t, "ElasticsearchPrunable", dataSource.Name) + }) }) } diff --git a/pkg/services/provisioning/datasources/config_reader_test.go b/pkg/services/provisioning/datasources/config_reader_test.go index 028c32cdd9d..e35820da35d 100644 --- a/pkg/services/provisioning/datasources/config_reader_test.go +++ b/pkg/services/provisioning/datasources/config_reader_test.go @@ -28,6 +28,8 @@ var ( multipleOrgsWithDefault = "testdata/multiple-org-default" withoutDefaults = "testdata/appliedDefaults" invalidAccess = "testdata/invalid-access" + beforeAutoDeletion = "testdata/before-auto-deletion" + afterAutoDeletion = "testdata/after-auto-deletion" oneDatasourceWithTwoCorrelations = "testdata/one-datasource-two-correlations" correlationsDifferentOrganizations = "testdata/correlations-different-organizations" @@ -169,6 +171,39 @@ func TestDatasourceAsConfig(t *testing.T) { require.Equal(t, len(store.updated), 1) }) + t.Run("Delete data sources when removing them from provision files", func(t *testing.T) { + store := &spyStore{} + orgFake := &orgtest.FakeOrgService{} + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgFake) + if err := dc.applyChanges(context.Background(), beforeAutoDeletion); err != nil { + t.Fatalf("applyChanges return an error %v", err) + } + + require.Equal(t, len(store.deleted), 0) + require.Equal(t, len(store.inserted), 5) + require.Equal(t, len(store.updated), 0) + + if err := dc.applyChanges(context.Background(), afterAutoDeletion); err != nil { + t.Fatalf("applyChanges return an error %v", err) + } + + require.Equal(t, len(store.deleted), 2) + + remainingDataSourceNames := make([]string, 3) + + for i, ds := range store.items { + remainingDataSourceNames[i] = ds.Name + } + + require.Contains(t, remainingDataSourceNames, "test_graphite_without_prune") + require.Contains(t, remainingDataSourceNames, "test_prometheus_without_prune") + require.Contains(t, remainingDataSourceNames, "test_graphite_with_prune") + + require.NotContains(t, remainingDataSourceNames, "testdata_with_prune") + require.NotContains(t, remainingDataSourceNames, "test_prometheus_with_prune") + }) + t.Run("broken yaml should return error", func(t *testing.T) { reader := &configReader{} _, err := reader.readConfig(context.Background(), brokenYaml) @@ -429,6 +464,16 @@ func (s *spyStore) GetDataSource(ctx context.Context, query *datasources.GetData return nil, datasources.ErrDataSourceNotFound } +func (s *spyStore) GetPrunableProvisionedDataSources(ctx context.Context) ([]*datasources.DataSource, error) { + prunableProvisionedDataSources := []*datasources.DataSource{} + for _, item := range s.items { + if item.IsPrunable { + prunableProvisionedDataSources = append(prunableProvisionedDataSources, item) + } + } + return prunableProvisionedDataSources, nil +} + func (s *spyStore) DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error { s.deleted = append(s.deleted, cmd) for i, v := range s.items { @@ -443,7 +488,7 @@ func (s *spyStore) DeleteDataSource(ctx context.Context, cmd *datasources.Delete func (s *spyStore) AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) (*datasources.DataSource, error) { s.inserted = append(s.inserted, cmd) - newDataSource := &datasources.DataSource{UID: cmd.UID, Name: cmd.Name, OrgID: cmd.OrgID} + newDataSource := &datasources.DataSource{UID: cmd.UID, Name: cmd.Name, OrgID: cmd.OrgID, IsPrunable: cmd.IsPrunable} s.items = append(s.items, newDataSource) return newDataSource, nil } diff --git a/pkg/services/provisioning/datasources/datasources.go b/pkg/services/provisioning/datasources/datasources.go index 147d32e9e9f..e0468725def 100644 --- a/pkg/services/provisioning/datasources/datasources.go +++ b/pkg/services/provisioning/datasources/datasources.go @@ -14,6 +14,7 @@ import ( type Store interface { GetDataSource(ctx context.Context, query *datasources.GetDataSourceQuery) (*datasources.DataSource, error) + GetPrunableProvisionedDataSources(ctx context.Context) ([]*datasources.DataSource, error) AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) (*datasources.DataSource, error) UpdateDataSource(ctx context.Context, cmd *datasources.UpdateDataSourceCommand) (*datasources.DataSource, error) DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error @@ -153,6 +154,28 @@ func (dc *DatasourceProvisioner) applyChanges(ctx context.Context, configPath st } } + prunableProvisionedDataSources, err := dc.store.GetPrunableProvisionedDataSources(ctx) + if err != nil { + return err + } + + staleProvisionedDataSources := []*deleteDatasourceConfig{} + + for _, prunableProvisionedDataSource := range prunableProvisionedDataSources { + key := DataSourceMapKey{ + OrgId: prunableProvisionedDataSource.OrgID, + Name: prunableProvisionedDataSource.Name, + } + if _, ok := willExistAfterProvisioning[key]; !ok { + staleProvisionedDataSources = append(staleProvisionedDataSources, &deleteDatasourceConfig{OrgID: prunableProvisionedDataSource.OrgID, Name: prunableProvisionedDataSource.Name}) + willExistAfterProvisioning[key] = false + } + } + + if err := dc.deleteDatasources(ctx, staleProvisionedDataSources, willExistAfterProvisioning); err != nil { + return err + } + for _, cfg := range configs { if err := dc.provisionDataSources(ctx, cfg, willExistAfterProvisioning); err != nil { return err diff --git a/pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision1.yaml b/pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision1.yaml new file mode 100644 index 00000000000..136292c9024 --- /dev/null +++ b/pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision1.yaml @@ -0,0 +1,6 @@ +apiVersion: 1 +datasources: + - name: test_graphite_without_prune + type: graphite + access: proxy + url: http://localhost:8080 diff --git a/pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision3.yaml b/pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision3.yaml new file mode 100644 index 00000000000..2c496ef3291 --- /dev/null +++ b/pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision3.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 +prune: true +datasources: + - name: test_graphite_with_prune + type: graphite + access: proxy + url: http://localhost:8080 diff --git a/pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision1.yaml b/pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision1.yaml new file mode 100644 index 00000000000..51e0e8bd881 --- /dev/null +++ b/pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision1.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 +datasources: + - name: test_graphite_without_prune + type: graphite + access: proxy + url: http://localhost:8080 + - name: test_prometheus_without_prune + type: prometheus + access: proxy + url: http://localhost:9090 diff --git a/pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision2.yaml b/pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision2.yaml new file mode 100644 index 00000000000..6f9b85e675b --- /dev/null +++ b/pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision2.yaml @@ -0,0 +1,5 @@ +apiVersion: 1 +prune: true +datasources: + - name: testdata_with_prune + type: testdata diff --git a/pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision3.yaml b/pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision3.yaml new file mode 100644 index 00000000000..fed47dbb562 --- /dev/null +++ b/pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision3.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 +prune: true +datasources: + - name: test_graphite_with_prune + type: graphite + access: proxy + url: http://localhost:8080 + - name: test_prometheus_with_prune + type: prometheus + access: proxy + url: http://localhost:9090 diff --git a/pkg/services/provisioning/datasources/types.go b/pkg/services/provisioning/datasources/types.go index 5088278cc85..004006ed1e0 100644 --- a/pkg/services/provisioning/datasources/types.go +++ b/pkg/services/provisioning/datasources/types.go @@ -18,6 +18,7 @@ type configVersion struct { type configs struct { APIVersion int64 + Prune bool Datasources []*upsertDataSourceFromConfig DeleteDatasources []*deleteDatasourceConfig @@ -47,6 +48,7 @@ type upsertDataSourceFromConfig struct { SecureJSONData map[string]string Editable bool UID string + IsPrunable bool } type configsV0 struct { @@ -58,7 +60,8 @@ type configsV0 struct { type configsV1 struct { configVersion - log log.Logger + log log.Logger + Prune bool Datasources []*upsertDataSourceFromConfigV1 `json:"datasources" yaml:"datasources"` DeleteDatasources []*deleteDatasourceConfigV1 `json:"deleteDatasources" yaml:"deleteDatasources"` @@ -111,6 +114,7 @@ type upsertDataSourceFromConfigV1 struct { SecureJSONData values.StringMapValue `json:"secureJsonData" yaml:"secureJsonData"` Editable values.BoolValue `json:"editable" yaml:"editable"` UID values.StringValue `json:"uid" yaml:"uid"` + IsPrunable values.BoolValue } func (cfg *configsV1) mapToDatasourceFromConfig(apiVersion int64) *configs { @@ -141,6 +145,7 @@ func (cfg *configsV1) mapToDatasourceFromConfig(apiVersion int64) *configs { Editable: ds.Editable.Value(), Version: ds.Version.Value(), UID: ds.UID.Value(), + IsPrunable: cfg.Prune, }) } @@ -218,6 +223,7 @@ func createInsertCommand(ds *upsertDataSourceFromConfig) *datasources.AddDataSou SecureJsonData: ds.SecureJSONData, ReadOnly: !ds.Editable, UID: ds.UID, + IsPrunable: ds.IsPrunable, } if cmd.UID == "" { @@ -260,5 +266,6 @@ func createUpdateCommand(ds *upsertDataSourceFromConfig, id int64) *datasources. SecureJsonData: ds.SecureJSONData, ReadOnly: !ds.Editable, IgnoreOldSecureJsonData: true, + IsPrunable: ds.IsPrunable, } } diff --git a/pkg/services/sqlstore/migrations/datasource_mig.go b/pkg/services/sqlstore/migrations/datasource_mig.go index b08a6d3f052..0d0567d61df 100644 --- a/pkg/services/sqlstore/migrations/datasource_mig.go +++ b/pkg/services/sqlstore/migrations/datasource_mig.go @@ -134,4 +134,8 @@ func addDataSourceMigration(mg *Migrator) { mg.AddMigration("add unique index datasource_org_id_is_default", NewAddIndexMigration(tableV2, &Index{ Cols: []string{"org_id", "is_default"}})) + + mg.AddMigration("Add is_prunable column", NewAddColumnMigration(tableV2, &Column{ + Name: "is_prunable", Type: DB_Bool, Nullable: true, Default: "0", + })) }