public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Provisioning: datasources auto deletion (#83034)

Browse Source
This commit is contained in:
Mikel Vuka
2024-04-08 11:45:39 +02:00
committed by GitHub
parent 4b4bdc7c33
commit 724517dc40
16 changed files with 205 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/provisioning/datasources/config_reader_test.go
+46 -1
View File
@@ -28,6 +28,8 @@ var (
multipleOrgsWithDefault = "testdata/multiple-org-default"
withoutDefaults = "testdata/appliedDefaults"
invalidAccess = "testdata/invalid-access"
beforeAutoDeletion = "testdata/before-auto-deletion"
afterAutoDeletion = "testdata/after-auto-deletion"
oneDatasourceWithTwoCorrelations = "testdata/one-datasource-two-correlations"
correlationsDifferentOrganizations = "testdata/correlations-different-organizations"
@@ -169,6 +171,39 @@ func TestDatasourceAsConfig(t *testing.T) {
require.Equal(t, len(store.updated), 1)
})
t.Run("Delete data sources when removing them from provision files", func(t *testing.T) {
store := &spyStore{}
orgFake := &orgtest.FakeOrgService{}
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgFake)
if err := dc.applyChanges(context.Background(), beforeAutoDeletion); err != nil {
t.Fatalf("applyChanges return an error %v", err)
}
require.Equal(t, len(store.deleted), 0)
require.Equal(t, len(store.inserted), 5)
require.Equal(t, len(store.updated), 0)
if err := dc.applyChanges(context.Background(), afterAutoDeletion); err != nil {
t.Fatalf("applyChanges return an error %v", err)
}
require.Equal(t, len(store.deleted), 2)
remainingDataSourceNames := make([]string, 3)
for i, ds := range store.items {
remainingDataSourceNames[i] = ds.Name
}
require.Contains(t, remainingDataSourceNames, "test_graphite_without_prune")
require.Contains(t, remainingDataSourceNames, "test_prometheus_without_prune")
require.Contains(t, remainingDataSourceNames, "test_graphite_with_prune")
require.NotContains(t, remainingDataSourceNames, "testdata_with_prune")
require.NotContains(t, remainingDataSourceNames, "test_prometheus_with_prune")
})
t.Run("broken yaml should return error", func(t *testing.T) {
reader := &configReader{}
_, err := reader.readConfig(context.Background(), brokenYaml)
@@ -429,6 +464,16 @@ func (s *spyStore) GetDataSource(ctx context.Context, query *datasources.GetData
return nil, datasources.ErrDataSourceNotFound
}
func (s *spyStore) GetPrunableProvisionedDataSources(ctx context.Context) ([]*datasources.DataSource, error) {
prunableProvisionedDataSources := []*datasources.DataSource{}
for _, item := range s.items {
if item.IsPrunable {
prunableProvisionedDataSources = append(prunableProvisionedDataSources, item)
}
}
return prunableProvisionedDataSources, nil
}
func (s *spyStore) DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error {
s.deleted = append(s.deleted, cmd)
for i, v := range s.items {
@@ -443,7 +488,7 @@ func (s *spyStore) DeleteDataSource(ctx context.Context, cmd *datasources.Delete
func (s *spyStore) AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) (*datasources.DataSource, error) {
s.inserted = append(s.inserted, cmd)
newDataSource := &datasources.DataSource{UID: cmd.UID, Name: cmd.Name, OrgID: cmd.OrgID}
newDataSource := &datasources.DataSource{UID: cmd.UID, Name: cmd.Name, OrgID: cmd.OrgID, IsPrunable: cmd.IsPrunable}
s.items = append(s.items, newDataSource)
return newDataSource, nil
}
pkg/services/provisioning/datasources/datasources.go
+23
View File
@@ -14,6 +14,7 @@ import (
type Store interface {
GetDataSource(ctx context.Context, query *datasources.GetDataSourceQuery) (*datasources.DataSource, error)
GetPrunableProvisionedDataSources(ctx context.Context) ([]*datasources.DataSource, error)
AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) (*datasources.DataSource, error)
UpdateDataSource(ctx context.Context, cmd *datasources.UpdateDataSourceCommand) (*datasources.DataSource, error)
DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error
@@ -153,6 +154,28 @@ func (dc *DatasourceProvisioner) applyChanges(ctx context.Context, configPath st
}
}
prunableProvisionedDataSources, err := dc.store.GetPrunableProvisionedDataSources(ctx)
if err != nil {
return err
}
staleProvisionedDataSources := []*deleteDatasourceConfig{}
for _, prunableProvisionedDataSource := range prunableProvisionedDataSources {
key := DataSourceMapKey{
OrgId: prunableProvisionedDataSource.OrgID,
Name: prunableProvisionedDataSource.Name,
}
if _, ok := willExistAfterProvisioning[key]; !ok {
staleProvisionedDataSources = append(staleProvisionedDataSources, &deleteDatasourceConfig{OrgID: prunableProvisionedDataSource.OrgID, Name: prunableProvisionedDataSource.Name})
willExistAfterProvisioning[key] = false
}
}
if err := dc.deleteDatasources(ctx, staleProvisionedDataSources, willExistAfterProvisioning); err != nil {
return err
}
for _, cfg := range configs {
if err := dc.provisionDataSources(ctx, cfg, willExistAfterProvisioning); err != nil {
return err
pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision1.yaml
Vendored
+6
View File
@@ -0,0 +1,6 @@
apiVersion: 1
datasources:
- name: test_graphite_without_prune
type: graphite
access: proxy
url: http://localhost:8080
pkg/services/provisioning/datasources/testdata/after-auto-deletion/provision3.yaml
Vendored
+7
View File
@@ -0,0 +1,7 @@
apiVersion: 1
prune: true
datasources:
- name: test_graphite_with_prune
type: graphite
access: proxy
url: http://localhost:8080
pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision1.yaml
Vendored
+10
View File
@@ -0,0 +1,10 @@
apiVersion: 1
datasources:
- name: test_graphite_without_prune
type: graphite
access: proxy
url: http://localhost:8080
- name: test_prometheus_without_prune
type: prometheus
access: proxy
url: http://localhost:9090
pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision2.yaml
Vendored
+5
View File
@@ -0,0 +1,5 @@
apiVersion: 1
prune: true
datasources:
- name: testdata_with_prune
type: testdata
pkg/services/provisioning/datasources/testdata/before-auto-deletion/provision3.yaml
Vendored
+11
View File
@@ -0,0 +1,11 @@
apiVersion: 1
prune: true
datasources:
- name: test_graphite_with_prune
type: graphite
access: proxy
url: http://localhost:8080
- name: test_prometheus_with_prune
type: prometheus
access: proxy
url: http://localhost:9090
pkg/services/provisioning/datasources/types.go
+8 -1
View File
@@ -18,6 +18,7 @@ type configVersion struct {
type configs struct {
APIVersion int64
Prune bool
Datasources []*upsertDataSourceFromConfig
DeleteDatasources []*deleteDatasourceConfig
@@ -47,6 +48,7 @@ type upsertDataSourceFromConfig struct {
SecureJSONData map[string]string
Editable bool
UID string
IsPrunable bool
}
type configsV0 struct {
@@ -58,7 +60,8 @@ type configsV0 struct {
type configsV1 struct {
configVersion
log log.Logger
log log.Logger
Prune bool
Datasources []*upsertDataSourceFromConfigV1 `json:"datasources" yaml:"datasources"`
DeleteDatasources []*deleteDatasourceConfigV1 `json:"deleteDatasources" yaml:"deleteDatasources"`
@@ -111,6 +114,7 @@ type upsertDataSourceFromConfigV1 struct {
SecureJSONData values.StringMapValue `json:"secureJsonData" yaml:"secureJsonData"`
Editable values.BoolValue `json:"editable" yaml:"editable"`
UID values.StringValue `json:"uid" yaml:"uid"`
IsPrunable values.BoolValue
}
func (cfg *configsV1) mapToDatasourceFromConfig(apiVersion int64) *configs {
@@ -141,6 +145,7 @@ func (cfg *configsV1) mapToDatasourceFromConfig(apiVersion int64) *configs {
Editable: ds.Editable.Value(),
Version: ds.Version.Value(),
UID: ds.UID.Value(),
IsPrunable: cfg.Prune,
})
}
@@ -218,6 +223,7 @@ func createInsertCommand(ds *upsertDataSourceFromConfig) *datasources.AddDataSou
SecureJsonData: ds.SecureJSONData,
ReadOnly: !ds.Editable,
UID: ds.UID,
IsPrunable: ds.IsPrunable,
}
if cmd.UID == "" {
@@ -260,5 +266,6 @@ func createUpdateCommand(ds *upsertDataSourceFromConfig, id int64) *datasources.
SecureJsonData: ds.SecureJSONData,
ReadOnly: !ds.Editable,
IgnoreOldSecureJsonData: true,
IsPrunable: ds.IsPrunable,
}
}