Correlations: Add CreateCorrelation HTTP API (#51630)

* Correlations: add migration

* Correlations: Add CreateCorrelation API

* Correlations: Make correlations work with provisioning

* Handle version changes

* Fix lining error

* lint fixes

* rebuild betterer results

* add a UID to each correlation

* Fix lint errors

* add docs

* better wording in API docs

* remove leftover comment

* handle ds updates

* Fix error message typo

* add bad data test

* make correlations a separate table

* skip readonly check when provisioning correlations

* delete stale correlations when datasources are deleted

* restore provisioned readonly ds

* publish deletion event with full data

* generate swagger and HTTP API docs

* apply source datasource permission to create correlation API

* Fix tests & lint errors

* ignore empty deletion events

* fix last lint errors

* fix more lint error

* Only publish deletion event if datasource was actually deleted

* delete DS provisioning deletes correlations, added & fixed tests

* Fix unmarshalling tests

* Fix linting errors

* Fix deltion event tests

* fix small linting error

* fix lint errors

* update betterer

* fix test

* make path singular

* Revert "make path singular"

This reverts commit 420c3d315e.

* add integration tests

* remove unneeded id from correlations table

* update spec

* update leftover references to CorrelationDTO

* fix tests

* cleanup tests

* fix lint error
This commit is contained in:
Giordano Ricci
2022-07-25 15:19:07 +01:00
committed by GitHub
parent dbc2171401
commit 5ce4baf6f5
27 changed files with 1326 additions and 48 deletions
@@ -9,6 +9,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/correlations"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/util"
)
@@ -26,13 +27,16 @@ var (
multipleOrgsWithDefault = "testdata/multiple-org-default"
withoutDefaults = "testdata/appliedDefaults"
invalidAccess = "testdata/invalid-access"
oneDatasourceWithTwoCorrelations = "testdata/one-datasource-two-correlations"
)
func TestDatasourceAsConfig(t *testing.T) {
t.Run("when some values missing should apply default on insert", func(t *testing.T) {
store := &spyStore{}
orgStore := &mockOrgStore{ExpectedOrg: &models.Org{Id: 1}}
dc := newDatasourceProvisioner(logger, store, orgStore)
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), withoutDefaults)
if err != nil {
t.Fatalf("applyChanges return an error %v", err)
@@ -50,7 +54,8 @@ func TestDatasourceAsConfig(t *testing.T) {
items: []*datasources.DataSource{{Name: "My datasource name", OrgId: 1, Id: 1, Uid: util.GenerateShortUID()}},
}
orgStore := &mockOrgStore{}
dc := newDatasourceProvisioner(logger, store, orgStore)
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), withoutDefaults)
if err != nil {
t.Fatalf("applyChanges return an error %v", err)
@@ -65,7 +70,8 @@ func TestDatasourceAsConfig(t *testing.T) {
t.Run("no datasource in database", func(t *testing.T) {
store := &spyStore{}
orgStore := &mockOrgStore{}
dc := newDatasourceProvisioner(logger, store, orgStore)
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), twoDatasourcesConfig)
if err != nil {
t.Fatalf("applyChanges return an error %v", err)
@@ -79,7 +85,8 @@ func TestDatasourceAsConfig(t *testing.T) {
t.Run("One datasource in database with same name should update one datasource", func(t *testing.T) {
store := &spyStore{items: []*datasources.DataSource{{Name: "Graphite", OrgId: 1, Id: 1}}}
orgStore := &mockOrgStore{}
dc := newDatasourceProvisioner(logger, store, orgStore)
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), twoDatasourcesConfig)
if err != nil {
t.Fatalf("applyChanges return an error %v", err)
@@ -93,7 +100,8 @@ func TestDatasourceAsConfig(t *testing.T) {
t.Run("Two datasources with is_default should raise error", func(t *testing.T) {
store := &spyStore{}
orgStore := &mockOrgStore{}
dc := newDatasourceProvisioner(logger, store, orgStore)
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), doubleDatasourcesConfig)
require.Equal(t, err, ErrInvalidConfigToManyDefault)
})
@@ -101,7 +109,8 @@ func TestDatasourceAsConfig(t *testing.T) {
t.Run("Multiple datasources in different organizations with isDefault in each organization should not raise error", func(t *testing.T) {
store := &spyStore{}
orgStore := &mockOrgStore{}
dc := newDatasourceProvisioner(logger, store, orgStore)
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), multipleOrgsWithDefault)
require.NoError(t, err)
require.Equal(t, len(store.inserted), 4)
@@ -114,7 +123,8 @@ func TestDatasourceAsConfig(t *testing.T) {
t.Run("Remove one datasource should have removed old datasource", func(t *testing.T) {
store := &spyStore{}
orgStore := &mockOrgStore{}
dc := newDatasourceProvisioner(logger, store, orgStore)
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), deleteOneDatasource)
if err != nil {
t.Fatalf("applyChanges return an error %v", err)
@@ -130,7 +140,8 @@ func TestDatasourceAsConfig(t *testing.T) {
t.Run("Two configured datasource and purge others", func(t *testing.T) {
store := &spyStore{items: []*datasources.DataSource{{Name: "old-graphite", OrgId: 1, Id: 1}, {Name: "old-graphite2", OrgId: 1, Id: 2}}}
orgStore := &mockOrgStore{}
dc := newDatasourceProvisioner(logger, store, orgStore)
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), twoDatasourcesConfigPurgeOthers)
if err != nil {
t.Fatalf("applyChanges return an error %v", err)
@@ -144,7 +155,8 @@ func TestDatasourceAsConfig(t *testing.T) {
t.Run("Two configured datasource and purge others = false", func(t *testing.T) {
store := &spyStore{items: []*datasources.DataSource{{Name: "Graphite", OrgId: 1, Id: 1}, {Name: "old-graphite2", OrgId: 1, Id: 2}}}
orgStore := &mockOrgStore{}
dc := newDatasourceProvisioner(logger, store, orgStore)
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), twoDatasourcesConfig)
if err != nil {
t.Fatalf("applyChanges return an error %v", err)
@@ -224,6 +236,53 @@ func TestDatasourceAsConfig(t *testing.T) {
validateDatasource(t, dsCfg)
validateDeleteDatasources(t, dsCfg)
})
t.Run("Correlations", func(t *testing.T) {
t.Run("Creates two correlations", func(t *testing.T) {
store := &spyStore{}
orgStore := &mockOrgStore{}
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), oneDatasourceWithTwoCorrelations)
if err != nil {
t.Fatalf("applyChanges return an error %v", err)
}
require.Equal(t, 2, len(correlationsStore.created))
require.Equal(t, 0, len(correlationsStore.deletedBySourceUID))
require.Equal(t, 0, len(correlationsStore.deletedByTargetUID))
})
t.Run("Updating existing datasource deletes existing correlations and creates two", func(t *testing.T) {
store := &spyStore{items: []*datasources.DataSource{{Name: "Graphite", OrgId: 1, Id: 1}}}
orgStore := &mockOrgStore{}
correlationsStore := &mockCorrelationsStore{}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), oneDatasourceWithTwoCorrelations)
if err != nil {
t.Fatalf("applyChanges return an error %v", err)
}
require.Equal(t, 2, len(correlationsStore.created))
require.Equal(t, 1, len(correlationsStore.deletedBySourceUID))
require.Equal(t, 0, len(correlationsStore.deletedByTargetUID))
})
t.Run("Deleting datasource deletes existing correlations", func(t *testing.T) {
store := &spyStore{items: []*datasources.DataSource{{Name: "old-data-source", OrgId: 1, Id: 1, Uid: "some-uid"}}}
orgStore := &mockOrgStore{}
correlationsStore := &mockCorrelationsStore{items: []correlations.Correlation{{UID: "some-uid", SourceUID: "some-uid", TargetUID: "target-uid"}}}
dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore)
err := dc.applyChanges(context.Background(), deleteOneDatasource)
if err != nil {
t.Fatalf("applyChanges return an error %v", err)
}
require.Equal(t, 0, len(correlationsStore.created))
require.Equal(t, 1, len(correlationsStore.deletedBySourceUID))
require.Equal(t, 1, len(correlationsStore.deletedByTargetUID))
})
})
}
func validateDeleteDatasources(t *testing.T, dsCfg *configs) {
@@ -249,6 +308,12 @@ func validateDatasource(t *testing.T, dsCfg *configs) {
require.True(t, ds.Editable)
require.Equal(t, ds.Version, 10)
require.Equal(t, []map[string]interface{}{{
"targetUid": "a target",
"label": "a label",
"description": "a description",
}}, ds.Correlations)
require.Greater(t, len(ds.JSONData), 2)
require.Equal(t, ds.JSONData["graphiteVersion"], "1.1")
require.Equal(t, ds.JSONData["tlsAuth"], true)
@@ -273,6 +338,28 @@ func (m *mockOrgStore) GetOrgById(c context.Context, cmd *models.GetOrgByIdQuery
return nil
}
type mockCorrelationsStore struct {
created []correlations.CreateCorrelationCommand
deletedBySourceUID []correlations.DeleteCorrelationsBySourceUIDCommand
deletedByTargetUID []correlations.DeleteCorrelationsByTargetUIDCommand
items []correlations.Correlation
}
func (m *mockCorrelationsStore) CreateCorrelation(c context.Context, cmd correlations.CreateCorrelationCommand) (correlations.Correlation, error) {
m.created = append(m.created, cmd)
return correlations.Correlation{}, nil
}
func (m *mockCorrelationsStore) DeleteCorrelationsBySourceUID(c context.Context, cmd correlations.DeleteCorrelationsBySourceUIDCommand) error {
m.deletedBySourceUID = append(m.deletedBySourceUID, cmd)
return nil
}
func (m *mockCorrelationsStore) DeleteCorrelationsByTargetUID(c context.Context, cmd correlations.DeleteCorrelationsByTargetUIDCommand) error {
m.deletedByTargetUID = append(m.deletedByTargetUID, cmd)
return nil
}
type spyStore struct {
inserted []*datasources.AddDataSourceCommand
deleted []*datasources.DeleteDataSourceCommand
@@ -292,11 +379,20 @@ func (s *spyStore) GetDataSource(ctx context.Context, query *datasources.GetData
func (s *spyStore) DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error {
s.deleted = append(s.deleted, cmd)
for _, v := range s.items {
if cmd.Name == v.Name && cmd.OrgID == v.OrgId {
cmd.DeletedDatasourcesCount = 1
return nil
}
}
return nil
}
func (s *spyStore) AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) error {
s.inserted = append(s.inserted, cmd)
cmd.Result = &datasources.DataSource{
Uid: cmd.Uid,
}
return nil
}
@@ -3,8 +3,10 @@ package datasources
import (
"context"
"errors"
"fmt"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/correlations"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/provisioning/utils"
)
@@ -16,6 +18,12 @@ type Store interface {
DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error
}
type CorrelationsStore interface {
DeleteCorrelationsByTargetUID(ctx context.Context, cmd correlations.DeleteCorrelationsByTargetUIDCommand) error
DeleteCorrelationsBySourceUID(ctx context.Context, cmd correlations.DeleteCorrelationsBySourceUIDCommand) error
CreateCorrelation(ctx context.Context, cmd correlations.CreateCorrelationCommand) (correlations.Correlation, error)
}
var (
// ErrInvalidConfigToManyDefault indicates that multiple datasource in the provisioning files
// contains more than one datasource marked as default.
@@ -24,24 +32,26 @@ var (
// Provision scans a directory for provisioning config files
// and provisions the datasource in those files.
func Provision(ctx context.Context, configDirectory string, store Store, orgStore utils.OrgStore) error {
dc := newDatasourceProvisioner(log.New("provisioning.datasources"), store, orgStore)
func Provision(ctx context.Context, configDirectory string, store Store, correlationsStore CorrelationsStore, orgStore utils.OrgStore) error {
dc := newDatasourceProvisioner(log.New("provisioning.datasources"), store, correlationsStore, orgStore)
return dc.applyChanges(ctx, configDirectory)
}
// DatasourceProvisioner is responsible for provisioning datasources based on
// configuration read by the `configReader`
type DatasourceProvisioner struct {
log log.Logger
cfgProvider *configReader
store Store
log log.Logger
cfgProvider *configReader
store Store
correlationsStore CorrelationsStore
}
func newDatasourceProvisioner(log log.Logger, store Store, orgStore utils.OrgStore) DatasourceProvisioner {
func newDatasourceProvisioner(log log.Logger, store Store, correlationsStore CorrelationsStore, orgStore utils.OrgStore) DatasourceProvisioner {
return DatasourceProvisioner{
log: log,
cfgProvider: &configReader{log: log, orgStore: orgStore},
store: store,
log: log,
cfgProvider: &configReader{log: log, orgStore: orgStore},
store: store,
correlationsStore: correlationsStore,
}
}
@@ -50,6 +60,8 @@ func (dc *DatasourceProvisioner) apply(ctx context.Context, cfg *configs) error
return err
}
correlationsToInsert := make([]correlations.CreateCorrelationCommand, 0)
for _, ds := range cfg.Datasources {
cmd := &datasources.GetDataSourceQuery{OrgId: ds.OrgID, Name: ds.Name}
err := dc.store.GetDataSource(ctx, cmd)
@@ -63,12 +75,44 @@ func (dc *DatasourceProvisioner) apply(ctx context.Context, cfg *configs) error
if err := dc.store.AddDataSource(ctx, insertCmd); err != nil {
return err
}
for _, correlation := range ds.Correlations {
if insertCorrelationCmd, err := makeCreateCorrelationCommand(correlation, insertCmd.Result.Uid, insertCmd.OrgId); err == nil {
correlationsToInsert = append(correlationsToInsert, insertCorrelationCmd)
} else {
dc.log.Error("failed to parse correlation", "correlation", correlation)
return err
}
}
} else {
updateCmd := createUpdateCommand(ds, cmd.Result.Id)
dc.log.Debug("updating datasource from configuration", "name", updateCmd.Name, "uid", updateCmd.Uid)
if err := dc.store.UpdateDataSource(ctx, updateCmd); err != nil {
return err
}
if len(ds.Correlations) > 0 {
if err := dc.correlationsStore.DeleteCorrelationsBySourceUID(ctx, correlations.DeleteCorrelationsBySourceUIDCommand{
SourceUID: cmd.Result.Uid,
}); err != nil {
return err
}
}
for _, correlation := range ds.Correlations {
if insertCorrelationCmd, err := makeCreateCorrelationCommand(correlation, cmd.Result.Uid, updateCmd.OrgId); err == nil {
correlationsToInsert = append(correlationsToInsert, insertCorrelationCmd)
} else {
dc.log.Error("failed to parse correlation", "correlation", correlation)
return err
}
}
}
}
for _, createCorrelationCmd := range correlationsToInsert {
if _, err := dc.correlationsStore.CreateCorrelation(ctx, createCorrelationCmd); err != nil {
return fmt.Errorf("err=%s source=%s", err.Error(), createCorrelationCmd.SourceUID)
}
}
@@ -90,13 +134,50 @@ func (dc *DatasourceProvisioner) applyChanges(ctx context.Context, configPath st
return nil
}
func makeCreateCorrelationCommand(correlation map[string]interface{}, SourceUid string, OrgId int64) (correlations.CreateCorrelationCommand, error) {
targetUid, ok := correlation["targetUid"].(string)
if !ok {
return correlations.CreateCorrelationCommand{}, fmt.Errorf("correlation missing targetUid")
}
return correlations.CreateCorrelationCommand{
SourceUID: SourceUid,
TargetUID: targetUid,
Label: correlation["label"].(string),
Description: correlation["description"].(string),
OrgId: OrgId,
SkipReadOnlyCheck: true,
}, nil
}
func (dc *DatasourceProvisioner) deleteDatasources(ctx context.Context, dsToDelete []*deleteDatasourceConfig) error {
for _, ds := range dsToDelete {
cmd := &datasources.DeleteDataSourceCommand{OrgID: ds.OrgID, Name: ds.Name}
getDsQuery := &datasources.GetDataSourceQuery{Name: ds.Name, OrgId: ds.OrgID}
if err := dc.store.GetDataSource(ctx, getDsQuery); err != nil && !errors.Is(err, datasources.ErrDataSourceNotFound) {
return err
}
if err := dc.store.DeleteDataSource(ctx, cmd); err != nil {
return err
}
if getDsQuery.Result != nil {
if err := dc.correlationsStore.DeleteCorrelationsBySourceUID(ctx, correlations.DeleteCorrelationsBySourceUIDCommand{
SourceUID: getDsQuery.Result.Uid,
}); err != nil {
return err
}
if err := dc.correlationsStore.DeleteCorrelationsByTargetUID(ctx, correlations.DeleteCorrelationsByTargetUIDCommand{
TargetUID: getDsQuery.Result.Uid,
}); err != nil {
return err
}
dc.log.Info("deleted correlations based on configuration", "ds_name", ds.Name)
}
if cmd.DeletedDatasourcesCount > 0 {
dc.log.Info("deleted datasource based on configuration", "name", ds.Name)
}
@@ -12,6 +12,10 @@ datasources:
basicAuthUser: basic_auth_user
withCredentials: true
isDefault: true
correlations:
- targetUid: a target
label: a label
description: a description
jsonData:
graphiteVersion: "1.1"
tlsAuth: true
@@ -0,0 +1,15 @@
apiVersion: 1
datasources:
- name: Graphite
type: graphite
uid: graphite
access: proxy
url: http://localhost:8080
correlations:
- targetUid: graphite
label: a label
description: a description
- targetUid: graphite
label: a second label
description: a second description
@@ -10,6 +10,10 @@ datasources:
basic_auth_user: basic_auth_user
with_credentials: true
is_default: true
correlations:
- targetUid: a target
label: a label
description: a description
json_data:
graphiteVersion: "1.1"
tlsAuth: true
+20 -15
View File
@@ -42,6 +42,7 @@ type upsertDataSourceFromConfig struct {
BasicAuthUser string
WithCredentials bool
IsDefault bool
Correlations []map[string]interface{}
JSONData map[string]interface{}
SecureJSONData map[string]string
Editable bool
@@ -74,21 +75,22 @@ type deleteDatasourceConfigV1 struct {
}
type upsertDataSourceFromConfigV0 struct {
OrgID int64 `json:"org_id" yaml:"org_id"`
Version int `json:"version" yaml:"version"`
Name string `json:"name" yaml:"name"`
Type string `json:"type" yaml:"type"`
Access string `json:"access" yaml:"access"`
URL string `json:"url" yaml:"url"`
User string `json:"user" yaml:"user"`
Database string `json:"database" yaml:"database"`
BasicAuth bool `json:"basic_auth" yaml:"basic_auth"`
BasicAuthUser string `json:"basic_auth_user" yaml:"basic_auth_user"`
WithCredentials bool `json:"with_credentials" yaml:"with_credentials"`
IsDefault bool `json:"is_default" yaml:"is_default"`
JSONData map[string]interface{} `json:"json_data" yaml:"json_data"`
SecureJSONData map[string]string `json:"secure_json_data" yaml:"secure_json_data"`
Editable bool `json:"editable" yaml:"editable"`
OrgID int64 `json:"org_id" yaml:"org_id"`
Version int `json:"version" yaml:"version"`
Name string `json:"name" yaml:"name"`
Type string `json:"type" yaml:"type"`
Access string `json:"access" yaml:"access"`
URL string `json:"url" yaml:"url"`
User string `json:"user" yaml:"user"`
Database string `json:"database" yaml:"database"`
BasicAuth bool `json:"basic_auth" yaml:"basic_auth"`
BasicAuthUser string `json:"basic_auth_user" yaml:"basic_auth_user"`
WithCredentials bool `json:"with_credentials" yaml:"with_credentials"`
IsDefault bool `json:"is_default" yaml:"is_default"`
Correlations []map[string]interface{} `json:"correlations" yaml:"correlations"`
JSONData map[string]interface{} `json:"json_data" yaml:"json_data"`
SecureJSONData map[string]string `json:"secure_json_data" yaml:"secure_json_data"`
Editable bool `json:"editable" yaml:"editable"`
}
type upsertDataSourceFromConfigV1 struct {
@@ -104,6 +106,7 @@ type upsertDataSourceFromConfigV1 struct {
BasicAuthUser values.StringValue `json:"basicAuthUser" yaml:"basicAuthUser"`
WithCredentials values.BoolValue `json:"withCredentials" yaml:"withCredentials"`
IsDefault values.BoolValue `json:"isDefault" yaml:"isDefault"`
Correlations values.JSONSliceValue `json:"correlations" yaml:"correlations"`
JSONData values.JSONValue `json:"jsonData" yaml:"jsonData"`
SecureJSONData values.StringMapValue `json:"secureJsonData" yaml:"secureJsonData"`
Editable values.BoolValue `json:"editable" yaml:"editable"`
@@ -132,6 +135,7 @@ func (cfg *configsV1) mapToDatasourceFromConfig(apiVersion int64) *configs {
BasicAuthUser: ds.BasicAuthUser.Value(),
WithCredentials: ds.WithCredentials.Value(),
IsDefault: ds.IsDefault.Value(),
Correlations: ds.Correlations.Value(),
JSONData: ds.JSONData.Value(),
SecureJSONData: ds.SecureJSONData.Value(),
Editable: ds.Editable.Value(),
@@ -172,6 +176,7 @@ func (cfg *configsV0) mapToDatasourceFromConfig(apiVersion int64) *configs {
BasicAuthUser: ds.BasicAuthUser,
WithCredentials: ds.WithCredentials,
IsDefault: ds.IsDefault,
Correlations: ds.Correlations,
JSONData: ds.JSONData,
SecureJSONData: ds.SecureJSONData,
Editable: ds.Editable,