public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Correlations: Allow creating correlations for provisioned data sources (#73737)

Browse Source 
* Allow creating correlations for provisioned data sources

* Update docs

* Fix linting

* Add missing props

* Add missing props

* Fix linting

* Fix linting

* Clarify error name

* Removed error handling for a non-existing use case

* Create a list of deleted data datasources based on all configs

* Add org_id to correlations

* Add tests

* Allow org_id to be null in case org_id=0 is used

* Create organization to ensure stable id is generated

* Fix linting

* Ensure backwards compatibility

* Add deprecation information

* Update comments

* Override existing datasSource variable so the UID is retrieved correctly

* Migrate correlations indices

* Default org_id when migrating

* Remove redundant default

* Make PK non-nullable

* Post merge fixes

* Separate data sources / correlations provisioning

* Adjust comments

* Store new data sources in spy store so it can be used to test correlations as well

* Fix linting

* Update tests

* Ensure response is closed

* Avoid creating duplicates during provisioning

* Fix updating provisioned column and update tests

* Rename error message

* Fix linting errors

* Fix linting errors and rename variable

* Update test

* Update pkg/services/sqlstore/migrations/correlations_mig.go

Co-authored-by: Giordano Ricci <me@giordanoricci.com>

* Remove unused error

* Fix lining

---------

Co-authored-by: Giordano Ricci <me@giordanoricci.com>
This commit is contained in:
Piotr Jamróz
2023-09-13 15:10:09 +02:00
committed by GitHub
co-authored by Giordano Ricci
parent 38c3483594
commit 946da57b6a
21 changed files with 464 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tests/api/correlations/common_test.go
+7
View File
@@ -182,3 +182,10 @@ func (c TestContext) createCorrelation(cmd correlations.CreateCorrelationCommand
require.NoError(c.t, err)
return correlation
}
func (c TestContext) createOrUpdateCorrelation(cmd correlations.CreateCorrelationCommand) {
c.t.Helper()
err := c.env.Server.HTTPServer.CorrelationsService.CreateOrUpdateCorrelation(context.Background(), cmd)
require.NoError(c.t, err)
}
pkg/tests/api/correlations/correlations_create_test.go
+8 -5
View File
@@ -166,7 +166,7 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
require.NoError(t, res.Body.Close())
})
t.Run("creating a correlation originating from a read-only data source should result in a 403", func(t *testing.T) {
t.Run("creating a correlation originating from a read-only data source should work", func(t *testing.T) {
res := ctx.Post(PostParams{
url: fmt.Sprintf("/api/datasources/uid/%s/correlations", readOnlyDS),
body: fmt.Sprintf(`{
@@ -179,17 +179,20 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
}`, readOnlyDS),
user: adminUser,
})
require.Equal(t, http.StatusForbidden, res.StatusCode)
require.Equal(t, http.StatusOK, res.StatusCode)
responseBody, err := io.ReadAll(res.Body)
require.NoError(t, err)
var response errorResponseBody
var response correlations.CreateCorrelationResponseBody
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Equal(t, "Data source is read only", response.Message)
require.Equal(t, correlations.ErrSourceDataSourceReadOnly.Error(), response.Error)
require.Equal(t, "Correlation created", response.Message)
require.Equal(t, readOnlyDS, response.Result.SourceUID)
require.Equal(t, readOnlyDS, *response.Result.TargetUID)
require.Equal(t, "", response.Result.Description)
require.Equal(t, "", response.Result.Label)
require.NoError(t, res.Body.Close())
})
pkg/tests/api/correlations/correlations_delete_test.go
+53 -4
View File
@@ -50,6 +50,7 @@ func TestIntegrationDeleteCorrelation(t *testing.T) {
}
dataSource = ctx.createDs(createDsCommand)
writableDs := dataSource.UID
writableDsId := dataSource.ID
writableDsOrgId := dataSource.OrgID
t.Run("Unauthenticated users shouldn't be able to delete correlations", func(t *testing.T) {
@@ -130,9 +131,16 @@ func TestIntegrationDeleteCorrelation(t *testing.T) {
require.NoError(t, res.Body.Close())
})
t.Run("deleting a correlation originating from a read-only data source should result in a 403", func(t *testing.T) {
t.Run("deleting a read-only correlation should result in a 403", func(t *testing.T) {
correlation := ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: writableDs,
TargetUID: &writableDs,
OrgId: writableDsOrgId,
Provisioned: true,
})
res := ctx.Delete(DeleteParams{
url: fmt.Sprintf("/api/datasources/uid/%s/correlations/%s", readOnlyDS, "nonexistent-correlation-uid"),
url: fmt.Sprintf("/api/datasources/uid/%s/correlations/%s", correlation.SourceUID, correlation.UID),
user: adminUser,
})
require.Equal(t, http.StatusForbidden, res.StatusCode)
@@ -144,8 +152,8 @@ func TestIntegrationDeleteCorrelation(t *testing.T) {
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Equal(t, "Data source is read only", response.Message)
require.Equal(t, correlations.ErrSourceDataSourceReadOnly.Error(), response.Error)
require.Equal(t, "Correlation can only be edited via provisioning", response.Message)
require.Equal(t, correlations.ErrCorrelationReadOnly.Error(), response.Error)
require.NoError(t, res.Body.Close())
})
@@ -213,4 +221,45 @@ func TestIntegrationDeleteCorrelation(t *testing.T) {
require.NoError(t, res.Body.Close())
require.Equal(t, http.StatusNotFound, res.StatusCode)
})
t.Run("deleting data source removes related correlations", func(t *testing.T) {
ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: writableDs,
TargetUID: &readOnlyDS,
OrgId: writableDsOrgId,
Provisioned: false,
})
ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: writableDs,
TargetUID: &readOnlyDS,
OrgId: writableDsOrgId,
Provisioned: true,
})
res := ctx.Delete(DeleteParams{
url: fmt.Sprintf("/api/datasources/%d", writableDsId),
user: adminUser,
})
require.Equal(t, http.StatusOK, res.StatusCode)
require.NoError(t, res.Body.Close())
res = ctx.Get(GetParams{
url: "/api/datasources/correlations",
user: adminUser,
page: "0",
})
require.Equal(t, http.StatusOK, res.StatusCode)
responseBody, err := io.ReadAll(res.Body)
require.NoError(t, err)
var response correlations.GetCorrelationsResponseBody
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Len(t, response.Correlations, 0)
require.NoError(t, res.Body.Close())
})
}
pkg/tests/api/correlations/correlations_provisioning_api_test.go
+120
View File
@@ -0,0 +1,120 @@
package correlations
import (
"encoding/json"
"io"
"net/http"
"testing"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/services/correlations"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/user"
)
func TestIntegrationCreateOrUpdateCorrelation(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
ctx := NewTestEnv(t)
adminUser := ctx.createUser(user.CreateUserCommand{
DefaultOrgRole: string(org.RoleAdmin),
Password: "admin",
Login: "admin",
})
createDsCommand := &datasources.AddDataSourceCommand{
Name: "loki",
Type: "loki",
OrgID: adminUser.User.OrgID,
}
dataSource := ctx.createDs(createDsCommand)
needsMigration := ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: dataSource.UID,
TargetUID: &dataSource.UID,
OrgId: dataSource.OrgID,
Label: "needs migration",
Config: correlations.CorrelationConfig{
Type: correlations.ConfigTypeQuery,
Field: "foo",
Target: map[string]any{},
Transformations: []correlations.Transformation{
{Type: "logfmt"},
},
},
Provisioned: false,
})
ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: dataSource.UID,
TargetUID: &dataSource.UID,
OrgId: dataSource.OrgID,
Label: "existing",
Config: correlations.CorrelationConfig{
Type: correlations.ConfigTypeQuery,
Field: "foo",
Target: map[string]any{},
Transformations: []correlations.Transformation{
{Type: "logfmt"},
},
},
Provisioned: false,
})
t.Run("Correctly marks existing correlations as provisioned", func(t *testing.T) {
// should be updated
ctx.createOrUpdateCorrelation(correlations.CreateCorrelationCommand{
SourceUID: needsMigration.SourceUID,
OrgId: needsMigration.OrgID,
TargetUID: needsMigration.TargetUID,
Label: needsMigration.Label,
Description: needsMigration.Description,
Config: needsMigration.Config,
Provisioned: true,
})
// should be added
ctx.createOrUpdateCorrelation(correlations.CreateCorrelationCommand{
SourceUID: needsMigration.SourceUID,
OrgId: needsMigration.OrgID,
TargetUID: needsMigration.TargetUID,
Label: "different",
Description: needsMigration.Description,
Config: needsMigration.Config,
Provisioned: true,
})
res := ctx.Get(GetParams{
url: "/api/datasources/correlations",
user: adminUser,
})
require.Equal(t, http.StatusOK, res.StatusCode)
responseBody, err := io.ReadAll(res.Body)
require.NoError(t, err)
var response correlations.GetCorrelationsResponseBody
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Len(t, response.Correlations, 3)
unordered := make(map[string]correlations.Correlation)
for _, v := range response.Correlations {
unordered[v.Label] = v
}
// existing correlation is updated
require.EqualValues(t, true, unordered["needs migration"].Provisioned)
// other existing correlations are not changed
require.EqualValues(t, false, unordered["existing"].Provisioned)
// new correlation is added
require.EqualValues(t, true, unordered["different"].Provisioned)
require.NoError(t, res.Body.Close())
})
}
pkg/tests/api/correlations/correlations_update_test.go
+12 -14
View File
@@ -35,20 +35,11 @@ func TestIntegrationUpdateCorrelation(t *testing.T) {
})
createDsCommand := &datasources.AddDataSourceCommand{
Name: "read-only",
Type: "loki",
ReadOnly: true,
OrgID: adminUser.User.OrgID,
}
dataSource := ctx.createDs(createDsCommand)
readOnlyDS := dataSource.UID
createDsCommand = &datasources.AddDataSourceCommand{
Name: "writable",
Type: "loki",
OrgID: adminUser.User.OrgID,
}
dataSource = ctx.createDs(createDsCommand)
dataSource := ctx.createDs(createDsCommand)
writableDs := dataSource.UID
writableDsOrgId := dataSource.OrgID
@@ -137,9 +128,16 @@ func TestIntegrationUpdateCorrelation(t *testing.T) {
require.NoError(t, res.Body.Close())
})
t.Run("updating a correlation originating from a read-only data source should result in a 403", func(t *testing.T) {
t.Run("updating a read-only correlation should result in a 403", func(t *testing.T) {
correlation := ctx.createCorrelation(correlations.CreateCorrelationCommand{
SourceUID: writableDs,
TargetUID: &writableDs,
OrgId: writableDsOrgId,
Provisioned: true,
})
res := ctx.Patch(PatchParams{
url: fmt.Sprintf("/api/datasources/uid/%s/correlations/%s", readOnlyDS, "nonexistent-correlation-uid"),
url: fmt.Sprintf("/api/datasources/uid/%s/correlations/%s", correlation.SourceUID, correlation.UID),
user: adminUser,
body: `{
"label": "some-label"
@@ -154,8 +152,8 @@ func TestIntegrationUpdateCorrelation(t *testing.T) {
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Equal(t, "Data source is read only", response.Message)
require.Equal(t, correlations.ErrSourceDataSourceReadOnly.Error(), response.Error)
require.Equal(t, "Correlation can only be edited via provisioning", response.Message)
require.Equal(t, correlations.ErrCorrelationReadOnly.Error(), response.Error)
require.NoError(t, res.Body.Close())
})