public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Provisioning: Allows specifying uid for datasource and use that in derived fields (#23585)

Browse Source 
* Add uid to datasource

* Fix uid passing when provisioning

* Better error handling and Uid column type change

* Fix test and strict null error counts

* Add backend tests

* Add tests

* Fix strict null checks

* Update test

* Improve tests

* Update pkg/services/sqlstore/datasource.go

Co-Authored-By: Arve Knudsen <arve.knudsen@gmail.com>

* Variable rename

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
This commit is contained in:
Andrej Ocenas
2020-04-20 15:48:38 +02:00
committed by GitHub
co-authored by Arve Knudsen
parent d5f8d976f0
commit e5dd7efdee
34 changed files with 446 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/sqlstore/datasource.go
+32
View File
@@ -1,6 +1,8 @@
package sqlstore
import (
"github.com/grafana/grafana/pkg/util/errutil"
"strings"
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
@@ -101,6 +103,14 @@ func AddDataSource(cmd *models.AddDataSourceCommand) error {
cmd.JsonData = simplejson.New()
}
if cmd.Uid == "" {
uid, err := generateNewDatasourceUid(sess, cmd.OrgId)
if err != nil {
return errutil.Wrapf(err, "Failed to generate UID for datasource %q", cmd.Name)
}
cmd.Uid = uid
}
ds := &models.DataSource{
OrgId: cmd.OrgId,
Name: cmd.Name,
@@ -121,9 +131,13 @@ func AddDataSource(cmd *models.AddDataSourceCommand) error {
Updated: time.Now(),
Version: 1,
ReadOnly: cmd.ReadOnly,
Uid: cmd.Uid,
}
if _, err := sess.Insert(ds); err != nil {
if dialect.IsUniqueConstraintViolation(err) && strings.Contains(strings.ToLower(dialect.ErrorMessage(err)), "uid") {
return models.ErrDataSourceUidExists
}
return err
}
if err := updateIsDefaultFlag(ds, sess); err != nil {
@@ -172,6 +186,7 @@ func UpdateDataSource(cmd *models.UpdateDataSourceCommand) error {
Updated: time.Now(),
ReadOnly: cmd.ReadOnly,
Version: cmd.Version + 1,
Uid: cmd.Uid,
}
sess.UseBool("is_default")
@@ -209,3 +224,20 @@ func UpdateDataSource(cmd *models.UpdateDataSourceCommand) error {
return err
})
}
func generateNewDatasourceUid(sess *DBSession, orgId int64) (string, error) {
for i := 0; i < 3; i++ {
uid := generateNewUid()
exists, err := sess.Where("org_id=? AND uid=?", orgId, uid).Get(&models.DataSource{})
if err != nil {
return "", err
}
if !exists {
return uid, nil
}
}
return "", models.ErrDataSourceFailedGenerateUniqueUid
}
pkg/services/sqlstore/datasource_test.go
+176 -136
View File
@@ -3,173 +3,213 @@ package sqlstore
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
"github.com/grafana/grafana/pkg/models"
"github.com/stretchr/testify/require"
)
type Test struct {
Id int64
Name string
}
func TestDataAccess(t *testing.T) {
Convey("Testing DB", t, func() {
InitTestDB(t)
Convey("Can add datasource", func() {
defaultAddDatasourceCommand := models.AddDataSourceCommand{
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_DIRECT,
Url: "http://test",
}
defaultUpdateDatasourceCommand := models.UpdateDataSourceCommand{
OrgId: 10,
Name: "nisse_updated",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_DIRECT,
Url: "http://test",
}
initDatasource := func() *models.DataSource {
cmd := defaultAddDatasourceCommand
err := AddDataSource(&cmd)
require.NoError(t, err)
query := models.GetDataSourcesQuery{OrgId: 10}
err = GetDataSources(&query)
require.NoError(t, err)
require.Equal(t, 1, len(query.Result))
return query.Result[0]
}
t.Run("AddDataSource", func(t *testing.T) {
t.Run("Can add datasource", func(t *testing.T) {
InitTestDB(t)
err := AddDataSource(&models.AddDataSourceCommand{
OrgId: 10,
Name: "laban",
Type: models.DS_INFLUXDB,
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_DIRECT,
Url: "http://test",
Database: "site",
ReadOnly: true,
})
So(err, ShouldBeNil)
require.NoError(t, err)
query := models.GetDataSourcesQuery{OrgId: 10}
err = GetDataSources(&query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 1)
require.NoError(t, err)
require.Equal(t, 1, len(query.Result))
ds := query.Result[0]
So(ds.OrgId, ShouldEqual, 10)
So(ds.Database, ShouldEqual, "site")
So(ds.ReadOnly, ShouldBeTrue)
require.EqualValues(t, 10, ds.OrgId)
require.Equal(t, "site", ds.Database)
require.True(t, ds.ReadOnly)
})
Convey("Given a datasource", func() {
err := AddDataSource(&models.AddDataSourceCommand{
t.Run("generates uid if not specified", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
require.NotEmpty(t, ds.Uid)
})
t.Run("fails to insert ds with same uid", func(t *testing.T) {
InitTestDB(t)
cmd1 := defaultAddDatasourceCommand
cmd2 := defaultAddDatasourceCommand
cmd1.Uid = "test"
cmd2.Uid = "test"
err := AddDataSource(&cmd1)
require.NoError(t, err)
err = AddDataSource(&cmd2)
require.Error(t, err)
require.IsType(t, models.ErrDataSourceUidExists, err)
})
})
t.Run("UpdateDataSource", func(t *testing.T) {
t.Run("updates datasource with version", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
cmd := defaultUpdateDatasourceCommand
cmd.Id = ds.Id
cmd.Version = ds.Version
err := UpdateDataSource(&cmd)
require.NoError(t, err)
})
t.Run("does not overwrite Uid if not specified", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
require.NotEmpty(t, ds.Uid)
cmd := defaultUpdateDatasourceCommand
cmd.Id = ds.Id
err := UpdateDataSource(&cmd)
require.NoError(t, err)
query := models.GetDataSourceByIdQuery{Id: ds.Id}
err = GetDataSourceById(&query)
require.NoError(t, err)
require.Equal(t, ds.Uid, query.Result.Uid)
})
t.Run("prevents update if version changed", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
cmd := models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: ds.Version,
}
// Make a copy as UpdateDataSource modifies it
cmd2 := cmd
err := UpdateDataSource(&cmd)
require.NoError(t, err)
err = UpdateDataSource(&cmd2)
require.Error(t, err)
})
t.Run("updates ds without version specified", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
cmd := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_DIRECT,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
})
So(err, ShouldBeNil)
}
err := UpdateDataSource(cmd)
require.NoError(t, err)
})
t.Run("updates ds without higher version", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
cmd := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: 90000,
}
err := UpdateDataSource(cmd)
require.NoError(t, err)
})
})
t.Run("DeleteDataSourceById", func(t *testing.T) {
t.Run("can delete datasource", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
err := DeleteDataSourceById(&models.DeleteDataSourceByIdCommand{Id: ds.Id, OrgId: ds.OrgId})
require.NoError(t, err)
query := models.GetDataSourcesQuery{OrgId: 10}
err = GetDataSources(&query)
So(err, ShouldBeNil)
require.NoError(t, err)
ds := query.Result[0]
require.Equal(t, 0, len(query.Result))
})
Convey(" updated ", func() {
cmd := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: ds.Version,
}
t.Run("Can not delete datasource with wrong orgId", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
Convey("with same version as source", func() {
err := UpdateDataSource(cmd)
So(err, ShouldBeNil)
})
err := DeleteDataSourceById(&models.DeleteDataSourceByIdCommand{Id: ds.Id, OrgId: 123123})
require.NoError(t, err)
query := models.GetDataSourcesQuery{OrgId: 10}
err = GetDataSources(&query)
require.NoError(t, err)
Convey("when someone else updated between read and update", func() {
query := models.GetDataSourcesQuery{OrgId: 10}
err = GetDataSources(&query)
So(err, ShouldBeNil)
ds := query.Result[0]
intendedUpdate := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: ds.Version,
}
updateFromOtherUser := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: ds.Version,
}
err := UpdateDataSource(updateFromOtherUser)
So(err, ShouldBeNil)
err = UpdateDataSource(intendedUpdate)
So(err, ShouldNotBeNil)
})
Convey("updating datasource without version", func() {
cmd := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
}
Convey("should not raise errors", func() {
err := UpdateDataSource(cmd)
So(err, ShouldBeNil)
})
})
Convey("updating datasource without higher version", func() {
cmd := &models.UpdateDataSourceCommand{
Id: ds.Id,
OrgId: 10,
Name: "nisse",
Type: models.DS_GRAPHITE,
Access: models.DS_ACCESS_PROXY,
Url: "http://test",
Version: 90000,
}
Convey("should not raise errors", func() {
err := UpdateDataSource(cmd)
So(err, ShouldBeNil)
})
})
})
Convey("Can delete datasource by id", func() {
err := DeleteDataSourceById(&models.DeleteDataSourceByIdCommand{Id: ds.Id, OrgId: ds.OrgId})
So(err, ShouldBeNil)
err = GetDataSources(&query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 0)
})
Convey("Can delete datasource by name", func() {
err := DeleteDataSourceByName(&models.DeleteDataSourceByNameCommand{Name: ds.Name, OrgId: ds.OrgId})
So(err, ShouldBeNil)
err = GetDataSources(&query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 0)
})
Convey("Can not delete datasource with wrong orgId", func() {
err := DeleteDataSourceById(&models.DeleteDataSourceByIdCommand{Id: ds.Id, OrgId: 123123})
So(err, ShouldBeNil)
err = GetDataSources(&query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 1)
})
require.Equal(t, 1, len(query.Result))
})
})
t.Run("DeleteDataSourceByName", func(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
query := models.GetDataSourcesQuery{OrgId: 10}
err := DeleteDataSourceByName(&models.DeleteDataSourceByNameCommand{Name: ds.Name, OrgId: ds.OrgId})
require.NoError(t, err)
err = GetDataSources(&query)
require.NoError(t, err)
require.Equal(t, 0, len(query.Result))
})
}
pkg/services/sqlstore/migrations/datasource_mig.go
+18
View File
@@ -133,4 +133,22 @@ func addDataSourceMigration(mg *Migrator) {
const setEmptyJSONWhereNullJSON = `UPDATE data_source SET json_data = '{}' WHERE json_data is null`
mg.AddMigration("Update json_data with nulls", NewRawSqlMigration(setEmptyJSONWhereNullJSON))
// add column uid for linking
mg.AddMigration("Add uid column", NewAddColumnMigration(tableV2, &Column{
Name: "uid", Type: DB_NVarchar, Length: 40, Nullable: false, Default: "0",
}))
// Initialize as id as that is unique already
mg.AddMigration(
"Update uid value",
NewRawSqlMigration("").
Sqlite("UPDATE data_source SET uid=printf('%09d',id);").
Postgres("UPDATE data_source SET uid=lpad('' || id::text,9,'0');").
Mysql("UPDATE data_source SET uid=lpad(id,9,'0');"),
)
mg.AddMigration("Add unique index datasource_org_id_uid", NewAddIndexMigration(tableV2, &Index{
Cols: []string{"org_id", "uid"}, Type: UniqueIndex,
}))
}
pkg/services/sqlstore/migrator/dialect.go
+1
View File
@@ -48,6 +48,7 @@ type Dialect interface {
NoOpSql() string
IsUniqueConstraintViolation(err error) bool
ErrorMessage(err error) string
IsDeadlock(err error) bool
}
pkg/services/sqlstore/migrator/mysql_dialect.go
+7
View File
@@ -148,6 +148,13 @@ func (db *Mysql) IsUniqueConstraintViolation(err error) bool {
return db.isThisError(err, mysqlerr.ER_DUP_ENTRY)
}
func (db *Mysql) ErrorMessage(err error) string {
if driverErr, ok := err.(*mysql.MySQLError); ok {
return driverErr.Message
}
return ""
}
func (db *Mysql) IsDeadlock(err error) bool {
return db.isThisError(err, mysqlerr.ER_LOCK_DEADLOCK)
}
pkg/services/sqlstore/migrator/postgres_dialect.go
+8 -1
View File
@@ -2,11 +2,11 @@ package migrator
import (
"fmt"
"github.com/lib/pq"
"strconv"
"strings"
"github.com/grafana/grafana/pkg/util/errutil"
"github.com/lib/pq"
"xorm.io/xorm"
)
@@ -149,6 +149,13 @@ func (db *Postgres) isThisError(err error, errcode string) bool {
return false
}
func (db *Postgres) ErrorMessage(err error) string {
if driverErr, ok := err.(*pq.Error); ok {
return driverErr.Message
}
return ""
}
func (db *Postgres) IsUniqueConstraintViolation(err error) bool {
return db.isThisError(err, "23505")
}
pkg/services/sqlstore/migrator/sqlite_dialect.go
+7 -1
View File
@@ -2,7 +2,6 @@ package migrator
import (
"fmt"
"github.com/mattn/go-sqlite3"
"xorm.io/xorm"
)
@@ -95,6 +94,13 @@ func (db *Sqlite3) isThisError(err error, errcode int) bool {
return false
}
func (db *Sqlite3) ErrorMessage(err error) string {
if driverErr, ok := err.(sqlite3.Error); ok {
return driverErr.Error()
}
return ""
}
func (db *Sqlite3) IsUniqueConstraintViolation(err error) bool {
return db.isThisError(err, int(sqlite3.ErrConstraintUnique))
}