public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

API: Add by UID routes for data sources (#29884)

Browse Source 
- also add Get by UID+OrgID to datasource cache
- Refactor backend commands for Delete and Get to be unified
This commit is contained in:
Kyle Brandt
2021-01-13 13:16:27 -05:00
committed by GitHub
parent d5cbb17666
commit 1c1a800bc0
18 changed files with 295 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/alerting/alerting_usage.go
+1 -1
View File
@@ -61,7 +61,7 @@ func (e *AlertEngine) mapRulesToUsageStats(rules []*models.Alert) (DatasourceAle
// map of datsource types and frequency
result := map[string]int{}
for k, v := range typeCount {
query := &models.GetDataSourceByIdQuery{Id: k}
query := &models.GetDataSourceQuery{Id: k}
err := e.Bus.Dispatch(query)
if err != nil {
return map[string]int{}, nil
pkg/services/alerting/alerting_usage_test.go
+1 -1
View File
@@ -38,7 +38,7 @@ func TestAlertingUsageStats(t *testing.T) {
return nil
})
ae.Bus.AddHandler(func(query *models.GetDataSourceByIdQuery) error {
ae.Bus.AddHandler(func(query *models.GetDataSourceQuery) error {
ds := map[int64]*models.DataSource{
1: {Type: "influxdb"},
2: {Type: "graphite"},
pkg/services/alerting/conditions/query.go
+1 -1
View File
@@ -110,7 +110,7 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) (*alerting.Conditio
}
func (c *QueryCondition) executeQuery(context *alerting.EvalContext, timeRange *tsdb.TimeRange) (tsdb.TimeSeriesSlice, error) {
getDsInfo := &models.GetDataSourceByIdQuery{
getDsInfo := &models.GetDataSourceQuery{
Id: c.Query.DatasourceID,
OrgId: context.Rule.OrgID,
}
pkg/services/alerting/conditions/query_test.go
+1 -1
View File
@@ -228,7 +228,7 @@ func (ctx *queryConditionTestContext) exec() (*alerting.ConditionResult, error)
func queryConditionScenario(desc string, fn queryConditionScenarioFunc) {
Convey(desc, func() {
bus.AddHandler("test", func(query *models.GetDataSourceByIdQuery) error {
bus.AddHandler("test", func(query *models.GetDataSourceQuery) error {
query.Result = &models.DataSource{Id: 1, Type: "graphite"}
return nil
})
pkg/services/alerting/extractor.go
+1 -1
View File
@@ -39,7 +39,7 @@ func (e *DashAlertExtractor) lookupDatasourceID(dsName string) (*models.DataSour
return query.Result, nil
}
query := &models.GetDataSourceByNameQuery{Name: dsName, OrgId: e.OrgID}
query := &models.GetDataSourceQuery{Name: dsName, OrgId: e.OrgID}
if err := bus.Dispatch(query); err != nil {
return nil, err
}
pkg/services/alerting/extractor_test.go
+1 -1
View File
@@ -29,7 +29,7 @@ func TestAlertRuleExtraction(t *testing.T) {
return nil
})
bus.AddHandler("test", func(query *models.GetDataSourceByNameQuery) error {
bus.AddHandler("test", func(query *models.GetDataSourceQuery) error {
if query.Name == defaultDs.Name {
query.Result = defaultDs
}
pkg/services/datasources/cache.go
+47 -2
View File
@@ -12,6 +12,7 @@ import (
type CacheService interface {
GetDatasource(datasourceID int64, user *models.SignedInUser, skipCache bool) (*models.DataSource, error)
GetDatasourceByUID(datasourceUID string, user *models.SignedInUser, skipCache bool) (*models.DataSource, error)
}
type CacheServiceImpl struct {
@@ -36,7 +37,7 @@ func (dc *CacheServiceImpl) GetDatasource(
user *models.SignedInUser,
skipCache bool,
) (*models.DataSource, error) {
cacheKey := fmt.Sprintf("ds-%d", datasourceID)
cacheKey := idKey(datasourceID)
if !skipCache {
if cached, found := dc.CacheService.Get(cacheKey); found {
@@ -48,11 +49,55 @@ func (dc *CacheServiceImpl) GetDatasource(
}
plog.Debug("Querying for data source via SQL store", "id", datasourceID, "orgId", user.OrgId)
ds, err := dc.SQLStore.GetDataSourceByID(datasourceID, user.OrgId)
ds, err := dc.SQLStore.GetDataSource("", datasourceID, "", user.OrgId)
if err != nil {
return nil, err
}
if ds.Uid != "" {
dc.CacheService.Set(uidKey(ds.OrgId, ds.Uid), ds, time.Second*5)
}
dc.CacheService.Set(cacheKey, ds, time.Second*5)
return ds, nil
}
func (dc *CacheServiceImpl) GetDatasourceByUID(
datasourceUID string,
user *models.SignedInUser,
skipCache bool,
) (*models.DataSource, error) {
if datasourceUID == "" {
return nil, fmt.Errorf("can not get data source by uid, uid is empty")
}
if user.OrgId == 0 {
return nil, fmt.Errorf("can not get data source by uid, orgId is missing")
}
uidCacheKey := uidKey(user.OrgId, datasourceUID)
if !skipCache {
if cached, found := dc.CacheService.Get(uidCacheKey); found {
ds := cached.(*models.DataSource)
if ds.OrgId == user.OrgId {
return ds, nil
}
}
}
plog.Debug("Querying for data source via SQL store", "uid", datasourceUID, "orgId", user.OrgId)
ds, err := dc.SQLStore.GetDataSource(datasourceUID, 0, "", user.OrgId)
if err != nil {
return nil, err
}
dc.CacheService.Set(uidCacheKey, ds, time.Second*5)
dc.CacheService.Set(idKey(ds.Id), ds, time.Second*5)
return ds, nil
}
func idKey(id int64) string {
return fmt.Sprintf("ds-%d", id)
}
func uidKey(orgID int64, uid string) string {
return fmt.Sprintf("ds-orgid-uid-%d-%s", orgID, uid)
}
pkg/services/provisioning/datasources/config_reader_test.go
+3 -3
View File
@@ -260,13 +260,13 @@ func validateDatasourceV1(dsCfg *configs) {
type fakeRepository struct {
inserted []*models.AddDataSourceCommand
deleted []*models.DeleteDataSourceByNameCommand
deleted []*models.DeleteDataSourceCommand
updated []*models.UpdateDataSourceCommand
loadAll []*models.DataSource
}
func mockDelete(cmd *models.DeleteDataSourceByNameCommand) error {
func mockDelete(cmd *models.DeleteDataSourceCommand) error {
fakeRepo.deleted = append(fakeRepo.deleted, cmd)
return nil
}
@@ -281,7 +281,7 @@ func mockInsert(cmd *models.AddDataSourceCommand) error {
return nil
}
func mockGet(cmd *models.GetDataSourceByNameQuery) error {
func mockGet(cmd *models.GetDataSourceQuery) error {
for _, v := range fakeRepo.loadAll {
if cmd.Name == v.Name && cmd.OrgId == v.OrgId {
cmd.Result = v
pkg/services/provisioning/datasources/datasources.go
+2 -2
View File
@@ -43,7 +43,7 @@ func (dc *DatasourceProvisioner) apply(cfg *configs) error {
}
for _, ds := range cfg.Datasources {
cmd := &models.GetDataSourceByNameQuery{OrgId: ds.OrgID, Name: ds.Name}
cmd := &models.GetDataSourceQuery{OrgId: ds.OrgID, Name: ds.Name}
err := bus.Dispatch(cmd)
if err != nil && !errors.Is(err, models.ErrDataSourceNotFound) {
return err
@@ -84,7 +84,7 @@ func (dc *DatasourceProvisioner) applyChanges(configPath string) error {
func (dc *DatasourceProvisioner) deleteDatasources(dsToDelete []*deleteDatasourceConfig) error {
for _, ds := range dsToDelete {
cmd := &models.DeleteDataSourceByNameCommand{OrgId: ds.OrgID, Name: ds.Name}
cmd := &models.DeleteDataSourceCommand{OrgID: ds.OrgID, Name: ds.Name}
if err := bus.Dispatch(cmd); err != nil {
return err
}
pkg/services/sqlstore/datasource.go
+68 -44
View File
@@ -18,53 +18,50 @@ import (
func init() {
bus.AddHandler("sql", GetDataSources)
bus.AddHandler("sql", GetDataSource)
bus.AddHandler("sql", AddDataSource)
bus.AddHandler("sql", DeleteDataSourceById)
bus.AddHandler("sql", DeleteDataSourceByName)
bus.AddHandler("sql", DeleteDataSource)
bus.AddHandler("sql", UpdateDataSource)
bus.AddHandler("sql", GetDataSourceById)
bus.AddHandler("sql", GetDataSourceByName)
bus.AddHandler("sql", GetDefaultDataSource)
}
func getDataSourceByID(id, orgID int64, engine *xorm.Engine) (*models.DataSource, error) {
metrics.MDBDataSourceQueryByID.Inc()
// GetDataSource returns a datasource by org_id and either uid (preferred), id, or name.
// Zero values (0, or "") should be used for the parameters that will not be queried.
func (ss *SQLStore) GetDataSource(uid string, id int64, name string, orgID int64) (*models.DataSource, error) {
query := &models.GetDataSourceQuery{
Id: id,
Uid: uid,
Name: name,
OrgId: orgID,
}
datasource := models.DataSource{OrgId: orgID, Id: id}
has, err := engine.Get(&datasource)
if err != nil {
sqlog.Error("Failed getting data source", "err", err, "id", id, "orgId", orgID)
if err := GetDataSource(query); err != nil {
return nil, err
}
if !has {
sqlog.Debug("Failed to find data source", "id", id, "orgId", orgID)
return nil, models.ErrDataSourceNotFound
return query.Result, nil
}
// GetDataSource adds a datasource to the query model by querying by org_id as well as
// either uid (preferred), id, or name and is added to the bus.
func GetDataSource(query *models.GetDataSourceQuery) error {
metrics.MDBDataSourceQueryByID.Inc()
if query.OrgId == 0 || (query.Id == 0 && len(query.Name) == 0 && len(query.Uid) == 0) {
return models.ErrDataSourceIdentifierNotSet
}
return &datasource, nil
}
func (ss *SQLStore) GetDataSourceByID(id, orgID int64) (*models.DataSource, error) {
return getDataSourceByID(id, orgID, ss.engine)
}
func GetDataSourceById(query *models.GetDataSourceByIdQuery) error {
ds, err := getDataSourceByID(query.Id, query.OrgId, x)
query.Result = ds
return err
}
func GetDataSourceByName(query *models.GetDataSourceByNameQuery) error {
datasource := models.DataSource{OrgId: query.OrgId, Name: query.Name}
datasource := models.DataSource{Name: query.Name, OrgId: query.OrgId, Id: query.Id, Uid: query.Uid}
has, err := x.Get(&datasource)
if !has {
if err != nil {
sqlog.Error("Failed getting data source", "err", err, "uid", query.Uid, "id", query.Id, "name", query.Name, "orgId", query.OrgId)
return err
} else if !has {
return models.ErrDataSourceNotFound
}
query.Result = &datasource
return err
return nil
}
func GetDataSources(query *models.GetDataSourcesQuery) error {
@@ -92,22 +89,49 @@ func GetDefaultDataSource(query *models.GetDefaultDataSourceQuery) error {
return err
}
func DeleteDataSourceById(cmd *models.DeleteDataSourceByIdCommand) error {
return inTransaction(func(sess *DBSession) error {
var rawSQL = "DELETE FROM data_source WHERE id=? and org_id=?"
result, err := sess.Exec(rawSQL, cmd.Id, cmd.OrgId)
affected, _ := result.RowsAffected()
cmd.DeletedDatasourcesCount = affected
return err
})
// DeleteDataSource deletes a datasource by org_id and either uid (preferred), id, or name.
// Zero values (0, or "") should be used for the parameters that will not be queried.
func (ss *SQLStore) DeleteDataSource(uid string, id int64, name string, orgID int64) (int64, error) {
cmd := &models.DeleteDataSourceCommand{
ID: id,
UID: uid,
Name: name,
OrgID: orgID,
}
if err := DeleteDataSource(cmd); err != nil {
return 0, err
}
return cmd.DeletedDatasourcesCount, nil
}
func DeleteDataSourceByName(cmd *models.DeleteDataSourceByNameCommand) error {
// DeleteDataSource removes a datasource by org_id as well as either uid (preferred), id, or name
// and is added to the bus.
func DeleteDataSource(cmd *models.DeleteDataSourceCommand) error {
params := make([]interface{}, 0)
makeQuery := func(sql string, p ...interface{}) {
params = append(params, sql)
params = append(params, p...)
}
switch {
case cmd.OrgID == 0:
return models.ErrDataSourceIdentifierNotSet
case cmd.UID != "":
makeQuery("DELETE FROM data_source WHERE uid=? and org_id=?", cmd.UID, cmd.OrgID)
case cmd.ID != 0:
makeQuery("DELETE FROM data_source WHERE id=? and org_id=?", cmd.ID, cmd.OrgID)
case cmd.Name != "":
makeQuery("DELETE FROM data_source WHERE name=? and org_id=?", cmd.Name, cmd.OrgID)
default:
return models.ErrDataSourceIdentifierNotSet
}
return inTransaction(func(sess *DBSession) error {
var rawSQL = "DELETE FROM data_source WHERE name=? and org_id=?"
result, err := sess.Exec(rawSQL, cmd.Name, cmd.OrgId)
affected, _ := result.RowsAffected()
cmd.DeletedDatasourcesCount = affected
result, err := sess.Exec(params...)
cmd.DeletedDatasourcesCount, _ = result.RowsAffected()
return err
})
}
pkg/services/sqlstore/datasource_test.go
+5 -5
View File
@@ -110,8 +110,8 @@ func TestDataAccess(t *testing.T) {
err := UpdateDataSource(&cmd)
require.NoError(t, err)
query := models.GetDataSourceByIdQuery{Id: ds.Id}
err = GetDataSourceById(&query)
query := models.GetDataSourceQuery{Id: ds.Id, OrgId: 10}
err = GetDataSource(&query)
require.NoError(t, err)
require.Equal(t, ds.Uid, query.Result.Uid)
})
@@ -180,7 +180,7 @@ func TestDataAccess(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
err := DeleteDataSourceById(&models.DeleteDataSourceByIdCommand{Id: ds.Id, OrgId: ds.OrgId})
err := DeleteDataSource(&models.DeleteDataSourceCommand{ID: ds.Id, OrgID: ds.OrgId})
require.NoError(t, err)
query := models.GetDataSourcesQuery{OrgId: 10}
@@ -194,7 +194,7 @@ func TestDataAccess(t *testing.T) {
InitTestDB(t)
ds := initDatasource()
err := DeleteDataSourceById(&models.DeleteDataSourceByIdCommand{Id: ds.Id, OrgId: 123123})
err := DeleteDataSource(&models.DeleteDataSourceCommand{ID: ds.Id, OrgID: 123123})
require.NoError(t, err)
query := models.GetDataSourcesQuery{OrgId: 10}
err = GetDataSources(&query)
@@ -209,7 +209,7 @@ func TestDataAccess(t *testing.T) {
ds := initDatasource()
query := models.GetDataSourcesQuery{OrgId: 10}
err := DeleteDataSourceByName(&models.DeleteDataSourceByNameCommand{Name: ds.Name, OrgId: ds.OrgId})
err := DeleteDataSource(&models.DeleteDataSourceCommand{Name: ds.Name, OrgID: ds.OrgId})
require.NoError(t, err)
err = GetDataSources(&query)