public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Chore: Refactor quota service (#58643)

Browse Source 
Chore: Refactor quota service (#57586)

* Chore: refactore quota service

* Apply suggestions from code review
This commit is contained in:
Sofia Papagiannaki
2022-11-14 21:08:10 +02:00
committed by GitHub
parent dd0d034796
commit 9855e74b92
99 changed files with 2596 additions and 1398 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/datasources/models.go
+6
View File
@@ -4,6 +4,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/services/quota"
"github.com/grafana/grafana/pkg/services/user"
)
@@ -193,3 +194,8 @@ type DatasourcesPermissionFilterQuery struct {
Datasources []*DataSource
Result []*DataSource
}
const (
QuotaTargetSrv quota.TargetSrv = "data_source"
QuotaTarget quota.Target = "data_source"
)
pkg/services/datasources/service/datasource.go
+41 -2
View File
@@ -20,6 +20,7 @@ import (
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/quota"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
"github.com/grafana/grafana/pkg/setting"
@@ -52,7 +53,8 @@ type cachedRoundTripper struct {
func ProvideService(
db db.DB, secretsService secrets.Service, secretsStore kvstore.SecretsKVStore, cfg *setting.Cfg,
features featuremgmt.FeatureToggles, ac accesscontrol.AccessControl, datasourcePermissionsService accesscontrol.DatasourcePermissionsService,
) *Service {
quotaService quota.Service,
) (*Service, error) {
dslogger := log.New("datasources")
store := &SqlStore{db: db, logger: dslogger}
s := &Service{
@@ -73,7 +75,23 @@ func ProvideService(
ac.RegisterScopeAttributeResolver(NewNameScopeResolver(store))
ac.RegisterScopeAttributeResolver(NewIDScopeResolver(store))
return s
defaultLimits, err := readQuotaConfig(cfg)
if err != nil {
return nil, err
}
if err := quotaService.RegisterQuotaReporter(&quota.NewUsageReporter{
TargetSrv: datasources.QuotaTargetSrv,
DefaultLimits: defaultLimits,
Reporter: s.Usage,
}); err != nil {
return nil, err
}
return s, nil
}
func (s *Service) Usage(ctx context.Context, scopeParams *quota.ScopeParameters) (*quota.Map, error) {
return s.SQLStore.Count(ctx, scopeParams)
}
// DataSourceRetriever interface for retrieving a datasource.
@@ -591,3 +609,24 @@ func (s *Service) fillWithSecureJSONData(ctx context.Context, cmd *datasources.U
return nil
}
func readQuotaConfig(cfg *setting.Cfg) (*quota.Map, error) {
limits := &quota.Map{}
if cfg == nil {
return limits, nil
}
globalQuotaTag, err := quota.NewTag(datasources.QuotaTargetSrv, datasources.QuotaTarget, quota.GlobalScope)
if err != nil {
return limits, err
}
orgQuotaTag, err := quota.NewTag(datasources.QuotaTargetSrv, datasources.QuotaTarget, quota.OrgScope)
if err != nil {
return limits, err
}
limits.Set(globalQuotaTag, cfg.Quota.Global.DataSource)
limits.Set(orgQuotaTag, cfg.Quota.Org.DataSource)
return limits, nil
}
pkg/services/datasources/service/datasource_test.go
+34 -11
View File
@@ -21,6 +21,7 @@ import (
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/quota/quotatest"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/secrets/fakes"
secretskvs "github.com/grafana/grafana/pkg/services/secrets/kvstore"
@@ -200,7 +201,9 @@ func TestService_GetHttpTransport(t *testing.T) {
sqlStore := db.InitTestDB(t)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
quotaService := quotatest.New(false, nil)
dsService, err := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService(), quotaService)
require.NoError(t, err)
rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err)
@@ -235,7 +238,9 @@ func TestService_GetHttpTransport(t *testing.T) {
sqlStore := db.InitTestDB(t)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
quotaService := quotatest.New(false, nil)
dsService, err := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService(), quotaService)
require.NoError(t, err)
ds := datasources.DataSource{
Id: 1,
@@ -284,7 +289,9 @@ func TestService_GetHttpTransport(t *testing.T) {
sqlStore := db.InitTestDB(t)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
quotaService := quotatest.New(false, nil)
dsService, err := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService(), quotaService)
require.NoError(t, err)
ds := datasources.DataSource{
Id: 1,
@@ -330,7 +337,9 @@ func TestService_GetHttpTransport(t *testing.T) {
sqlStore := db.InitTestDB(t)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
quotaService := quotatest.New(false, nil)
dsService, err := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService(), quotaService)
require.NoError(t, err)
ds := datasources.DataSource{
Id: 1,
@@ -373,7 +382,9 @@ func TestService_GetHttpTransport(t *testing.T) {
sqlStore := db.InitTestDB(t)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
quotaService := quotatest.New(false, nil)
dsService, err := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService(), quotaService)
require.NoError(t, err)
ds := datasources.DataSource{
Id: 1,
@@ -406,7 +417,9 @@ func TestService_GetHttpTransport(t *testing.T) {
sqlStore := db.InitTestDB(t)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
quotaService := quotatest.New(false, nil)
dsService, err := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService(), quotaService)
require.NoError(t, err)
ds := datasources.DataSource{
Id: 1,
@@ -473,7 +486,9 @@ func TestService_GetHttpTransport(t *testing.T) {
sqlStore := db.InitTestDB(t)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
quotaService := quotatest.New(false, nil)
dsService, err := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService(), quotaService)
require.NoError(t, err)
ds := datasources.DataSource{
Id: 1,
Url: "http://k8s:8001",
@@ -507,7 +522,9 @@ func TestService_GetHttpTransport(t *testing.T) {
sqlStore := db.InitTestDB(t)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
quotaService := quotatest.New(false, nil)
dsService, err := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService(), quotaService)
require.NoError(t, err)
ds := datasources.DataSource{
Type: datasources.DS_ES,
@@ -544,7 +561,9 @@ func TestService_getTimeout(t *testing.T) {
sqlStore := db.InitTestDB(t)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
dsService := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
quotaService := quotatest.New(false, nil)
dsService, err := ProvideService(sqlStore, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService(), quotaService)
require.NoError(t, err)
for _, tc := range testCases {
ds := &datasources.DataSource{
@@ -565,7 +584,9 @@ func TestService_GetDecryptedValues(t *testing.T) {
sqlStore := db.InitTestDB(t)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
dsService := ProvideService(sqlStore, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
quotaService := quotatest.New(false, nil)
dsService, err := ProvideService(sqlStore, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService(), quotaService)
require.NoError(t, err)
jsonData := map[string]string{
"password": "securePassword",
@@ -591,7 +612,9 @@ func TestService_GetDecryptedValues(t *testing.T) {
sqlStore := db.InitTestDB(t)
secretsService := secretsmng.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsStore := secretskvs.NewSQLSecretsKVStore(sqlStore, secretsService, log.New("test.logger"))
dsService := ProvideService(sqlStore, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService())
quotaService := quotatest.New(false, nil)
dsService, err := ProvideService(sqlStore, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewMockedPermissionsService(), quotaService)
require.NoError(t, err)
jsonData := map[string]string{
"password": "securePassword",
pkg/services/datasources/service/store.go
+48
View File
@@ -16,6 +16,8 @@ import (
"github.com/grafana/grafana/pkg/infra/metrics"
ac "github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/quota"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/util"
)
@@ -29,6 +31,8 @@ type Store interface {
AddDataSource(context.Context, *datasources.AddDataSourceCommand) error
UpdateDataSource(context.Context, *datasources.UpdateDataSourceCommand) error
GetAllDataSources(ctx context.Context, query *datasources.GetAllDataSourcesQuery) error
Count(context.Context, *quota.ScopeParameters) (*quota.Map, error)
}
type SqlStore struct {
@@ -171,6 +175,50 @@ func (ss *SqlStore) DeleteDataSource(ctx context.Context, cmd *datasources.Delet
})
}
func (ss *SqlStore) Count(ctx context.Context, scopeParams *quota.ScopeParameters) (*quota.Map, error) {
u := &quota.Map{}
type result struct {
Count int64
}
r := result{}
if err := ss.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
rawSQL := "SELECT COUNT(*) AS count FROM data_source"
if _, err := sess.SQL(rawSQL).Get(&r); err != nil {
return err
}
return nil
}); err != nil {
return u, err
} else {
tag, err := quota.NewTag(datasources.QuotaTargetSrv, datasources.QuotaTarget, quota.GlobalScope)
if err != nil {
return u, err
}
u.Set(tag, r.Count)
}
if scopeParams.OrgID != 0 {
if err := ss.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
rawSQL := "SELECT COUNT(*) AS count FROM data_source WHERE org_id=?"
if _, err := sess.SQL(rawSQL, scopeParams.OrgID).Get(&r); err != nil {
return err
}
return nil
}); err != nil {
return u, err
} else {
tag, err := quota.NewTag(datasources.QuotaTargetSrv, datasources.QuotaTarget, quota.OrgScope)
if err != nil {
return u, err
}
u.Set(tag, r.Count)
}
}
return u, nil
}
func (ss *SqlStore) AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) error {
return ss.db.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
existing := datasources.DataSource{OrgId: cmd.OrgId, Name: cmd.Name}