public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split Delete expired dashboard version store (#49610)

Browse Source 
* Split Delete expired dashboard version store

* Add method to fakes

* Fix lint

* Fix lint 2

* Use split store method in cleanup

* Add tests

* Remove DeleteExpiredVersions from sqlstore

* Fix lint

* Fix integration tests
This commit is contained in:
idafurjes
2022-05-31 11:56:05 +02:00
committed by GitHub
parent 72367cf1ad
commit f69c9bd704
12 changed files with 177 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/dashboardversion/dashverimpl/dashver.go
+36 -1
View File
@@ -8,11 +8,16 @@ import (
"github.com/grafana/grafana/pkg/setting"
)
const (
maxVersionsToDeletePerBatch = 100
maxVersionDeletionBatches = 50
)
type Service struct {
store store
}
func ProvideService(db db.DB, cfg *setting.Cfg) dashver.Service {
func ProvideService(db db.DB) dashver.Service {
return &Service{
store: &sqlStore{
db: db,
@@ -28,3 +33,33 @@ func (s *Service) Get(ctx context.Context, query *dashver.GetDashboardVersionQue
version.Data.Set("id", version.DashboardID)
return version, nil
}
func (s *Service) DeleteExpired(ctx context.Context, cmd *dashver.DeleteExpiredVersionsCommand) error {
versionsToKeep := setting.DashboardVersionsToKeep
if versionsToKeep < 1 {
versionsToKeep = 1
}
for batch := 0; batch < maxVersionDeletionBatches; batch++ {
versionIdsToDelete, batchErr := s.store.GetBatch(ctx, cmd, maxVersionsToDeletePerBatch, versionsToKeep)
if batchErr != nil {
return batchErr
}
if len(versionIdsToDelete) < 1 {
return nil
}
deleted, err := s.store.DeleteBatch(ctx, cmd, versionIdsToDelete)
if err != nil {
return err
}
cmd.DeletedRows += deleted
if deleted < int64(maxVersionsToDeletePerBatch) {
break
}
}
return nil
}
pkg/services/dashboardversion/dashverimpl/dashver_test.go
+42 -4
View File
@@ -2,10 +2,12 @@ package dashverimpl
import (
"context"
"errors"
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
dashver "github.com/grafana/grafana/pkg/services/dashboardversion"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/require"
)
@@ -25,15 +27,51 @@ func TestDashboardVersionService(t *testing.T) {
})
}
type FakeDashboardVersionStroe struct {
func TestDeleteExpiredVersions(t *testing.T) {
versionsToKeep := 5
setting.DashboardVersionsToKeep = versionsToKeep
dashboardVersionStore := newDashboardVersionStoreFake()
dashboardVersionService := Service{store: dashboardVersionStore}
t.Run("Don't delete anything if there are no expired versions", func(t *testing.T) {
err := dashboardVersionService.DeleteExpired(context.Background(), &dashver.DeleteExpiredVersionsCommand{DeletedRows: 4})
require.Nil(t, err)
})
t.Run("Clean up old dashboard versions successfully", func(t *testing.T) {
dashboardVersionStore.ExptectedDeletedVersions = 4
dashboardVersionStore.ExpectedVersions = []interface{}{1, 2, 3, 4}
err := dashboardVersionService.DeleteExpired(context.Background(), &dashver.DeleteExpiredVersionsCommand{DeletedRows: 4})
require.Nil(t, err)
})
t.Run("Clean up old dashboard versions with error", func(t *testing.T) {
dashboardVersionStore.ExpectedError = errors.New("some error")
err := dashboardVersionService.DeleteExpired(context.Background(), &dashver.DeleteExpiredVersionsCommand{DeletedRows: 4})
require.NotNil(t, err)
})
}
type FakeDashboardVersionStore struct {
ExpectedDashboardVersion *dashver.DashboardVersion
ExptectedDeletedVersions int64
ExpectedVersions []interface{}
ExpectedError error
}
func newDashboardVersionStoreFake() *FakeDashboardVersionStroe {
return &FakeDashboardVersionStroe{}
func newDashboardVersionStoreFake() *FakeDashboardVersionStore {
return &FakeDashboardVersionStore{}
}
func (f *FakeDashboardVersionStroe) Get(ctx context.Context, query *dashver.GetDashboardVersionQuery) (*dashver.DashboardVersion, error) {
func (f *FakeDashboardVersionStore) Get(ctx context.Context, query *dashver.GetDashboardVersionQuery) (*dashver.DashboardVersion, error) {
return f.ExpectedDashboardVersion, f.ExpectedError
}
func (f *FakeDashboardVersionStore) GetBatch(ctx context.Context, cmd *dashver.DeleteExpiredVersionsCommand, perBatch int, versionsToKeep int) ([]interface{}, error) {
return f.ExpectedVersions, f.ExpectedError
}
func (f *FakeDashboardVersionStore) DeleteBatch(ctx context.Context, cmd *dashver.DeleteExpiredVersionsCommand, versionIdsToDelete []interface{}) (int64, error) {
return f.ExptectedDeletedVersions, f.ExpectedError
}
pkg/services/dashboardversion/dashverimpl/store.go
+42 -5
View File
@@ -2,24 +2,26 @@ package dashverimpl
import (
"context"
"strings"
"github.com/grafana/grafana/pkg/models"
dashver "github.com/grafana/grafana/pkg/services/dashboardversion"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/sqlstore/db"
)
type store interface {
Get(ctx context.Context, query *dashver.GetDashboardVersionQuery) (*dashver.DashboardVersion, error)
Get(context.Context, *dashver.GetDashboardVersionQuery) (*dashver.DashboardVersion, error)
GetBatch(context.Context, *dashver.DeleteExpiredVersionsCommand, int, int) ([]interface{}, error)
DeleteBatch(context.Context, *dashver.DeleteExpiredVersionsCommand, []interface{}) (int64, error)
}
type sqlStore struct {
db db.DB
}
func (s *sqlStore) Get(ctx context.Context, query *dashver.GetDashboardVersionQuery) (*dashver.DashboardVersion, error) {
func (ss *sqlStore) Get(ctx context.Context, query *dashver.GetDashboardVersionQuery) (*dashver.DashboardVersion, error) {
var version dashver.DashboardVersion
err := s.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
err := ss.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
has, err := sess.Where("dashboard_version.dashboard_id=? AND dashboard_version.version=? AND dashboard.org_id=?", query.DashboardID, query.Version, query.OrgID).
Join("LEFT", "dashboard", `dashboard.id = dashboard_version.dashboard_id`).
Get(&version)
@@ -29,7 +31,7 @@ func (s *sqlStore) Get(ctx context.Context, query *dashver.GetDashboardVersionQu
}
if !has {
return models.ErrDashboardVersionNotFound
return dashver.ErrDashboardVersionNotFound
}
return nil
})
@@ -38,3 +40,38 @@ func (s *sqlStore) Get(ctx context.Context, query *dashver.GetDashboardVersionQu
}
return &version, nil
}
func (ss *sqlStore) GetBatch(ctx context.Context, cmd *dashver.DeleteExpiredVersionsCommand, perBatch int, versionsToKeep int) ([]interface{}, error) {
var versionIds []interface{}
err := ss.db.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error {
versionIdsToDeleteQuery := `SELECT id
FROM dashboard_version, (
SELECT dashboard_id, count(version) as count, min(version) as min
FROM dashboard_version
GROUP BY dashboard_id
) AS vtd
WHERE dashboard_version.dashboard_id=vtd.dashboard_id
AND version < vtd.min + vtd.count - ?
LIMIT ?`
err := sess.SQL(versionIdsToDeleteQuery, versionsToKeep, perBatch).Find(&versionIds)
return err
})
return versionIds, err
}
func (ss *sqlStore) DeleteBatch(ctx context.Context, cmd *dashver.DeleteExpiredVersionsCommand, versionIdsToDelete []interface{}) (int64, error) {
var deleted int64
err := ss.db.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error {
deleteExpiredSQL := `DELETE FROM dashboard_version WHERE id IN (?` + strings.Repeat(",?", len(versionIdsToDelete)-1) + `)`
sqlOrArgs := append([]interface{}{deleteExpiredSQL}, versionIdsToDelete...)
expiredResponse, err := sess.Exec(sqlOrArgs...)
if err != nil {
return err
}
deleted, err = expiredResponse.RowsAffected()
return err
})
return deleted, err
}
pkg/services/dashboardversion/dashverimpl/store_test.go
+23 -1
View File
@@ -14,10 +14,11 @@ import (
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestIntegrationDashboardVersion(t *testing.T) {
func TestIntegrationGetDashboardVersion(t *testing.T) {
ss := sqlstore.InitTestDB(t)
dashVerStore := sqlStore{db: ss}
@@ -60,6 +61,27 @@ func TestIntegrationDashboardVersion(t *testing.T) {
})
}
func TestIntegrationDeleteExpiredVersions(t *testing.T) {
versionsToWrite := 10
ss := sqlstore.InitTestDB(t)
dashVerStore := sqlStore{db: ss}
for i := 0; i < versionsToWrite-1; i++ {
insertTestDashboard(t, ss, "test dash 53", 1, int64(i), false, "diff-all")
}
t.Run("Clean up old dashboard versions", func(t *testing.T) {
versionIDsToDelete := []interface{}{1, 2, 3, 4}
res, err := dashVerStore.DeleteBatch(
context.Background(),
&dashver.DeleteExpiredVersionsCommand{DeletedRows: 4},
versionIDsToDelete,
)
require.Nil(t, err)
assert.EqualValues(t, 4, res)
})
}
func getDashboard(t *testing.T, sqlStore *sqlstore.SQLStore, dashboard *models.Dashboard) error {
t.Helper()
return sqlStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {