From 08e2bbef832644fa8234086a854a71d7cca2f038 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 14 Nov 2017 13:34:27 +0300 Subject: [PATCH] dashboard history: clean up dashboard version history --- conf/defaults.ini | 9 +- pkg/models/dashboard_version.go | 7 ++ pkg/services/cleanup/cleanup.go | 5 ++ pkg/services/sqlstore/dashboard_version.go | 97 ++++++++++++++++++++++ pkg/setting/setting.go | 7 ++ 5 files changed, 124 insertions(+), 1 deletion(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index a4c87356a55..e6728580fb9 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -174,6 +174,7 @@ disable_gravatar = false # data source proxy whitelist (ip_or_domain:port separated by spaces) data_source_proxy_whitelist = +#################################### Snapshots ########################### [snapshots] # snapshot sharing options external_enabled = true @@ -186,7 +187,13 @@ snapshot_remove_expired = true # remove snapshots after 90 days snapshot_TTL_days = 90 -#################################### Users #################################### +#################################### Dashboards History ################## +[dashboards.history] +# A setting of 20 (default) means only the last 20 versions will be stored and older versions removed. +# To keep all dashboard versions you can set this value to 2147483647. +versions_to_keep = 20 + +#################################### Users ############################### [users] # disable user signup / registration allow_sign_up = false diff --git a/pkg/models/dashboard_version.go b/pkg/models/dashboard_version.go index 06b5797e57c..4acb4282a58 100644 --- a/pkg/models/dashboard_version.go +++ b/pkg/models/dashboard_version.go @@ -69,3 +69,10 @@ type GetDashboardVersionsQuery struct { Result []*DashboardVersionDTO } + +// +// Commands +// + +type DeleteExpiredVersionsCommand struct { +} diff --git a/pkg/services/cleanup/cleanup.go b/pkg/services/cleanup/cleanup.go index ffaa75de9cc..1b85a1819e0 100644 --- a/pkg/services/cleanup/cleanup.go +++ b/pkg/services/cleanup/cleanup.go @@ -45,6 +45,7 @@ func (service *CleanUpService) start(ctx context.Context) error { case <-ticker.C: service.cleanUpTmpFiles() service.deleteExpiredSnapshots() + service.deleteExpiredDashboardVersions() case <-ctx.Done(): return ctx.Err() } @@ -83,3 +84,7 @@ func (service *CleanUpService) cleanUpTmpFiles() { func (service *CleanUpService) deleteExpiredSnapshots() { bus.Dispatch(&m.DeleteExpiredSnapshotsCommand{}) } + +func (service *CleanUpService) deleteExpiredDashboardVersions() { + bus.Dispatch(&m.DeleteExpiredVersionsCommand{}) +} diff --git a/pkg/services/sqlstore/dashboard_version.go b/pkg/services/sqlstore/dashboard_version.go index 0b296650b4b..d55674da1b2 100644 --- a/pkg/services/sqlstore/dashboard_version.go +++ b/pkg/services/sqlstore/dashboard_version.go @@ -1,13 +1,21 @@ package sqlstore import ( + "fmt" + "math" + "sort" + "strconv" + "strings" + "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" ) func init() { bus.AddHandler("sql", GetDashboardVersion) bus.AddHandler("sql", GetDashboardVersions) + bus.AddHandler("sql", DeleteExpiredVersions) } // GetDashboardVersion gets the dashboard version for the given dashboard ID and version number. @@ -58,3 +66,92 @@ func GetDashboardVersions(query *m.GetDashboardVersionsQuery) error { } return nil } + +func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error { + return inTransaction(func(sess *DBSession) error { + var expiredCount int64 = 0 + var versions []DashboardVersionExp + + // Don't clean up if user set versions_to_keep to 2147483647 (MaxInt32) + if versionsToKeep := setting.DashboardVersionsToKeep; versionsToKeep < math.MaxInt32 { + // Get dashboard ids to clean up + affectedDashboardsQuery := fmt.Sprintf(`SELECT dashboard_id FROM dashboard_version + GROUP BY dashboard_id HAVING COUNT(dashboard_version.id)>%d`, versionsToKeep) + + err := x.Table("dashboard_version"). + Select("dashboard_version.id, dashboard_version.version, dashboard_version.dashboard_id"). + Where(fmt.Sprintf("dashboard_id IN (%s)", affectedDashboardsQuery)). + Find(&versions) + + if err != nil { + return err + } + + // Keep last versionsToKeep versions and delete other + versionIdsToDelete := getVersionIDsToDelete(versions, versionsToKeep) + versionIdsToDeleteStr := getVersionIDsToDeleteStr(versionIdsToDelete) + deleteExpiredSql := fmt.Sprintf("DELETE FROM dashboard_version WHERE id IN (%v)", strings.Join(versionIdsToDeleteStr, ", ")) + expiredResponse, err := x.Exec(deleteExpiredSql) + if err != nil { + return err + } + expiredCount, _ = expiredResponse.RowsAffected() + } + + sqlog.Debug("Deleted old/expired dashboard versions", "expired", expiredCount) + return nil + }) +} + +// Short version of DashboardVersion for getting expired versions +type DashboardVersionExp struct { + Id int64 `json:"id"` + DashboardId int64 `json:"dashboardId"` + Version int `json:"version"` +} + +// Implement sort.Interface for []DashboardVersionExp (sort by Version field) +type ByVersion []DashboardVersionExp + +func (v ByVersion) Len() int { + return len(v) +} + +func (v ByVersion) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +func (v ByVersion) Less(i, j int) bool { + return v[i].Version < v[j].Version +} + +func getVersionIDsToDelete(versions []DashboardVersionExp, versionsToKeep int) []int64 { + dashboards := make(map[int64][]DashboardVersionExp) + for _, v := range versions { + elem, present := dashboards[v.DashboardId] + if present { + dashboards[v.DashboardId] = append(elem, v) + } else { + dashboards[v.DashboardId] = []DashboardVersionExp{v} + } + } + + versionIds := make([]int64, 0) + for dashboard_id, versions := range dashboards { + sort.Sort(sort.Reverse(ByVersion(versions))) + dashboards[dashboard_id] = versions[versionsToKeep:] + for _, ver := range dashboards[dashboard_id] { + versionIds = append(versionIds, ver.Id) + } + } + + return versionIds +} + +func getVersionIDsToDeleteStr(versionIds []int64) []string { + var versionIdsToDeleteStr []string + for _, versionId := range versionIds { + versionIdsToDeleteStr = append(versionIdsToDeleteStr, strconv.FormatInt(versionId, 10)) + } + return versionIdsToDeleteStr +} diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index ca65fe581af..20c9b8703d4 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -89,6 +89,9 @@ var ( SnapShotTTLDays int SnapShotRemoveExpired bool + // Dashboard history + DashboardVersionsToKeep int + // User settings AllowUserSignUp bool AllowUserOrgCreate bool @@ -518,6 +521,10 @@ func NewConfigContext(args *CommandLineArgs) error { SnapShotRemoveExpired = snapshots.Key("snapshot_remove_expired").MustBool(true) SnapShotTTLDays = snapshots.Key("snapshot_TTL_days").MustInt(90) + // read dashboard settings + dashboards := Cfg.Section("dashboards") + DashboardVersionsToKeep = dashboards.Key("snapshot_TTL_days").MustInt(20) + // read data source proxy white list DataProxyWhiteList = make(map[string]bool) for _, hostAndIp := range util.SplitString(security.Key("data_source_proxy_whitelist").String()) {