diff --git a/circle.yml b/.circleci/config.yml similarity index 100% rename from circle.yml rename to .circleci/config.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 001433fa652..d601469be0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * **MSSQL**: New Microsoft SQL Server data source [#10093](https://github.com/grafana/grafana/pull/10093), [#11298](https://github.com/grafana/grafana/pull/11298), thx [@linuxchips](https://github.com/linuxchips) * **Prometheus**: The heatmap panel now support Prometheus histograms [#10009](https://github.com/grafana/grafana/issues/10009) * **Postgres/MySQL**: Ability to insert 0s or nulls for missing intervals [#9487](https://github.com/grafana/grafana/issues/9487), thanks [@svenklemm](https://github.com/svenklemm) +* **Graph**: Align left and right Y-axes to one level [#1271](https://github.com/grafana/grafana/issues/1271) & [#2740](https://github.com/grafana/grafana/issues/2740) thx [@ilgizar](https://github.com/ilgizar) * **Graph**: Thresholds for Right Y axis [#7107](https://github.com/grafana/grafana/issues/7107), thx [@ilgizar](https://github.com/ilgizar) * **Graph**: Support multiple series stacking in histogram mode [#8151](https://github.com/grafana/grafana/issues/8151), thx [@mtanda](https://github.com/mtanda) * **Alerting**: Pausing/un alerts now updates new_state_date [#10942](https://github.com/grafana/grafana/pull/10942) @@ -17,6 +18,7 @@ * **Cloudwatch**: Add dimension filtering to CloudWatch `dimension_values()` [#10029](https://github.com/grafana/grafana/issues/10029), thx [@willyhutw](https://github.com/willyhutw) * **Units**: Second to HH:mm:ss formatter [#11107](https://github.com/grafana/grafana/issues/11107), thx [@gladdiologist](https://github.com/gladdiologist) * **Singlestat**: Add color to prefix and postfix in singlestat panel [#11143](https://github.com/grafana/grafana/pull/11143), thx [@ApsOps](https://github.com/ApsOps) +* **Dashboards**: Version cleanup fails on old databases with many entries [#11278](https://github.com/grafana/grafana/issues/11278) # 5.0.4 (unreleased) * **Dashboard** Fixed bug where collapsed panels could not be directly linked to/renderer [#11114](https://github.com/grafana/grafana/issues/11114) & [#11086](https://github.com/grafana/grafana/issues/11086) diff --git a/docs/sources/administration/provisioning.md b/docs/sources/administration/provisioning.md index a8cb9fe3023..2946b17c0ac 100644 --- a/docs/sources/administration/provisioning.md +++ b/docs/sources/administration/provisioning.md @@ -11,6 +11,8 @@ weight = 8 # Provisioning Grafana +In previous versions of Grafana, you could only use the API for provisioning data sources and dashboards. But that required the service to be running before you started creating dashboards and you also needed to set up credentials for the HTTP API. In v5.0 we decided to improve this experience by adding a new active provisioning system that uses config files. This will make GitOps more natural as data sources and dashboards can be defined via files that can be version controlled. We hope to extend this system to later add support for users, orgs and alerts as well. + ## Config file Checkout the [configuration](/installation/configuration) page for more information on what you can configure in `grafana.ini` @@ -77,9 +79,11 @@ Saltstack | [https://github.com/salt-formulas/salt-formula-grafana](https://gith It's possible to manage datasources in Grafana by adding one or more yaml config files in the [`provisioning/datasources`](/installation/configuration/#provisioning) directory. Each config file can contain a list of `datasources` that will be added or updated during start up. If the datasource already exists, Grafana will update it to match the configuration file. The config file can also contain a list of datasources that should be deleted. That list is called `delete_datasources`. Grafana will delete datasources listed in `delete_datasources` before inserting/updating those in the `datasource` list. ### Running multiple Grafana instances. + If you are running multiple instances of Grafana you might run into problems if they have different versions of the `datasource.yaml` configuration file. The best way to solve this problem is to add a version number to each datasource in the configuration and increase it when you update the config. Grafana will only update datasources with the same or lower version number than specified in the config. That way, old configs cannot overwrite newer configs if they restart at the same time. ### Example datasource config file + ```yaml # config file version apiVersion: 1 @@ -137,7 +141,7 @@ datasources: | Datasource | Misc | | ---- | ---- | -| Elasticserach | Elasticsearch uses the `database` property to configure the index for a datasource | +| Elasticsearch | Elasticsearch uses the `database` property to configure the index for a datasource | #### Json data @@ -146,7 +150,7 @@ Since not all datasources have the same configuration settings we only have the | Name | Type | Datasource | Description | | ---- | ---- | ---- | ---- | | tlsAuth | boolean | *All* | Enable TLS authentication using client cert configured in secure json data | -| tlsAuthWithCACert | boolean | *All* | Enable TLS authtication using CA cert | +| tlsAuthWithCACert | boolean | *All* | Enable TLS authentication using CA cert | | tlsSkipVerify | boolean | *All* | Controls whether a client verifies the server's certificate chain and host name. | | graphiteVersion | string | Graphite | Graphite version | | timeInterval | string | Elastic, Influxdb & Prometheus | Lowest interval/step value that should be used for this data source | @@ -161,7 +165,6 @@ Since not all datasources have the same configuration settings we only have the | tsdbResolution | string | OpenTsdb | Resolution | | sslmode | string | Postgre | SSLmode. 'disable', 'require', 'verify-ca' or 'verify-full' | - #### Secure Json data `{"authType":"keys","defaultRegion":"us-west-2","timeField":"@timestamp"}` diff --git a/pkg/services/sqlstore/dashboard_version.go b/pkg/services/sqlstore/dashboard_version.go index 547f62628f3..1f2850b2021 100644 --- a/pkg/services/sqlstore/dashboard_version.go +++ b/pkg/services/sqlstore/dashboard_version.go @@ -67,30 +67,39 @@ func GetDashboardVersions(query *m.GetDashboardVersionsQuery) error { return nil } +const MAX_VERSIONS_TO_DELETE = 100 + func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error { return inTransaction(func(sess *DBSession) error { - versions := []DashboardVersionExp{} versionsToKeep := setting.DashboardVersionsToKeep - if versionsToKeep < 1 { versionsToKeep = 1 } - err := sess.Table("dashboard_version"). - Select("dashboard_version.id, dashboard_version.version, dashboard_version.dashboard_id"). - Where(`dashboard_id IN ( - SELECT dashboard_id FROM dashboard_version - GROUP BY dashboard_id HAVING COUNT(dashboard_version.id) > ? - )`, versionsToKeep). - Desc("dashboard_version.dashboard_id", "dashboard_version.version"). - Find(&versions) + // Idea of this query is finding version IDs to delete based on formula: + // min_version_to_keep = min_version + (versions_count - versions_to_keep) + // where version stats is processed for each dashboard. This guarantees that we keep at least versions_to_keep + // versions, but in some cases (when versions are sparse) this number may be more. + versionIdsToDeleteQuery := `SELECT id + FROM dashboard_version, ( + SELECT dashboard_id, count(version) as count, min(version) as min + FROM dashboard_version + GROUP BY dashboard_id + ) AS vtd + WHERE dashboard_version.dashboard_id=vtd.dashboard_id + AND version < vtd.min + vtd.count - ?` + var versionIdsToDelete []interface{} + err := sess.SQL(versionIdsToDeleteQuery, versionsToKeep).Find(&versionIdsToDelete) if err != nil { return err } - // Keep last versionsToKeep versions and delete other - versionIdsToDelete := getVersionIDsToDelete(versions, versionsToKeep) + // Don't delete more than MAX_VERSIONS_TO_DELETE version per time + if len(versionIdsToDelete) > MAX_VERSIONS_TO_DELETE { + versionIdsToDelete = versionIdsToDelete[:MAX_VERSIONS_TO_DELETE] + } + if len(versionIdsToDelete) > 0 { deleteExpiredSql := `DELETE FROM dashboard_version WHERE id IN (?` + strings.Repeat(",?", len(versionIdsToDelete)-1) + `)` expiredResponse, err := sess.Exec(deleteExpiredSql, versionIdsToDelete...) @@ -103,34 +112,3 @@ func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error { return nil }) } - -// Short version of DashboardVersion for getting expired versions -type DashboardVersionExp struct { - Id int64 `json:"id"` - DashboardId int64 `json:"dashboardId"` - Version int `json:"version"` -} - -func getVersionIDsToDelete(versions []DashboardVersionExp, versionsToKeep int) []interface{} { - versionIds := make([]interface{}, 0) - - if len(versions) == 0 { - return versionIds - } - - currentDashboard := versions[0].DashboardId - count := 0 - for _, v := range versions { - if v.DashboardId == currentDashboard { - count++ - } else { - count = 1 - currentDashboard = v.DashboardId - } - if count > versionsToKeep { - versionIds = append(versionIds, v.Id) - } - } - - return versionIds -} diff --git a/pkg/services/sqlstore/dashboard_version_test.go b/pkg/services/sqlstore/dashboard_version_test.go index 1b74e7847c4..a6403755d05 100644 --- a/pkg/services/sqlstore/dashboard_version_test.go +++ b/pkg/services/sqlstore/dashboard_version_test.go @@ -136,10 +136,30 @@ func TestDeleteExpiredVersions(t *testing.T) { err := DeleteExpiredVersions(&m.DeleteExpiredVersionsCommand{}) So(err, ShouldBeNil) - query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1} + query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1, Limit: versionsToWrite} GetDashboardVersions(&query) So(len(query.Result), ShouldEqual, versionsToWrite) }) + + Convey("Don't delete more than MAX_VERSIONS_TO_DELETE per iteration", func() { + versionsToWriteBigNumber := MAX_VERSIONS_TO_DELETE + versionsToWrite + for i := 0; i < versionsToWriteBigNumber-versionsToWrite; i++ { + updateTestDashboard(savedDash, map[string]interface{}{ + "tags": "different-tag", + }) + } + + err := DeleteExpiredVersions(&m.DeleteExpiredVersionsCommand{}) + So(err, ShouldBeNil) + + query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1, Limit: versionsToWriteBigNumber} + GetDashboardVersions(&query) + + // Ensure we have at least versionsToKeep versions + So(len(query.Result), ShouldBeGreaterThanOrEqualTo, versionsToKeep) + // Ensure we haven't deleted more than MAX_VERSIONS_TO_DELETE rows + So(versionsToWriteBigNumber-len(query.Result), ShouldBeLessThanOrEqualTo, MAX_VERSIONS_TO_DELETE) + }) }) } diff --git a/pkg/services/sqlstore/user.go b/pkg/services/sqlstore/user.go index 73ea07f031f..f42ff5fb2ed 100644 --- a/pkg/services/sqlstore/user.go +++ b/pkg/services/sqlstore/user.go @@ -315,6 +315,7 @@ func GetUserProfile(query *m.GetUserProfileQuery) error { } query.Result = m.UserProfileDTO{ + Id: user.Id, Name: user.Name, Email: user.Email, Login: user.Login, diff --git a/public/app/containers/ManageDashboards/FolderSettings.tsx b/public/app/containers/ManageDashboards/FolderSettings.tsx index 586a8f05b4c..e56659ad789 100644 --- a/public/app/containers/ManageDashboards/FolderSettings.tsx +++ b/public/app/containers/ManageDashboards/FolderSettings.tsx @@ -143,7 +143,7 @@ export class FolderSettings extends React.Component { className="btn btn-success" disabled={!folder.folder.canSave || !folder.folder.hasChanged} > - Save + Save