diff --git a/public/app/plugins/datasource/cloudwatch/utils/logsRetry.ts b/public/app/plugins/datasource/cloudwatch/utils/logsRetry.ts
index 9ff34b37737..eea4b7c2077 100644
--- a/public/app/plugins/datasource/cloudwatch/utils/logsRetry.ts
+++ b/public/app/plugins/datasource/cloudwatch/utils/logsRetry.ts
@@ -10,9 +10,9 @@ type Result = { frames: DataFrameJSON[]; error?: string };
/**
* A retry strategy specifically for cloud watch logs query. Cloud watch logs queries need first starting the query
* and the polling for the results. The start query can fail because of the concurrent queries rate limit,
- * and so we hove to retry the start query call if there is already lot of queries running.
+ * and so we have to retry the start query call if there is already lot of queries running.
*
- * As we send multiple queries in single request some can fail and some can succeed and we have to also handle those
+ * As we send multiple queries in a single request some can fail and some can succeed and we have to also handle those
* cases by only retrying the failed queries. We retry the failed queries until we hit the time limit or all queries
* succeed and only then we pass the data forward. This means we wait longer but makes the code a bit simpler as we
* can treat starting the query and polling as steps in a pipeline.
From 53e9bf47db21bace6783f2d0eb777209bcc725c6 Mon Sep 17 00:00:00 2001
From: Guilherme Caulada
Date: Mon, 25 Apr 2022 15:12:44 -0300
Subject: [PATCH 06/10] Secrets: Implement tests and debug log improvements on
unified secrets (#48213)
* Add test for decrypted values on datasource service
* Add debug log when fail to parse secure json fields
* Fix minor import issue
* Refactor encJson to json and simplejson to sjson on tests
---
pkg/api/datasources.go | 2 +
.../service/datasource_service_test.go | 100 ++++++++++++++----
2 files changed, 79 insertions(+), 23 deletions(-)
diff --git a/pkg/api/datasources.go b/pkg/api/datasources.go
index f1d1e7c1870..7eb21d0562c 100644
--- a/pkg/api/datasources.go
+++ b/pkg/api/datasources.go
@@ -487,6 +487,8 @@ func (hs *HTTPServer) convertModelToDtos(ctx context.Context, ds *models.DataSou
dto.SecureJsonFields[k] = true
}
}
+ } else {
+ datasourcesLogger.Debug("Failed to retrieve datasource secrets to parse secure json fields", "error", err)
}
return dto
diff --git a/pkg/services/datasources/service/datasource_service_test.go b/pkg/services/datasources/service/datasource_service_test.go
index 747a34be797..e250315fa73 100644
--- a/pkg/services/datasources/service/datasource_service_test.go
+++ b/pkg/services/datasources/service/datasource_service_test.go
@@ -2,7 +2,7 @@ package service
import (
"context"
- encJson "encoding/json"
+ "encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
@@ -11,6 +11,7 @@ import (
"github.com/grafana/grafana-azure-sdk-go/azsettings"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+ "github.com/grafana/grafana/pkg/services/secrets"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/components/simplejson"
@@ -224,8 +225,8 @@ func TestService_GetHttpTransport(t *testing.T) {
setting.SecretKey = "password"
- json := simplejson.New()
- json.Set("tlsAuthWithCACert", true)
+ sjson := simplejson.New()
+ sjson.Set("tlsAuthWithCACert", true)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
@@ -272,8 +273,8 @@ func TestService_GetHttpTransport(t *testing.T) {
setting.SecretKey = "password"
- json := simplejson.New()
- json.Set("tlsAuth", true)
+ sjson := simplejson.New()
+ sjson.Set("tlsAuth", true)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
@@ -285,10 +286,10 @@ func TestService_GetHttpTransport(t *testing.T) {
Name: "kubernetes",
Url: "http://k8s:8001",
Type: "Kubernetes",
- JsonData: json,
+ JsonData: sjson,
}
- secureJsonData, err := encJson.Marshal(map[string]string{
+ secureJsonData, err := json.Marshal(map[string]string{
"tlsClientCert": clientCert,
"tlsClientKey": clientKey,
})
@@ -316,9 +317,9 @@ func TestService_GetHttpTransport(t *testing.T) {
setting.SecretKey = "password"
- json := simplejson.New()
- json.Set("tlsAuthWithCACert", true)
- json.Set("serverName", "server-name")
+ sjson := simplejson.New()
+ sjson.Set("tlsAuthWithCACert", true)
+ sjson.Set("serverName", "server-name")
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
@@ -330,10 +331,10 @@ func TestService_GetHttpTransport(t *testing.T) {
Name: "kubernetes",
Url: "http://k8s:8001",
Type: "Kubernetes",
- JsonData: json,
+ JsonData: sjson,
}
- secureJsonData, err := encJson.Marshal(map[string]string{
+ secureJsonData, err := json.Marshal(map[string]string{
"tlsCACert": caCert,
})
require.NoError(t, err)
@@ -359,8 +360,8 @@ func TestService_GetHttpTransport(t *testing.T) {
},
})
- json := simplejson.New()
- json.Set("tlsSkipVerify", true)
+ sjson := simplejson.New()
+ sjson.Set("tlsSkipVerify", true)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
@@ -370,7 +371,7 @@ func TestService_GetHttpTransport(t *testing.T) {
Id: 1,
Url: "http://k8s:8001",
Type: "Kubernetes",
- JsonData: json,
+ JsonData: sjson,
}
rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
@@ -390,7 +391,7 @@ func TestService_GetHttpTransport(t *testing.T) {
t.Run("Should set custom headers if configured in JsonData", func(t *testing.T) {
provider := httpclient.NewProvider()
- json := simplejson.NewFromAny(map[string]interface{}{
+ sjson := simplejson.NewFromAny(map[string]interface{}{
"httpHeaderName1": "Authorization",
})
@@ -404,10 +405,10 @@ func TestService_GetHttpTransport(t *testing.T) {
Name: "kubernetes",
Url: "http://k8s:8001",
Type: "Kubernetes",
- JsonData: json,
+ JsonData: sjson,
}
- secureJsonData, err := encJson.Marshal(map[string]string{
+ secureJsonData, err := json.Marshal(map[string]string{
"httpHeaderValue1": "Bearer xf5yhfkpsnmgo",
})
require.NoError(t, err)
@@ -415,7 +416,7 @@ func TestService_GetHttpTransport(t *testing.T) {
err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData))
require.NoError(t, err)
- headers := dsService.getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"})
+ headers := dsService.getCustomHeaders(sjson, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"})
require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"])
// 1. Start HTTP test server which checks the request headers
@@ -456,7 +457,7 @@ func TestService_GetHttpTransport(t *testing.T) {
t.Run("Should use request timeout if configured in JsonData", func(t *testing.T) {
provider := httpclient.NewProvider()
- json := simplejson.NewFromAny(map[string]interface{}{
+ sjson := simplejson.NewFromAny(map[string]interface{}{
"timeout": 19,
})
@@ -468,7 +469,7 @@ func TestService_GetHttpTransport(t *testing.T) {
Id: 1,
Url: "http://k8s:8001",
Type: "Kubernetes",
- JsonData: json,
+ JsonData: sjson,
}
client, err := dsService.GetHTTPClient(context.Background(), &ds, provider)
@@ -491,7 +492,7 @@ func TestService_GetHttpTransport(t *testing.T) {
setting.SigV4AuthEnabled = origSigV4Enabled
})
- json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`))
+ sjson, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`))
require.NoError(t, err)
secretsStore := kvstore.SetupTestService(t)
@@ -500,7 +501,7 @@ func TestService_GetHttpTransport(t *testing.T) {
ds := models.DataSource{
Type: models.DS_ES,
- JsonData: json,
+ JsonData: sjson,
}
_, err = dsService.GetHTTPTransport(context.Background(), &ds, provider)
@@ -706,6 +707,59 @@ func TestService_HTTPClientOptions(t *testing.T) {
})
}
+func TestService_GetDecryptedValues(t *testing.T) {
+ t.Run("should migrate and retrieve values from secure json data", func(t *testing.T) {
+ ds := &models.DataSource{
+ Id: 1,
+ Url: "https://api.example.com",
+ Type: "prometheus",
+ }
+
+ secretsStore := kvstore.SetupTestService(t)
+ secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
+ dsService := ProvideService(nil, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
+
+ jsonData := map[string]string{
+ "password": "securePassword",
+ }
+ secureJsonData, err := dsService.SecretsService.EncryptJsonData(context.Background(), jsonData, secrets.WithoutScope())
+
+ require.NoError(t, err)
+ ds.SecureJsonData = secureJsonData
+
+ values, err := dsService.DecryptedValues(context.Background(), ds)
+ require.NoError(t, err)
+
+ require.Equal(t, jsonData, values)
+ })
+
+ t.Run("should retrieve values from secret store", func(t *testing.T) {
+ ds := &models.DataSource{
+ Id: 1,
+ Url: "https://api.example.com",
+ Type: "prometheus",
+ }
+
+ secretsStore := kvstore.SetupTestService(t)
+ secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
+ dsService := ProvideService(nil, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
+
+ jsonData := map[string]string{
+ "password": "securePassword",
+ }
+ jsonString, err := json.Marshal(jsonData)
+ require.NoError(t, err)
+
+ err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(jsonString))
+ require.NoError(t, err)
+
+ values, err := dsService.DecryptedValues(context.Background(), ds)
+ require.NoError(t, err)
+
+ require.Equal(t, jsonData, values)
+ })
+}
+
const caCert string = `-----BEGIN CERTIFICATE-----
MIIDATCCAemgAwIBAgIJAMQ5hC3CPDTeMA0GCSqGSIb3DQEBCwUAMBcxFTATBgNV
BAMMDGNhLWs4cy1zdGhsbTAeFw0xNjEwMjcwODQyMjdaFw00NDAzMTQwODQyMjda
From 7311c9757ab4441a25ccc9950c54caa0e1f7ecd9 Mon Sep 17 00:00:00 2001
From: achatterjee-grafana
<70489351+achatterjee-grafana@users.noreply.github.com>
Date: Mon, 25 Apr 2022 15:53:09 -0400
Subject: [PATCH 07/10] Docs: Break down alerting HA topics (#48143)
* Initial commit
* Added some refinement to the alerting HA topics.
* Update docs/sources/administration/set-up-for-high-availability.md
Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com>
* Updates from Chris's review. Also fixed a couple of broken relrefs
* Ran prettier
Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com>
---
.../set-up-for-high-availability.md | 10 ++---
.../alerting/unified-alerting/_index.md | 2 +-
.../unified-alerting/high-availability.md | 44 -------------------
.../high-availability/_index.md | 25 +++++++++++
.../high-availability/enable-alerting-ha.md | 36 +++++++++++++++
docs/sources/dashboards/_index.md | 4 +-
...hboard_folders.md => dashboard-folders.md} | 0
...hboard_history.md => dashboard-history.md} | 0
.../enterprise/saml/set-up-saml-with-okta.md | 2 +-
docs/sources/whatsnew/whats-new-in-v7-0.md | 2 +-
docs/sources/whatsnew/whats-new-in-v7-4.md | 2 +-
11 files changed, 71 insertions(+), 56 deletions(-)
delete mode 100644 docs/sources/alerting/unified-alerting/high-availability.md
create mode 100644 docs/sources/alerting/unified-alerting/high-availability/_index.md
create mode 100644 docs/sources/alerting/unified-alerting/high-availability/enable-alerting-ha.md
rename docs/sources/dashboards/{dashboard_folders.md => dashboard-folders.md} (100%)
rename docs/sources/dashboards/{dashboard_history.md => dashboard-history.md} (100%)
diff --git a/docs/sources/administration/set-up-for-high-availability.md b/docs/sources/administration/set-up-for-high-availability.md
index 818986df7c8..9dd11efcc41 100644
--- a/docs/sources/administration/set-up-for-high-availability.md
+++ b/docs/sources/administration/set-up-for-high-availability.md
@@ -20,17 +20,15 @@ First, you need to set up MySQL or Postgres on another server and configure Graf
You can find the configuration for doing that in the [[database]]({{< relref "../administration/configuration.md#database" >}}) section in the Grafana config.
Grafana will now persist all long term data in the database. How to configure the database for high availability is out of scope for this guide. We recommend finding an expert on the database you're using.
-## Alerting
+## Alerting high availability
-**Grafana 8 alerts**
+Grafana alerting provides a new [highly-available model]({{< relref "../alerting/unified-alerting/high-availability/_index.md" >}}). It also preserves the semantics of legacy dashboard alerting by executing all alerts on every server and by sending notifications only once per alert. Load distribution between servers is not supported at this time.
-Grafana 8 Alerts provides a new highly-available model under the hood. It preserves the previous semantics by executing all alerts on every server and notifications are sent only once per alert. There is no support for load distribution between servers at this time.
-
-For configuration, [follow the guide]({{< relref "../alerting/unified-alerting/high-availability.md" >}}).
+For instructions on setting up alerting high availability, see [enable alerting high availability]({{< relref "../alerting/unified-alerting/high-availability/enable-alerting-ha.md" >}}).
**Legacy dashboard alerts**
-Legacy Grafana alerting supports a limited form of high availability. [Alert notifications]({{< relref "../alerting/old-alerting/notifications.md" >}}) are deduplicated when running multiple servers. This means all alerts are executed on every server but alert notifications are only sent once per alert. Grafana does not support load distribution between servers.
+Legacy Grafana alerting supports a limited form of high availability. In this model, [alert notifications]({{< relref "../alerting/old-alerting/notifications.md" >}}) are deduplicated when running multiple servers. This means all alerts are executed on every server, but alert notifications are only sent once per alert. Grafana does not support load distribution between servers.
## Grafana Live
diff --git a/docs/sources/alerting/unified-alerting/_index.md b/docs/sources/alerting/unified-alerting/_index.md
index de360455048..0dc5778ef3d 100644
--- a/docs/sources/alerting/unified-alerting/_index.md
+++ b/docs/sources/alerting/unified-alerting/_index.md
@@ -8,7 +8,7 @@ weight = 113
Grafana 8.0 has new and improved alerting that centralizes alerting information in a single, searchable view. It is enabled by default for all new OSS instances, and is an [opt-in]({{< relref "./opt-in.md" >}}) feature for older installations that still use legacy dashboard alerting. We encourage you to create issues in the Grafana GitHub repository for bugs found while testing Grafana alerting. See also, [What's New with Grafana alerting]({{< relref "./difference-old-new.md" >}}).
-> Refer to [Fine-grained access control]({{< relref "../enterprise/access-control/_index.md" >}}) in Grafana Enterprise to learn more about controlling access to alerts using fine-grained permissions.
+> Refer to [Fine-grained access control]({{< relref "../../enterprise/access-control/_index.md" >}}) in Grafana Enterprise to learn more about controlling access to alerts using fine-grained permissions.
When Grafana alerting is enabled, you can:
diff --git a/docs/sources/alerting/unified-alerting/high-availability.md b/docs/sources/alerting/unified-alerting/high-availability.md
deleted file mode 100644
index d85ddd9a8f4..00000000000
--- a/docs/sources/alerting/unified-alerting/high-availability.md
+++ /dev/null
@@ -1,44 +0,0 @@
-+++
-title = " High availability"
-description = "High Availability"
-keywords = ["grafana", "alerting", "tutorials", "ha", "high availability"]
-weight = 450
-+++
-
-# High availability
-
-The Grafana alerting system has two main components: a `Scheduler` and an internal `Alertmanager`. The `Scheduler` is responsible for the evaluation of your [alert rules]({{< relref "./fundamentals/evaluate-grafana-alerts.md" >}}) while the internal Alertmanager takes care of the **routing** and **grouping**.
-
-When it comes to running Grafana alerting in high availability the operational mode of the scheduler is unaffected such that all alerts continue be evaluated in each Grafana instance. Rather the operational change happens in the Alertmanager which **deduplicates** alert notifications across Grafana instances.
-
-{{< figure src="/static/img/docs/alerting/unified/high-availability-ua.png" class="docs-image--no-shadow" max-width= "750px" caption="High availability" >}}
-
-The coordination between Grafana instances happens via [a Gossip protocol](https://en.wikipedia.org/wiki/Gossip_protocol). Alerts are not gossiped between instances. It is expected that each scheduler delivers the same alerts to each Alertmanager.
-
-The two types of messages that are gossiped between instances are:
-
-- Notification logs: Who (which instance) notified what (which alert)
-- Silences: If an alert should fire or not
-
-These two states are persisted in the database periodically and when Grafana is gracefully shutdown.
-
-## Enable high availability
-
-To enable high availability support you need to add at least 1 Grafana instance to the [`[ha_peer]` configuration option]({{}}) within the `[unified_alerting]` section:
-
-1. In your custom configuration file ($WORKING_DIR/conf/custom.ini), go to the `[unified_alerting]` section.
-2. Set `[ha_peers]` to the number of hosts for each grafana instance in the cluster (using a format of host:port) e.g. `ha_peers=10.0.0.5:9094,10.0.0.6:9094,10.0.0.7:9094`
-3. Gossiping of notifications and silences uses both TCP and UDP port 9094. Each Grafana instance will need to be able to accept incoming connections on these ports.
-4. Set `[ha_listen_address]` to the instance IP address using a format of host:port (or the [Pod's](https://kubernetes.io/docs/concepts/workloads/pods/) IP in the case of using Kubernetes) by default it is set to listen to all interfaces (`0.0.0.0`).
-
-## Kubernetes
-
-If you are using Kubernetes, you can expose the pod IP [through an environment variable](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/) via the container definition such as:
-
-```bash
-env:
-- name: POD_IP
- valueFrom:
- fieldRef:
- fieldPath: status.podIP
-```
diff --git a/docs/sources/alerting/unified-alerting/high-availability/_index.md b/docs/sources/alerting/unified-alerting/high-availability/_index.md
new file mode 100644
index 00000000000..8b04bee271e
--- /dev/null
+++ b/docs/sources/alerting/unified-alerting/high-availability/_index.md
@@ -0,0 +1,25 @@
++++
+title = " About alerting high availability"
+description = "High availability"
+keywords = ["grafana", "alerting", "tutorials", "ha", "high availability"]
+weight = 450
++++
+
+# About alerting high availability
+
+The Grafana alerting system has two main components: a `Scheduler` and an internal `Alertmanager`. The `Scheduler` evaluates your [alert rules]({{< relref "../fundamentals/evaluate-grafana-alerts.md" >}}), while the internal Alertmanager manages **routing** and **grouping**.
+
+When running Grafana alerting in high availability, the operational mode of the scheduler remains unaffected, and each Grafana instance evaluates all alerts. The operational change happens in the Alertmanager when it deduplicates alert notifications across Grafana instances.
+
+{{< figure src="/static/img/docs/alerting/unified/high-availability-ua.png" class="docs-image--no-shadow" max-width= "750px" caption="High availability" >}}
+
+The coordination between Grafana instances happens via [a Gossip protocol](https://en.wikipedia.org/wiki/Gossip_protocol). Alerts are not gossiped between instances and each scheduler delivers the same volume of alerts to each Alertmanager.
+
+The two types of messages gossiped between Grafana instances are:
+
+- Notification logs: Who (which instance) notified what (which alert).
+- Silences: If an alert should fire or not.
+
+The notification logs and silences are persisted in the database periodically and during a graceful Grafana shut down.
+
+For configuration instructions, refer to [enable alerting high availability]({{< relref "./enable-alerting-ha.md" >}}).
diff --git a/docs/sources/alerting/unified-alerting/high-availability/enable-alerting-ha.md b/docs/sources/alerting/unified-alerting/high-availability/enable-alerting-ha.md
new file mode 100644
index 00000000000..9a6c9e12afe
--- /dev/null
+++ b/docs/sources/alerting/unified-alerting/high-availability/enable-alerting-ha.md
@@ -0,0 +1,36 @@
++++
+title = "Enable alerting high availability"
+description = "Enable alerting high availability"
+keywords = ["grafana", "alerting", "tutorials", "ha", "high availability"]
+weight = 450
++++
+
+# Enable alerting high availability
+
+You can enable [alerting high availability]({{< relref "./_index.md" >}}) support by updating the Grafana configuration file. On Kubernetes, you can enable alerting high availability by updating the Kubernetes container definition.
+
+## Update Grafana configuration file
+
+### Before you begin
+
+Since gossiping of notifications and silences uses both TCP and UDP port `9094`, ensure that each Grafana instance is able to accept incoming connections on these ports.
+
+**To enable high availability support:**
+
+1. In your custom configuration file ($WORKING_DIR/conf/custom.ini), go to the `[unified_alerting]` section.
+2. Set `[ha_peers]` to the number of hosts for each Grafana instance in the cluster (using a format of host:port), for example, `ha_peers=10.0.0.5:9094,10.0.0.6:9094,10.0.0.7:9094`.
+ You must have at least one (1) Grafana instance added to the [`[ha_peer]` section.
+3. Set `[ha_listen_address]` to the instance IP address using a format of `host:port` (or the [Pod's](https://kubernetes.io/docs/concepts/workloads/pods/) IP in the case of using Kubernetes).
+ By default, it is set to listen to all interfaces (`0.0.0.0`).
+
+## Update Kubernetes container definition
+
+If you are using Kubernetes, you can expose the pod IP [through an environment variable](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/) via the container definition such as:
+
+```bash
+env:
+- name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+```
diff --git a/docs/sources/dashboards/_index.md b/docs/sources/dashboards/_index.md
index 0c73e1a8696..672c34883df 100644
--- a/docs/sources/dashboards/_index.md
+++ b/docs/sources/dashboards/_index.md
@@ -13,7 +13,7 @@ Dashboard snapshots are static . Queries and expressions cannot be re-executed f
Before you begin, ensure that you have configured a data source. See also:
- [Working with Grafana dashboard UI]({{< relref "./dashboard-ui/_index.md" >}})
-- [Dashboard folders]({{< relref "./dashboard_folders.md" >}})
+- [Dashboard folders]({{< relref "./dashboard-folders.md" >}})
- [Create dashboard]({{< relref "./dashboard-create" >}})
- [Manage dashboards]({{< relref "./dashboard-manage.md" >}})
- [Annotations]({{< relref "./annotations.md" >}})
@@ -22,7 +22,7 @@ Before you begin, ensure that you have configured a data source. See also:
- [Keyboard shortcuts]({{< relref "./shortcuts.md" >}})
- [Reporting]({{< relref "./reporting.md" >}})
- [Time range controls]({{< relref "./time-range-controls.md" >}})
-- [Dashboard version history]({{< relref "./dashboard_history.md" >}})
+- [Dashboard version history]({{< relref "./dashboard-history.md" >}})
- [Dashboard export and import]({{< relref "./export-import.md" >}})
- [Dashboard JSON model]({{< relref "./json-model.md" >}})
- [Scripted dashboards]({{< relref "./scripted-dashboards.md" >}})
diff --git a/docs/sources/dashboards/dashboard_folders.md b/docs/sources/dashboards/dashboard-folders.md
similarity index 100%
rename from docs/sources/dashboards/dashboard_folders.md
rename to docs/sources/dashboards/dashboard-folders.md
diff --git a/docs/sources/dashboards/dashboard_history.md b/docs/sources/dashboards/dashboard-history.md
similarity index 100%
rename from docs/sources/dashboards/dashboard_history.md
rename to docs/sources/dashboards/dashboard-history.md
diff --git a/docs/sources/enterprise/saml/set-up-saml-with-okta.md b/docs/sources/enterprise/saml/set-up-saml-with-okta.md
index 4edd3de7a7b..827739f6480 100644
--- a/docs/sources/enterprise/saml/set-up-saml-with-okta.md
+++ b/docs/sources/enterprise/saml/set-up-saml-with-okta.md
@@ -13,7 +13,7 @@ Grafana supports user authentication through Okta, which is useful when you want
## Before you begin
- To configure SAML integration with Okta, create integration inside the Okta organization first. [Add integration in Okta](https://help.okta.com/en/prod/Content/Topics/Apps/apps-overview-add-apps.htm)
-- Ensure you have permission to administer SAML authentication. For more information about permissions, refer to [About users and permissions]({{< relref "../manage-users-and-permissions/about-users-and-permissions.md#">}}).
+- Ensure you have permission to administer SAML authentication. For more information about permissions, refer to [About users and permissions]({{< relref "../../administration/manage-users-and-permissions/about-users-and-permissions.md#">}}).
**To set up SAML with Okta:**
diff --git a/docs/sources/whatsnew/whats-new-in-v7-0.md b/docs/sources/whatsnew/whats-new-in-v7-0.md
index 0aeb5342e61..ac172c4d711 100644
--- a/docs/sources/whatsnew/whats-new-in-v7-0.md
+++ b/docs/sources/whatsnew/whats-new-in-v7-0.md
@@ -214,7 +214,7 @@ This release includes a series of features that build on our new usage analytics
### SAML Role and Team Sync
-SAML support in Grafana Enterprise is improved by adding Role and Team Sync. Read more about how to use these features in the [SAML team sync documentation]({{< relref "../enterprise/saml.md#configure-team-sync" >}}).
+SAML support in Grafana Enterprise is improved by adding Role and Team Sync. Read more about how to use these features in the [SAML team sync documentation]({{< relref "../enterprise/saml/configure-saml.md#configure-team-sync" >}}).
### Okta OAuth Team Sync
diff --git a/docs/sources/whatsnew/whats-new-in-v7-4.md b/docs/sources/whatsnew/whats-new-in-v7-4.md
index aa2908ec258..267efa72fd4 100644
--- a/docs/sources/whatsnew/whats-new-in-v7-4.md
+++ b/docs/sources/whatsnew/whats-new-in-v7-4.md
@@ -202,7 +202,7 @@ For more information, refer to [Export logs of usage insights]({{< relref "../en
### New audit log events
-New log out events are logged based on when a token expires or is revoked, as well as [SAML Single Logout]({{< relref "../enterprise/saml.md#single-logout" >}}). A `tokenId` field was added to all audit logs to help understand which session was logged out of.
+New log out events are logged based on when a token expires or is revoked, as well as [SAML Single Logout]({{< relref "../enterprise/saml/configure-saml.md#single-logout" >}}). A `tokenId` field was added to all audit logs to help understand which session was logged out of.
Also, a counter for audit log writing actions with status (success / failure) and logger (loki / file / console) labels was added.
From e0aeb83786731769e870d79c0d8a21ef2c33b073 Mon Sep 17 00:00:00 2001
From: Ryan McKinley
Date: Mon, 25 Apr 2022 16:59:18 -0700
Subject: [PATCH 08/10] Export: introduce export plumbing (behind dev feature
flag) (#48091)
---
.github/CODEOWNERS | 1 +
.../src/types/featureToggles.gen.ts | 1 +
pkg/api/api.go | 5 +
pkg/api/http_server.go | 5 +-
pkg/server/wire.go | 2 +
pkg/services/export/dummy_job.go | 103 ++++++++++++++++++
pkg/services/export/service.go | 93 ++++++++++++++++
pkg/services/export/stopped_job.go | 19 ++++
pkg/services/export/stub.go | 20 ++++
pkg/services/export/types.go | 36 ++++++
pkg/services/featuremgmt/registry.go | 6 +
pkg/services/featuremgmt/toggles_gen.go | 4 +
.../app/features/admin/ExportStartButton.tsx | 62 +++++++++++
public/app/features/admin/ExportStatus.tsx | 82 ++++++++++++++
public/app/features/admin/ServerStats.tsx | 2 +
15 files changed, 440 insertions(+), 1 deletion(-)
create mode 100644 pkg/services/export/dummy_job.go
create mode 100644 pkg/services/export/service.go
create mode 100644 pkg/services/export/stopped_job.go
create mode 100644 pkg/services/export/stub.go
create mode 100644 pkg/services/export/types.go
create mode 100644 public/app/features/admin/ExportStartButton.tsx
create mode 100644 public/app/features/admin/ExportStatus.tsx
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index b421e6f3663..8f364c8f8bc 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -58,6 +58,7 @@ go.sum @grafana/backend-platform
/pkg/services/live/ @grafana/grafana-edge-squad
/pkg/services/searchV2/ @grafana/grafana-edge-squad
/pkg/services/store/ @grafana/grafana-edge-squad
+/pkg/services/export/ @grafana/grafana-edge-squad
/pkg/infra/filestore/ @grafana/grafana-edge-squad
pkg/tsdb/testdatasource/sims/ @grafana/grafana-edge-squad
diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts
index 080394dd1a9..e2bdfadde9d 100644
--- a/packages/grafana-data/src/types/featureToggles.gen.ts
+++ b/packages/grafana-data/src/types/featureToggles.gen.ts
@@ -50,6 +50,7 @@ export interface FeatureToggles {
saveDashboardDrawer?: boolean;
storage?: boolean;
alertProvisioning?: boolean;
+ export?: boolean;
storageLocalUpload?: boolean;
azureMonitorResourcePickerForMetrics?: boolean;
explore2Dashboard?: boolean;
diff --git a/pkg/api/api.go b/pkg/api/api.go
index 24883640e05..030c920ada2 100644
--- a/pkg/api/api.go
+++ b/pkg/api/api.go
@@ -523,6 +523,11 @@ func (hs *HTTPServer) registerRoutes() {
adminRoute.Get("/crawler/status", reqGrafanaAdmin, routing.Wrap(hs.ThumbService.CrawlerStatus))
}
+ if hs.Features.IsEnabled(featuremgmt.FlagExport) {
+ adminRoute.Get("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleGetStatus))
+ adminRoute.Post("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleRequestExport))
+ }
+
adminRoute.Post("/provisioning/dashboards/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDashboards)), routing.Wrap(hs.AdminProvisioningReloadDashboards))
adminRoute.Post("/provisioning/plugins/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersPlugins)), routing.Wrap(hs.AdminProvisioningReloadPlugins))
adminRoute.Post("/provisioning/datasources/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDatasources)), routing.Wrap(hs.AdminProvisioningReloadDatasources))
diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go
index 5491b86d316..0d4e4241744 100644
--- a/pkg/api/http_server.go
+++ b/pkg/api/http_server.go
@@ -39,6 +39,7 @@ import (
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/datasources/permissions"
"github.com/grafana/grafana/pkg/services/encryption"
+ "github.com/grafana/grafana/pkg/services/export"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/hooks"
"github.com/grafana/grafana/pkg/services/ldap"
@@ -112,6 +113,7 @@ type HTTPServer struct {
Live *live.GrafanaLive
LivePushGateway *pushhttp.Gateway
ThumbService thumbs.Service
+ ExportService export.ExportService
StorageService store.HTTPStorageService
ContextHandler *contexthandler.ContextHandler
SQLStore sqlstore.Store
@@ -170,7 +172,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
contextHandler *contexthandler.ContextHandler, features *featuremgmt.FeatureManager,
schemaService *schemaloader.SchemaLoaderService, alertNG *ngalert.AlertNG,
libraryPanelService librarypanels.Service, libraryElementService libraryelements.Service,
- quotaService *quota.QuotaService, socialService social.Service, tracer tracing.Tracer,
+ quotaService *quota.QuotaService, socialService social.Service, tracer tracing.Tracer, exportService export.ExportService,
encryptionService encryption.Internal, grafanaUpdateChecker *updatechecker.GrafanaService,
pluginsUpdateChecker *updatechecker.PluginsService, searchUsersService searchusers.Service,
dataSourcesService datasources.DataSourceService, secretsService secrets.Service, queryDataService *query.Service,
@@ -217,6 +219,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
AccessControl: accessControl,
DataProxy: dataSourceProxy,
SearchService: searchService,
+ ExportService: exportService,
Live: live,
LivePushGateway: livePushGateway,
PluginContextProvider: plugCtxProvider,
diff --git a/pkg/server/wire.go b/pkg/server/wire.go
index 47e2d1eee5d..4461854bd5b 100644
--- a/pkg/server/wire.go
+++ b/pkg/server/wire.go
@@ -50,6 +50,7 @@ import (
"github.com/grafana/grafana/pkg/services/datasourceproxy"
"github.com/grafana/grafana/pkg/services/datasources"
datasourceservice "github.com/grafana/grafana/pkg/services/datasources/service"
+ "github.com/grafana/grafana/pkg/services/export"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/guardian"
"github.com/grafana/grafana/pkg/services/hooks"
@@ -175,6 +176,7 @@ var wireBasicSet = wire.NewSet(
searchV2.ProvideService,
store.ProvideService,
store.ProvideHTTPService,
+ export.ProvideService,
live.ProvideService,
pushhttp.ProvideService,
plugincontext.ProvideService,
diff --git a/pkg/services/export/dummy_job.go b/pkg/services/export/dummy_job.go
new file mode 100644
index 00000000000..029bc8edabb
--- /dev/null
+++ b/pkg/services/export/dummy_job.go
@@ -0,0 +1,103 @@
+package export
+
+import (
+ "errors"
+ "fmt"
+ "math"
+ "math/rand"
+ "sync"
+ "time"
+
+ "github.com/grafana/grafana/pkg/infra/log"
+)
+
+var _ Job = new(dummyExportJob)
+
+type dummyExportJob struct {
+ logger log.Logger
+
+ statusMu sync.Mutex
+ status ExportStatus
+ cfg ExportConfig
+ broadcaster statusBroadcaster
+}
+
+func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job, error) {
+ if cfg.Format != "git" {
+ return nil, errors.New("only git format is supported")
+ }
+
+ job := &dummyExportJob{
+ logger: log.New("dummy_export_job"),
+ cfg: cfg,
+ broadcaster: broadcaster,
+ status: ExportStatus{
+ Running: true,
+ Target: "git export",
+ Started: time.Now().UnixMilli(),
+ Count: int64(math.Round(10 + rand.Float64()*20)),
+ Current: 0,
+ },
+ }
+
+ broadcaster(job.status)
+ go job.start()
+ return job, nil
+}
+
+func (e *dummyExportJob) start() {
+ defer func() {
+ e.logger.Info("Finished dummy export job")
+
+ e.statusMu.Lock()
+ defer e.statusMu.Unlock()
+ s := e.status
+ if err := recover(); err != nil {
+ e.logger.Error("export panic", "error", err)
+ s.Status = fmt.Sprintf("ERROR: %v", err)
+ }
+ // Make sure it finishes OK
+ if s.Finished < 10 {
+ s.Finished = time.Now().UnixMilli()
+ }
+ s.Running = false
+ if s.Status == "" {
+ s.Status = "done"
+ }
+ e.status = s
+ e.broadcaster(s)
+ }()
+
+ e.logger.Info("Starting dummy export job")
+
+ ticker := time.NewTicker(1 * time.Second)
+ for t := range ticker.C {
+ e.statusMu.Lock()
+ e.status.Changed = t.UnixMilli()
+ e.status.Current++
+ e.status.Last = fmt.Sprintf("ITEM: %d", e.status.Current)
+ e.statusMu.Unlock()
+
+ // Wait till we are done
+ shouldStop := e.status.Current >= e.status.Count
+ e.broadcaster(e.status)
+
+ if shouldStop {
+ break
+ }
+ }
+}
+
+func (e *dummyExportJob) getStatus() ExportStatus {
+ e.statusMu.Lock()
+ defer e.statusMu.Unlock()
+
+ return e.status
+}
+
+func (e *dummyExportJob) getConfig() ExportConfig {
+ e.statusMu.Lock()
+ defer e.statusMu.Unlock()
+
+ return e.cfg
+}
diff --git a/pkg/services/export/service.go b/pkg/services/export/service.go
new file mode 100644
index 00000000000..ee4f1b040ea
--- /dev/null
+++ b/pkg/services/export/service.go
@@ -0,0 +1,93 @@
+package export
+
+import (
+ "encoding/json"
+ "net/http"
+ "sync"
+
+ "github.com/grafana/grafana/pkg/api/response"
+ "github.com/grafana/grafana/pkg/infra/log"
+ "github.com/grafana/grafana/pkg/models"
+ "github.com/grafana/grafana/pkg/services/featuremgmt"
+ "github.com/grafana/grafana/pkg/services/live"
+ "github.com/grafana/grafana/pkg/services/sqlstore"
+)
+
+type ExportService interface {
+ // List folder contents
+ HandleGetStatus(c *models.ReqContext) response.Response
+
+ // Read raw file contents out of the store
+ HandleRequestExport(c *models.ReqContext) response.Response
+}
+
+type StandardExport struct {
+ logger log.Logger
+ sql *sqlstore.SQLStore
+ glive *live.GrafanaLive
+ mutex sync.Mutex
+
+ // updated with mutex
+ exportJob Job
+}
+
+func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, gl *live.GrafanaLive) ExportService {
+ if !features.IsEnabled(featuremgmt.FlagExport) {
+ return &StubExport{}
+ }
+
+ return &StandardExport{
+ sql: sql,
+ glive: gl,
+ logger: log.New("export_service"),
+ exportJob: &stoppedJob{},
+ }
+}
+
+func (ex *StandardExport) HandleGetStatus(c *models.ReqContext) response.Response {
+ ex.mutex.Lock()
+ defer ex.mutex.Unlock()
+
+ return response.JSON(http.StatusOK, ex.exportJob.getStatus())
+}
+
+func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Response {
+ var cfg ExportConfig
+ err := json.NewDecoder(c.Req.Body).Decode(&cfg)
+ if err != nil {
+ return response.Error(http.StatusBadRequest, "unable to read config", err)
+ }
+
+ ex.mutex.Lock()
+ defer ex.mutex.Unlock()
+
+ status := ex.exportJob.getStatus()
+ if status.Running {
+ ex.logger.Error("export already running")
+ return response.Error(http.StatusLocked, "export already running", nil)
+ }
+
+ job, err := startDummyExportJob(cfg, func(s ExportStatus) {
+ ex.broadcastStatus(c.OrgId, s)
+ })
+ if err != nil {
+ ex.logger.Error("failed to start export job", "err", err)
+ return response.Error(http.StatusBadRequest, "failed to start export job", err)
+ }
+
+ ex.exportJob = job
+ return response.JSON(http.StatusOK, ex.exportJob.getStatus())
+}
+
+func (ex *StandardExport) broadcastStatus(orgID int64, s ExportStatus) {
+ msg, err := json.Marshal(s)
+ if err != nil {
+ ex.logger.Warn("Error making message", "err", err)
+ return
+ }
+ err = ex.glive.Publish(orgID, "grafana/broadcast/export", msg)
+ if err != nil {
+ ex.logger.Warn("Error Publish message", "err", err)
+ return
+ }
+}
diff --git a/pkg/services/export/stopped_job.go b/pkg/services/export/stopped_job.go
new file mode 100644
index 00000000000..b9756f9d72f
--- /dev/null
+++ b/pkg/services/export/stopped_job.go
@@ -0,0 +1,19 @@
+package export
+
+import "time"
+
+var _ Job = new(stoppedJob)
+
+type stoppedJob struct {
+}
+
+func (e *stoppedJob) getStatus() ExportStatus {
+ return ExportStatus{
+ Running: false,
+ Changed: time.Now().UnixMilli(),
+ }
+}
+
+func (e *stoppedJob) getConfig() ExportConfig {
+ return ExportConfig{}
+}
diff --git a/pkg/services/export/stub.go b/pkg/services/export/stub.go
new file mode 100644
index 00000000000..551bb730f5c
--- /dev/null
+++ b/pkg/services/export/stub.go
@@ -0,0 +1,20 @@
+package export
+
+import (
+ "net/http"
+
+ "github.com/grafana/grafana/pkg/api/response"
+ "github.com/grafana/grafana/pkg/models"
+)
+
+var _ ExportService = new(StubExport)
+
+type StubExport struct{}
+
+func (ex *StubExport) HandleGetStatus(c *models.ReqContext) response.Response {
+ return response.Error(http.StatusForbidden, "feature not enabled", nil)
+}
+
+func (ex *StubExport) HandleRequestExport(c *models.ReqContext) response.Response {
+ return response.Error(http.StatusForbidden, "feature not enabled", nil)
+}
diff --git a/pkg/services/export/types.go b/pkg/services/export/types.go
new file mode 100644
index 00000000000..20a1ec49e3d
--- /dev/null
+++ b/pkg/services/export/types.go
@@ -0,0 +1,36 @@
+package export
+
+// Export status. Only one running at a time
+type ExportStatus struct {
+ Running bool `json:"running"`
+ Target string `json:"target"` // description of where it is going (no secrets)
+ Started int64 `json:"started,omitempty"`
+ Finished int64 `json:"finished,omitempty"`
+ Changed int64 `json:"update,omitempty"`
+ Count int64 `json:"count,omitempty"`
+ Current int64 `json:"current,omitempty"`
+ Last string `json:"last,omitempty"`
+ Status string `json:"status"` // ERROR, SUCCESS, ETC
+}
+
+// Basic export config (for now)
+type ExportConfig struct {
+ Format string `json:"format"`
+ Git GitExportConfig `json:"git"`
+}
+
+type GitExportConfig struct {
+ // General folder is either at the root or as a subfolder
+ GeneralAtRoot bool `json:"generalAtRoot"`
+
+ // Keeping all history is nice, but much slower
+ ExcludeHistory bool `json:"excludeHistory"`
+}
+
+type Job interface {
+ getStatus() ExportStatus
+ getConfig() ExportConfig
+}
+
+// Will broadcast the live status
+type statusBroadcaster func(s ExportStatus)
diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go
index a96c4249061..1266e6f75f1 100644
--- a/pkg/services/featuremgmt/registry.go
+++ b/pkg/services/featuremgmt/registry.go
@@ -190,6 +190,12 @@ var (
Description: "Provisioning-friendly routes for alerting",
State: FeatureStateAlpha,
},
+ {
+ Name: "export",
+ Description: "Export grafana instance (to git, etc)",
+ State: FeatureStateAlpha,
+ RequiresDevMode: true,
+ },
{
Name: "storageLocalUpload",
Description: "allow uploads to local storage",
diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go
index 31978faca12..1966db6a137 100644
--- a/pkg/services/featuremgmt/toggles_gen.go
+++ b/pkg/services/featuremgmt/toggles_gen.go
@@ -143,6 +143,10 @@ const (
// Provisioning-friendly routes for alerting
FlagAlertProvisioning = "alertProvisioning"
+ // FlagExport
+ // Export grafana instance (to git, etc)
+ FlagExport = "export"
+
// FlagStorageLocalUpload
// allow uploads to local storage
FlagStorageLocalUpload = "storageLocalUpload"
diff --git a/public/app/features/admin/ExportStartButton.tsx b/public/app/features/admin/ExportStartButton.tsx
new file mode 100644
index 00000000000..f94d4f5b081
--- /dev/null
+++ b/public/app/features/admin/ExportStartButton.tsx
@@ -0,0 +1,62 @@
+import { css } from '@emotion/css';
+import React, { useState } from 'react';
+
+import { GrafanaTheme2 } from '@grafana/data';
+import { getBackendSrv } from '@grafana/runtime';
+import { Button, CodeEditor, Modal, useTheme2 } from '@grafana/ui';
+
+export const ExportStartButton = () => {
+ const styles = getStyles(useTheme2());
+ const [open, setOpen] = useState(false);
+ const [body, setBody] = useState({
+ format: 'git',
+ git: {},
+ });
+ const onDismiss = () => setOpen(false);
+ const doStart = () => {
+ getBackendSrv()
+ .post('/api/admin/export', body)
+ .then((v) => {
+ console.log('GOT', v);
+ onDismiss();
+ });
+ };
+
+ return (
+ <>
+
+