diff --git a/.drone.yml b/.drone.yml index 249df7599f2..7554714ed6b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1222,7 +1222,7 @@ depends_on: - main-build-e2e-publish - main-integration-tests kind: pipeline -name: windows-main +name: main-windows platform: arch: amd64 os: windows @@ -1310,9 +1310,9 @@ depends_on: - main-test - main-build-e2e-publish - main-integration-tests -- windows-main +- main-windows kind: pipeline -name: publish-main +name: main-publish node: type: no-parallel platform: @@ -1364,10 +1364,10 @@ depends_on: - main-test - main-build-e2e-publish - main-integration-tests -- windows-main -- publish-main +- main-windows +- main-publish kind: pipeline -name: notify-main +name: main-notify platform: arch: amd64 os: linux @@ -4536,6 +4536,6 @@ kind: secret name: gcp_upload_artifacts_key --- kind: signature -hmac: 9380beb722ab1276e9219ae41fa07c84db1e7dc249f69e42386f868c026e1795 +hmac: 48f6a3ade75e5993b4a49d8f1a5b7d184a63767a108b16c5f71eca6c986da9c7 ... diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b421e6f3663..8f364c8f8bc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -58,6 +58,7 @@ go.sum @grafana/backend-platform /pkg/services/live/ @grafana/grafana-edge-squad /pkg/services/searchV2/ @grafana/grafana-edge-squad /pkg/services/store/ @grafana/grafana-edge-squad +/pkg/services/export/ @grafana/grafana-edge-squad /pkg/infra/filestore/ @grafana/grafana-edge-squad pkg/tsdb/testdatasource/sims/ @grafana/grafana-edge-squad diff --git a/conf/defaults.ini b/conf/defaults.ini index 4bb5376c233..bb99b8a70b4 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -608,7 +608,7 @@ allow_sign_up = true # LDAP background sync (Enterprise only) # At 1 am every day -sync_cron = "0 0 1 * * *" +sync_cron = "0 1 * * *" active_sync_enabled = true #################################### AWS ########################### @@ -1176,3 +1176,22 @@ default_baselayer_config = # Enable or disable loading other base map layers enable_custom_baselayers = true + +#################################### Dashboard previews ##################################### + +[dashboard_previews.crawler] +# Number of dashboards rendered in parallel. Default is 6. +thread_count = + +# Timeout passed down to the Image Renderer plugin. It is used in two separate places within a single rendering request: +# First during the initial navigation to the dashboard and then when waiting for all the panels to load. Default is 20s. +# This setting should be expressed as a duration. Examples: 10s (seconds), 1m (minutes). +rendering_timeout = + +# Maximum duration of a single crawl. Default is 1h. +# This setting should be expressed as a duration. Examples: 10s (seconds), 1m (minutes). +max_crawl_duration = + +# Minimum interval between two subsequent scheduler runs. Default is 12h. +# This setting should be expressed as a duration. Examples: 10s (seconds), 1m (minutes). +scheduler_interval = diff --git a/conf/sample.ini b/conf/sample.ini index fbe7b6b7f8d..7807f03976b 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -591,7 +591,7 @@ # LDAP background sync (Enterprise only) # At 1 am every day -;sync_cron = "0 0 1 * * *" +;sync_cron = "0 1 * * *" ;active_sync_enabled = true #################################### AWS ########################### diff --git a/docs/sources/administration/configuration.md b/docs/sources/administration/configuration.md index 4a9106482d3..6ac1f0f6015 100644 --- a/docs/sources/administration/configuration.md +++ b/docs/sources/administration/configuration.md @@ -1371,6 +1371,14 @@ Configures the Profile section. Enable or disable the Profile section. Default is `enabled`. +## [query_history] + +Configures Query history in Explore. + +### enabled + +Enable or disable the Query history. Default is `disabled`. + ## [metrics] For detailed instructions, refer to [Internal Grafana metrics]({{< relref "view-server/internal-metrics.md" >}}). diff --git a/docs/sources/administration/set-up-for-high-availability.md b/docs/sources/administration/set-up-for-high-availability.md index 818986df7c8..9dd11efcc41 100644 --- a/docs/sources/administration/set-up-for-high-availability.md +++ b/docs/sources/administration/set-up-for-high-availability.md @@ -20,17 +20,15 @@ First, you need to set up MySQL or Postgres on another server and configure Graf You can find the configuration for doing that in the [[database]]({{< relref "../administration/configuration.md#database" >}}) section in the Grafana config. Grafana will now persist all long term data in the database. How to configure the database for high availability is out of scope for this guide. We recommend finding an expert on the database you're using. -## Alerting +## Alerting high availability -**Grafana 8 alerts** +Grafana alerting provides a new [highly-available model]({{< relref "../alerting/unified-alerting/high-availability/_index.md" >}}). It also preserves the semantics of legacy dashboard alerting by executing all alerts on every server and by sending notifications only once per alert. Load distribution between servers is not supported at this time. -Grafana 8 Alerts provides a new highly-available model under the hood. It preserves the previous semantics by executing all alerts on every server and notifications are sent only once per alert. There is no support for load distribution between servers at this time. - -For configuration, [follow the guide]({{< relref "../alerting/unified-alerting/high-availability.md" >}}). +For instructions on setting up alerting high availability, see [enable alerting high availability]({{< relref "../alerting/unified-alerting/high-availability/enable-alerting-ha.md" >}}). **Legacy dashboard alerts** -Legacy Grafana alerting supports a limited form of high availability. [Alert notifications]({{< relref "../alerting/old-alerting/notifications.md" >}}) are deduplicated when running multiple servers. This means all alerts are executed on every server but alert notifications are only sent once per alert. Grafana does not support load distribution between servers. +Legacy Grafana alerting supports a limited form of high availability. In this model, [alert notifications]({{< relref "../alerting/old-alerting/notifications.md" >}}) are deduplicated when running multiple servers. This means all alerts are executed on every server, but alert notifications are only sent once per alert. Grafana does not support load distribution between servers. ## Grafana Live diff --git a/docs/sources/alerting/unified-alerting/_index.md b/docs/sources/alerting/unified-alerting/_index.md index de360455048..0dc5778ef3d 100644 --- a/docs/sources/alerting/unified-alerting/_index.md +++ b/docs/sources/alerting/unified-alerting/_index.md @@ -8,7 +8,7 @@ weight = 113 Grafana 8.0 has new and improved alerting that centralizes alerting information in a single, searchable view. It is enabled by default for all new OSS instances, and is an [opt-in]({{< relref "./opt-in.md" >}}) feature for older installations that still use legacy dashboard alerting. We encourage you to create issues in the Grafana GitHub repository for bugs found while testing Grafana alerting. See also, [What's New with Grafana alerting]({{< relref "./difference-old-new.md" >}}). -> Refer to [Fine-grained access control]({{< relref "../enterprise/access-control/_index.md" >}}) in Grafana Enterprise to learn more about controlling access to alerts using fine-grained permissions. +> Refer to [Fine-grained access control]({{< relref "../../enterprise/access-control/_index.md" >}}) in Grafana Enterprise to learn more about controlling access to alerts using fine-grained permissions. When Grafana alerting is enabled, you can: diff --git a/docs/sources/alerting/unified-alerting/high-availability.md b/docs/sources/alerting/unified-alerting/high-availability.md deleted file mode 100644 index d85ddd9a8f4..00000000000 --- a/docs/sources/alerting/unified-alerting/high-availability.md +++ /dev/null @@ -1,44 +0,0 @@ -+++ -title = " High availability" -description = "High Availability" -keywords = ["grafana", "alerting", "tutorials", "ha", "high availability"] -weight = 450 -+++ - -# High availability - -The Grafana alerting system has two main components: a `Scheduler` and an internal `Alertmanager`. The `Scheduler` is responsible for the evaluation of your [alert rules]({{< relref "./fundamentals/evaluate-grafana-alerts.md" >}}) while the internal Alertmanager takes care of the **routing** and **grouping**. - -When it comes to running Grafana alerting in high availability the operational mode of the scheduler is unaffected such that all alerts continue be evaluated in each Grafana instance. Rather the operational change happens in the Alertmanager which **deduplicates** alert notifications across Grafana instances. - -{{< figure src="/static/img/docs/alerting/unified/high-availability-ua.png" class="docs-image--no-shadow" max-width= "750px" caption="High availability" >}} - -The coordination between Grafana instances happens via [a Gossip protocol](https://en.wikipedia.org/wiki/Gossip_protocol). Alerts are not gossiped between instances. It is expected that each scheduler delivers the same alerts to each Alertmanager. - -The two types of messages that are gossiped between instances are: - -- Notification logs: Who (which instance) notified what (which alert) -- Silences: If an alert should fire or not - -These two states are persisted in the database periodically and when Grafana is gracefully shutdown. - -## Enable high availability - -To enable high availability support you need to add at least 1 Grafana instance to the [`[ha_peer]` configuration option]({{}}) within the `[unified_alerting]` section: - -1. In your custom configuration file ($WORKING_DIR/conf/custom.ini), go to the `[unified_alerting]` section. -2. Set `[ha_peers]` to the number of hosts for each grafana instance in the cluster (using a format of host:port) e.g. `ha_peers=10.0.0.5:9094,10.0.0.6:9094,10.0.0.7:9094` -3. Gossiping of notifications and silences uses both TCP and UDP port 9094. Each Grafana instance will need to be able to accept incoming connections on these ports. -4. Set `[ha_listen_address]` to the instance IP address using a format of host:port (or the [Pod's](https://kubernetes.io/docs/concepts/workloads/pods/) IP in the case of using Kubernetes) by default it is set to listen to all interfaces (`0.0.0.0`). - -## Kubernetes - -If you are using Kubernetes, you can expose the pod IP [through an environment variable](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/) via the container definition such as: - -```bash -env: -- name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP -``` diff --git a/docs/sources/alerting/unified-alerting/high-availability/_index.md b/docs/sources/alerting/unified-alerting/high-availability/_index.md new file mode 100644 index 00000000000..8b04bee271e --- /dev/null +++ b/docs/sources/alerting/unified-alerting/high-availability/_index.md @@ -0,0 +1,25 @@ ++++ +title = " About alerting high availability" +description = "High availability" +keywords = ["grafana", "alerting", "tutorials", "ha", "high availability"] +weight = 450 ++++ + +# About alerting high availability + +The Grafana alerting system has two main components: a `Scheduler` and an internal `Alertmanager`. The `Scheduler` evaluates your [alert rules]({{< relref "../fundamentals/evaluate-grafana-alerts.md" >}}), while the internal Alertmanager manages **routing** and **grouping**. + +When running Grafana alerting in high availability, the operational mode of the scheduler remains unaffected, and each Grafana instance evaluates all alerts. The operational change happens in the Alertmanager when it deduplicates alert notifications across Grafana instances. + +{{< figure src="/static/img/docs/alerting/unified/high-availability-ua.png" class="docs-image--no-shadow" max-width= "750px" caption="High availability" >}} + +The coordination between Grafana instances happens via [a Gossip protocol](https://en.wikipedia.org/wiki/Gossip_protocol). Alerts are not gossiped between instances and each scheduler delivers the same volume of alerts to each Alertmanager. + +The two types of messages gossiped between Grafana instances are: + +- Notification logs: Who (which instance) notified what (which alert). +- Silences: If an alert should fire or not. + +The notification logs and silences are persisted in the database periodically and during a graceful Grafana shut down. + +For configuration instructions, refer to [enable alerting high availability]({{< relref "./enable-alerting-ha.md" >}}). diff --git a/docs/sources/alerting/unified-alerting/high-availability/enable-alerting-ha.md b/docs/sources/alerting/unified-alerting/high-availability/enable-alerting-ha.md new file mode 100644 index 00000000000..9a6c9e12afe --- /dev/null +++ b/docs/sources/alerting/unified-alerting/high-availability/enable-alerting-ha.md @@ -0,0 +1,36 @@ ++++ +title = "Enable alerting high availability" +description = "Enable alerting high availability" +keywords = ["grafana", "alerting", "tutorials", "ha", "high availability"] +weight = 450 ++++ + +# Enable alerting high availability + +You can enable [alerting high availability]({{< relref "./_index.md" >}}) support by updating the Grafana configuration file. On Kubernetes, you can enable alerting high availability by updating the Kubernetes container definition. + +## Update Grafana configuration file + +### Before you begin + +Since gossiping of notifications and silences uses both TCP and UDP port `9094`, ensure that each Grafana instance is able to accept incoming connections on these ports. + +**To enable high availability support:** + +1. In your custom configuration file ($WORKING_DIR/conf/custom.ini), go to the `[unified_alerting]` section. +2. Set `[ha_peers]` to the number of hosts for each Grafana instance in the cluster (using a format of host:port), for example, `ha_peers=10.0.0.5:9094,10.0.0.6:9094,10.0.0.7:9094`. + You must have at least one (1) Grafana instance added to the [`[ha_peer]` section. +3. Set `[ha_listen_address]` to the instance IP address using a format of `host:port` (or the [Pod's](https://kubernetes.io/docs/concepts/workloads/pods/) IP in the case of using Kubernetes). + By default, it is set to listen to all interfaces (`0.0.0.0`). + +## Update Kubernetes container definition + +If you are using Kubernetes, you can expose the pod IP [through an environment variable](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/) via the container definition such as: + +```bash +env: +- name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP +``` diff --git a/docs/sources/best-practices/dashboard-management-maturity-levels.md b/docs/sources/best-practices/dashboard-management-maturity-levels.md index 268cd83d7dd..17ca84905e6 100644 --- a/docs/sources/best-practices/dashboard-management-maturity-levels.md +++ b/docs/sources/best-practices/dashboard-management-maturity-levels.md @@ -46,7 +46,7 @@ How can you tell you are here? - Compare like to like: split service dashboards when the magnitude differs. Make sure aggregated metrics don't drown out important information. - Expressive charts with meaningful use of color and normalizing axes where you can. - - Example of meaningful color: Blue means it's good, red means it's bad. [Thresholds]({{< relref "../panels/specify-thresholds/about-thresholds.md" >}}) can help with that. + - Example of meaningful color: Blue means it's good, red means it's bad. [Thresholds]({{< relref "../panels/configure-thresholds" >}}) can help with that. - Example of normalizing axes: When comparing CPU usage, measure by percentage rather than raw number, because machines can have a different number of cores. Normalizing CPU usage by the number of cores reduces cognitive load because the viewer can trust that at 100% all cores are being used, without having to know the number of CPUs. - Directed browsing cuts down on "guessing." - Template variables make it harder to “just browse” randomly or aimlessly. diff --git a/docs/sources/dashboards/dashboard_folders.md b/docs/sources/dashboards/dashboard-folders.md similarity index 100% rename from docs/sources/dashboards/dashboard_folders.md rename to docs/sources/dashboards/dashboard-folders.md diff --git a/docs/sources/dashboards/dashboard_history.md b/docs/sources/dashboards/dashboard-history.md similarity index 100% rename from docs/sources/dashboards/dashboard_history.md rename to docs/sources/dashboards/dashboard-history.md diff --git a/docs/sources/enterprise/enhanced_ldap.md b/docs/sources/enterprise/enhanced_ldap.md index f66ae04c00c..d07f1b37750 100644 --- a/docs/sources/enterprise/enhanced_ldap.md +++ b/docs/sources/enterprise/enhanced_ldap.md @@ -48,9 +48,9 @@ Removed users are automatically logged out and their account disabled. These acc # @weekly | Run once a week, midnight between Sat/Sun | 0 0 0 * * 0 # @daily (or @midnight) | Run once a day, midnight | 0 0 0 * * * # @hourly | Run once an hour, beginning of hour | 0 0 * * * * -sync_cron = "0 0 1 * * *" # This is default value (At 1 am every day) -# This cron expression format uses 6 space-separated fields (including seconds), for example -# sync_cron = "* */10 * * * *" +sync_cron = "0 1 * * *" # This is default value (At 1 am every day) +# This cron expression format uses 5 space-separated fields, for example +# sync_cron = "*/10 * * * *" # This will run the LDAP Synchronization every 10th minute, which is also the minimal interval between the Grafana sync times i.e. you cannot set it for every 9th minute # You can also disable active LDAP synchronization diff --git a/docs/sources/enterprise/saml/set-up-saml-with-okta.md b/docs/sources/enterprise/saml/set-up-saml-with-okta.md index 4edd3de7a7b..827739f6480 100644 --- a/docs/sources/enterprise/saml/set-up-saml-with-okta.md +++ b/docs/sources/enterprise/saml/set-up-saml-with-okta.md @@ -13,7 +13,7 @@ Grafana supports user authentication through Okta, which is useful when you want ## Before you begin - To configure SAML integration with Okta, create integration inside the Okta organization first. [Add integration in Okta](https://help.okta.com/en/prod/Content/Topics/Apps/apps-overview-add-apps.htm) -- Ensure you have permission to administer SAML authentication. For more information about permissions, refer to [About users and permissions]({{< relref "../manage-users-and-permissions/about-users-and-permissions.md#">}}). +- Ensure you have permission to administer SAML authentication. For more information about permissions, refer to [About users and permissions]({{< relref "../../administration/manage-users-and-permissions/about-users-and-permissions.md#">}}). **To set up SAML with Okta:** diff --git a/docs/sources/panels/configure-thresholds/_index.md b/docs/sources/panels/configure-thresholds/_index.md new file mode 100644 index 00000000000..65ff85242f0 --- /dev/null +++ b/docs/sources/panels/configure-thresholds/_index.md @@ -0,0 +1,90 @@ +--- +title: 'Configure visualization thresholds' +menuTitle: 'Configure visualization thresholds' +description: 'This section includes information about using thresholds in your visualizations.' +weight: 300 +aliases: + [ + docs/grafana/latest/panels/thresholds/, + docs/grafana/latest/panels/, + docs/grafana/latest/panels/specify-thresholds/about-thresholds/, + docs/grafana/latest/panels/specify-thresholds/add-a-threshold/, + docs/grafana/latest/panels/specify-thresholds/add-threshold-to-graph/, + docs/grafana/latest/panels/specify-thresholds/delete-a-threshold/, + ] +--- + +# Configure visualization thresholds + +This section includes information about using thresholds in your visualizations. You'll learn about thresholds and their defaults, how to add or delete a threshold, and adding a threshold to a legacy panel. + +## About thresholds + +A threshold is a value that you specify for a metric that is visually reflected in a dashboard when the threshold value is met or exceeded. + +Thresholds provide one method for you to conditionally style and color your visualizations based on query results. You can apply thresholds to most, but not all, visualizations. For more information about visualizations, refer to [Visualization panels]({{< relref "../../visualizations" >}}). + +You can use thresholds to: + +- Color grid lines or grid ares areas in the [Time-series visualization]({{< relref "../../visualizations/time-series" >}}) +- Color lines in the [Time-series visualization]({{< relref "../../visualizations/time-series/graph-color-scheme/#from-thresholds" >}}) +- Color the background or value text in the [Stat visualization]({{< relref "../../visualizations/stat-panel" >}}) +- Color the gauge and threshold markers in the [Gauge visualization]({{< relref "../../visualizations/gauge-panel" >}}) +- Color markers in the [Geomap visualization]({{< relref "../../visualizations/geomap" >}}) +- Color cell text or background in the [Table visualization]({{< relref "../../visualizations/table" >}}) +- Define regions and region colors in the [State timeline visualization]({{< relref "../../visualizations/state-timeline" >}}) + +There are two types of thresholds: + +- **Absolute** thresholds are defined by a number. For example, 80 on a scale of 1 to 150. +- **Percentage** thresholds are defined relative to minimum or maximum. For example, 80 percent. + +### Default thresholds + +On visualizations that support it, Grafana sets default threshold values of: + +- 80 = red +- Base = green +- Mode = Absolute + +The **Base** value represents minus infinity. It is generally the “good” color. + +## Add or delete a threshold + +You can add as many thresholds to a panel as you want. Grafana automatically sorts thresholds values from highest to lowest. + +Delete a threshold when it is no longer relevant to your business operations. When you delete a threshold, the system removes the threshold from all visualizations that include the threshold. + +1. To add a threshold: + + a. Edit the panel to which you want to add a threshold. + + b. In the options side pane, locate the **Thresholds** section and click **+ Add threshold**. + + c. Select a threshold color, number, and mode. + Threshold mode applies to all thresholds on this panel. + + d. For a time-series panel, select a **Show thresholds** option. + +1. To delete a threshold, navigate to the panel that contains the threshold and click the trash icon next to the threshold you want to remove. + +## Add a threshold to a legacy graph panel + +In the Graph panel visualization, thresholds enable you to add lines or sections to a graph to make it easier to recognize when the graph crosses a threshold. + +1. Navigate to the graph panel to which you want to add a threshold. +1. On the **Panel** tab, click **Thresholds**. +1. Click **Add threshold**. +1. Complete the following fields: + - **T1 -** Both values are required to display a threshold. + - **lt** or **gt** - Select **lt** for less than or **gt** for greater than to indicate what the threshold applies to. + - **Value -** Enter a threshold value. Grafana draws a threshold line along the Y-axis at that value. + - **Color -** Choose a condition that corresponds to a color, or define your own color. + - **custom -** You define the fill color and line color. + - **critical -** Fill and line color are red. + - **warning -** Fill and line color are yellow. + - **ok -** Fill and line color are green. + - **Fill -** Controls whether the threshold fill is displayed. + - **Line -** Controls whether the threshold line is displayed. + - **Y-Axis -** Choose **left** or **right**. +1. Click **Save** to save the changes in the dashboard. diff --git a/docs/sources/panels/specify-thresholds/_index.md b/docs/sources/panels/specify-thresholds/_index.md deleted file mode 100644 index 7c879fc2116..00000000000 --- a/docs/sources/panels/specify-thresholds/_index.md +++ /dev/null @@ -1,11 +0,0 @@ -+++ -title = "Modify visualization text and background colors" -aliases = ["/docs/grafana/next/panels/thresholds/", "/docs/sources/panels/specify-thresholds/"] -weight = 300 -+++ - -# Modify visualization text and background colors - -Use thresholds to set the color of a visualization text and background. - -{{< section >}} diff --git a/docs/sources/panels/specify-thresholds/about-thresholds.md b/docs/sources/panels/specify-thresholds/about-thresholds.md deleted file mode 100644 index c99ae25c479..00000000000 --- a/docs/sources/panels/specify-thresholds/about-thresholds.md +++ /dev/null @@ -1,26 +0,0 @@ -+++ -title = "About thresholds" -weight = 10 -aliases = ["/docs/sources/panels/specify-thresholds/about-thresholds/"] -+++ - -# About thresholds - -Thresholds set the color of either the value text or the background based on conditions that you define. - -There are two types of thresholds: - -- **Absolute** thresholds are defined based on a number. For example, 80 on a scale of 1 to 150. -- **Percentage** thresholds are defined relative to minimum or maximum. For example, 80 percent. - -You can apply thresholds to most, but not all, visualizations. - -## Default thresholds - -On visualizations that support it, Grafana sets default threshold values of: - -- 80 = red -- Base = green -- Mode = Absolute - -The **Base** value represents minus infinity. It is generally the “good” color. diff --git a/docs/sources/panels/specify-thresholds/add-a-threshold.md b/docs/sources/panels/specify-thresholds/add-a-threshold.md deleted file mode 100644 index 27ce198f93a..00000000000 --- a/docs/sources/panels/specify-thresholds/add-a-threshold.md +++ /dev/null @@ -1,29 +0,0 @@ -+++ -title = "Add a threshold" -weight = 20 -aliases = ["/docs/sources/panels/specify-thresholds/add-a-threshold/"] -+++ - -# Add a threshold - -You can add as many thresholds to a panel as you want. Grafana automatically sorts thresholds from highest value to lowest. - -## Before you begin - -- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). - -**To add a threshold**: - -1. Edit the panel to which you want to add a threshold. -1. On the panel display options, locate the **Thresholds** section. -1. Click **+ Add threshold**. - - Grafana adds a threshold value and color. - -1. Accept the recommendations or edit the threshold. - - **Edit color:** To select a color, click the color dot. - - **Edit number:** To change the threshold value, click in the field and enter a number. -1. Select a **Threshold mode**. - Threshold mode applies to all thresholds on this panel. -1. In the **Show thresholds** drop-down, select a threshold display option. -1. Click **Save**. diff --git a/docs/sources/panels/specify-thresholds/add-threshold-to-graph.md b/docs/sources/panels/specify-thresholds/add-threshold-to-graph.md deleted file mode 100644 index edc6f2044a0..00000000000 --- a/docs/sources/panels/specify-thresholds/add-threshold-to-graph.md +++ /dev/null @@ -1,32 +0,0 @@ -+++ -title = "Add a threshold to a legacy graph panel" -weight = 40 -aliases = ["/docs/sources/panels/specify-thresholds/add-threshold-to-graph/"] -+++ - -# Add a threshold to a legacy graph panel - -In the Graph panel visualization, thresholds enable you to add lines or sections to a graph to make it easier to recognize when the graph crosses a threshold. - -## Before you begin - -- [Add a panel to a dashboard]({{< relref "../working-with-panels/add-panel.md" >}}). - -**To add a threshold to a graph panel**: - -1. Navigate to the graph panel to which you want to add a threshold. -1. On the **Panel** tab, click **Thresholds**. -1. Click **Add threshold**. -1. Complete the following fields: - - **T1 -** Both values are required to display a threshold. - - **lt** or **gt** - Select **lt** for less than or **gt** for greater than to indicate what the threshold applies to. - - **Value -** Enter a threshold value. Grafana draws a threshold line along the Y-axis at that value. - - **Color -** Choose a condition that corresponds to a color, or define your own color. - - **custom -** You define the fill color and line color. - - **critical -** Fill and line color are red. - - **warning -** Fill and line color are yellow. - - **ok -** Fill and line color are green. - - **Fill -** Controls whether the threshold fill is displayed. - - **Line -** Controls whether the threshold line is displayed. - - **Y-Axis -** Choose **left** or **right**. -1. Click **Save** to save the changes in the dashboard. diff --git a/docs/sources/panels/specify-thresholds/delete-a-threshold.md b/docs/sources/panels/specify-thresholds/delete-a-threshold.md deleted file mode 100644 index c673e211fc6..00000000000 --- a/docs/sources/panels/specify-thresholds/delete-a-threshold.md +++ /dev/null @@ -1,20 +0,0 @@ -+++ -title = "Delete a threshold" -weight = 30 -aliases = ["/docs/sources/panels/specify-thresholds/delete-a-threshold/"] -+++ - -# Delete a threshold - -Delete a threshold when it is no longer relevant to your business operations. When you delete a threshold, the system removes the threshold from all visualizations that include the threshold. - -## Before you begin - -- [Add a threshold]({{< relref "./add-a-threshold.md" >}}). - -**To delete a threshold**: - -1. Navigate to the panel to which you want to add a threshold. -1. Click the **Field** tab. (Or **Panel** tab for a graph panel.) -1. Click the trash can icon next to the threshold you want to remove. -1. Click **Save** to save the changes in the dashboard. diff --git a/docs/sources/panels/working-with-panels/add-panel.md b/docs/sources/panels/working-with-panels/add-panel.md index b35c960887c..7a09be1b9eb 100644 --- a/docs/sources/panels/working-with-panels/add-panel.md +++ b/docs/sources/panels/working-with-panels/add-panel.md @@ -47,7 +47,7 @@ Panels allow you to show your data in visual form. Each panel needs at least one - [Format data using value mapping]({{< relref "../format-data/about-value-mapping.md" >}}) - [Visualization-specific options]({{< relref "../../visualizations/_index.md" >}}) - [Override field values]({{< relref "../override-field-values/about-field-overrides.md" >}}) - - [Specify thresholds to set the color of visualization text and background]({{< relref "../specify-thresholds/about-thresholds.md" >}}) + - [Configure thresholds]({{< relref "../configure-thresholds/" >}}) - [Apply color to series and fields]({{< relref "./apply-color-to-series.md" >}}) 1. Add a note to describe the visualization (or describe your changes) and then click **Save** in the upper-right corner of the page. diff --git a/docs/sources/visualizations/graph-panel.md b/docs/sources/visualizations/graph-panel.md index 61e869246cb..6832bca406e 100644 --- a/docs/sources/visualizations/graph-panel.md +++ b/docs/sources/visualizations/graph-panel.md @@ -18,7 +18,7 @@ Graph visualizations allow you to apply: - [Alerts]({{< relref "../alerting/_index.md" >}}) - This is the only type of visualization that allows you to set alerts. - [Transform data]({{< relref "../panels/transform-data/add-transformation-to-data.md" >}}) - [Add a field override]({{< relref "../panels/override-field-values/add-a-field-override.md" >}}) -- [Add a threshold]({{< relref "../panels/specify-thresholds/add-a-threshold.md" >}}) +- [Configure thresholds]({{< relref "../panels/configure-thresholds/" >}}) ## Display options diff --git a/docs/sources/whatsnew/whats-new-in-v7-0.md b/docs/sources/whatsnew/whats-new-in-v7-0.md index 0aeb5342e61..ac172c4d711 100644 --- a/docs/sources/whatsnew/whats-new-in-v7-0.md +++ b/docs/sources/whatsnew/whats-new-in-v7-0.md @@ -214,7 +214,7 @@ This release includes a series of features that build on our new usage analytics ### SAML Role and Team Sync -SAML support in Grafana Enterprise is improved by adding Role and Team Sync. Read more about how to use these features in the [SAML team sync documentation]({{< relref "../enterprise/saml.md#configure-team-sync" >}}). +SAML support in Grafana Enterprise is improved by adding Role and Team Sync. Read more about how to use these features in the [SAML team sync documentation]({{< relref "../enterprise/saml/configure-saml.md#configure-team-sync" >}}). ### Okta OAuth Team Sync diff --git a/docs/sources/whatsnew/whats-new-in-v7-4.md b/docs/sources/whatsnew/whats-new-in-v7-4.md index aa2908ec258..267efa72fd4 100644 --- a/docs/sources/whatsnew/whats-new-in-v7-4.md +++ b/docs/sources/whatsnew/whats-new-in-v7-4.md @@ -202,7 +202,7 @@ For more information, refer to [Export logs of usage insights]({{< relref "../en ### New audit log events -New log out events are logged based on when a token expires or is revoked, as well as [SAML Single Logout]({{< relref "../enterprise/saml.md#single-logout" >}}). A `tokenId` field was added to all audit logs to help understand which session was logged out of. +New log out events are logged based on when a token expires or is revoked, as well as [SAML Single Logout]({{< relref "../enterprise/saml/configure-saml.md#single-logout" >}}). A `tokenId` field was added to all audit logs to help understand which session was logged out of. Also, a counter for audit log writing actions with status (success / failure) and logger (loki / file / console) labels was added. diff --git a/go.mod b/go.mod index 55c4cabade7..19e8e6a6d86 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,6 @@ require ( github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.32.1 github.com/prometheus/prometheus v1.8.2-0.20211011171444-354d8d2ecfac - github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 github.com/robfig/cron/v3 v3.0.1 github.com/russellhaering/goxmldsig v1.1.1 github.com/stretchr/testify v1.7.1 diff --git a/package.json b/package.json index f29a1f170c9..359c944c8c7 100644 --- a/package.json +++ b/package.json @@ -250,7 +250,7 @@ "@grafana/aws-sdk": "0.0.35", "@grafana/data": "workspace:*", "@grafana/e2e-selectors": "workspace:*", - "@grafana/experimental": "^0.0.2-canary.25", + "@grafana/experimental": "^0.0.2-canary.30", "@grafana/google-sdk": "0.0.3", "@grafana/lezer-logql": "^0.0.11", "@grafana/runtime": "workspace:*", diff --git a/packages/grafana-data/src/themes/createComponents.ts b/packages/grafana-data/src/themes/createComponents.ts index fefd3dbf101..51603f036c7 100644 --- a/packages/grafana-data/src/themes/createComponents.ts +++ b/packages/grafana-data/src/themes/createComponents.ts @@ -39,6 +39,12 @@ export interface ThemeComponents { sidemenu: { width: number; }; + menuTabs: { + height: number; + }; + horizontalDrawer: { + defaultHeight: number; + }; } export function createComponents(colors: ThemeColors, shadows: ThemeShadows): ThemeComponents { @@ -82,5 +88,11 @@ export function createComponents(colors: ThemeColors, shadows: ThemeShadows): Th sidemenu: { width: 48, }, + menuTabs: { + height: 41, + }, + horizontalDrawer: { + defaultHeight: 400, + }, }; } diff --git a/packages/grafana-data/src/transformations/transformers/renameByRegex.test.ts b/packages/grafana-data/src/transformations/transformers/renameByRegex.test.ts index aa630829a58..170811442db 100644 --- a/packages/grafana-data/src/transformations/transformers/renameByRegex.test.ts +++ b/packages/grafana-data/src/transformations/transformers/renameByRegex.test.ts @@ -80,6 +80,59 @@ describe('Rename By Regex Transformer', () => { }); }); + it('should be able to replace globally', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.renameByRegex, + options: { + regex: '/e/g', + renamePattern: 'E', + }, + }; + await expect(transformDataFrame([cfg], [data])).toEmitValuesWith((received) => { + const data = received[0]; + const frame = data[0]; + expect(frame.fields).toMatchInlineSnapshot(` + Array [ + Object { + "config": Object { + "displayName": "TimE", + "name": "Time", + }, + "name": "Time", + "state": Object { + "displayName": "TimE", + "multipleFrames": false, + }, + "type": "time", + "values": Array [ + 3000, + 4000, + 5000, + 6000, + ], + }, + Object { + "config": Object { + "displayName": "wEb-01.ExamplE.com", + }, + "name": "Value", + "state": Object { + "displayName": "wEb-01.ExamplE.com", + "multipleFrames": false, + }, + "type": "number", + "values": Array [ + 10000.3, + 10000.4, + 10000.5, + 10000.6, + ], + }, + ] + `); + }); + }); + it('should not rename misses', async () => { const cfg: DataTransformerConfig = { id: DataTransformerID.renameByRegex, @@ -147,7 +200,6 @@ describe('Rename By Regex Transformer', () => { Array [ Object { "config": Object { - "displayName": "Time", "name": "Time", }, "name": "Time", diff --git a/packages/grafana-data/src/transformations/transformers/renameByRegex.ts b/packages/grafana-data/src/transformations/transformers/renameByRegex.ts index 05426c988c0..ad6e2d39942 100644 --- a/packages/grafana-data/src/transformations/transformers/renameByRegex.ts +++ b/packages/grafana-data/src/transformations/transformers/renameByRegex.ts @@ -1,6 +1,7 @@ import { map } from 'rxjs/operators'; import { getFieldDisplayName } from '../../field/fieldState'; +import { stringToJsRegex } from '../../text/string'; import { DataFrame } from '../../types/dataFrame'; import { DataTransformerInfo } from '../../types/transformations'; @@ -47,7 +48,7 @@ export const renameByRegexTransformer: DataTransformerInfo (frame: DataFrame) => { - const regex = new RegExp(options.regex); + const regex = stringToJsRegex(options.regex); const fields = frame.fields.map((field) => { const displayName = getFieldDisplayName(field, frame); if (!regex.test(displayName)) { diff --git a/packages/grafana-data/src/types/config.ts b/packages/grafana-data/src/types/config.ts index bc7c0f13a15..f1233175bc0 100644 --- a/packages/grafana-data/src/types/config.ts +++ b/packages/grafana-data/src/types/config.ts @@ -153,6 +153,7 @@ export interface GrafanaConfig { alertingMinInterval: number; authProxyEnabled: boolean; exploreEnabled: boolean; + queryHistoryEnabled: boolean; helpEnabled: boolean; profileEnabled: boolean; ldapEnabled: boolean; diff --git a/packages/grafana-data/src/types/datasource.ts b/packages/grafana-data/src/types/datasource.ts index c3bf11b2ef2..d05fc6985e6 100644 --- a/packages/grafana-data/src/types/datasource.ts +++ b/packages/grafana-data/src/types/datasource.ts @@ -416,6 +416,7 @@ export type ExploreQueryFieldProps< export interface QueryEditorHelpProps { datasource: DataSourceApi; + query: TQuery; onClickExample: (query: TQuery) => void; exploreId?: any; } diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index e5b4cb3e532..61741409adc 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -22,7 +22,6 @@ export interface FeatureToggles { serviceAccounts?: boolean; database_metrics?: boolean; dashboardPreviews?: boolean; - dashboardPreviewsScheduler?: boolean; dashboardPreviewsAdmin?: boolean; ['live-config']?: boolean; ['live-pipeline']?: boolean; @@ -51,9 +50,12 @@ export interface FeatureToggles { saveDashboardDrawer?: boolean; storage?: boolean; alertProvisioning?: boolean; + export?: boolean; storageLocalUpload?: boolean; azureMonitorResourcePickerForMetrics?: boolean; explore2Dashboard?: boolean; + tracing?: boolean; persistNotifications?: boolean; commandPalette?: boolean; + savedItems?: boolean; } diff --git a/packages/grafana-e2e/src/flows/configurePanel.ts b/packages/grafana-e2e/src/flows/configurePanel.ts index 275561809d1..384bef3200c 100644 --- a/packages/grafana-e2e/src/flows/configurePanel.ts +++ b/packages/grafana-e2e/src/flows/configurePanel.ts @@ -1,4 +1,4 @@ -import { e2e } from '../index'; +import { e2e } from '..'; import { getScenarioContext } from '../support/scenarioContext'; import { selectOption } from './selectOption'; @@ -148,7 +148,7 @@ export const configurePanel = (config: PartialAddPanelConfig | PartialEditPanelC //e2e().wait('@chartData'); // Avoid annotations flakiness - e2e.components.RefreshPicker.runButton().should('be.visible').click(); + e2e.components.RefreshPicker.runButtonV2().first().should('be.visible').click({ force: true }); e2e().wait('@chartData'); diff --git a/packages/grafana-runtime/src/config.ts b/packages/grafana-runtime/src/config.ts index a37143b9de8..9aef491b0c8 100644 --- a/packages/grafana-runtime/src/config.ts +++ b/packages/grafana-runtime/src/config.ts @@ -47,6 +47,7 @@ export class GrafanaBootConfig implements GrafanaConfig { angularSupportEnabled = false; authProxyEnabled = false; exploreEnabled = false; + queryHistoryEnabled = false; helpEnabled = false; profileEnabled = false; ldapEnabled = false; diff --git a/packages/grafana-ui/src/components/Forms/Field.tsx b/packages/grafana-ui/src/components/Forms/Field.tsx index 82ab1d5d949..a7c7b406c08 100644 --- a/packages/grafana-ui/src/components/Forms/Field.tsx +++ b/packages/grafana-ui/src/components/Forms/Field.tsx @@ -97,11 +97,12 @@ export const Field: React.FC = ({ label ); + const childProps = deleteUndefinedProps({ invalid, disabled, loading }); return (
{labelElement}
- {React.cloneElement(children, { invalid, disabled, loading })} + {React.cloneElement(children, childProps)} {invalid && error && !horizontal && (
= ({
); }; + +function deleteUndefinedProps(obj: T): Partial { + for (const key in obj) { + if (obj[key] === undefined) { + delete obj[key]; + } + } + + return obj; +} diff --git a/packages/grafana-ui/src/components/GraphNG/utils.test.ts b/packages/grafana-ui/src/components/GraphNG/utils.test.ts index 7395c859226..23efa7a7726 100644 --- a/packages/grafana-ui/src/components/GraphNG/utils.test.ts +++ b/packages/grafana-ui/src/components/GraphNG/utils.test.ts @@ -1,6 +1,8 @@ import { + ArrayVector, createTheme, DashboardCursorSync, + DataFrame, DefaultTimeZone, EventBusSrv, FieldConfig, @@ -204,4 +206,301 @@ describe('GraphNG utils', () => { }).getConfig(); expect(result).toMatchSnapshot(); }); + + test('preparePlotFrame appends min bar spaced nulls when > 1 bar series', () => { + const df1: DataFrame = { + name: 'A', + length: 5, + fields: [ + { + name: 'time', + type: FieldType.time, + config: {}, + values: new ArrayVector([1, 2, 4, 6, 100]), // should find smallest delta === 1 from here + }, + { + name: 'value', + type: FieldType.number, + config: { + custom: { + drawStyle: GraphDrawStyle.Bars, + }, + }, + values: new ArrayVector([1, 1, 1, 1, 1]), + }, + ], + }; + + const df2: DataFrame = { + name: 'B', + length: 5, + fields: [ + { + name: 'time', + type: FieldType.time, + config: {}, + values: new ArrayVector([30, 40, 50, 90, 100]), // should be appended with two smallest-delta increments + }, + { + name: 'value', + type: FieldType.number, + config: { + custom: { + drawStyle: GraphDrawStyle.Bars, + }, + }, + values: new ArrayVector([2, 2, 2, 2, 2]), // bar series should be appended with nulls + }, + { + name: 'value', + type: FieldType.number, + config: { + custom: { + drawStyle: GraphDrawStyle.Line, + }, + }, + values: new ArrayVector([3, 3, 3, 3, 3]), // line series should be appended with undefineds + }, + ], + }; + + const df3: DataFrame = { + name: 'C', + length: 2, + fields: [ + { + name: 'time', + type: FieldType.time, + config: {}, + values: new ArrayVector([1, 1.1]), // should not trip up on smaller deltas of non-bars + }, + { + name: 'value', + type: FieldType.number, + config: { + custom: { + drawStyle: GraphDrawStyle.Line, + }, + }, + values: new ArrayVector([4, 4]), + }, + { + name: 'value', + type: FieldType.number, + config: { + custom: { + drawStyle: GraphDrawStyle.Bars, + hideFrom: { + viz: true, // should ignore hidden bar series + }, + }, + }, + values: new ArrayVector([4, 4]), + }, + ], + }; + + let aligndFrame = preparePlotFrame([df1, df2, df3], { + x: fieldMatchers.get(FieldMatcherID.firstTimeField).get({}), + y: fieldMatchers.get(FieldMatcherID.numeric).get({}), + }); + + expect(aligndFrame).toMatchInlineSnapshot(` + Object { + "fields": Array [ + Object { + "config": Object {}, + "name": "time", + "state": Object { + "origin": Object { + "fieldIndex": 0, + "frameIndex": 0, + }, + }, + "type": "time", + "values": Array [ + 1, + 1.1, + 2, + 4, + 6, + 30, + 40, + 50, + 90, + 100, + 101, + 102, + ], + }, + Object { + "config": Object { + "custom": Object { + "drawStyle": "bars", + "spanNulls": -1, + }, + }, + "labels": Object { + "name": "A", + }, + "name": "value", + "state": Object { + "origin": Object { + "fieldIndex": 1, + "frameIndex": 0, + }, + }, + "type": "number", + "values": Array [ + 1, + undefined, + 1, + 1, + 1, + undefined, + undefined, + undefined, + undefined, + 1, + null, + null, + ], + }, + Object { + "config": Object { + "custom": Object { + "drawStyle": "bars", + "spanNulls": -1, + }, + }, + "labels": Object { + "name": "B", + }, + "name": "value", + "state": Object { + "origin": Object { + "fieldIndex": 1, + "frameIndex": 1, + }, + }, + "type": "number", + "values": Array [ + undefined, + undefined, + undefined, + undefined, + undefined, + 2, + 2, + 2, + 2, + 2, + null, + null, + ], + }, + Object { + "config": Object { + "custom": Object { + "drawStyle": "line", + }, + }, + "labels": Object { + "name": "B", + }, + "name": "value", + "state": Object { + "origin": Object { + "fieldIndex": 2, + "frameIndex": 1, + }, + }, + "type": "number", + "values": Array [ + undefined, + undefined, + undefined, + undefined, + undefined, + 3, + 3, + 3, + 3, + 3, + undefined, + undefined, + ], + }, + Object { + "config": Object { + "custom": Object { + "drawStyle": "line", + }, + }, + "labels": Object { + "name": "C", + }, + "name": "value", + "state": Object { + "origin": Object { + "fieldIndex": 1, + "frameIndex": 2, + }, + }, + "type": "number", + "values": Array [ + 4, + 4, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + ], + }, + Object { + "config": Object { + "custom": Object { + "drawStyle": "bars", + "hideFrom": Object { + "viz": true, + }, + }, + }, + "labels": Object { + "name": "C", + }, + "name": "value", + "state": Object { + "origin": Object { + "fieldIndex": 2, + "frameIndex": 2, + }, + }, + "type": "number", + "values": Array [ + 4, + 4, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + ], + }, + ], + "length": 12, + } + `); + }); }); diff --git a/packages/grafana-ui/src/components/GraphNG/utils.ts b/packages/grafana-ui/src/components/GraphNG/utils.ts index 41558d9bd3c..5d35a8e3926 100644 --- a/packages/grafana-ui/src/components/GraphNG/utils.ts +++ b/packages/grafana-ui/src/components/GraphNG/utils.ts @@ -1,4 +1,4 @@ -import { ArrayVector, DataFrame, FieldConfig, FieldType, outerJoinDataFrames, TimeRange } from '@grafana/data'; +import { ArrayVector, DataFrame, Field, FieldConfig, FieldType, outerJoinDataFrames, TimeRange } from '@grafana/data'; import { AxisPlacement, GraphDrawStyle, @@ -12,6 +12,12 @@ import { applyNullInsertThreshold } from './nullInsertThreshold'; import { nullToUndefThreshold } from './nullToUndefThreshold'; import { XYFieldMatchers } from './types'; +function isVisibleBarField(f: Field) { + return ( + f.type === FieldType.number && f.config.custom?.drawStyle === GraphDrawStyle.Bars && !f.config.custom?.hideFrom?.viz + ); +} + // will mutate the DataFrame's fields' values function applySpanNullsThresholds(frame: DataFrame) { let refField = frame.fields.find((field) => field.type === FieldType.time); // this doesnt need to be time, just any numeric/asc join field @@ -20,7 +26,7 @@ function applySpanNullsThresholds(frame: DataFrame) { for (let i = 0; i < frame.fields.length; i++) { let field = frame.fields[i]; - if (field === refField) { + if (field === refField || isVisibleBarField(field)) { continue; } @@ -37,30 +43,78 @@ function applySpanNullsThresholds(frame: DataFrame) { } export function preparePlotFrame(frames: DataFrame[], dimFields: XYFieldMatchers, timeRange?: TimeRange | null) { - let alignedFrame = outerJoinDataFrames({ - frames: frames.map((frame) => { - let fr = applyNullInsertThreshold(frame, null, timeRange?.to.valueOf()); + // apply null insertions at interval + frames = frames.map((frame) => applyNullInsertThreshold(frame, null, timeRange?.to.valueOf())); - // prevent minesweeper-expansion of nulls (gaps) when joining bars - // since bar width is determined from the minimum distance between non-undefined values - // (this strategy will still retain any original pre-join nulls, though) - fr.fields.forEach((f) => { - if (f.type === FieldType.number && f.config.custom?.drawStyle === GraphDrawStyle.Bars) { - f.config.custom = { - ...f.config.custom, - spanNulls: -1, - }; + let numBarSeries = 0; + + frames.forEach((frame) => { + frame.fields.forEach((f) => { + if (isVisibleBarField(f)) { + // prevent minesweeper-expansion of nulls (gaps) when joining bars + // since bar width is determined from the minimum distance between non-undefined values + // (this strategy will still retain any original pre-join nulls, though) + f.config.custom = { + ...f.config.custom, + spanNulls: -1, + }; + + numBarSeries++; + } + }); + }); + + // to make bar widths of all series uniform (equal to narrowest bar series), find smallest distance between x points + let minXDelta = Infinity; + + if (numBarSeries > 1) { + frames.forEach((frame) => { + if (!frame.fields.some(isVisibleBarField)) { + return; + } + + const xVals = frame.fields[0].values.toArray(); + + for (let i = 0; i < xVals.length; i++) { + if (i > 0) { + minXDelta = Math.min(minXDelta, xVals[i] - xVals[i - 1]); } - }); + } + }); + } - return fr; - }), + let alignedFrame = outerJoinDataFrames({ + frames, joinBy: dimFields.x, keep: dimFields.y, keepOriginIndices: true, }); - return alignedFrame && applySpanNullsThresholds(alignedFrame); + if (alignedFrame) { + alignedFrame = applySpanNullsThresholds(alignedFrame); + + // append 2 null vals at minXDelta to bar series + if (minXDelta !== Infinity) { + alignedFrame.fields.forEach((f, fi) => { + let vals = f.values.toArray(); + + if (fi === 0) { + let lastVal = vals[vals.length - 1]; + vals.push(lastVal + minXDelta, lastVal + 2 * minXDelta); + } else if (isVisibleBarField(f)) { + vals.push(null, null); + } else { + vals.push(undefined, undefined); + } + }); + + alignedFrame.length += 2; + } + + return alignedFrame; + } + + return null; } export function buildScaleKey(config: FieldConfig) { diff --git a/packages/grafana-ui/src/components/TabbedContainer/TabbedContainer.tsx b/packages/grafana-ui/src/components/TabbedContainer/TabbedContainer.tsx index 0baad2778c7..f4a14cfe11e 100644 --- a/packages/grafana-ui/src/components/TabbedContainer/TabbedContainer.tsx +++ b/packages/grafana-ui/src/components/TabbedContainer/TabbedContainer.tsx @@ -28,7 +28,7 @@ const getStyles = stylesFactory((theme: GrafanaTheme2) => { tabContent: css` padding: ${theme.spacing(2)}; background-color: ${theme.colors.background.primary}; - height: 100%; + height: calc(100% - ${theme.components.menuTabs.height}px); `, close: css` position: absolute; diff --git a/packages/grafana-ui/src/components/Table/Filter.tsx b/packages/grafana-ui/src/components/Table/Filter.tsx index 824bcdfc3b2..cae40aea9a2 100644 --- a/packages/grafana-ui/src/components/Table/Filter.tsx +++ b/packages/grafana-ui/src/components/Table/Filter.tsx @@ -32,6 +32,7 @@ export const Filter: FC = ({ column, field, tableStyles }) => { diff --git a/packages/grafana-ui/src/components/Table/Table.test.tsx b/packages/grafana-ui/src/components/Table/Table.test.tsx index 1eb1e91be58..271fd6180b6 100644 --- a/packages/grafana-ui/src/components/Table/Table.test.tsx +++ b/packages/grafana-ui/src/components/Table/Table.test.tsx @@ -177,4 +177,35 @@ describe('Table', () => { ]); }); }); + + describe('on filtering', () => { + it('the rows should be filtered', async () => { + getTestContext({ + data: toDataFrame({ + name: 'A', + fields: [ + { + name: 'number', + type: FieldType.number, + values: [1, 1, 1, 2, 2, 3, 4, 5], + config: { + custom: { + filterable: true, + }, + }, + }, + ], + }), + }); + + expect(within(getTable()).getAllByRole('row')).toHaveLength(9); + + await userEvent.click(within(getColumnHeader(/number/)).getByRole('filterIcon')); + await userEvent.click(screen.getByLabelText('1')); + await userEvent.click(screen.getByText('Ok')); + + // 3 + header row + expect(within(getTable()).getAllByRole('row')).toHaveLength(4); + }); + }); }); diff --git a/packages/grafana-ui/src/components/Table/Table.tsx b/packages/grafana-ui/src/components/Table/Table.tsx index 7e407d12e04..1ca8e3a9ab8 100644 --- a/packages/grafana-ui/src/components/Table/Table.tsx +++ b/packages/grafana-ui/src/components/Table/Table.tsx @@ -251,7 +251,7 @@ export const Table: FC = memo((props: Props) => { [gotoPage] ); - const itemCount = enablePagination ? page.length : data.length; + const itemCount = enablePagination ? page.length : rows.length; let paginationEl = null; if (enablePagination) { const itemsRangeStart = state.pageIndex * state.pageSize + 1; diff --git a/packages/grafana-ui/src/components/Tabs/TabsBar.tsx b/packages/grafana-ui/src/components/Tabs/TabsBar.tsx index 73acd2a67bc..6cfb03409be 100644 --- a/packages/grafana-ui/src/components/Tabs/TabsBar.tsx +++ b/packages/grafana-ui/src/components/Tabs/TabsBar.tsx @@ -23,7 +23,7 @@ const getTabsBarStyles = stylesFactory((theme: GrafanaTheme2, hideBorder = false tabs: css` position: relative; display: flex; - height: 41px; + height: ${theme.components.menuTabs.height}px; `, }; }); diff --git a/packages/grafana-ui/src/components/uPlot/utils.test.ts b/packages/grafana-ui/src/components/uPlot/utils.test.ts index 09df64ff81c..6075349a62d 100644 --- a/packages/grafana-ui/src/components/uPlot/utils.test.ts +++ b/packages/grafana-ui/src/components/uPlot/utils.test.ts @@ -230,29 +230,29 @@ describe('preparePlotData2', () => { ], }); expect(preparePlotData2(df, getStackingGroups(df))).toMatchInlineSnapshot(` - Array [ - Array [ - 9997, - 9998, - 9999, - ], - Array [ - -10, - 20, - 10, - ], - Array [ - 10, - 10, - 10, - ], - Array [ - 20, - 20, - 20, - ], - ] - `); + Array [ + Array [ + 9997, + 9998, + 9999, + ], + Array [ + -10, + 20, + 10, + ], + Array [ + 10, + 10, + 10, + ], + Array [ + 20, + 20, + 20, + ], + ] + `); }); it('standard', () => { @@ -289,14 +289,14 @@ describe('preparePlotData2', () => { 10, ], Array [ - 0, - 30, - 20, + 10, + 10, + 10, ], Array [ - 20, - 50, - 40, + 30, + 30, + 30, ], ] `); @@ -345,19 +345,19 @@ describe('preparePlotData2', () => { 10, ], Array [ + 10, + 10, + 10, + ], + Array [ + -30, 0, - 30, - 20, + -10, ], Array [ + -40, + -10, -20, - -20, - -20, - ], - Array [ - -30, - -30, - -30, ], ] `); @@ -413,14 +413,14 @@ describe('preparePlotData2', () => { 10, ], Array [ - 0, - 30, - 20, + 10, + 10, + 10, ], Array [ - 20, - 50, - 40, + 30, + 30, + 30, ], Array [ 1, @@ -580,13 +580,13 @@ describe('auto stacking groups', () => { "dir": -1, "series": Array [ 1, - 3, ], }, Object { "dir": 1, "series": Array [ 2, + 3, ], }, ] @@ -622,11 +622,6 @@ describe('auto stacking groups', () => { "series": Array [ 1, 2, - ], - }, - Object { - "dir": -1, - "series": Array [ 3, ], }, diff --git a/packages/grafana-ui/src/components/uPlot/utils.ts b/packages/grafana-ui/src/components/uPlot/utils.ts index 9c76ed929a3..0e8923bb387 100755 --- a/packages/grafana-ui/src/components/uPlot/utils.ts +++ b/packages/grafana-ui/src/components/uPlot/utils.ts @@ -115,16 +115,17 @@ export function getStackingGroups(frame: DataFrame) { // will this be stacked up or down after any transforms applied let vals = values.toArray(); let transform = custom.transform; + let firstValue = vals.find((v) => v != null); let stackDir = transform === GraphTransform.Constant - ? vals[0] > 0 + ? firstValue >= 0 ? StackDirection.Pos : StackDirection.Neg : transform === GraphTransform.NegativeY - ? vals.some((v) => v > 0) + ? firstValue >= 0 ? StackDirection.Neg : StackDirection.Pos - : vals.some((v) => v > 0) + : firstValue >= 0 ? StackDirection.Pos : StackDirection.Neg; diff --git a/pkg/api/annotations.go b/pkg/api/annotations.go index fee82d3fbb8..7aa1a2a3a2e 100644 --- a/pkg/api/annotations.go +++ b/pkg/api/annotations.go @@ -12,7 +12,6 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/annotations" - "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/guardian" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" @@ -275,7 +274,7 @@ func (hs *HTTPServer) MassDeleteAnnotations(c *models.ReqContext) response.Respo // validations only for RBAC. A user can mass delete all annotations in a (dashboard + panel) or a specific annotation // if has access to that dashboard. - if hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if !hs.AccessControl.IsDisabled() { var dashboardId int64 if cmd.AnnotationId != 0 { @@ -351,7 +350,7 @@ func (hs *HTTPServer) canSaveAnnotation(c *models.ReqContext, annotation *annota if annotation.GetType() == annotations.Dashboard { return canEditDashboard(c, annotation.DashboardId) } else { - if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if hs.AccessControl.IsDisabled() { return c.SignedInUser.HasRole(models.ROLE_EDITOR), nil } return true, nil @@ -446,7 +445,7 @@ func AnnotationTypeScopeResolver() (string, accesscontrol.AttributeScopeResolveF func (hs *HTTPServer) canCreateAnnotation(c *models.ReqContext, dashboardId int64) (bool, error) { if dashboardId != 0 { - if hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if !hs.AccessControl.IsDisabled() { evaluator := accesscontrol.EvalPermission(accesscontrol.ActionAnnotationsCreate, accesscontrol.ScopeAnnotationsTypeDashboard) if canSave, err := hs.AccessControl.Evaluate(c.Req.Context(), c.SignedInUser, evaluator); err != nil || !canSave { return canSave, err @@ -454,7 +453,7 @@ func (hs *HTTPServer) canCreateAnnotation(c *models.ReqContext, dashboardId int6 } return canEditDashboard(c, dashboardId) } else { // organization annotations - if hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if !hs.AccessControl.IsDisabled() { evaluator := accesscontrol.EvalPermission(accesscontrol.ActionAnnotationsCreate, accesscontrol.ScopeAnnotationsTypeOrganization) return hs.AccessControl.Evaluate(c.Req.Context(), c.SignedInUser, evaluator) } else { diff --git a/pkg/api/api.go b/pkg/api/api.go index 24883640e05..030c920ada2 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -523,6 +523,11 @@ func (hs *HTTPServer) registerRoutes() { adminRoute.Get("/crawler/status", reqGrafanaAdmin, routing.Wrap(hs.ThumbService.CrawlerStatus)) } + if hs.Features.IsEnabled(featuremgmt.FlagExport) { + adminRoute.Get("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleGetStatus)) + adminRoute.Post("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleRequestExport)) + } + adminRoute.Post("/provisioning/dashboards/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDashboards)), routing.Wrap(hs.AdminProvisioningReloadDashboards)) adminRoute.Post("/provisioning/plugins/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersPlugins)), routing.Wrap(hs.AdminProvisioningReloadPlugins)) adminRoute.Post("/provisioning/datasources/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDatasources)), routing.Wrap(hs.AdminProvisioningReloadDatasources)) diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index fccd521269f..8809c9fccfb 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -409,7 +409,7 @@ func (hs *HTTPServer) postDashboard(c *models.ReqContext, cmd models.SaveDashboa // GetHomeDashboard returns the home dashboard. func (hs *HTTPServer) GetHomeDashboard(c *models.ReqContext) response.Response { - prefsQuery := pref.GetPreferenceWithDefaultsQuery{UserID: c.SignedInUser.UserId} + prefsQuery := pref.GetPreferenceWithDefaultsQuery{OrgID: c.OrgId, UserID: c.SignedInUser.UserId} homePage := hs.Cfg.HomePage preference, err := hs.preferenceService.GetWithDefaults(c.Req.Context(), &prefsQuery) diff --git a/pkg/api/dashboard_permission.go b/pkg/api/dashboard_permission.go index 45d105c9e4d..184c7c53c58 100644 --- a/pkg/api/dashboard_permission.go +++ b/pkg/api/dashboard_permission.go @@ -11,7 +11,6 @@ import ( "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" - "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/guardian" "github.com/grafana/grafana/pkg/web" ) @@ -133,7 +132,7 @@ func (hs *HTTPServer) UpdateDashboardPermissions(c *models.ReqContext) response. return response.Error(403, "Cannot remove own admin permission for a folder", nil) } - if hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if !hs.AccessControl.IsDisabled() { old, err := g.GetAcl() if err != nil { return response.Error(500, "Error while checking dashboard permissions", err) diff --git a/pkg/api/dashboard_permission_test.go b/pkg/api/dashboard_permission_test.go index 18c9a90e0ca..136bb08e8a8 100644 --- a/pkg/api/dashboard_permission_test.go +++ b/pkg/api/dashboard_permission_test.go @@ -38,6 +38,7 @@ func TestDashboardPermissionAPIEndpoint(t *testing.T) { dashboardService: dashboardservice.ProvideDashboardService( settings, dashboardStore, nil, features, accesscontrolmock.NewPermissionsServicesMock(), ), + AccessControl: accesscontrolmock.New().WithDisabled(), } t.Run("Given user has no admin permissions", func(t *testing.T) { diff --git a/pkg/api/datasources.go b/pkg/api/datasources.go index 61a25517107..7eb21d0562c 100644 --- a/pkg/api/datasources.go +++ b/pkg/api/datasources.go @@ -97,7 +97,7 @@ func (hs *HTTPServer) GetDataSourceById(c *models.ReqContext) response.Response return response.Error(404, "Data source not found", err) } - dto := convertModelToDtos(filtered[0]) + dto := hs.convertModelToDtos(c.Req.Context(), filtered[0]) // Add accesscontrol metadata dto.AccessControl = hs.getAccessControlMetadata(c, c.OrgId, datasources.ScopePrefix, dto.UID) @@ -128,7 +128,7 @@ func (hs *HTTPServer) DeleteDataSourceById(c *models.ReqContext) response.Respon return response.Error(403, "Cannot delete read-only data source", nil) } - cmd := &models.DeleteDataSourceCommand{ID: id, OrgID: c.OrgId} + cmd := &models.DeleteDataSourceCommand{ID: id, OrgID: c.OrgId, Name: ds.Name} err = hs.DataSourcesService.DeleteDataSource(c.Req.Context(), cmd) if err != nil { @@ -156,7 +156,7 @@ func (hs *HTTPServer) GetDataSourceByUID(c *models.ReqContext) response.Response return response.Error(404, "Data source not found", err) } - dto := convertModelToDtos(filtered[0]) + dto := hs.convertModelToDtos(c.Req.Context(), filtered[0]) // Add accesscontrol metadata dto.AccessControl = hs.getAccessControlMetadata(c, c.OrgId, datasources.ScopePrefix, dto.UID) @@ -184,7 +184,7 @@ func (hs *HTTPServer) DeleteDataSourceByUID(c *models.ReqContext) response.Respo return response.Error(403, "Cannot delete read-only data source", nil) } - cmd := &models.DeleteDataSourceCommand{UID: uid, OrgID: c.OrgId} + cmd := &models.DeleteDataSourceCommand{UID: uid, OrgID: c.OrgId, Name: ds.Name} err = hs.DataSourcesService.DeleteDataSource(c.Req.Context(), cmd) if err != nil { @@ -265,7 +265,7 @@ func (hs *HTTPServer) AddDataSource(c *models.ReqContext) response.Response { return response.Error(500, "Failed to add datasource", err) } - ds := convertModelToDtos(cmd.Result) + ds := hs.convertModelToDtos(c.Req.Context(), cmd.Result) return response.JSON(http.StatusOK, util.DynMap{ "message": "Datasource added", "id": cmd.Result.Id, @@ -327,7 +327,7 @@ func (hs *HTTPServer) UpdateDataSource(c *models.ReqContext) response.Response { return response.Error(500, "Failed to query datasource", err) } - datasourceDTO := convertModelToDtos(query.Result) + datasourceDTO := hs.convertModelToDtos(c.Req.Context(), query.Result) hs.Live.HandleDatasourceUpdate(c.OrgId, datasourceDTO.UID) @@ -408,7 +408,7 @@ func (hs *HTTPServer) GetDataSourceByName(c *models.ReqContext) response.Respons return response.Error(404, "Data source not found", err) } - dto := convertModelToDtos(filtered[0]) + dto := hs.convertModelToDtos(c.Req.Context(), filtered[0]) return response.JSON(http.StatusOK, &dto) } @@ -457,7 +457,7 @@ func (hs *HTTPServer) CallDatasourceResource(c *models.ReqContext) { hs.callPluginResource(c, plugin.ID, ds.Uid) } -func convertModelToDtos(ds *models.DataSource) dtos.DataSource { +func (hs *HTTPServer) convertModelToDtos(ctx context.Context, ds *models.DataSource) dtos.DataSource { dto := dtos.DataSource{ Id: ds.Id, UID: ds.Uid, @@ -480,10 +480,15 @@ func convertModelToDtos(ds *models.DataSource) dtos.DataSource { ReadOnly: ds.ReadOnly, } - for k, v := range ds.SecureJsonData { - if len(v) > 0 { - dto.SecureJsonFields[k] = true + secrets, err := hs.DataSourcesService.DecryptedValues(ctx, ds) + if err == nil { + for k, v := range secrets { + if len(v) > 0 { + dto.SecureJsonFields[k] = true + } } + } else { + datasourcesLogger.Debug("Failed to retrieve datasource secrets to parse secure json fields", "error", err) } return dto @@ -510,7 +515,7 @@ func (hs *HTTPServer) CheckDatasourceHealth(c *models.ReqContext) response.Respo return response.Error(http.StatusInternalServerError, "Unable to find datasource plugin", err) } - dsInstanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn()) + dsInstanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn(c.Req.Context())) if err != nil { return response.Error(http.StatusInternalServerError, "Unable to get datasource model", err) } @@ -561,9 +566,9 @@ func (hs *HTTPServer) CheckDatasourceHealth(c *models.ReqContext) response.Respo return response.JSON(http.StatusOK, payload) } -func (hs *HTTPServer) decryptSecureJsonDataFn() func(map[string][]byte) map[string]string { - return func(m map[string][]byte) map[string]string { - decryptedJsonData, err := hs.SecretsService.DecryptJsonData(context.Background(), m) +func (hs *HTTPServer) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string { + return func(ds *models.DataSource) map[string]string { + decryptedJsonData, err := hs.DataSourcesService.DecryptedValues(ctx, ds) if err != nil { hs.log.Error("Failed to decrypt secure json data", "error", err) } diff --git a/pkg/api/datasources_test.go b/pkg/api/datasources_test.go index c490e2a0142..e4640009d67 100644 --- a/pkg/api/datasources_test.go +++ b/pkg/api/datasources_test.go @@ -583,3 +583,8 @@ func (m *dataSourcesServiceMock) UpdateDataSource(ctx context.Context, cmd *mode cmd.Result = m.expectedDatasource return m.expectedError } + +func (m *dataSourcesServiceMock) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) { + decryptedValues := make(map[string]string) + return decryptedValues, m.expectedError +} diff --git a/pkg/api/folder.go b/pkg/api/folder.go index 3a14be06d33..9efde7251f9 100644 --- a/pkg/api/folder.go +++ b/pkg/api/folder.go @@ -52,7 +52,7 @@ func (hs *HTTPServer) GetFolderByID(c *models.ReqContext) response.Response { if err != nil { return response.Error(http.StatusBadRequest, "id is invalid", err) } - folder, err := hs.folderService.GetFolderByID(c.Req.Context(), c.SignedInUser, c.OrgId, id) + folder, err := hs.folderService.GetFolderByID(c.Req.Context(), c.SignedInUser, id, c.OrgId) if err != nil { return apierrors.ToFolderErrorResponse(err) } diff --git a/pkg/api/folder_permission.go b/pkg/api/folder_permission.go index 830c71b7afd..edaff924528 100644 --- a/pkg/api/folder_permission.go +++ b/pkg/api/folder_permission.go @@ -9,7 +9,6 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/guardian" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" @@ -115,7 +114,7 @@ func (hs *HTTPServer) UpdateFolderPermissions(c *models.ReqContext) response.Res return response.Error(403, "Cannot remove own admin permission for a folder", nil) } - if hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if !hs.AccessControl.IsDisabled() { old, err := g.GetAcl() if err != nil { return response.Error(500, "Error while checking dashboard permissions", err) diff --git a/pkg/api/folder_permission_test.go b/pkg/api/folder_permission_test.go index 9198f15e55b..24556b120d7 100644 --- a/pkg/api/folder_permission_test.go +++ b/pkg/api/folder_permission_test.go @@ -44,6 +44,7 @@ func TestFolderPermissionAPIEndpoint(t *testing.T) { dashboardService: service.ProvideDashboardService( settings, dashboardStore, nil, features, permissionsServices, ), + AccessControl: accesscontrolmock.New().WithDisabled(), } t.Run("Given folder not exists", func(t *testing.T) { diff --git a/pkg/api/frontendsettings.go b/pkg/api/frontendsettings.go index 2c341d53979..3eff68dea68 100644 --- a/pkg/api/frontendsettings.go +++ b/pkg/api/frontendsettings.go @@ -248,9 +248,14 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins Enab if ds.Access == models.DS_ACCESS_DIRECT { if ds.BasicAuth { + password, err := hs.DataSourcesService.DecryptedBasicAuthPassword(c.Req.Context(), ds) + if err != nil { + return nil, err + } + dsDTO.BasicAuth = util.GetBasicAuthHeader( ds.BasicAuthUser, - hs.DataSourcesService.DecryptedBasicAuthPassword(ds), + password, ) } if ds.WithCredentials { @@ -258,14 +263,24 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins Enab } if ds.Type == models.DS_INFLUXDB_08 { + password, err := hs.DataSourcesService.DecryptedPassword(c.Req.Context(), ds) + if err != nil { + return nil, err + } + dsDTO.Username = ds.User - dsDTO.Password = hs.DataSourcesService.DecryptedPassword(ds) + dsDTO.Password = password dsDTO.URL = url + "/db/" + ds.Database } if ds.Type == models.DS_INFLUXDB { + password, err := hs.DataSourcesService.DecryptedPassword(c.Req.Context(), ds) + if err != nil { + return nil, err + } + dsDTO.Username = ds.User - dsDTO.Password = hs.DataSourcesService.DecryptedPassword(ds) + dsDTO.Password = password dsDTO.URL = url } } diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index 5491b86d316..0d4e4241744 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -39,6 +39,7 @@ import ( "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/datasources/permissions" "github.com/grafana/grafana/pkg/services/encryption" + "github.com/grafana/grafana/pkg/services/export" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/hooks" "github.com/grafana/grafana/pkg/services/ldap" @@ -112,6 +113,7 @@ type HTTPServer struct { Live *live.GrafanaLive LivePushGateway *pushhttp.Gateway ThumbService thumbs.Service + ExportService export.ExportService StorageService store.HTTPStorageService ContextHandler *contexthandler.ContextHandler SQLStore sqlstore.Store @@ -170,7 +172,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi contextHandler *contexthandler.ContextHandler, features *featuremgmt.FeatureManager, schemaService *schemaloader.SchemaLoaderService, alertNG *ngalert.AlertNG, libraryPanelService librarypanels.Service, libraryElementService libraryelements.Service, - quotaService *quota.QuotaService, socialService social.Service, tracer tracing.Tracer, + quotaService *quota.QuotaService, socialService social.Service, tracer tracing.Tracer, exportService export.ExportService, encryptionService encryption.Internal, grafanaUpdateChecker *updatechecker.GrafanaService, pluginsUpdateChecker *updatechecker.PluginsService, searchUsersService searchusers.Service, dataSourcesService datasources.DataSourceService, secretsService secrets.Service, queryDataService *query.Service, @@ -217,6 +219,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi AccessControl: accessControl, DataProxy: dataSourceProxy, SearchService: searchService, + ExportService: exportService, Live: live, LivePushGateway: livePushGateway, PluginContextProvider: plugCtxProvider, diff --git a/pkg/api/index.go b/pkg/api/index.go index bc3b400037e..e0f0707ea26 100644 --- a/pkg/api/index.go +++ b/pkg/api/index.go @@ -168,12 +168,12 @@ func (hs *HTTPServer) ReqCanAdminTeams(c *models.ReqContext) bool { return c.OrgRole == models.ROLE_ADMIN || (hs.Cfg.EditorsCanAdmin && c.OrgRole == models.ROLE_EDITOR) } -func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool) ([]*dtos.NavLink, error) { +func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool, prefs *pref.Preference) ([]*dtos.NavLink, error) { hasAccess := ac.HasAccess(hs.AccessControl, c) navTree := []*dtos.NavLink{} - if hs.Features.IsEnabled(featuremgmt.FlagNewNavigation) { - savedItemsLinks, err := hs.buildSavedItemsNavLinks(c) + if hs.Features.IsEnabled(featuremgmt.FlagSavedItems) { + savedItemsLinks, err := hs.buildSavedItemsNavLinks(c, prefs) if err != nil { return nil, err } @@ -411,16 +411,11 @@ func (hs *HTTPServer) addHelpLinks(navTree []*dtos.NavLink, c *models.ReqContext return navTree } -func (hs *HTTPServer) buildSavedItemsNavLinks(c *models.ReqContext) ([]*dtos.NavLink, error) { +func (hs *HTTPServer) buildSavedItemsNavLinks(c *models.ReqContext, prefs *pref.Preference) ([]*dtos.NavLink, error) { savedItemsChildNavs := []*dtos.NavLink{} // query preferences table for any saved items - prefsQuery := pref.GetPreferenceWithDefaultsQuery{UserID: c.SignedInUser.UserId} - preference, err := hs.preferenceService.GetWithDefaults(c.Req.Context(), &prefsQuery) - if err != nil { - return nil, err - } - savedItems := preference.JSONData.Navbar.SavedItems + savedItems := prefs.JSONData.Navbar.SavedItems if len(savedItems) > 0 { for _, savedItem := range savedItems { @@ -665,7 +660,7 @@ func (hs *HTTPServer) setIndexViewData(c *models.ReqContext) (*dtos.IndexViewDat settings["dateFormats"] = hs.Cfg.DateFormats - prefsQuery := pref.GetPreferenceWithDefaultsQuery{UserID: c.SignedInUser.UserId} + prefsQuery := pref.GetPreferenceWithDefaultsQuery{UserID: c.UserId, OrgID: c.OrgId, Teams: c.Teams} prefs, err := hs.preferenceService.GetWithDefaults(c.Req.Context(), &prefsQuery) if err != nil { return nil, err @@ -690,7 +685,7 @@ func (hs *HTTPServer) setIndexViewData(c *models.ReqContext) (*dtos.IndexViewDat settings["appSubUrl"] = "" } - navTree, err := hs.getNavTree(c, hasEditPerm) + navTree, err := hs.getNavTree(c, hasEditPerm, prefs) if err != nil { return nil, err } @@ -738,7 +733,7 @@ func (hs *HTTPServer) setIndexViewData(c *models.ReqContext) (*dtos.IndexViewDat LoadingLogo: "public/img/grafana_icon.svg", } - if hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if !hs.AccessControl.IsDisabled() { userPermissions, err := hs.AccessControl.GetUserPermissions(c.Req.Context(), c.SignedInUser, ac.Options{ReloadCache: false}) if err != nil { return nil, err diff --git a/pkg/api/metrics_test.go b/pkg/api/metrics_test.go index 5a32d5cfbba..0b4472b0d52 100644 --- a/pkg/api/metrics_test.go +++ b/pkg/api/metrics_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/sqlstore/mockstore" @@ -18,8 +19,11 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + datasources "github.com/grafana/grafana/pkg/services/datasources/service" "github.com/grafana/grafana/pkg/services/query" "github.com/grafana/grafana/pkg/services/secrets/fakes" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" + secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/stretchr/testify/assert" ) @@ -193,13 +197,17 @@ type dashboardFakePluginClient struct { func TestAPIEndpoint_Metrics_QueryMetricsFromDashboard(t *testing.T) { sc := setupHTTPServerWithMockDb(t, false, false) + secretsStore := kvstore.SetupTestService(t) + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) + ds := datasources.ProvideService(nil, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + setInitCtxSignedInViewer(sc.initCtx) sc.hs.queryDataService = query.ProvideService( nil, nil, nil, &fakePluginRequestValidator{}, - fakes.NewFakeSecretsService(), + ds, &dashboardFakePluginClient{}, &fakeOAuthTokenService{}, ) diff --git a/pkg/api/org.go b/pkg/api/org.go index d6cd5b3799b..2512938837b 100644 --- a/pkg/api/org.go +++ b/pkg/api/org.go @@ -10,8 +10,6 @@ import ( "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" @@ -90,13 +88,13 @@ func (hs *HTTPServer) CreateOrg(c *models.ReqContext) response.Response { if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - acEnabled := hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) + acEnabled := !hs.AccessControl.IsDisabled() if !acEnabled && !(setting.AllowUserOrgCreate || c.IsGrafanaAdmin) { return response.Error(403, "Access denied", nil) } cmd.UserId = c.UserId - if err := sqlstore.CreateOrg(c.Req.Context(), &cmd); err != nil { + if err := hs.SQLStore.CreateOrg(c.Req.Context(), &cmd); err != nil { if errors.Is(err, models.ErrOrgNameTaken) { return response.Error(409, "Organization name taken", err) } diff --git a/pkg/api/pluginproxy/ds_proxy.go b/pkg/api/pluginproxy/ds_proxy.go index 239249cf620..75e93fd513b 100644 --- a/pkg/api/pluginproxy/ds_proxy.go +++ b/pkg/api/pluginproxy/ds_proxy.go @@ -19,7 +19,6 @@ import ( "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/oauthtoken" - "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/util/proxyutil" @@ -43,7 +42,6 @@ type DataSourceProxy struct { oAuthTokenService oauthtoken.OAuthTokenService dataSourcesService datasources.DataSourceService tracer tracing.Tracer - secretsService secrets.Service } type httpClient interface { @@ -54,7 +52,7 @@ type httpClient interface { func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ctx *models.ReqContext, proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider, oAuthTokenService oauthtoken.OAuthTokenService, dsService datasources.DataSourceService, - tracer tracing.Tracer, secretsService secrets.Service) (*DataSourceProxy, error) { + tracer tracing.Tracer) (*DataSourceProxy, error) { targetURL, err := datasource.ValidateURL(ds.Type, ds.Url) if err != nil { return nil, err @@ -71,7 +69,6 @@ func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ct oAuthTokenService: oAuthTokenService, dataSourcesService: dsService, tracer: tracer, - secretsService: secretsService, }, nil } @@ -97,7 +94,7 @@ func (proxy *DataSourceProxy) HandleRequest() { "referer", proxy.ctx.Req.Referer(), ) - transport, err := proxy.dataSourcesService.GetHTTPTransport(proxy.ds, proxy.clientProvider) + transport, err := proxy.dataSourcesService.GetHTTPTransport(proxy.ctx.Req.Context(), proxy.ds, proxy.clientProvider) if err != nil { proxy.ctx.JsonApiErr(400, "Unable to load TLS certificate", err) return @@ -169,17 +166,28 @@ func (proxy *DataSourceProxy) director(req *http.Request) { switch proxy.ds.Type { case models.DS_INFLUXDB_08: + password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds) + if err != nil { + logger.Error("Error interpolating proxy url", "error", err) + return + } + req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, "db/"+proxy.ds.Database+"/"+proxy.proxyPath) reqQueryVals.Add("u", proxy.ds.User) - reqQueryVals.Add("p", proxy.dataSourcesService.DecryptedPassword(proxy.ds)) + reqQueryVals.Add("p", password) req.URL.RawQuery = reqQueryVals.Encode() case models.DS_INFLUXDB: + password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds) + if err != nil { + logger.Error("Error interpolating proxy url", "error", err) + return + } req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, proxy.proxyPath) req.URL.RawQuery = reqQueryVals.Encode() if !proxy.ds.BasicAuth { req.Header.Set( "Authorization", - util.GetBasicAuthHeader(proxy.ds.User, proxy.dataSourcesService.DecryptedPassword(proxy.ds)), + util.GetBasicAuthHeader(proxy.ds.User, password), ) } default: @@ -195,8 +203,13 @@ func (proxy *DataSourceProxy) director(req *http.Request) { req.URL.Path = unescapedPath if proxy.ds.BasicAuth { + password, err := proxy.dataSourcesService.DecryptedBasicAuthPassword(req.Context(), proxy.ds) + if err != nil { + logger.Error("Error interpolating proxy url", "error", err) + return + } req.Header.Set("Authorization", util.GetBasicAuthHeader(proxy.ds.BasicAuthUser, - proxy.dataSourcesService.DecryptedBasicAuthPassword(proxy.ds))) + password)) } dsAuth := req.Header.Get("X-DS-Authorization") @@ -226,23 +239,23 @@ func (proxy *DataSourceProxy) director(req *http.Request) { } } - secureJsonData, err := proxy.secretsService.DecryptJsonData(req.Context(), proxy.ds.SecureJsonData) - if err != nil { - logger.Error("Error interpolating proxy url", "error", err) - return - } - if proxy.matchedRoute != nil { - ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, DSInfo{ + decryptedValues, err := proxy.dataSourcesService.DecryptedValues(req.Context(), proxy.ds) + if err != nil { + logger.Error("Error interpolating proxy url", "error", err) + return + } + + ApplyRoute(req.Context(), req, proxy.proxyPath, proxy.matchedRoute, DSInfo{ ID: proxy.ds.Id, Updated: proxy.ds.Updated, JSONData: jsonData, - DecryptedSecureJSONData: secureJsonData, + DecryptedSecureJSONData: decryptedValues, }, proxy.cfg) } if proxy.oAuthTokenService.IsOAuthPassThruEnabled(proxy.ds) { - if token := proxy.oAuthTokenService.GetCurrentOAuthToken(proxy.ctx.Req.Context(), proxy.ctx.SignedInUser); token != nil { + if token := proxy.oAuthTokenService.GetCurrentOAuthToken(req.Context(), proxy.ctx.SignedInUser); token != nil { req.Header.Set("Authorization", fmt.Sprintf("%s %s", token.Type(), token.AccessToken)) idToken, ok := token.Extra("id_token").(string) diff --git a/pkg/api/pluginproxy/ds_proxy_test.go b/pkg/api/pluginproxy/ds_proxy_test.go index d0784e839eb..544b0997e74 100644 --- a/pkg/api/pluginproxy/ds_proxy_test.go +++ b/pkg/api/pluginproxy/ds_proxy_test.go @@ -3,6 +3,7 @@ package pluginproxy import ( "bytes" "context" + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -24,6 +25,7 @@ import ( "github.com/grafana/grafana/pkg/services/oauthtoken" "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/secrets/fakes" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/web" @@ -90,6 +92,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { }) setting.SecretKey = "password" //nolint:goconst + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) key, err := secretsService.Encrypt(context.Background(), []byte("123"), secrets.WithoutScope()) require.NoError(t, err) @@ -128,9 +131,9 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path", func(t *testing.T) { ctx, req := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, - &oauthtoken.Service{}, dsService, tracer, secretsService) + &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[0] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -141,8 +144,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path and has dynamic url", func(t *testing.T) { ctx, req := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/common/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/common/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[3] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -153,8 +156,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path with no url", func(t *testing.T) { ctx, req := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[4] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -164,8 +167,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path and has dynamic body", func(t *testing.T) { ctx, req := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/body", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/body", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[5] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -178,8 +181,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("Validating request", func(t *testing.T) { t.Run("plugin route with valid role", func(t *testing.T) { ctx, _ := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) err = proxy.validateRequest() require.NoError(t, err) @@ -187,8 +190,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("plugin route with admin role and user is editor", func(t *testing.T) { ctx, _ := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) err = proxy.validateRequest() require.Error(t, err) @@ -197,8 +200,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("plugin route with admin role and user is admin", func(t *testing.T) { ctx, _ := setUp() ctx.SignedInUser.OrgRole = models.ROLE_ADMIN - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) err = proxy.validateRequest() require.NoError(t, err) @@ -242,6 +245,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { }) setting.SecretKey = "password" + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) key, err := secretsService.Encrypt(context.Background(), []byte("123"), secrets.WithoutScope()) require.NoError(t, err) @@ -286,8 +290,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { }, } - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg) @@ -302,8 +306,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { req, err := http.NewRequest("GET", "http://localhost/asd", nil) require.NoError(t, err) client = newFakeHTTPClient(t, json2) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken2", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken2", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[1], dsInfo, cfg) @@ -319,8 +323,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { require.NoError(t, err) client = newFakeHTTPClient(t, []byte{}) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg) @@ -340,9 +344,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ds := &models.DataSource{Url: "htttp://graphite:8080", Type: models.DS_GRAPHITE} ctx := &models.ReqContext{} + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -366,9 +371,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ctx := &models.ReqContext{} var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) @@ -390,9 +396,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ctx := &models.ReqContext{} var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) requestURL, err := url.Parse("http://grafana.com/sub") @@ -418,9 +425,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ctx := &models.ReqContext{} var pluginRoutes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, pluginRoutes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, pluginRoutes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) requestURL, err := url.Parse("http://grafana.com/sub") @@ -441,9 +449,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { } ctx := &models.ReqContext{} var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) req.Header.Set("Origin", "grafana.com") @@ -490,9 +499,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { } var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &mockAuthToken, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &mockAuthToken, dsService, tracer) require.NoError(t, err) req, err = http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -543,24 +553,25 @@ func TestDataSourceProxy_routeRule(t *testing.T) { }) t.Run("When proxying data source proxy should handle authentication", func(t *testing.T) { + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) tests := []*testCase{ - createAuthTest(t, secretsService, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, false), - createAuthTest(t, secretsService, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, true), - createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, true), - createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, false), - createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, true), - createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, false), + createAuthTest(t, secretsStore, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, false), + createAuthTest(t, secretsStore, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, true), + createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, true), + createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, false), + createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, true), + createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, false), // These two should be enough for any other datasource at the moment. Proxy has special handling // only for Influx, others have the same path and only BasicAuth. Non BasicAuth datasources // do not go through proxy but through TSDB API which is not tested here. - createAuthTest(t, secretsService, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, false), - createAuthTest(t, secretsService, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, true), + createAuthTest(t, secretsStore, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, false), + createAuthTest(t, secretsStore, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, true), } for _, test := range tests { - runDatasourceAuthTest(t, secretsService, cfg, test) + runDatasourceAuthTest(t, secretsService, secretsStore, cfg, test) } }) } @@ -624,9 +635,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { t.Run("When response header Set-Cookie is not set should remove proxied Set-Cookie header", func(t *testing.T) { ctx, ds := setUp(t) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -642,9 +654,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { }, }) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -656,9 +669,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { t.Run("When response should set Content-Security-Policy header", func(t *testing.T) { ctx, ds := setUp(t) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -678,9 +692,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { }, }) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -703,9 +718,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -727,9 +743,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -752,9 +769,10 @@ func TestNewDataSourceProxy_InvalidURL(t *testing.T) { tracer, err := tracing.InitializeTracerForTest() require.NoError(t, err) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.Error(t, err) assert.True(t, strings.HasPrefix(err.Error(), `validation of data source URL "://host/root" failed`)) } @@ -773,9 +791,10 @@ func TestNewDataSourceProxy_ProtocolLessURL(t *testing.T) { require.NoError(t, err) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) } @@ -816,9 +835,10 @@ func TestNewDataSourceProxy_MSSQL(t *testing.T) { } var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - p, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + p, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) if tc.err == nil { require.NoError(t, err) assert.Equal(t, &url.URL{ @@ -843,9 +863,10 @@ func getDatasourceProxiedRequest(t *testing.T, ctx *models.ReqContext, cfg *sett require.NoError(t, err) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -897,15 +918,15 @@ const ( authCheckHeader = "header" ) -func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, url string, authType string, authCheck string, useSecureJsonData bool) *testCase { - ctx := context.Background() - +func createAuthTest(t *testing.T, secretsStore kvstore.SecretsKVStore, dsType string, url string, authType string, authCheck string, useSecureJsonData bool) *testCase { // Basic user:password base64AuthHeader := "Basic dXNlcjpwYXNzd29yZA==" test := &testCase{ datasource: &models.DataSource{ Id: 1, + OrgId: 1, + Name: fmt.Sprintf("%s,%s,%s,%s,%t", dsType, url, authType, authCheck, useSecureJsonData), Type: dsType, JsonData: simplejson.New(), Url: url, @@ -917,11 +938,13 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, message = fmt.Sprintf("%v should add username and password", dsType) test.datasource.User = "user" if useSecureJsonData { - test.datasource.SecureJsonData, err = secretsService.EncryptJsonData( - ctx, - map[string]string{ - "password": "password", - }, secrets.WithoutScope()) + secureJsonData, err := json.Marshal(map[string]string{ + "password": "password", + }) + require.NoError(t, err) + + err = secretsStore.Set(context.Background(), test.datasource.OrgId, test.datasource.Name, "datasource", string(secureJsonData)) + require.NoError(t, err) } else { test.datasource.Password = "password" } @@ -930,11 +953,13 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, test.datasource.BasicAuth = true test.datasource.BasicAuthUser = "user" if useSecureJsonData { - test.datasource.SecureJsonData, err = secretsService.EncryptJsonData( - ctx, - map[string]string{ - "basicAuthPassword": "password", - }, secrets.WithoutScope()) + secureJsonData, err := json.Marshal(map[string]string{ + "basicAuthPassword": "password", + }) + require.NoError(t, err) + + err = secretsStore.Set(context.Background(), test.datasource.OrgId, test.datasource.Name, "datasource", string(secureJsonData)) + require.NoError(t, err) } else { test.datasource.BasicAuthPassword = "password" } @@ -962,14 +987,14 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, return test } -func runDatasourceAuthTest(t *testing.T, secretsService secrets.Service, cfg *setting.Cfg, test *testCase) { +func runDatasourceAuthTest(t *testing.T, secretsService secrets.Service, secretsStore kvstore.SecretsKVStore, cfg *setting.Cfg, test *testCase) { ctx := &models.ReqContext{} tracer, err := tracing.InitializeTracerForTest() require.NoError(t, err) var routes []*plugins.Route - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(test.datasource, routes, ctx, "", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(test.datasource, routes, ctx, "", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) @@ -1010,9 +1035,10 @@ func Test_PathCheck(t *testing.T) { return ctx, req } ctx, _ := setUp() + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(&models.DataSource{}, routes, ctx, "b", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(&models.DataSource{}, routes, ctx, "b", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) require.Nil(t, proxy.validateRequest()) diff --git a/pkg/api/team.go b/pkg/api/team.go index 79c0eac9982..cfd44400e06 100644 --- a/pkg/api/team.go +++ b/pkg/api/team.go @@ -8,7 +8,6 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" ) @@ -19,7 +18,7 @@ func (hs *HTTPServer) CreateTeam(c *models.ReqContext) response.Response { if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - accessControlEnabled := hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) + accessControlEnabled := !hs.AccessControl.IsDisabled() if !accessControlEnabled && c.OrgRole == models.ROLE_VIEWER { return response.Error(403, "Not allowed to create team.", nil) } @@ -63,7 +62,7 @@ func (hs *HTTPServer) UpdateTeam(c *models.ReqContext) response.Response { return response.Error(http.StatusBadRequest, "teamId is invalid", err) } - if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if hs.AccessControl.IsDisabled() { if err := hs.teamGuardian.CanAdmin(c.Req.Context(), cmd.OrgId, cmd.Id, c.SignedInUser); err != nil { return response.Error(403, "Not allowed to update team", err) } @@ -88,7 +87,7 @@ func (hs *HTTPServer) DeleteTeamByID(c *models.ReqContext) response.Response { } user := c.SignedInUser - if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if hs.AccessControl.IsDisabled() { if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, user); err != nil { return response.Error(403, "Not allowed to delete team", err) } @@ -116,7 +115,7 @@ func (hs *HTTPServer) SearchTeams(c *models.ReqContext) response.Response { // Using accesscontrol the filtering is done based on user permissions userIdFilter := models.FilterIgnoreUser - if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if hs.AccessControl.IsDisabled() { userIdFilter = userFilter(c) } @@ -174,7 +173,7 @@ func (hs *HTTPServer) GetTeamByID(c *models.ReqContext) response.Response { // Using accesscontrol the filtering has already been performed at middleware layer userIdFilter := models.FilterIgnoreUser - if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if hs.AccessControl.IsDisabled() { userIdFilter = userFilter(c) } @@ -210,7 +209,7 @@ func (hs *HTTPServer) GetTeamPreferences(c *models.ReqContext) response.Response orgId := c.OrgId - if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if hs.AccessControl.IsDisabled() { if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { return response.Error(403, "Not allowed to view team preferences.", err) } @@ -233,7 +232,7 @@ func (hs *HTTPServer) UpdateTeamPreferences(c *models.ReqContext) response.Respo orgId := c.OrgId - if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if hs.AccessControl.IsDisabled() { if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { return response.Error(403, "Not allowed to update team preferences.", err) } diff --git a/pkg/api/team_members.go b/pkg/api/team_members.go index a8c44cde66a..f0a7576da64 100644 --- a/pkg/api/team_members.go +++ b/pkg/api/team_members.go @@ -11,7 +11,6 @@ import ( "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" - "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" ) @@ -27,7 +26,7 @@ func (hs *HTTPServer) GetTeamMembers(c *models.ReqContext) response.Response { // With accesscontrol the permission check has been done at middleware layer // and the membership filtering will be done at DB layer based on user permissions - if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if hs.AccessControl.IsDisabled() { if err := hs.teamGuardian.CanAdmin(c.Req.Context(), query.OrgId, query.TeamId, c.SignedInUser); err != nil { return response.Error(403, "Not allowed to list team members", err) } @@ -70,7 +69,7 @@ func (hs *HTTPServer) AddTeamMember(c *models.ReqContext) response.Response { return response.Error(http.StatusBadRequest, "teamId is invalid", err) } - if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if hs.AccessControl.IsDisabled() { if err := hs.teamGuardian.CanAdmin(c.Req.Context(), cmd.OrgId, cmd.TeamId, c.SignedInUser); err != nil { return response.Error(403, "Not allowed to add team member", err) } @@ -110,7 +109,7 @@ func (hs *HTTPServer) UpdateTeamMember(c *models.ReqContext) response.Response { } orgId := c.OrgId - if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if hs.AccessControl.IsDisabled() { if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { return response.Error(403, "Not allowed to update team member", err) } @@ -153,7 +152,7 @@ func (hs *HTTPServer) RemoveTeamMember(c *models.ReqContext) response.Response { return response.Error(http.StatusBadRequest, "userId is invalid", err) } - if !hs.Features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if hs.AccessControl.IsDisabled() { if err := hs.teamGuardian.CanAdmin(c.Req.Context(), orgId, teamId, c.SignedInUser); err != nil { return response.Error(403, "Not allowed to remove team member", err) } diff --git a/pkg/expr/service.go b/pkg/expr/service.go index 4e276683004..d01cc386e9f 100644 --- a/pkg/expr/service.go +++ b/pkg/expr/service.go @@ -7,7 +7,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/setting" ) @@ -36,16 +36,16 @@ func IsDataSource(uid string) bool { // Service is service representation for expression handling. type Service struct { - cfg *setting.Cfg - dataService backend.QueryDataHandler - secretsService secrets.Service + cfg *setting.Cfg + dataService backend.QueryDataHandler + dataSourceService datasources.DataSourceService } -func ProvideService(cfg *setting.Cfg, pluginClient plugins.Client, secretsService secrets.Service) *Service { +func ProvideService(cfg *setting.Cfg, pluginClient plugins.Client, dataSourceService datasources.DataSourceService) *Service { return &Service{ - cfg: cfg, - dataService: pluginClient, - secretsService: secretsService, + cfg: cfg, + dataService: pluginClient, + dataSourceService: dataSourceService, } } diff --git a/pkg/expr/service_test.go b/pkg/expr/service_test.go index 6741174baad..c47944c283a 100644 --- a/pkg/expr/service_test.go +++ b/pkg/expr/service_test.go @@ -11,8 +11,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/secrets/fakes" - secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" + datasources "github.com/grafana/grafana/pkg/services/datasources/fakes" "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/require" ) @@ -28,12 +27,10 @@ func TestService(t *testing.T) { cfg := setting.NewCfg() - secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - s := Service{ - cfg: cfg, - dataService: me, - secretsService: secretsService, + cfg: cfg, + dataService: me, + dataSourceService: &datasources.FakeDataSourceService{}, } queries := []Query{ diff --git a/pkg/expr/transform.go b/pkg/expr/transform.go index fb327613fe9..5e309d84c30 100644 --- a/pkg/expr/transform.go +++ b/pkg/expr/transform.go @@ -126,9 +126,9 @@ func hiddenRefIDs(queries []Query) (map[string]struct{}, error) { return hidden, nil } -func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(map[string][]byte) map[string]string { - return func(m map[string][]byte) map[string]string { - decryptedJsonData, err := s.secretsService.DecryptJsonData(ctx, m) +func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string { + return func(ds *models.DataSource) map[string]string { + decryptedJsonData, err := s.dataSourceService.DecryptedValues(ctx, ds) if err != nil { logger.Error("Failed to decrypt secure json data", "error", err) } diff --git a/pkg/extensions/main.go b/pkg/extensions/main.go index 65fde070218..adcaff8ca6f 100644 --- a/pkg/extensions/main.go +++ b/pkg/extensions/main.go @@ -1,9 +1,6 @@ package extensions import ( - // Upgrade ldapsync from cron to cron.v3 and - // remove the cron (v1) dependency - _ "cloud.google.com/go/kms/apiv1" _ "github.com/Azure/azure-sdk-for-go/sdk/azidentity" _ "github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys" @@ -24,7 +21,6 @@ import ( _ "github.com/linkedin/goavro/v2" _ "github.com/m3db/prometheus_remote_client_golang/promremote" _ "github.com/pkg/errors" - _ "github.com/robfig/cron" _ "github.com/robfig/cron/v3" _ "github.com/russellhaering/goxmldsig" _ "github.com/stretchr/testify/require" diff --git a/pkg/infra/usagestats/statscollector/prometheus_flavor.go b/pkg/infra/usagestats/statscollector/prometheus_flavor.go index 5d79e9f2a2f..652b312ea6a 100644 --- a/pkg/infra/usagestats/statscollector/prometheus_flavor.go +++ b/pkg/infra/usagestats/statscollector/prometheus_flavor.go @@ -60,7 +60,7 @@ func (s *Service) detectPrometheusVariant(ctx context.Context, ds *models.DataSo } `json:"data"` } - c, err := s.datasources.GetHTTPTransport(ds, s.httpClientProvider) + c, err := s.datasources.GetHTTPTransport(ctx, ds, s.httpClientProvider) if err != nil { s.log.Error("Failed to get HTTP client for Prometheus data source", "error", err) return "", err diff --git a/pkg/infra/usagestats/statscollector/service_test.go b/pkg/infra/usagestats/statscollector/service_test.go index 91419ac5cbc..86320488e03 100644 --- a/pkg/infra/usagestats/statscollector/service_test.go +++ b/pkg/infra/usagestats/statscollector/service_test.go @@ -395,6 +395,6 @@ func (s mockDatasourceService) GetDataSourcesByType(ctx context.Context, query * return nil } -func (s mockDatasourceService) GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) { +func (s mockDatasourceService) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) { return provider.GetTransport() } diff --git a/pkg/plugins/adapters/adapters.go b/pkg/plugins/adapters/adapters.go index 00c7933e6b3..96827e4f439 100644 --- a/pkg/plugins/adapters/adapters.go +++ b/pkg/plugins/adapters/adapters.go @@ -9,7 +9,7 @@ import ( ) // ModelToInstanceSettings converts a models.DataSource to a backend.DataSourceInstanceSettings. -func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(map[string][]byte) map[string]string, +func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(ds *models.DataSource) map[string]string, ) (*backend.DataSourceInstanceSettings, error) { var jsonDataBytes json.RawMessage if ds.JsonData != nil { @@ -30,7 +30,7 @@ func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(map[string][] BasicAuthEnabled: ds.BasicAuth, BasicAuthUser: ds.BasicAuthUser, JSONData: jsonDataBytes, - DecryptedSecureJSONData: decryptFn(ds.SecureJsonData), + DecryptedSecureJSONData: decryptFn(ds), Updated: ds.Updated, }, nil } diff --git a/pkg/plugins/plugincontext/plugincontext.go b/pkg/plugins/plugincontext/plugincontext.go index 652417534b7..e06a7821d4a 100644 --- a/pkg/plugins/plugincontext/plugincontext.go +++ b/pkg/plugins/plugincontext/plugincontext.go @@ -15,18 +15,17 @@ import ( "github.com/grafana/grafana/pkg/plugins/adapters" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/pluginsettings" - "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/util/errutil" ) func ProvideService(cacheService *localcache.CacheService, pluginStore plugins.Store, - dataSourceCache datasources.CacheService, secretsService secrets.Service, + dataSourceCache datasources.CacheService, dataSourceService datasources.DataSourceService, pluginSettingsService pluginsettings.Service) *Provider { return &Provider{ cacheService: cacheService, pluginStore: pluginStore, dataSourceCache: dataSourceCache, - secretsService: secretsService, + dataSourceService: dataSourceService, pluginSettingsService: pluginSettingsService, logger: log.New("plugincontext"), } @@ -36,7 +35,7 @@ type Provider struct { cacheService *localcache.CacheService pluginStore plugins.Store dataSourceCache datasources.CacheService - secretsService secrets.Service + dataSourceService datasources.DataSourceService pluginSettingsService pluginsettings.Service logger log.Logger } @@ -87,7 +86,7 @@ func (p *Provider) Get(ctx context.Context, pluginID string, datasourceUID strin if err != nil { return pc, false, errutil.Wrap("Failed to get datasource", err) } - datasourceSettings, err := adapters.ModelToInstanceSettings(ds, p.decryptSecureJsonDataFn()) + datasourceSettings, err := adapters.ModelToInstanceSettings(ds, p.decryptSecureJsonDataFn(ctx)) if err != nil { return pc, false, errutil.Wrap("Failed to convert datasource", err) } @@ -122,9 +121,9 @@ func (p *Provider) getCachedPluginSettings(ctx context.Context, pluginID string, return ps, nil } -func (p *Provider) decryptSecureJsonDataFn() func(map[string][]byte) map[string]string { - return func(m map[string][]byte) map[string]string { - decryptedJsonData, err := p.secretsService.DecryptJsonData(context.Background(), m) +func (p *Provider) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string { + return func(ds *models.DataSource) map[string]string { + decryptedJsonData, err := p.dataSourceService.DecryptedValues(ctx, ds) if err != nil { p.logger.Error("Failed to decrypt secure json data", "error", err) } diff --git a/pkg/server/wire.go b/pkg/server/wire.go index ca33eb3eac2..4461854bd5b 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -50,6 +50,7 @@ import ( "github.com/grafana/grafana/pkg/services/datasourceproxy" "github.com/grafana/grafana/pkg/services/datasources" datasourceservice "github.com/grafana/grafana/pkg/services/datasources/service" + "github.com/grafana/grafana/pkg/services/export" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/guardian" "github.com/grafana/grafana/pkg/services/hooks" @@ -79,6 +80,7 @@ import ( "github.com/grafana/grafana/pkg/services/searchV2" "github.com/grafana/grafana/pkg/services/secrets" secretsDatabase "github.com/grafana/grafana/pkg/services/secrets/database" + secretsStore "github.com/grafana/grafana/pkg/services/secrets/kvstore" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/services/serviceaccounts" serviceaccountsmanager "github.com/grafana/grafana/pkg/services/serviceaccounts/manager" @@ -174,6 +176,7 @@ var wireBasicSet = wire.NewSet( searchV2.ProvideService, store.ProvideService, store.ProvideHTTPService, + export.ProvideService, live.ProvideService, pushhttp.ProvideService, plugincontext.ProvideService, @@ -239,6 +242,7 @@ var wireBasicSet = wire.NewSet( wire.Bind(new(alerting.DashAlertExtractor), new(*alerting.DashAlertExtractorService)), comments.ProvideService, guardian.ProvideService, + secretsStore.ProvideService, avatar.ProvideAvatarCacheServer, authproxy.ProvideAuthProxy, statscollector.ProvideService, diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 59add0c5eb2..d55894be29d 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -93,7 +93,8 @@ func ProvideAlertEngine(renderer rendering.Service, requestValidator models.Plug // Run starts the alerting service background process. func (e *AlertEngine) Run(ctx context.Context) error { - e.ticker = NewTicker(clock.New(), 1*time.Second, metrics.NewTickerMetrics(prometheus.DefaultRegisterer)) + reg := prometheus.WrapRegistererWithPrefix("legacy_", prometheus.DefaultRegisterer) + e.ticker = NewTicker(clock.New(), 1*time.Second, metrics.NewTickerMetrics(reg)) alertGroup, ctx := errgroup.WithContext(ctx) alertGroup.Go(func() error { return e.alertingTicker(ctx) }) alertGroup.Go(func() error { return e.runJobDispatcher(ctx) }) diff --git a/pkg/services/alerting/ticker_test.go b/pkg/services/alerting/ticker_test.go index 8eb4061e14d..b642067ea63 100644 --- a/pkg/services/alerting/ticker_test.go +++ b/pkg/services/alerting/ticker_test.go @@ -137,7 +137,14 @@ func TestTicker(t *testing.T) { expectedMetric := fmt.Sprintf(expectedMetricFmt, interval.Seconds(), 0, float64(expectedTick.UnixNano())/1e9) - require.NoError(t, testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetric), "grafana_alerting_ticker_last_consumed_tick_timestamp_seconds", "grafana_alerting_ticker_next_tick_timestamp_seconds", "grafana_alerting_ticker_interval_seconds")) + errs := make(map[string]error, 1) + require.Eventuallyf(t, func() bool { + err := testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetric), "grafana_alerting_ticker_last_consumed_tick_timestamp_seconds", "grafana_alerting_ticker_next_tick_timestamp_seconds", "grafana_alerting_ticker_interval_seconds") + if err != nil { + errs["error"] = err + } + return err == nil + }, 1*time.Second, 100*time.Millisecond, "failed to wait for metrics to match expected values:\n%v", errs) clk.Add(interval) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -147,6 +154,13 @@ func TestTicker(t *testing.T) { actual := readChanOrFail(t, ctx, ticker.C) expectedMetric = fmt.Sprintf(expectedMetricFmt, interval.Seconds(), float64(actual.UnixNano())/1e9, float64(expectedTick.Add(interval).UnixNano())/1e9) - require.NoError(t, testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetric), "grafana_alerting_ticker_last_consumed_tick_timestamp_seconds", "grafana_alerting_ticker_next_tick_timestamp_seconds", "grafana_alerting_ticker_interval_seconds")) + + require.Eventuallyf(t, func() bool { + err := testutil.GatherAndCompare(registry, bytes.NewBufferString(expectedMetric), "grafana_alerting_ticker_last_consumed_tick_timestamp_seconds", "grafana_alerting_ticker_next_tick_timestamp_seconds", "grafana_alerting_ticker_interval_seconds") + if err != nil { + errs["error"] = err + } + return err == nil + }, 1*time.Second, 100*time.Millisecond, "failed to wait for metrics to match expected values:\n%v", errs) }) } diff --git a/pkg/services/comments/commentmodel/permissions.go b/pkg/services/comments/commentmodel/permissions.go index b1bcbee82d1..33c17b3eca4 100644 --- a/pkg/services/comments/commentmodel/permissions.go +++ b/pkg/services/comments/commentmodel/permissions.go @@ -105,7 +105,7 @@ func (c *PermissionChecker) CheckWritePermissions(ctx context.Context, orgId int if !c.features.IsEnabled(featuremgmt.FlagAnnotationComments) { return false, nil } - if c.features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if !c.accessControl.IsDisabled() { evaluator := accesscontrol.EvalPermission(accesscontrol.ActionAnnotationsWrite, accesscontrol.ScopeAnnotationsTypeDashboard) if canEdit, err := c.accessControl.Evaluate(ctx, signedInUser, evaluator); err != nil || !canEdit { return canEdit, err diff --git a/pkg/services/datasourceproxy/datasourceproxy.go b/pkg/services/datasourceproxy/datasourceproxy.go index 75d5f506ea1..c40c4360e06 100644 --- a/pkg/services/datasourceproxy/datasourceproxy.go +++ b/pkg/services/datasourceproxy/datasourceproxy.go @@ -115,7 +115,7 @@ func (p *DataSourceProxyService) proxyDatasourceRequest(c *models.ReqContext, ds proxyPath := getProxyPath(c) proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin.Routes, c, proxyPath, p.Cfg, p.HTTPClientProvider, - p.OAuthTokenService, p.DataSourcesService, p.tracer, p.secretsService) + p.OAuthTokenService, p.DataSourcesService, p.tracer) if err != nil { if errors.Is(err, datasource.URLValidationError{}) { c.JsonApiErr(http.StatusBadRequest, fmt.Sprintf("Invalid data source URL: %q", ds.Url), err) diff --git a/pkg/services/datasources/datasources.go b/pkg/services/datasources/datasources.go index cfbc2095548..b6212794328 100644 --- a/pkg/services/datasources/datasources.go +++ b/pkg/services/datasources/datasources.go @@ -33,23 +33,23 @@ type DataSourceService interface { GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error // GetHTTPTransport gets a datasource specific HTTP transport. - GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) + GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) // DecryptedValues decrypts the encrypted secureJSONData of the provided datasource and // returns the decrypted values. - DecryptedValues(ds *models.DataSource) map[string]string + DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) // DecryptedValue decrypts the encrypted datasource secureJSONData identified by key // and returns the decryped value. - DecryptedValue(ds *models.DataSource, key string) (string, bool) + DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) // DecryptedBasicAuthPassword decrypts the encrypted datasource basic authentication // password and returns the decryped value. - DecryptedBasicAuthPassword(ds *models.DataSource) string + DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) // DecryptedPassword decrypts the encrypted datasource password and returns the // decryped value. - DecryptedPassword(ds *models.DataSource) string + DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) } // CacheService interface for retrieving a cached datasource. diff --git a/pkg/services/datasources/fake_cache_service.go b/pkg/services/datasources/fakes/fake_cache_service.go similarity index 87% rename from pkg/services/datasources/fake_cache_service.go rename to pkg/services/datasources/fakes/fake_cache_service.go index e4c2ed9193b..7d7fe4c9a97 100644 --- a/pkg/services/datasources/fake_cache_service.go +++ b/pkg/services/datasources/fakes/fake_cache_service.go @@ -4,13 +4,14 @@ import ( "context" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/datasources" ) type FakeCacheService struct { DataSources []*models.DataSource } -var _ CacheService = &FakeCacheService{} +var _ datasources.CacheService = &FakeCacheService{} func (c *FakeCacheService) GetDatasource(ctx context.Context, datasourceID int64, user *models.SignedInUser, skipCache bool) (*models.DataSource, error) { for _, datasource := range c.DataSources { diff --git a/pkg/services/datasources/fakes/fake_datasource_service.go b/pkg/services/datasources/fakes/fake_datasource_service.go new file mode 100644 index 00000000000..a75b40163c8 --- /dev/null +++ b/pkg/services/datasources/fakes/fake_datasource_service.go @@ -0,0 +1,124 @@ +package datasources + +import ( + "context" + "net/http" + + sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/infra/httpclient" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/datasources" +) + +type FakeDataSourceService struct { + lastId int64 + DataSources []*models.DataSource +} + +var _ datasources.DataSourceService = &FakeDataSourceService{} + +func (s *FakeDataSourceService) GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error { + for _, datasource := range s.DataSources { + idMatch := query.Id != 0 && query.Id == datasource.Id + uidMatch := query.Uid != "" && query.Uid == datasource.Uid + nameMatch := query.Name != "" && query.Name == datasource.Name + if idMatch || nameMatch || uidMatch { + query.Result = datasource + + return nil + } + } + return models.ErrDataSourceNotFound +} + +func (s *FakeDataSourceService) GetDataSources(ctx context.Context, query *models.GetDataSourcesQuery) error { + for _, datasource := range s.DataSources { + orgMatch := query.OrgId != 0 && query.OrgId == datasource.OrgId + if orgMatch { + query.Result = append(query.Result, datasource) + } + } + return nil +} + +func (s *FakeDataSourceService) GetDataSourcesByType(ctx context.Context, query *models.GetDataSourcesByTypeQuery) error { + for _, datasource := range s.DataSources { + typeMatch := query.Type != "" && query.Type == datasource.Type + if typeMatch { + query.Result = append(query.Result, datasource) + } + } + return nil +} + +func (s *FakeDataSourceService) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error { + if s.lastId == 0 { + s.lastId = int64(len(s.DataSources) - 1) + } + cmd.Result = &models.DataSource{ + Id: s.lastId + 1, + Name: cmd.Name, + Type: cmd.Type, + Uid: cmd.Uid, + OrgId: cmd.OrgId, + } + s.DataSources = append(s.DataSources, cmd.Result) + return nil +} + +func (s *FakeDataSourceService) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error { + for i, datasource := range s.DataSources { + idMatch := cmd.ID != 0 && cmd.ID == datasource.Id + uidMatch := cmd.UID != "" && cmd.UID == datasource.Uid + nameMatch := cmd.Name != "" && cmd.Name == datasource.Name + if idMatch || nameMatch || uidMatch { + s.DataSources = append(s.DataSources[:i], s.DataSources[i+1:]...) + return nil + } + } + return models.ErrDataSourceNotFound +} + +func (s *FakeDataSourceService) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error { + for _, datasource := range s.DataSources { + idMatch := cmd.Id != 0 && cmd.Id == datasource.Id + uidMatch := cmd.Uid != "" && cmd.Uid == datasource.Uid + nameMatch := cmd.Name != "" && cmd.Name == datasource.Name + if idMatch || nameMatch || uidMatch { + if cmd.Name != "" { + datasource.Name = cmd.Name + } + return nil + } + } + return models.ErrDataSourceNotFound +} + +func (s *FakeDataSourceService) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error { + return nil +} + +func (s *FakeDataSourceService) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) { + rt, err := provider.GetTransport(sdkhttpclient.Options{}) + if err != nil { + return nil, err + } + return rt, nil +} + +func (s *FakeDataSourceService) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) { + values := make(map[string]string) + return values, nil +} + +func (s *FakeDataSourceService) DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) { + return "", false, nil +} + +func (s *FakeDataSourceService) DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) { + return "", nil +} + +func (s *FakeDataSourceService) DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) { + return "", nil +} diff --git a/pkg/services/datasources/service/datasource_service.go b/pkg/services/datasources/service/datasource_service.go index 914f5ed0b26..7f74a8bafa0 100644 --- a/pkg/services/datasources/service/datasource_service.go +++ b/pkg/services/datasources/service/datasource_service.go @@ -3,6 +3,7 @@ package service import ( "context" "crypto/tls" + "encoding/json" "fmt" "net/http" "net/url" @@ -23,19 +24,21 @@ import ( "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" ) type Service struct { SQLStore *sqlstore.SQLStore + SecretsStore kvstore.SecretsKVStore SecretsService secrets.Service cfg *setting.Cfg features featuremgmt.FeatureToggles permissionsService accesscontrol.PermissionsService + ac accesscontrol.AccessControl - ptc proxyTransportCache - dsDecryptionCache secureJSONDecryptionCache + ptc proxyTransportCache } type proxyTransportCache struct { @@ -48,32 +51,21 @@ type cachedRoundTripper struct { roundTripper http.RoundTripper } -type secureJSONDecryptionCache struct { - cache map[int64]cachedDecryptedJSON - sync.Mutex -} - -type cachedDecryptedJSON struct { - updated time.Time - json map[string]string -} - func ProvideService( - store *sqlstore.SQLStore, secretsService secrets.Service, cfg *setting.Cfg, features featuremgmt.FeatureToggles, - ac accesscontrol.AccessControl, permissionsServices accesscontrol.PermissionsServices, + store *sqlstore.SQLStore, secretsService secrets.Service, secretsStore kvstore.SecretsKVStore, cfg *setting.Cfg, + features featuremgmt.FeatureToggles, ac accesscontrol.AccessControl, permissionsServices accesscontrol.PermissionsServices, ) *Service { s := &Service{ SQLStore: store, + SecretsStore: secretsStore, SecretsService: secretsService, ptc: proxyTransportCache{ cache: make(map[int64]cachedRoundTripper), }, - dsDecryptionCache: secureJSONDecryptionCache{ - cache: make(map[int64]cachedDecryptedJSON), - }, cfg: cfg, features: features, permissionsService: permissionsServices.GetDataSourceService(), + ac: ac, } ac.RegisterAttributeScopeResolver(NewNameScopeResolver(store)) @@ -88,6 +80,8 @@ type DataSourceRetriever interface { GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error } +const secretType = "datasource" + // NewNameScopeResolver provides an AttributeScopeResolver able to // translate a scope prefixed with "datasources:name:" into an uid based scope. func NewNameScopeResolver(db DataSourceRetriever) (string, accesscontrol.AttributeScopeResolveFunc) { @@ -153,16 +147,21 @@ func (s *Service) GetDataSourcesByType(ctx context.Context, query *models.GetDat func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error { var err error - cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope()) - if err != nil { - return err - } - if err := s.SQLStore.AddDataSource(ctx, cmd); err != nil { return err } - if s.features.IsEnabled(featuremgmt.FlagAccesscontrol) { + secret, err := json.Marshal(cmd.SecureJsonData) + if err != nil { + return err + } + + err = s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret)) + if err != nil { + return err + } + + if !s.ac.IsDisabled() { // This belongs in Data source permissions, and we probably want // to do this with a hook in the store and rollback on fail. // We can't use events, because there's no way to communicate @@ -184,25 +183,50 @@ func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCo } func (s *Service) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error { - return s.SQLStore.DeleteDataSource(ctx, cmd) + err := s.SQLStore.DeleteDataSource(ctx, cmd) + if err != nil { + return err + } + return s.SecretsStore.Del(ctx, cmd.OrgID, cmd.Name, secretType) } func (s *Service) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error { var err error - cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope()) + secret, err := json.Marshal(cmd.SecureJsonData) if err != nil { return err } - return s.SQLStore.UpdateDataSource(ctx, cmd) + query := &models.GetDataSourceQuery{ + Id: cmd.Id, + OrgId: cmd.OrgId, + } + err = s.SQLStore.GetDataSource(ctx, query) + if err != nil { + return err + } + + err = s.SQLStore.UpdateDataSource(ctx, cmd) + if err != nil { + return err + } + + if query.Result.Name != cmd.Name { + err = s.SecretsStore.Rename(ctx, cmd.OrgId, query.Result.Name, secretType, cmd.Name) + if err != nil { + return err + } + } + + return s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret)) } func (s *Service) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error { return s.SQLStore.GetDefaultDataSource(ctx, query) } -func (s *Service) GetHTTPClient(ds *models.DataSource, provider httpclient.Provider) (*http.Client, error) { - transport, err := s.GetHTTPTransport(ds, provider) +func (s *Service) GetHTTPClient(ctx context.Context, ds *models.DataSource, provider httpclient.Provider) (*http.Client, error) { + transport, err := s.GetHTTPTransport(ctx, ds, provider) if err != nil { return nil, err } @@ -213,7 +237,7 @@ func (s *Service) GetHTTPClient(ds *models.DataSource, provider httpclient.Provi }, nil } -func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, +func (s *Service) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) { s.ptc.Lock() defer s.ptc.Unlock() @@ -222,7 +246,7 @@ func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Pr return t.roundTripper, nil } - opts, err := s.httpClientOptions(ds) + opts, err := s.httpClientOptions(ctx, ds) if err != nil { return nil, err } @@ -242,58 +266,84 @@ func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Pr return rt, nil } -func (s *Service) GetTLSConfig(ds *models.DataSource, httpClientProvider httpclient.Provider) (*tls.Config, error) { - opts, err := s.httpClientOptions(ds) +func (s *Service) GetTLSConfig(ctx context.Context, ds *models.DataSource, httpClientProvider httpclient.Provider) (*tls.Config, error) { + opts, err := s.httpClientOptions(ctx, ds) if err != nil { return nil, err } return httpClientProvider.GetTLSConfig(*opts) } -func (s *Service) DecryptedValues(ds *models.DataSource) map[string]string { - s.dsDecryptionCache.Lock() - defer s.dsDecryptionCache.Unlock() - - if item, present := s.dsDecryptionCache.cache[ds.Id]; present && ds.Updated.Equal(item.updated) { - return item.json - } - - json, err := s.SecretsService.DecryptJsonData(context.Background(), ds.SecureJsonData) +func (s *Service) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) { + decryptedValues := make(map[string]string) + secret, exist, err := s.SecretsStore.Get(ctx, ds.OrgId, ds.Name, secretType) if err != nil { - return map[string]string{} + return nil, err } - s.dsDecryptionCache.cache[ds.Id] = cachedDecryptedJSON{ - updated: ds.Updated, - json: json, + if exist { + err := json.Unmarshal([]byte(secret), &decryptedValues) + if err != nil { + return nil, err + } + } else if len(ds.SecureJsonData) > 0 { + decryptedValues, err = s.MigrateSecrets(ctx, ds) + if err != nil { + return nil, err + } } - return json + return decryptedValues, nil } -func (s *Service) DecryptedValue(ds *models.DataSource, key string) (string, bool) { - value, exists := s.DecryptedValues(ds)[key] - return value, exists -} - -func (s *Service) DecryptedBasicAuthPassword(ds *models.DataSource) string { - if value, ok := s.DecryptedValue(ds, "basicAuthPassword"); ok { - return value +func (s *Service) MigrateSecrets(ctx context.Context, ds *models.DataSource) (map[string]string, error) { + secureJsonData, err := s.SecretsService.DecryptJsonData(ctx, ds.SecureJsonData) + if err != nil { + return nil, err } - return ds.BasicAuthPassword -} - -func (s *Service) DecryptedPassword(ds *models.DataSource) string { - if value, ok := s.DecryptedValue(ds, "password"); ok { - return value + jsonData, err := json.Marshal(secureJsonData) + if err != nil { + return nil, err } - return ds.Password + err = s.SecretsStore.Set(ctx, ds.OrgId, ds.Name, secretType, string(jsonData)) + return secureJsonData, err } -func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Options, error) { - tlsOptions := s.dsTLSOptions(ds) +func (s *Service) DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) { + values, err := s.DecryptedValues(ctx, ds) + if err != nil { + return "", false, err + } + value, exists := values[key] + return value, exists, nil +} + +func (s *Service) DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) { + value, ok, err := s.DecryptedValue(ctx, ds, "basicAuthPassword") + if ok { + return value, nil + } + + return ds.BasicAuthPassword, err +} + +func (s *Service) DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) { + value, ok, err := s.DecryptedValue(ctx, ds, "password") + if ok { + return value, nil + } + + return ds.Password, err +} + +func (s *Service) httpClientOptions(ctx context.Context, ds *models.DataSource) (*sdkhttpclient.Options, error) { + tlsOptions, err := s.dsTLSOptions(ctx, ds) + if err != nil { + return nil, err + } + timeouts := &sdkhttpclient.TimeoutOptions{ Timeout: s.getTimeout(ds), DialTimeout: sdkhttpclient.DefaultTimeoutOptions.DialTimeout, @@ -305,9 +355,15 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio MaxIdleConnsPerHost: sdkhttpclient.DefaultTimeoutOptions.MaxIdleConnsPerHost, IdleConnTimeout: sdkhttpclient.DefaultTimeoutOptions.IdleConnTimeout, } + + decryptedValues, err := s.DecryptedValues(ctx, ds) + if err != nil { + return nil, err + } + opts := &sdkhttpclient.Options{ Timeouts: timeouts, - Headers: s.getCustomHeaders(ds.JsonData, s.DecryptedValues(ds)), + Headers: s.getCustomHeaders(ds.JsonData, decryptedValues), Labels: map[string]string{ "datasource_name": ds.Name, "datasource_uid": ds.Uid, @@ -318,22 +374,30 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio if ds.JsonData != nil { opts.CustomOptions = ds.JsonData.MustMap() } - if ds.BasicAuth { + password, err := s.DecryptedBasicAuthPassword(ctx, ds) + if err != nil { + return opts, err + } + opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{ User: ds.BasicAuthUser, - Password: s.DecryptedBasicAuthPassword(ds), + Password: password, } } else if ds.User != "" { + password, err := s.DecryptedPassword(ctx, ds) + if err != nil { + return opts, err + } + opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{ User: ds.User, - Password: s.DecryptedPassword(ds), + Password: password, } } - // Azure authentication if ds.JsonData != nil && s.features.IsEnabled(featuremgmt.FlagHttpclientproviderAzureAuth) { - credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), s.DecryptedValues(ds)) + credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), decryptedValues) if err != nil { err = fmt.Errorf("invalid Azure credentials: %s", err) return nil, err @@ -369,19 +433,27 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio Profile: ds.JsonData.Get("sigV4Profile").MustString(), } - if val, exists := s.DecryptedValue(ds, "sigV4AccessKey"); exists { - opts.SigV4.AccessKey = val + if val, exists, err := s.DecryptedValue(ctx, ds, "sigV4AccessKey"); err == nil { + if exists { + opts.SigV4.AccessKey = val + } + } else { + return opts, err } - if val, exists := s.DecryptedValue(ds, "sigV4SecretKey"); exists { - opts.SigV4.SecretKey = val + if val, exists, err := s.DecryptedValue(ctx, ds, "sigV4SecretKey"); err == nil { + if exists { + opts.SigV4.SecretKey = val + } + } else { + return opts, err } } return opts, nil } -func (s *Service) dsTLSOptions(ds *models.DataSource) sdkhttpclient.TLSOptions { +func (s *Service) dsTLSOptions(ctx context.Context, ds *models.DataSource) (sdkhttpclient.TLSOptions, error) { var tlsSkipVerify, tlsClientAuth, tlsAuthWithCACert bool var serverName string @@ -399,22 +471,35 @@ func (s *Service) dsTLSOptions(ds *models.DataSource) sdkhttpclient.TLSOptions { if tlsClientAuth || tlsAuthWithCACert { if tlsAuthWithCACert { - if val, exists := s.DecryptedValue(ds, "tlsCACert"); exists && len(val) > 0 { - opts.CACertificate = val + if val, exists, err := s.DecryptedValue(ctx, ds, "tlsCACert"); err == nil { + if exists && len(val) > 0 { + opts.CACertificate = val + } + } else { + return opts, err } } if tlsClientAuth { - if val, exists := s.DecryptedValue(ds, "tlsClientCert"); exists && len(val) > 0 { - opts.ClientCertificate = val + if val, exists, err := s.DecryptedValue(ctx, ds, "tlsClientCert"); err == nil { + fmt.Print("\n\n\n\n", val, exists, err, "\n\n\n\n") + if exists && len(val) > 0 { + opts.ClientCertificate = val + } + } else { + return opts, err } - if val, exists := s.DecryptedValue(ds, "tlsClientKey"); exists && len(val) > 0 { - opts.ClientKey = val + if val, exists, err := s.DecryptedValue(ctx, ds, "tlsClientKey"); err == nil { + if exists && len(val) > 0 { + opts.ClientKey = val + } + } else { + return opts, err } } } - return opts + return opts, nil } func (s *Service) getTimeout(ds *models.DataSource) time.Duration { diff --git a/pkg/services/datasources/service/datasource_service_test.go b/pkg/services/datasources/service/datasource_service_test.go index 511b0f8c9b2..e250315fa73 100644 --- a/pkg/services/datasources/service/datasource_service_test.go +++ b/pkg/services/datasources/service/datasource_service_test.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/json" "io/ioutil" "net/http" "net/http/httptest" @@ -10,6 +11,8 @@ import ( "github.com/grafana/grafana-azure-sdk-go/azsettings" sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/services/secrets" + secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/httpclient" @@ -17,59 +20,13 @@ import ( "github.com/grafana/grafana/pkg/services/accesscontrol" acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/secrets" - "github.com/grafana/grafana/pkg/services/secrets/database" "github.com/grafana/grafana/pkg/services/secrets/fakes" - secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" - "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestService(t *testing.T) { - cfg := &setting.Cfg{} - sqlStore := sqlstore.InitTestDB(t) - - origSecret := setting.SecretKey - setting.SecretKey = "datasources_service_test" - t.Cleanup(func() { - setting.SecretKey = origSecret - }) - - secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(sqlStore)) - s := ProvideService(sqlStore, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - var ds *models.DataSource - - t.Run("create datasource should encrypt the secure json data", func(t *testing.T) { - ctx := context.Background() - - sjd := map[string]string{"password": "12345"} - cmd := models.AddDataSourceCommand{SecureJsonData: sjd} - - err := s.AddDataSource(ctx, &cmd) - require.NoError(t, err) - - ds = cmd.Result - decrypted, err := s.SecretsService.DecryptJsonData(ctx, ds.SecureJsonData) - require.NoError(t, err) - require.Equal(t, sjd, decrypted) - }) - - t.Run("update datasource should encrypt the secure json data", func(t *testing.T) { - ctx := context.Background() - sjd := map[string]string{"password": "678910"} - cmd := models.UpdateDataSourceCommand{Id: ds.Id, OrgId: ds.OrgId, SecureJsonData: sjd} - err := s.UpdateDataSource(ctx, &cmd) - require.NoError(t, err) - - decrypted, err := s.SecretsService.DecryptJsonData(ctx, cmd.Result.SecureJsonData) - require.NoError(t, err) - require.Equal(t, sjd, decrypted) - }) -} - type dataSourceMockRetriever struct { res []*models.DataSource } @@ -237,15 +194,16 @@ func TestService_GetHttpTransport(t *testing.T) { Type: "Kubernetes", } + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - rt1, err := dsService.GetHTTPTransport(&ds, provider) + rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt1) tr1 := configuredTransport - rt2, err := dsService.GetHTTPTransport(&ds, provider) + rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt2) tr2 := configuredTransport @@ -267,24 +225,22 @@ func TestService_GetHttpTransport(t *testing.T) { setting.SecretKey = "password" - json := simplejson.New() - json.Set("tlsAuthWithCACert", true) + sjson := simplejson.New() + sjson.Set("tlsAuthWithCACert", true) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope()) - require.NoError(t, err) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Id: 1, Url: "http://k8s:8001", Type: "Kubernetes", - SecureJsonData: map[string][]byte{"tlsCACert": tlsCaCert}, + SecureJsonData: map[string][]byte{"tlsCACert": []byte(caCert)}, Updated: time.Now().Add(-2 * time.Minute), } - rt1, err := dsService.GetHTTPTransport(&ds, provider) + rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NotNil(t, rt1) require.NoError(t, err) @@ -298,7 +254,7 @@ func TestService_GetHttpTransport(t *testing.T) { ds.SecureJsonData = map[string][]byte{} ds.Updated = time.Now() - rt2, err := dsService.GetHTTPTransport(&ds, provider) + rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt2) tr2 := configuredTransport @@ -317,30 +273,32 @@ func TestService_GetHttpTransport(t *testing.T) { setting.SecretKey = "password" - json := simplejson.New() - json.Set("tlsAuth", true) + sjson := simplejson.New() + sjson.Set("tlsAuth", true) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - tlsClientCert, err := secretsService.Encrypt(context.Background(), []byte(clientCert), secrets.WithoutScope()) - require.NoError(t, err) - - tlsClientKey, err := secretsService.Encrypt(context.Background(), []byte(clientKey), secrets.WithoutScope()) - require.NoError(t, err) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Id: 1, + OrgId: 1, + Name: "kubernetes", Url: "http://k8s:8001", Type: "Kubernetes", - JsonData: json, - SecureJsonData: map[string][]byte{ - "tlsClientCert": tlsClientCert, - "tlsClientKey": tlsClientKey, - }, + JsonData: sjson, } - rt, err := dsService.GetHTTPTransport(&ds, provider) + secureJsonData, err := json.Marshal(map[string]string{ + "tlsClientCert": clientCert, + "tlsClientKey": clientKey, + }) + require.NoError(t, err) + + err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData)) + require.NoError(t, err) + + rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt) tr := configuredTransport @@ -359,27 +317,32 @@ func TestService_GetHttpTransport(t *testing.T) { setting.SecretKey = "password" - json := simplejson.New() - json.Set("tlsAuthWithCACert", true) - json.Set("serverName", "server-name") + sjson := simplejson.New() + sjson.Set("tlsAuthWithCACert", true) + sjson.Set("serverName", "server-name") + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope()) - require.NoError(t, err) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Id: 1, + OrgId: 1, + Name: "kubernetes", Url: "http://k8s:8001", Type: "Kubernetes", - JsonData: json, - SecureJsonData: map[string][]byte{ - "tlsCACert": tlsCaCert, - }, + JsonData: sjson, } - rt, err := dsService.GetHTTPTransport(&ds, provider) + secureJsonData, err := json.Marshal(map[string]string{ + "tlsCACert": caCert, + }) + require.NoError(t, err) + + err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData)) + require.NoError(t, err) + + rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt) tr := configuredTransport @@ -397,25 +360,26 @@ func TestService_GetHttpTransport(t *testing.T) { }, }) - json := simplejson.New() - json.Set("tlsSkipVerify", true) + sjson := simplejson.New() + sjson.Set("tlsSkipVerify", true) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Id: 1, Url: "http://k8s:8001", Type: "Kubernetes", - JsonData: json, + JsonData: sjson, } - rt1, err := dsService.GetHTTPTransport(&ds, provider) + rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt1) tr1 := configuredTransport - rt2, err := dsService.GetHTTPTransport(&ds, provider) + rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt2) tr2 := configuredTransport @@ -427,25 +391,32 @@ func TestService_GetHttpTransport(t *testing.T) { t.Run("Should set custom headers if configured in JsonData", func(t *testing.T) { provider := httpclient.NewProvider() - json := simplejson.NewFromAny(map[string]interface{}{ + sjson := simplejson.NewFromAny(map[string]interface{}{ "httpHeaderName1": "Authorization", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - encryptedData, err := secretsService.Encrypt(context.Background(), []byte(`Bearer xf5yhfkpsnmgo`), secrets.WithoutScope()) - require.NoError(t, err) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ - Id: 1, - Url: "http://k8s:8001", - Type: "Kubernetes", - JsonData: json, - SecureJsonData: map[string][]byte{"httpHeaderValue1": encryptedData}, + Id: 1, + OrgId: 1, + Name: "kubernetes", + Url: "http://k8s:8001", + Type: "Kubernetes", + JsonData: sjson, } - headers := dsService.getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"}) + secureJsonData, err := json.Marshal(map[string]string{ + "httpHeaderValue1": "Bearer xf5yhfkpsnmgo", + }) + require.NoError(t, err) + + err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData)) + require.NoError(t, err) + + headers := dsService.getCustomHeaders(sjson, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"}) require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"]) // 1. Start HTTP test server which checks the request headers @@ -465,7 +436,7 @@ func TestService_GetHttpTransport(t *testing.T) { // 2. Get HTTP transport from datasource which uses the test server as backend ds.Url = backend.URL - rt, err := dsService.GetHTTPTransport(&ds, provider) + rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt) @@ -486,21 +457,22 @@ func TestService_GetHttpTransport(t *testing.T) { t.Run("Should use request timeout if configured in JsonData", func(t *testing.T) { provider := httpclient.NewProvider() - json := simplejson.NewFromAny(map[string]interface{}{ + sjson := simplejson.NewFromAny(map[string]interface{}{ "timeout": 19, }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Id: 1, Url: "http://k8s:8001", Type: "Kubernetes", - JsonData: json, + JsonData: sjson, } - client, err := dsService.GetHTTPClient(&ds, provider) + client, err := dsService.GetHTTPClient(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, client) require.Equal(t, 19*time.Second, client.Timeout) @@ -520,18 +492,19 @@ func TestService_GetHttpTransport(t *testing.T) { setting.SigV4AuthEnabled = origSigV4Enabled }) - json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`)) + sjson, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`)) require.NoError(t, err) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Type: models.DS_ES, - JsonData: json, + JsonData: sjson, } - _, err = dsService.GetHTTPTransport(&ds, provider) + _, err = dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, configuredOpts) require.NotNil(t, configuredOpts.SigV4) @@ -558,8 +531,9 @@ func TestService_getTimeout(t *testing.T) { {jsonData: simplejson.NewFromAny(map[string]interface{}{"timeout": "2"}), expectedTimeout: 2 * time.Second}, } + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) for _, tc := range testCases { ds := &models.DataSource{ @@ -569,86 +543,6 @@ func TestService_getTimeout(t *testing.T) { } } -func TestService_DecryptedValue(t *testing.T) { - cfg := &setting.Cfg{} - - t.Run("When datasource hasn't been updated, encrypted JSON should be fetched from cache", func(t *testing.T) { - secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - encryptedJsonData, err := secretsService.EncryptJsonData( - context.Background(), - map[string]string{ - "password": "password", - }, secrets.WithoutScope()) - require.NoError(t, err) - - ds := models.DataSource{ - Id: 1, - Type: models.DS_INFLUXDB_08, - JsonData: simplejson.New(), - User: "user", - SecureJsonData: encryptedJsonData, - } - - // Populate cache - password, ok := dsService.DecryptedValue(&ds, "password") - require.True(t, ok) - require.Equal(t, "password", password) - - encryptedJsonData, err = secretsService.EncryptJsonData( - context.Background(), - map[string]string{ - "password": "", - }, secrets.WithoutScope()) - require.NoError(t, err) - - ds.SecureJsonData = encryptedJsonData - - password, ok = dsService.DecryptedValue(&ds, "password") - require.True(t, ok) - require.Equal(t, "password", password) - }) - - t.Run("When datasource is updated, encrypted JSON should not be fetched from cache", func(t *testing.T) { - secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - - encryptedJsonData, err := secretsService.EncryptJsonData( - context.Background(), - map[string]string{ - "password": "password", - }, secrets.WithoutScope()) - require.NoError(t, err) - - ds := models.DataSource{ - Id: 1, - Type: models.DS_INFLUXDB_08, - JsonData: simplejson.New(), - User: "user", - SecureJsonData: encryptedJsonData, - } - - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - // Populate cache - password, ok := dsService.DecryptedValue(&ds, "password") - require.True(t, ok) - require.Equal(t, "password", password) - - ds.SecureJsonData, err = secretsService.EncryptJsonData( - context.Background(), - map[string]string{ - "password": "", - }, secrets.WithoutScope()) - ds.Updated = time.Now() - require.NoError(t, err) - - password, ok = dsService.DecryptedValue(&ds, "password") - require.True(t, ok) - require.Empty(t, password) - }) -} - func TestService_HTTPClientOptions(t *testing.T) { cfg := &setting.Cfg{ Azure: &azsettings.AzureSettings{}, @@ -678,10 +572,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - opts, err := dsService.httpClientOptions(&ds) + opts, err := dsService.httpClientOptions(context.Background(), &ds) require.NoError(t, err) require.NotNil(t, opts.Middlewares) @@ -695,10 +590,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "httpMethod": "POST", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - opts, err := dsService.httpClientOptions(&ds) + opts, err := dsService.httpClientOptions(context.Background(), &ds) require.NoError(t, err) if opts.Middlewares != nil { @@ -714,10 +610,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureCredentials": "invalid", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - _, err := dsService.httpClientOptions(&ds) + _, err := dsService.httpClientOptions(context.Background(), &ds) assert.Error(t, err) }) @@ -732,10 +629,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - opts, err := dsService.httpClientOptions(&ds) + opts, err := dsService.httpClientOptions(context.Background(), &ds) require.NoError(t, err) require.NotNil(t, opts.Middlewares) @@ -750,10 +648,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - opts, err := dsService.httpClientOptions(&ds) + opts, err := dsService.httpClientOptions(context.Background(), &ds) require.NoError(t, err) if opts.Middlewares != nil { @@ -772,10 +671,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureEndpointResourceId": "invalid", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - _, err := dsService.httpClientOptions(&ds) + _, err := dsService.httpClientOptions(context.Background(), &ds) assert.Error(t, err) }) }) @@ -792,10 +692,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - opts, err := dsService.httpClientOptions(&ds) + opts, err := dsService.httpClientOptions(context.Background(), &ds) require.NoError(t, err) if opts.Middlewares != nil { @@ -806,6 +707,59 @@ func TestService_HTTPClientOptions(t *testing.T) { }) } +func TestService_GetDecryptedValues(t *testing.T) { + t.Run("should migrate and retrieve values from secure json data", func(t *testing.T) { + ds := &models.DataSource{ + Id: 1, + Url: "https://api.example.com", + Type: "prometheus", + } + + secretsStore := kvstore.SetupTestService(t) + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) + dsService := ProvideService(nil, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + + jsonData := map[string]string{ + "password": "securePassword", + } + secureJsonData, err := dsService.SecretsService.EncryptJsonData(context.Background(), jsonData, secrets.WithoutScope()) + + require.NoError(t, err) + ds.SecureJsonData = secureJsonData + + values, err := dsService.DecryptedValues(context.Background(), ds) + require.NoError(t, err) + + require.Equal(t, jsonData, values) + }) + + t.Run("should retrieve values from secret store", func(t *testing.T) { + ds := &models.DataSource{ + Id: 1, + Url: "https://api.example.com", + Type: "prometheus", + } + + secretsStore := kvstore.SetupTestService(t) + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) + dsService := ProvideService(nil, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + + jsonData := map[string]string{ + "password": "securePassword", + } + jsonString, err := json.Marshal(jsonData) + require.NoError(t, err) + + err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(jsonString)) + require.NoError(t, err) + + values, err := dsService.DecryptedValues(context.Background(), ds) + require.NoError(t, err) + + require.Equal(t, jsonData, values) + }) +} + const caCert string = `-----BEGIN CERTIFICATE----- MIIDATCCAemgAwIBAgIJAMQ5hC3CPDTeMA0GCSqGSIb3DQEBCwUAMBcxFTATBgNV BAMMDGNhLWs4cy1zdGhsbTAeFw0xNjEwMjcwODQyMjdaFw00NDAzMTQwODQyMjda diff --git a/pkg/services/export/dummy_job.go b/pkg/services/export/dummy_job.go new file mode 100644 index 00000000000..029bc8edabb --- /dev/null +++ b/pkg/services/export/dummy_job.go @@ -0,0 +1,103 @@ +package export + +import ( + "errors" + "fmt" + "math" + "math/rand" + "sync" + "time" + + "github.com/grafana/grafana/pkg/infra/log" +) + +var _ Job = new(dummyExportJob) + +type dummyExportJob struct { + logger log.Logger + + statusMu sync.Mutex + status ExportStatus + cfg ExportConfig + broadcaster statusBroadcaster +} + +func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job, error) { + if cfg.Format != "git" { + return nil, errors.New("only git format is supported") + } + + job := &dummyExportJob{ + logger: log.New("dummy_export_job"), + cfg: cfg, + broadcaster: broadcaster, + status: ExportStatus{ + Running: true, + Target: "git export", + Started: time.Now().UnixMilli(), + Count: int64(math.Round(10 + rand.Float64()*20)), + Current: 0, + }, + } + + broadcaster(job.status) + go job.start() + return job, nil +} + +func (e *dummyExportJob) start() { + defer func() { + e.logger.Info("Finished dummy export job") + + e.statusMu.Lock() + defer e.statusMu.Unlock() + s := e.status + if err := recover(); err != nil { + e.logger.Error("export panic", "error", err) + s.Status = fmt.Sprintf("ERROR: %v", err) + } + // Make sure it finishes OK + if s.Finished < 10 { + s.Finished = time.Now().UnixMilli() + } + s.Running = false + if s.Status == "" { + s.Status = "done" + } + e.status = s + e.broadcaster(s) + }() + + e.logger.Info("Starting dummy export job") + + ticker := time.NewTicker(1 * time.Second) + for t := range ticker.C { + e.statusMu.Lock() + e.status.Changed = t.UnixMilli() + e.status.Current++ + e.status.Last = fmt.Sprintf("ITEM: %d", e.status.Current) + e.statusMu.Unlock() + + // Wait till we are done + shouldStop := e.status.Current >= e.status.Count + e.broadcaster(e.status) + + if shouldStop { + break + } + } +} + +func (e *dummyExportJob) getStatus() ExportStatus { + e.statusMu.Lock() + defer e.statusMu.Unlock() + + return e.status +} + +func (e *dummyExportJob) getConfig() ExportConfig { + e.statusMu.Lock() + defer e.statusMu.Unlock() + + return e.cfg +} diff --git a/pkg/services/export/service.go b/pkg/services/export/service.go new file mode 100644 index 00000000000..ee4f1b040ea --- /dev/null +++ b/pkg/services/export/service.go @@ -0,0 +1,93 @@ +package export + +import ( + "encoding/json" + "net/http" + "sync" + + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/live" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +type ExportService interface { + // List folder contents + HandleGetStatus(c *models.ReqContext) response.Response + + // Read raw file contents out of the store + HandleRequestExport(c *models.ReqContext) response.Response +} + +type StandardExport struct { + logger log.Logger + sql *sqlstore.SQLStore + glive *live.GrafanaLive + mutex sync.Mutex + + // updated with mutex + exportJob Job +} + +func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, gl *live.GrafanaLive) ExportService { + if !features.IsEnabled(featuremgmt.FlagExport) { + return &StubExport{} + } + + return &StandardExport{ + sql: sql, + glive: gl, + logger: log.New("export_service"), + exportJob: &stoppedJob{}, + } +} + +func (ex *StandardExport) HandleGetStatus(c *models.ReqContext) response.Response { + ex.mutex.Lock() + defer ex.mutex.Unlock() + + return response.JSON(http.StatusOK, ex.exportJob.getStatus()) +} + +func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Response { + var cfg ExportConfig + err := json.NewDecoder(c.Req.Body).Decode(&cfg) + if err != nil { + return response.Error(http.StatusBadRequest, "unable to read config", err) + } + + ex.mutex.Lock() + defer ex.mutex.Unlock() + + status := ex.exportJob.getStatus() + if status.Running { + ex.logger.Error("export already running") + return response.Error(http.StatusLocked, "export already running", nil) + } + + job, err := startDummyExportJob(cfg, func(s ExportStatus) { + ex.broadcastStatus(c.OrgId, s) + }) + if err != nil { + ex.logger.Error("failed to start export job", "err", err) + return response.Error(http.StatusBadRequest, "failed to start export job", err) + } + + ex.exportJob = job + return response.JSON(http.StatusOK, ex.exportJob.getStatus()) +} + +func (ex *StandardExport) broadcastStatus(orgID int64, s ExportStatus) { + msg, err := json.Marshal(s) + if err != nil { + ex.logger.Warn("Error making message", "err", err) + return + } + err = ex.glive.Publish(orgID, "grafana/broadcast/export", msg) + if err != nil { + ex.logger.Warn("Error Publish message", "err", err) + return + } +} diff --git a/pkg/services/export/stopped_job.go b/pkg/services/export/stopped_job.go new file mode 100644 index 00000000000..b9756f9d72f --- /dev/null +++ b/pkg/services/export/stopped_job.go @@ -0,0 +1,19 @@ +package export + +import "time" + +var _ Job = new(stoppedJob) + +type stoppedJob struct { +} + +func (e *stoppedJob) getStatus() ExportStatus { + return ExportStatus{ + Running: false, + Changed: time.Now().UnixMilli(), + } +} + +func (e *stoppedJob) getConfig() ExportConfig { + return ExportConfig{} +} diff --git a/pkg/services/export/stub.go b/pkg/services/export/stub.go new file mode 100644 index 00000000000..551bb730f5c --- /dev/null +++ b/pkg/services/export/stub.go @@ -0,0 +1,20 @@ +package export + +import ( + "net/http" + + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/models" +) + +var _ ExportService = new(StubExport) + +type StubExport struct{} + +func (ex *StubExport) HandleGetStatus(c *models.ReqContext) response.Response { + return response.Error(http.StatusForbidden, "feature not enabled", nil) +} + +func (ex *StubExport) HandleRequestExport(c *models.ReqContext) response.Response { + return response.Error(http.StatusForbidden, "feature not enabled", nil) +} diff --git a/pkg/services/export/types.go b/pkg/services/export/types.go new file mode 100644 index 00000000000..20a1ec49e3d --- /dev/null +++ b/pkg/services/export/types.go @@ -0,0 +1,36 @@ +package export + +// Export status. Only one running at a time +type ExportStatus struct { + Running bool `json:"running"` + Target string `json:"target"` // description of where it is going (no secrets) + Started int64 `json:"started,omitempty"` + Finished int64 `json:"finished,omitempty"` + Changed int64 `json:"update,omitempty"` + Count int64 `json:"count,omitempty"` + Current int64 `json:"current,omitempty"` + Last string `json:"last,omitempty"` + Status string `json:"status"` // ERROR, SUCCESS, ETC +} + +// Basic export config (for now) +type ExportConfig struct { + Format string `json:"format"` + Git GitExportConfig `json:"git"` +} + +type GitExportConfig struct { + // General folder is either at the root or as a subfolder + GeneralAtRoot bool `json:"generalAtRoot"` + + // Keeping all history is nice, but much slower + ExcludeHistory bool `json:"excludeHistory"` +} + +type Job interface { + getStatus() ExportStatus + getConfig() ExportConfig +} + +// Will broadcast the live status +type statusBroadcaster func(s ExportStatus) diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 6c18fb2a947..8458c57093b 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -39,14 +39,10 @@ var ( State: FeatureStateAlpha, }, { - Name: "dashboardPreviewsScheduler", - Description: "Schedule automatic updates to dashboard previews", - State: FeatureStateAlpha, - }, - { - Name: "dashboardPreviewsAdmin", - Description: "Manage the dashboard previews crawler process from the UI", - State: FeatureStateAlpha, + Name: "dashboardPreviewsAdmin", + Description: "Manage the dashboard previews crawler process from the UI", + State: FeatureStateAlpha, + RequiresDevMode: true, }, { Name: "live-config", @@ -194,6 +190,12 @@ var ( Description: "Provisioning-friendly routes for alerting", State: FeatureStateAlpha, }, + { + Name: "export", + Description: "Export grafana instance (to git, etc)", + State: FeatureStateAlpha, + RequiresDevMode: true, + }, { Name: "storageLocalUpload", Description: "allow uploads to local storage", @@ -213,6 +215,12 @@ var ( State: FeatureStateBeta, FrontendOnly: true, }, + { + Name: "tracing", + Description: "Adds trace ID to error notifications", + State: FeatureStateAlpha, + FrontendOnly: true, + }, { Name: "persistNotifications", Description: "PoC Notifications page", @@ -224,5 +232,10 @@ var ( Description: "Enable command palette", State: FeatureStateAlpha, }, + { + Name: "savedItems", + Description: "Enable Saved Items in the navbar.", + State: FeatureStateAlpha, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 0d762f26555..18ebac24d06 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -31,10 +31,6 @@ const ( // Create and show thumbnails for dashboard search results FlagDashboardPreviews = "dashboardPreviews" - // FlagDashboardPreviewsScheduler - // Schedule automatic updates to dashboard previews - FlagDashboardPreviewsScheduler = "dashboardPreviewsScheduler" - // FlagDashboardPreviewsAdmin // Manage the dashboard previews crawler process from the UI FlagDashboardPreviewsAdmin = "dashboardPreviewsAdmin" @@ -147,6 +143,10 @@ const ( // Provisioning-friendly routes for alerting FlagAlertProvisioning = "alertProvisioning" + // FlagExport + // Export grafana instance (to git, etc) + FlagExport = "export" + // FlagStorageLocalUpload // allow uploads to local storage FlagStorageLocalUpload = "storageLocalUpload" @@ -159,6 +159,10 @@ const ( // Experimental Explore to Dashboard workflow FlagExplore2Dashboard = "explore2Dashboard" + // FlagTracing + // Adds trace ID to error notifications + FlagTracing = "tracing" + // FlagPersistNotifications // PoC Notifications page FlagPersistNotifications = "persistNotifications" @@ -166,4 +170,8 @@ const ( // FlagCommandPalette // Enable command palette FlagCommandPalette = "commandPalette" + + // FlagSavedItems + // Enable Saved Items in the navbar. + FlagSavedItems = "savedItems" ) diff --git a/pkg/services/guardian/provider.go b/pkg/services/guardian/provider.go index 5acccdf7774..3318fb45abb 100644 --- a/pkg/services/guardian/provider.go +++ b/pkg/services/guardian/provider.go @@ -12,7 +12,7 @@ import ( type Provider struct{} func ProvideService(store *sqlstore.SQLStore, ac accesscontrol.AccessControl, permissionsServices accesscontrol.PermissionsServices, features featuremgmt.FeatureToggles) *Provider { - if features.IsEnabled(featuremgmt.FlagAccesscontrol) { + if !ac.IsDisabled() { // TODO: Fix this hack, see https://github.com/grafana/grafana-enterprise/issues/2935 InitAcessControlGuardian(store, ac, permissionsServices) } else { diff --git a/pkg/services/ngalert/CHANGELOG.md b/pkg/services/ngalert/CHANGELOG.md index 6738445a117..83569bee7ae 100644 --- a/pkg/services/ngalert/CHANGELOG.md +++ b/pkg/services/ngalert/CHANGELOG.md @@ -46,9 +46,13 @@ Scopes must have an order to ensure consistency and ease of search, this helps u ## Grafana Alerting - main / unreleased - [CHANGE] Prometheus Compatible API: Use float-like values for `api/prometheus/grafana/api/v1/alerts` and `api/prometheus/grafana/api/v1/rules` instead of the evaluation string #47216 -- [BUGFIX] (Legacy) Templates: Parse notification templates using all the matches of the alert rule when going from `Alerting` to `OK` in legacy alerting #47355 -- [BUGFIX] Scheduler: Fix state manager to support OK option of `AlertRule.ExecErrState` #47670 -- [ENHANCEMENT] Templates: Enable the use of classic condition values in templates #46971 -- [ENHANCEMENT] Scheduler: ticker expose new metrics `grafana_alerting_ticker_last_consumed_tick_timestamp_seconds`, `grafana_alerting_ticker_next_tick_timestamp_seconds`, `grafana_alerting_ticker_interval_seconds` #47828 - [CHANGE] Notification URL points to alert view page instead of alert edit page. #47752 - [FEATURE] Indicate whether routes are provisioned when GETting Alertmanager configuration #47857 +- [BUGFIX] (Legacy) Templates: Parse notification templates using all the matches of the alert rule when going from `Alerting` to `OK` in legacy alerting #47355 +- [BUGFIX] Scheduler: Fix state manager to support OK option of `AlertRule.ExecErrState` #47670 +- [ENHANCEMENT] Templates: Enable the use of classic condition values in templates #46971 +- [ENHANCEMENT] Scheduler: Ticker expose new metrics. In legacy, metrics are prefixed with `legacy_` #47828, #48190 + - `grafana_alerting_ticker_last_consumed_tick_timestamp_seconds` + - `grafana_alerting_ticker_next_tick_timestamp_seconds` + - `grafana_alerting_ticker_interval_seconds` +- [ENHANCEMENT] Migration: Migrate each legacy notification channel to its own contact point, use nested routes to reproduce multi-channel alerts #47291 diff --git a/pkg/services/ngalert/api/api_alertmanager_test.go b/pkg/services/ngalert/api/api_alertmanager_test.go index 45c886cd1e4..d9af081ac64 100644 --- a/pkg/services/ngalert/api/api_alertmanager_test.go +++ b/pkg/services/ngalert/api/api_alertmanager_test.go @@ -505,8 +505,7 @@ func createRequestCtxInOrg(org int64) *models.ReqContext { // setRouteProvenance marks an org's routing tree as provisioned. func setRouteProvenance(t *testing.T, org int64, ps provisioning.ProvisioningStore) { t.Helper() - adp := provisioning.ProvenanceOrgAdapter{Inner: &apimodels.Route{}, OrgID: org} - err := ps.SetProvenance(context.Background(), adp, ngmodels.ProvenanceAPI) + err := ps.SetProvenance(context.Background(), &apimodels.Route{}, org, ngmodels.ProvenanceAPI) require.NoError(t, err) } diff --git a/pkg/services/ngalert/api/api_prometheus.go b/pkg/services/ngalert/api/api_prometheus.go index c7ee7b93b97..15759737c14 100644 --- a/pkg/services/ngalert/api/api_prometheus.go +++ b/pkg/services/ngalert/api/api_prometheus.go @@ -147,7 +147,7 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res DashboardUID: dashboardUID, PanelID: panelID, } - if err := srv.store.GetOrgAlertRules(c.Req.Context(), &alertRuleQuery); err != nil { + if err := srv.store.ListAlertRules(c.Req.Context(), &alertRuleQuery); err != nil { ruleResponse.DiscoveryBase.Status = "error" ruleResponse.DiscoveryBase.Error = fmt.Sprintf("failure getting rules: %s", err.Error()) ruleResponse.DiscoveryBase.ErrorType = apiv1.ErrServer diff --git a/pkg/services/ngalert/api/api_ruler.go b/pkg/services/ngalert/api/api_ruler.go index 963597e7e9c..f8f9a43a10f 100644 --- a/pkg/services/ngalert/api/api_ruler.go +++ b/pkg/services/ngalert/api/api_ruler.go @@ -55,9 +55,9 @@ func (srv RulerSrv) RouteDeleteAlertRules(c *models.ReqContext) response.Respons "namespace", namespace.Title, } - var ruleGroup *string + var ruleGroup string if group, ok := web.Params(c.Req)[":Groupname"]; ok { - ruleGroup = &group + ruleGroup = group loggerCtx = append(loggerCtx, "group", group) } logger := srv.log.New(loggerCtx...) @@ -68,12 +68,12 @@ func (srv RulerSrv) RouteDeleteAlertRules(c *models.ReqContext) response.Respons var canDelete, cannotDelete []string err = srv.xactManager.InTransaction(c.Req.Context(), func(ctx context.Context) error { - q := ngmodels.GetAlertRulesQuery{ - OrgID: c.SignedInUser.OrgId, - NamespaceUID: namespace.Uid, - RuleGroup: ruleGroup, + q := ngmodels.ListAlertRulesQuery{ + OrgID: c.SignedInUser.OrgId, + NamespaceUIDs: []string{namespace.Uid}, + RuleGroup: ruleGroup, } - if err = srv.store.GetAlertRules(ctx, &q); err != nil { + if err = srv.store.ListAlertRules(ctx, &q); err != nil { return err } @@ -128,11 +128,11 @@ func (srv RulerSrv) RouteGetNamespaceRulesConfig(c *models.ReqContext) response. return toNamespaceErrorResponse(err) } - q := ngmodels.GetAlertRulesQuery{ - OrgID: c.SignedInUser.OrgId, - NamespaceUID: namespace.Uid, + q := ngmodels.ListAlertRulesQuery{ + OrgID: c.SignedInUser.OrgId, + NamespaceUIDs: []string{namespace.Uid}, } - if err := srv.store.GetAlertRules(c.Req.Context(), &q); err != nil { + if err := srv.store.ListAlertRules(c.Req.Context(), &q); err != nil { return ErrResp(http.StatusInternalServerError, err, "failed to update rule group") } @@ -178,12 +178,12 @@ func (srv RulerSrv) RouteGetRulegGroupConfig(c *models.ReqContext) response.Resp } ruleGroup := web.Params(c.Req)[":Groupname"] - q := ngmodels.GetAlertRulesQuery{ - OrgID: c.SignedInUser.OrgId, - NamespaceUID: namespace.Uid, - RuleGroup: &ruleGroup, + q := ngmodels.ListAlertRulesQuery{ + OrgID: c.SignedInUser.OrgId, + NamespaceUIDs: []string{namespace.Uid}, + RuleGroup: ruleGroup, } - if err := srv.store.GetAlertRules(c.Req.Context(), &q); err != nil { + if err := srv.store.ListAlertRules(c.Req.Context(), &q); err != nil { return ErrResp(http.StatusInternalServerError, err, "failed to get group alert rules") } @@ -245,7 +245,7 @@ func (srv RulerSrv) RouteGetRulesConfig(c *models.ReqContext) response.Response PanelID: panelID, } - if err := srv.store.GetOrgAlertRules(c.Req.Context(), &q); err != nil { + if err := srv.store.ListAlertRules(c.Req.Context(), &q); err != nil { return ErrResp(http.StatusInternalServerError, err, "failed to get alert rules") } @@ -493,12 +493,12 @@ func (c *changes) isEmpty() bool { // calculateChanges calculates the difference between rules in the group in the database and the submitted rules. If a submitted rule has UID it tries to find it in the database (in other groups). // returns a list of rules that need to be added, updated and deleted. Deleted considered rules in the database that belong to the group but do not exist in the list of submitted rules. func calculateChanges(ctx context.Context, ruleStore store.RuleStore, orgId int64, namespace *models.Folder, ruleGroupName string, submittedRules []*ngmodels.AlertRule) (*changes, error) { - q := &ngmodels.GetAlertRulesQuery{ - OrgID: orgId, - NamespaceUID: namespace.Uid, - RuleGroup: &ruleGroupName, + q := &ngmodels.ListAlertRulesQuery{ + OrgID: orgId, + NamespaceUIDs: []string{namespace.Uid}, + RuleGroup: ruleGroupName, } - if err := ruleStore.GetAlertRules(ctx, q); err != nil { + if err := ruleStore.ListAlertRules(ctx, q); err != nil { return nil, fmt.Errorf("failed to query database for rules in the group %s: %w", ruleGroupName, err) } existingGroupRules := q.Result diff --git a/pkg/services/ngalert/api/api_ruler_test.go b/pkg/services/ngalert/api/api_ruler_test.go index 9a40a93dfbd..1d10b29c80d 100644 --- a/pkg/services/ngalert/api/api_ruler_test.go +++ b/pkg/services/ngalert/api/api_ruler_test.go @@ -232,7 +232,7 @@ func TestCalculateChanges(t *testing.T) { expectedErr := errors.New("TEST ERROR") fakeStore.Hook = func(cmd interface{}) error { switch cmd.(type) { - case models.GetAlertRulesQuery: + case models.ListAlertRulesQuery: return expectedErr } return nil diff --git a/pkg/services/ngalert/api/api_testing_test.go b/pkg/services/ngalert/api/api_testing_test.go index 91b976b7364..384aaf45ea5 100644 --- a/pkg/services/ngalert/api/api_testing_test.go +++ b/pkg/services/ngalert/api/api_testing_test.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana/pkg/services/accesscontrol" acMock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/datasources" + fakes "github.com/grafana/grafana/pkg/services/datasources/fakes" "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/eval" "github.com/grafana/grafana/pkg/services/ngalert/models" @@ -61,7 +62,7 @@ func TestRouteTestGrafanaRuleConfig(t *testing.T) { {Action: datasources.ActionQuery, Scope: datasources.ScopeProvider.GetResourceScopeUID(data2.DatasourceUID)}, }) - ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ + ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{ {Uid: data1.DatasourceUID}, {Uid: data2.DatasourceUID}, }} @@ -102,7 +103,7 @@ func TestRouteTestGrafanaRuleConfig(t *testing.T) { t.Run("should require user to be signed in", func(t *testing.T) { data1 := models.GenerateAlertQuery() - ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ + ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{ {Uid: data1.DatasourceUID}, }} @@ -182,7 +183,7 @@ func TestRouteEvalQueries(t *testing.T) { {Action: datasources.ActionQuery, Scope: datasources.ScopeProvider.GetResourceScopeUID(data2.DatasourceUID)}, }) - ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ + ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{ {Uid: data1.DatasourceUID}, {Uid: data2.DatasourceUID}, }} @@ -226,7 +227,7 @@ func TestRouteEvalQueries(t *testing.T) { t.Run("should require user to be signed in", func(t *testing.T) { data1 := models.GenerateAlertQuery() - ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ + ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{ {Uid: data1.DatasourceUID}, }} @@ -265,7 +266,7 @@ func TestRouteEvalQueries(t *testing.T) { }) } -func createTestingApiSrv(ds *datasources.FakeCacheService, ac *acMock.Mock, evaluator *eval.FakeEvaluator) *TestingApiSrv { +func createTestingApiSrv(ds *fakes.FakeCacheService, ac *acMock.Mock, evaluator *eval.FakeEvaluator) *TestingApiSrv { if ac == nil { ac = acMock.New().WithDisabled() } diff --git a/pkg/services/ngalert/models/alert_rule.go b/pkg/services/ngalert/models/alert_rule.go index e395f1f220f..6bf3c088421 100644 --- a/pkg/services/ngalert/models/alert_rule.go +++ b/pkg/services/ngalert/models/alert_rule.go @@ -232,6 +232,7 @@ type ListAlertRulesQuery struct { OrgID int64 NamespaceUIDs []string ExcludeOrgs []int64 + RuleGroup string // DashboardUID and PanelID are optional and allow filtering rules // to return just those for a dashboard and panel. @@ -250,21 +251,6 @@ type ListNamespaceAlertRulesQuery struct { Result []*AlertRule } -// GetAlertRulesQuery is the query for listing rule group alert rules -type GetAlertRulesQuery struct { - OrgID int64 - // Namespace is the folder slug - NamespaceUID string - RuleGroup *string - - // DashboardUID and PanelID are optional and allow filtering rules - // to return just those for a dashboard and panel. - DashboardUID string - PanelID int64 - - Result []*AlertRule -} - // ListRuleGroupsQuery is the query for listing unique rule groups // across all organizations type ListRuleGroupsQuery struct { diff --git a/pkg/services/ngalert/models/provisioning.go b/pkg/services/ngalert/models/provisioning.go index c3655edfc5b..1a68cd59aa6 100644 --- a/pkg/services/ngalert/models/provisioning.go +++ b/pkg/services/ngalert/models/provisioning.go @@ -14,11 +14,4 @@ const ( type Provisionable interface { ResourceType() string ResourceID() string - ResourceOrgID() int64 -} - -// ProvisionableInOrg represents a resource that can be provisioned, given external org-related information. -type ProvisionableInOrg interface { - ResourceType() string - ResourceID() string } diff --git a/pkg/services/ngalert/notifier/alertmanager_config.go b/pkg/services/ngalert/notifier/alertmanager_config.go index f3d873a9c98..7dd70647f2d 100644 --- a/pkg/services/ngalert/notifier/alertmanager_config.go +++ b/pkg/services/ngalert/notifier/alertmanager_config.go @@ -8,7 +8,6 @@ import ( "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/services/ngalert/provisioning" "github.com/grafana/grafana/pkg/services/ngalert/store" ) @@ -129,11 +128,7 @@ func (moa *MultiOrgAlertmanager) ApplyAlertmanagerConfiguration(ctx context.Cont func (moa *MultiOrgAlertmanager) mergeProvenance(ctx context.Context, config definitions.GettableApiAlertingConfig, org int64) (definitions.GettableApiAlertingConfig, error) { if config.Route != nil { - adp := provisioning.ProvenanceOrgAdapter{ - Inner: config.Route, - OrgID: org, - } - provenance, err := moa.ProvStore.GetProvenance(ctx, adp) + provenance, err := moa.ProvStore.GetProvenance(ctx, config.Route, org) if err != nil { return definitions.GettableApiAlertingConfig{}, err } diff --git a/pkg/services/ngalert/provisioning/contactpoints.go b/pkg/services/ngalert/provisioning/contactpoints.go index 83922021f0c..1bd94d74b78 100644 --- a/pkg/services/ngalert/provisioning/contactpoints.go +++ b/pkg/services/ngalert/provisioning/contactpoints.go @@ -177,11 +177,7 @@ func (ecp *ContactPointService) CreateContactPoint(ctx context.Context, orgID in if err != nil { return err } - adapter := ProvenanceOrgAdapter{ - Inner: &contactPoint, - OrgID: orgID, - } - err = ecp.provenanceStore.SetProvenance(ctx, adapter, provenance) + err = ecp.provenanceStore.SetProvenance(ctx, &contactPoint, orgID, provenance) if err != nil { return err } @@ -218,10 +214,7 @@ func (ecp *ContactPointService) UpdateContactPoint(ctx context.Context, orgID in return err } // check that provenance is not changed in a invalid way - storedProvenance, err := ecp.provenanceStore.GetProvenance(ctx, ProvenanceOrgAdapter{ - Inner: &contactPoint, - OrgID: orgID, - }) + storedProvenance, err := ecp.provenanceStore.GetProvenance(ctx, &contactPoint, orgID) if err != nil { return err } @@ -283,11 +276,7 @@ func (ecp *ContactPointService) UpdateContactPoint(ctx context.Context, orgID in if err != nil { return err } - adapter := ProvenanceOrgAdapter{ - Inner: &contactPoint, - OrgID: orgID, - } - err = ecp.provenanceStore.SetProvenance(ctx, adapter, provenance) + err = ecp.provenanceStore.SetProvenance(ctx, &contactPoint, orgID, provenance) if err != nil { return err } @@ -330,12 +319,10 @@ func (ecp *ContactPointService) DeleteContactPoint(ctx context.Context, orgID in return err } return ecp.xact.InTransaction(ctx, func(ctx context.Context) error { - err := ecp.provenanceStore.DeleteProvenance(ctx, ProvenanceOrgAdapter{ - Inner: &apimodels.EmbeddedContactPoint{ - UID: uid, - }, - OrgID: orgID, - }) + target := &apimodels.EmbeddedContactPoint{ + UID: uid, + } + err := ecp.provenanceStore.DeleteProvenance(ctx, target, orgID) if err != nil { return err } diff --git a/pkg/services/ngalert/provisioning/notification_policies.go b/pkg/services/ngalert/provisioning/notification_policies.go index 87582b012f8..f17e6a72e09 100644 --- a/pkg/services/ngalert/provisioning/notification_policies.go +++ b/pkg/services/ngalert/provisioning/notification_policies.go @@ -47,11 +47,7 @@ func (nps *NotificationPolicyService) GetPolicyTree(ctx context.Context, orgID i return definitions.Route{}, fmt.Errorf("no route present in current alertmanager config") } - adapter := ProvenanceOrgAdapter{ - Inner: cfg.AlertmanagerConfig.Route, - OrgID: orgID, - } - provenance, err := nps.provenanceStore.GetProvenance(ctx, adapter) + provenance, err := nps.provenanceStore.GetProvenance(ctx, cfg.AlertmanagerConfig.Route, orgID) if err != nil { return definitions.Route{}, err } @@ -95,11 +91,7 @@ func (nps *NotificationPolicyService) UpdatePolicyTree(ctx context.Context, orgI if err != nil { return err } - adapter := ProvenanceOrgAdapter{ - Inner: &tree, - OrgID: orgID, - } - err = nps.provenanceStore.SetProvenance(ctx, adapter, p) + err = nps.provenanceStore.SetProvenance(ctx, &tree, orgID, p) if err != nil { return err } diff --git a/pkg/services/ngalert/provisioning/persist.go b/pkg/services/ngalert/provisioning/persist.go index 0be6e093dd3..9368d79f0b7 100644 --- a/pkg/services/ngalert/provisioning/persist.go +++ b/pkg/services/ngalert/provisioning/persist.go @@ -14,30 +14,13 @@ type AMConfigStore interface { // ProvisioningStore is a store of provisioning data for arbitrary objects. type ProvisioningStore interface { - GetProvenance(ctx context.Context, o models.Provisionable) (models.Provenance, error) - GetProvenances(ctx context.Context, orgID int64, resourceType string) (map[string]models.Provenance, error) - SetProvenance(ctx context.Context, o models.Provisionable, p models.Provenance) error - DeleteProvenance(ctx context.Context, o models.Provisionable) error + GetProvenance(ctx context.Context, o models.Provisionable, org int64) (models.Provenance, error) + GetProvenances(ctx context.Context, org int64, resourceType string) (map[string]models.Provenance, error) + SetProvenance(ctx context.Context, o models.Provisionable, org int64, p models.Provenance) error + DeleteProvenance(ctx context.Context, o models.Provisionable, org int64) error } // TransactionManager represents the ability to issue and close transactions through contexts. type TransactionManager interface { InTransaction(ctx context.Context, work func(ctx context.Context) error) error } - -type ProvenanceOrgAdapter struct { - Inner models.ProvisionableInOrg - OrgID int64 -} - -func (a ProvenanceOrgAdapter) ResourceType() string { - return a.Inner.ResourceType() -} - -func (a ProvenanceOrgAdapter) ResourceID() string { - return a.Inner.ResourceID() -} - -func (a ProvenanceOrgAdapter) ResourceOrgID() int64 { - return a.OrgID -} diff --git a/pkg/services/ngalert/provisioning/testing.go b/pkg/services/ngalert/provisioning/testing.go index 4693460c1c8..ea8df1c666b 100644 --- a/pkg/services/ngalert/provisioning/testing.go +++ b/pkg/services/ngalert/provisioning/testing.go @@ -98,8 +98,8 @@ func NewFakeProvisioningStore() *fakeProvisioningStore { } } -func (f *fakeProvisioningStore) GetProvenance(ctx context.Context, o models.Provisionable) (models.Provenance, error) { - if val, ok := f.records[o.ResourceOrgID()]; ok { +func (f *fakeProvisioningStore) GetProvenance(ctx context.Context, o models.Provisionable, org int64) (models.Provenance, error) { + if val, ok := f.records[org]; ok { if prov, ok := val[o.ResourceID()+o.ResourceType()]; ok { return prov, nil } @@ -119,18 +119,17 @@ func (f *fakeProvisioningStore) GetProvenances(ctx context.Context, orgID int64, return results, nil } -func (f *fakeProvisioningStore) SetProvenance(ctx context.Context, o models.Provisionable, p models.Provenance) error { - orgID := o.ResourceOrgID() - if _, ok := f.records[orgID]; !ok { - f.records[orgID] = map[string]models.Provenance{} +func (f *fakeProvisioningStore) SetProvenance(ctx context.Context, o models.Provisionable, org int64, p models.Provenance) error { + if _, ok := f.records[org]; !ok { + f.records[org] = map[string]models.Provenance{} } - _ = f.DeleteProvenance(ctx, o) // delete old entries first - f.records[orgID][o.ResourceID()+o.ResourceType()] = p + _ = f.DeleteProvenance(ctx, o, org) // delete old entries first + f.records[org][o.ResourceID()+o.ResourceType()] = p return nil } -func (f *fakeProvisioningStore) DeleteProvenance(ctx context.Context, o models.Provisionable) error { - if val, ok := f.records[o.ResourceOrgID()]; ok { +func (f *fakeProvisioningStore) DeleteProvenance(ctx context.Context, o models.Provisionable, org int64) error { + if val, ok := f.records[org]; ok { delete(val, o.ResourceID()+o.ResourceType()) } return nil diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index a40f8fc0c82..5deb39d2555 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -76,7 +76,7 @@ func (st *Manager) Warm(ctx context.Context) { ruleCmd := ngModels.ListAlertRulesQuery{ OrgID: orgId, } - if err := st.ruleStore.GetOrgAlertRules(ctx, &ruleCmd); err != nil { + if err := st.ruleStore.ListAlertRules(ctx, &ruleCmd); err != nil { st.log.Error("unable to fetch previous state", "msg", err.Error()) } diff --git a/pkg/services/ngalert/store/alert_rule.go b/pkg/services/ngalert/store/alert_rule.go index 3d4822128a6..bc6a0f97c00 100644 --- a/pkg/services/ngalert/store/alert_rule.go +++ b/pkg/services/ngalert/store/alert_rule.go @@ -38,10 +38,9 @@ type RuleStore interface { DeleteAlertInstancesByRuleUID(ctx context.Context, orgID int64, ruleUID string) error GetAlertRuleByUID(ctx context.Context, query *ngmodels.GetAlertRuleByUIDQuery) error GetAlertRulesForScheduling(ctx context.Context, query *ngmodels.ListAlertRulesQuery) error - GetOrgAlertRules(ctx context.Context, query *ngmodels.ListAlertRulesQuery) error + ListAlertRules(ctx context.Context, query *ngmodels.ListAlertRulesQuery) error // GetRuleGroups returns the unique rule groups across all organizations. GetRuleGroups(ctx context.Context, query *ngmodels.ListRuleGroupsQuery) error - GetAlertRules(ctx context.Context, query *ngmodels.GetAlertRulesQuery) error GetUserVisibleNamespaces(context.Context, int64, *models.SignedInUser) (map[string]*models.Folder, error) GetNamespaceByTitle(context.Context, string, int64, *models.SignedInUser, bool) (*models.Folder, error) InsertAlertRules(ctx context.Context, rule []ngmodels.AlertRule) error @@ -228,33 +227,39 @@ func (st DBstore) UpdateAlertRules(ctx context.Context, rules []UpdateRule) erro } // GetOrgAlertRules is a handler for retrieving alert rules of specific organisation. -func (st DBstore) GetOrgAlertRules(ctx context.Context, query *ngmodels.ListAlertRulesQuery) error { +func (st DBstore) ListAlertRules(ctx context.Context, query *ngmodels.ListAlertRulesQuery) error { return st.SQLStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { - alertRules := make([]*ngmodels.AlertRule, 0) - q := "SELECT * FROM alert_rule WHERE org_id = ?" - params := []interface{}{query.OrgID} + q := sess.Table("alert_rule") - if len(query.NamespaceUIDs) > 0 { - placeholders := make([]string, 0, len(query.NamespaceUIDs)) - for _, folderUID := range query.NamespaceUIDs { - params = append(params, folderUID) - placeholders = append(placeholders, "?") - } - q = fmt.Sprintf("%s AND namespace_uid IN (%s)", q, strings.Join(placeholders, ",")) + if query.OrgID >= 0 { + q = q.Where("org_id = ?", query.OrgID) } if query.DashboardUID != "" { - params = append(params, query.DashboardUID) - q = fmt.Sprintf("%s AND dashboard_uid = ?", q) + q = q.Where("dashboard_uid = ?", query.DashboardUID) if query.PanelID != 0 { - params = append(params, query.PanelID) - q = fmt.Sprintf("%s AND panel_id = ?", q) + q = q.Where("panel_id = ?", query.PanelID) } } - q = fmt.Sprintf("%s ORDER BY id ASC", q) + if len(query.NamespaceUIDs) > 0 { + args := make([]interface{}, 0, len(query.NamespaceUIDs)) + in := make([]string, 0, len(query.NamespaceUIDs)) + for _, namespaceUID := range query.NamespaceUIDs { + args = append(args, namespaceUID) + in = append(in, "?") + } + q = q.Where(fmt.Sprintf("namespace_uid IN (%s)", strings.Join(in, ",")), args...) + } - if err := sess.SQL(q, params...).Find(&alertRules); err != nil { + if query.RuleGroup != "" { + q = q.Where("rule_group = ?", query.RuleGroup) + } + + q = q.OrderBy("id ASC") + + alertRules := make([]*ngmodels.AlertRule, 0) + if err := q.Find(&alertRules); err != nil { return err } @@ -274,30 +279,6 @@ func (st DBstore) GetRuleGroups(ctx context.Context, query *ngmodels.ListRuleGro }) } -// GetAlertRules is a handler for retrieving rule group alert rules of specific organisation. -func (st DBstore) GetAlertRules(ctx context.Context, query *ngmodels.GetAlertRulesQuery) error { - return st.SQLStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { - q := sess.Table("alert_rule").Where("org_id = ? AND namespace_uid = ?", query.OrgID, query.NamespaceUID) - if query.RuleGroup != nil { - q = q.Where("rule_group = ?", *query.RuleGroup) - } - if query.DashboardUID != "" { - q = q.Where("dashboard_uid = ?", query.DashboardUID) - if query.PanelID != 0 { - q = q.Where("panel_id = ?", query.PanelID) - } - } - - alertRules := make([]*ngmodels.AlertRule, 0) - if err := q.Find(&alertRules); err != nil { - return err - } - - query.Result = alertRules - return nil - }) -} - // GetNamespaces returns the folders that are visible to the user and have at least one alert in it func (st DBstore) GetUserVisibleNamespaces(ctx context.Context, orgID int64, user *models.SignedInUser) (map[string]*models.Folder, error) { namespaceMap := make(map[string]*models.Folder) diff --git a/pkg/services/ngalert/store/provisioning_store.go b/pkg/services/ngalert/store/provisioning_store.go index 6710b702a84..2e59d68de62 100644 --- a/pkg/services/ngalert/store/provisioning_store.go +++ b/pkg/services/ngalert/store/provisioning_store.go @@ -21,16 +21,15 @@ func (pr provenanceRecord) TableName() string { } // GetProvenance gets the provenance status for a provisionable object. -func (st DBstore) GetProvenance(ctx context.Context, o models.Provisionable) (models.Provenance, error) { +func (st DBstore) GetProvenance(ctx context.Context, o models.Provisionable, org int64) (models.Provenance, error) { recordType := o.ResourceType() recordKey := o.ResourceID() - orgID := o.ResourceOrgID() provenance := models.ProvenanceNone err := st.SQLStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { filter := "record_key = ? AND record_type = ? AND org_id = ?" var result models.Provenance - has, err := sess.Table(provenanceRecord{}).Where(filter, recordKey, recordType, orgID).Desc("id").Cols("provenance").Get(&result) + has, err := sess.Table(provenanceRecord{}).Where(filter, recordKey, recordType, org).Desc("id").Cols("provenance").Get(&result) if err != nil { return fmt.Errorf("failed to query for existing provenance status: %w", err) } @@ -46,11 +45,11 @@ func (st DBstore) GetProvenance(ctx context.Context, o models.Provisionable) (mo } // GetProvenance gets the provenance status for a provisionable object. -func (st DBstore) GetProvenances(ctx context.Context, orgID int64, resourceType string) (map[string]models.Provenance, error) { +func (st DBstore) GetProvenances(ctx context.Context, org int64, resourceType string) (map[string]models.Provenance, error) { resultMap := make(map[string]models.Provenance) err := st.SQLStore.WithDbSession(ctx, func(sess *sqlstore.DBSession) error { filter := "record_type = ? AND org_id = ?" - rawData, err := sess.Table(provenanceRecord{}).Where(filter, resourceType, orgID).Desc("id").Cols("record_key", "provenance").QueryString() + rawData, err := sess.Table(provenanceRecord{}).Where(filter, resourceType, org).Desc("id").Cols("record_key", "provenance").QueryString() if err != nil { return fmt.Errorf("failed to query for existing provenance status: %w", err) } @@ -63,17 +62,16 @@ func (st DBstore) GetProvenances(ctx context.Context, orgID int64, resourceType } // SetProvenance changes the provenance status for a provisionable object. -func (st DBstore) SetProvenance(ctx context.Context, o models.Provisionable, p models.Provenance) error { +func (st DBstore) SetProvenance(ctx context.Context, o models.Provisionable, org int64, p models.Provenance) error { recordType := o.ResourceType() recordKey := o.ResourceID() - orgID := o.ResourceOrgID() return st.SQLStore.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error { // TODO: Add a unit-of-work pattern, so updating objects + provenance will happen consistently with rollbacks across stores. // TODO: Need to make sure that writing a record where our concurrency key fails will also fail the whole transaction. That way, this gets rolled back too. can't just check that 0 updates happened inmemory. Check with jp. If not possible, we need our own concurrency key. // TODO: Clean up stale provenance records periodically. filter := "record_key = ? AND record_type = ? AND org_id = ?" - _, err := sess.Table(provenanceRecord{}).Where(filter, recordKey, recordType, orgID).Delete(provenanceRecord{}) + _, err := sess.Table(provenanceRecord{}).Where(filter, recordKey, recordType, org).Delete(provenanceRecord{}) if err != nil { return fmt.Errorf("failed to delete pre-existing provisioning status: %w", err) @@ -83,7 +81,7 @@ func (st DBstore) SetProvenance(ctx context.Context, o models.Provisionable, p m RecordKey: recordKey, RecordType: recordType, Provenance: p, - OrgID: orgID, + OrgID: org, } if _, err := sess.Insert(record); err != nil { @@ -95,12 +93,12 @@ func (st DBstore) SetProvenance(ctx context.Context, o models.Provisionable, p m } // DeleteProvenance deletes the provenance record from the table -func (st DBstore) DeleteProvenance(ctx context.Context, o models.Provisionable) error { +func (st DBstore) DeleteProvenance(ctx context.Context, o models.Provisionable, org int64) error { return st.SQLStore.WithTransactionalDbSession(ctx, func(sess *sqlstore.DBSession) error { _, err := sess.Delete(provenanceRecord{ RecordKey: o.ResourceID(), RecordType: o.ResourceType(), - OrgID: o.ResourceOrgID(), + OrgID: org, }) return err }) diff --git a/pkg/services/ngalert/store/provisioning_store_test.go b/pkg/services/ngalert/store/provisioning_store_test.go index 886ca7f6884..f8c8cac2cc6 100644 --- a/pkg/services/ngalert/store/provisioning_store_test.go +++ b/pkg/services/ngalert/store/provisioning_store_test.go @@ -22,7 +22,7 @@ func TestProvisioningStore(t *testing.T) { UID: "asdf", } - provenance, err := store.GetProvenance(context.Background(), &rule) + provenance, err := store.GetProvenance(context.Background(), &rule, 1) require.NoError(t, err) require.Equal(t, models.ProvenanceNone, provenance) @@ -32,10 +32,10 @@ func TestProvisioningStore(t *testing.T) { rule := models.AlertRule{ UID: "123", } - err := store.SetProvenance(context.Background(), &rule, models.ProvenanceFile) + err := store.SetProvenance(context.Background(), &rule, 1, models.ProvenanceFile) require.NoError(t, err) - p, err := store.GetProvenance(context.Background(), &rule) + p, err := store.GetProvenance(context.Background(), &rule, 1) require.NoError(t, err) require.Equal(t, models.ProvenanceFile, p) @@ -43,17 +43,15 @@ func TestProvisioningStore(t *testing.T) { t.Run("Store does not get provenance of record with different org ID", func(t *testing.T) { ruleOrg2 := models.AlertRule{ - UID: "456", - OrgID: 2, + UID: "456", } ruleOrg3 := models.AlertRule{ - UID: "456", - OrgID: 3, + UID: "456", } - err := store.SetProvenance(context.Background(), &ruleOrg2, models.ProvenanceFile) + err := store.SetProvenance(context.Background(), &ruleOrg2, 2, models.ProvenanceFile) require.NoError(t, err) - p, err := store.GetProvenance(context.Background(), &ruleOrg3) + p, err := store.GetProvenance(context.Background(), &ruleOrg3, 3) require.NoError(t, err) require.Equal(t, models.ProvenanceNone, p) @@ -68,42 +66,42 @@ func TestProvisioningStore(t *testing.T) { UID: "789", OrgID: 3, } - err := store.SetProvenance(context.Background(), &ruleOrg2, models.ProvenanceFile) + err := store.SetProvenance(context.Background(), &ruleOrg2, 2, models.ProvenanceFile) require.NoError(t, err) - err = store.SetProvenance(context.Background(), &ruleOrg3, models.ProvenanceFile) + err = store.SetProvenance(context.Background(), &ruleOrg3, 3, models.ProvenanceFile) require.NoError(t, err) - err = store.SetProvenance(context.Background(), &ruleOrg2, models.ProvenanceAPI) + err = store.SetProvenance(context.Background(), &ruleOrg2, 2, models.ProvenanceAPI) require.NoError(t, err) - p, err := store.GetProvenance(context.Background(), &ruleOrg2) + p, err := store.GetProvenance(context.Background(), &ruleOrg2, 2) require.NoError(t, err) require.Equal(t, models.ProvenanceAPI, p) - p, err = store.GetProvenance(context.Background(), &ruleOrg3) + p, err = store.GetProvenance(context.Background(), &ruleOrg3, 3) require.NoError(t, err) require.Equal(t, models.ProvenanceFile, p) }) t.Run("Store should return all provenances by type", func(t *testing.T) { const orgID = 123 - ruleOrg1 := models.AlertRule{ + rule1 := models.AlertRule{ UID: "789", OrgID: orgID, } - ruleOrg2 := models.AlertRule{ + rule2 := models.AlertRule{ UID: "790", OrgID: orgID, } - err := store.SetProvenance(context.Background(), &ruleOrg1, models.ProvenanceFile) + err := store.SetProvenance(context.Background(), &rule1, orgID, models.ProvenanceFile) require.NoError(t, err) - err = store.SetProvenance(context.Background(), &ruleOrg2, models.ProvenanceAPI) + err = store.SetProvenance(context.Background(), &rule2, orgID, models.ProvenanceAPI) require.NoError(t, err) - p, err := store.GetProvenances(context.Background(), orgID, ruleOrg1.ResourceType()) + p, err := store.GetProvenances(context.Background(), orgID, rule1.ResourceType()) require.NoError(t, err) require.Len(t, p, 2) - require.Equal(t, models.ProvenanceFile, p[ruleOrg1.UID]) - require.Equal(t, models.ProvenanceAPI, p[ruleOrg2.UID]) + require.Equal(t, models.ProvenanceFile, p[rule1.UID]) + require.Equal(t, models.ProvenanceAPI, p[rule2.UID]) }) t.Run("Store should delete provenance correctly", func(t *testing.T) { @@ -112,16 +110,16 @@ func TestProvisioningStore(t *testing.T) { UID: "7834539", OrgID: orgID, } - err := store.SetProvenance(context.Background(), &ruleOrg, models.ProvenanceFile) + err := store.SetProvenance(context.Background(), &ruleOrg, orgID, models.ProvenanceFile) require.NoError(t, err) - p, err := store.GetProvenance(context.Background(), &ruleOrg) + p, err := store.GetProvenance(context.Background(), &ruleOrg, orgID) require.NoError(t, err) require.Equal(t, models.ProvenanceFile, p) - err = store.DeleteProvenance(context.Background(), &ruleOrg) + err = store.DeleteProvenance(context.Background(), &ruleOrg, orgID) require.NoError(t, err) - p, err = store.GetProvenance(context.Background(), &ruleOrg) + p, err = store.GetProvenance(context.Background(), &ruleOrg, orgID) require.NoError(t, err) require.Equal(t, models.ProvenanceNone, p) }) diff --git a/pkg/services/ngalert/store/testing.go b/pkg/services/ngalert/store/testing.go index 4a340037ced..1a3af60bc30 100644 --- a/pkg/services/ngalert/store/testing.go +++ b/pkg/services/ngalert/store/testing.go @@ -166,16 +166,58 @@ func (f *FakeRuleStore) GetAlertRulesForScheduling(_ context.Context, q *models. return nil } -func (f *FakeRuleStore) GetOrgAlertRules(_ context.Context, q *models.ListAlertRulesQuery) error { +func (f *FakeRuleStore) ListAlertRules(_ context.Context, q *models.ListAlertRulesQuery) error { f.mtx.Lock() defer f.mtx.Unlock() f.RecordedOps = append(f.RecordedOps, *q) - rules, ok := f.Rules[q.OrgID] - if !ok { - return nil + if err := f.Hook(*q); err != nil { + return err } - q.Result = rules + + hasDashboard := func(r *models.AlertRule, dashboardUID string, panelID int64) bool { + if dashboardUID != "" { + if r.DashboardUID == nil || *r.DashboardUID != dashboardUID { + return false + } + if panelID > 0 { + if r.PanelID == nil || *r.PanelID != panelID { + return false + } + } + } + return true + } + + hasNamespace := func(r *models.AlertRule, namespaceUIDs []string) bool { + if len(namespaceUIDs) > 0 { + var ok bool + for _, uid := range q.NamespaceUIDs { + if uid == r.NamespaceUID { + ok = true + break + } + } + if !ok { + return false + } + } + return true + } + + for _, r := range f.Rules[q.OrgID] { + if !hasDashboard(r, q.DashboardUID, q.PanelID) { + continue + } + if !hasNamespace(r, q.NamespaceUIDs) { + continue + } + if q.RuleGroup != "" && r.RuleGroup != q.RuleGroup { + continue + } + q.Result = append(q.Result, r) + } + return nil } @@ -198,30 +240,6 @@ func (f *FakeRuleStore) GetRuleGroups(_ context.Context, q *models.ListRuleGroup return nil } -func (f *FakeRuleStore) GetAlertRules(_ context.Context, q *models.GetAlertRulesQuery) error { - f.mtx.Lock() - defer f.mtx.Unlock() - f.RecordedOps = append(f.RecordedOps, *q) - if err := f.Hook(*q); err != nil { - return err - } - rules, ok := f.Rules[q.OrgID] - if !ok { - return nil - } - var result []*models.AlertRule - for _, rule := range rules { - if q.NamespaceUID != rule.NamespaceUID { - continue - } - if q.RuleGroup != nil && *q.RuleGroup != rule.RuleGroup { - continue - } - result = append(result, rule) - } - q.Result = result - return nil -} func (f *FakeRuleStore) GetUserVisibleNamespaces(_ context.Context, orgID int64, _ *models2.SignedInUser) (map[string]*models2.Folder, error) { f.mtx.Lock() defer f.mtx.Unlock() @@ -238,6 +256,7 @@ func (f *FakeRuleStore) GetUserVisibleNamespaces(_ context.Context, orgID int64, } return namespacesMap, nil } + func (f *FakeRuleStore) GetNamespaceByTitle(_ context.Context, title string, orgID int64, _ *models2.SignedInUser, _ bool) (*models2.Folder, error) { folders := f.Folders[orgID] for _, folder := range folders { diff --git a/pkg/services/ngalert/tests/util.go b/pkg/services/ngalert/tests/util.go index 19de8f046a2..69f79d427ae 100644 --- a/pkg/services/ngalert/tests/util.go +++ b/pkg/services/ngalert/tests/util.go @@ -113,12 +113,12 @@ func CreateTestAlertRuleWithLabels(t *testing.T, ctx context.Context, dbstore *s }) require.NoError(t, err) - q := models.GetAlertRulesQuery{ - OrgID: orgID, - NamespaceUID: "namespace", - RuleGroup: &ruleGroup, + q := models.ListAlertRulesQuery{ + OrgID: orgID, + NamespaceUIDs: []string{"namespace"}, + RuleGroup: ruleGroup, } - err = dbstore.GetAlertRules(ctx, &q) + err = dbstore.ListAlertRules(ctx, &q) require.NoError(t, err) require.NotEmpty(t, q.Result) diff --git a/pkg/services/preference/prefimpl/pref.go b/pkg/services/preference/prefimpl/pref.go index bf78d8d34a4..d1ebdbe9c00 100644 --- a/pkg/services/preference/prefimpl/pref.go +++ b/pkg/services/preference/prefimpl/pref.go @@ -30,6 +30,7 @@ func (s *Service) GetWithDefaults(ctx context.Context, query *pref.GetPreference OrgID: query.OrgID, UserID: query.UserID, } + prefs, err := s.store.List(ctx, listQuery) if err != nil { return nil, err @@ -99,12 +100,14 @@ func (s *Service) Save(ctx context.Context, cmd *pref.SavePreferenceCommand) err } return err } + preference.Timezone = cmd.Timezone preference.WeekStart = cmd.WeekStart preference.Theme = cmd.Theme preference.Updated = time.Now() preference.Version += 1 preference.JSONData = &pref.PreferenceJSONData{} + preference.HomeDashboardID = cmd.HomeDashboardID if cmd.Navbar != nil { preference.JSONData.Navbar = *cmd.Navbar diff --git a/pkg/services/preference/prefimpl/pref_test.go b/pkg/services/preference/prefimpl/pref_test.go index 3c083c1cb68..2e6158ae56b 100644 --- a/pkg/services/preference/prefimpl/pref_test.go +++ b/pkg/services/preference/prefimpl/pref_test.go @@ -107,7 +107,7 @@ func TestPreferencesService(t *testing.T) { Theme: "light", Timezone: "UTC", } - query := &pref.GetPreferenceWithDefaultsQuery{} + query := &pref.GetPreferenceWithDefaultsQuery{OrgID: 1} preference, err := prefService.GetWithDefaults(context.Background(), query) require.NoError(t, err) expected := &pref.Preference{ diff --git a/pkg/services/provisioning/dashboards/config_reader_test.go b/pkg/services/provisioning/dashboards/config_reader_test.go index 7d04ae7d2d6..899654eda0d 100644 --- a/pkg/services/provisioning/dashboards/config_reader_test.go +++ b/pkg/services/provisioning/dashboards/config_reader_test.go @@ -36,7 +36,7 @@ func TestDashboardsAsConfig(t *testing.T) { for i := 1; i <= 2; i++ { orgCommand := models.CreateOrgCommand{Name: fmt.Sprintf("Main Org. %v", i)} - err := sqlstore.CreateOrg(context.Background(), &orgCommand) + err := store.CreateOrg(context.Background(), &orgCommand) require.NoError(t, err) } diff --git a/pkg/services/provisioning/notifiers/config_reader_test.go b/pkg/services/provisioning/notifiers/config_reader_test.go index c8cae8b6886..8c78a108a37 100644 --- a/pkg/services/provisioning/notifiers/config_reader_test.go +++ b/pkg/services/provisioning/notifiers/config_reader_test.go @@ -39,7 +39,7 @@ func TestNotificationAsConfig(t *testing.T) { for i := 1; i < 5; i++ { orgCommand := models.CreateOrgCommand{Name: fmt.Sprintf("Main Org. %v", i)} - err := sqlstore.CreateOrg(context.Background(), &orgCommand) + err := sqlStore.CreateOrg(context.Background(), &orgCommand) require.NoError(t, err) } diff --git a/pkg/services/query/query.go b/pkg/services/query/query.go index 95d3e7437c3..b449648ca22 100644 --- a/pkg/services/query/query.go +++ b/pkg/services/query/query.go @@ -15,7 +15,6 @@ import ( "github.com/grafana/grafana/pkg/plugins/adapters" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/oauthtoken" - "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/grafanads" "github.com/grafana/grafana/pkg/tsdb/legacydata" @@ -33,7 +32,7 @@ func ProvideService( dataSourceCache datasources.CacheService, expressionService *expr.Service, pluginRequestValidator models.PluginRequestValidator, - SecretsService secrets.Service, + dataSourceService datasources.DataSourceService, pluginClient plugins.Client, oAuthTokenService oauthtoken.OAuthTokenService, ) *Service { @@ -42,7 +41,7 @@ func ProvideService( dataSourceCache: dataSourceCache, expressionService: expressionService, pluginRequestValidator: pluginRequestValidator, - secretsService: SecretsService, + dataSourceService: dataSourceService, pluginClient: pluginClient, oAuthTokenService: oAuthTokenService, log: log.New("query_data"), @@ -56,7 +55,7 @@ type Service struct { dataSourceCache datasources.CacheService expressionService *expr.Service pluginRequestValidator models.PluginRequestValidator - secretsService secrets.Service + dataSourceService datasources.DataSourceService pluginClient plugins.Client oAuthTokenService oauthtoken.OAuthTokenService log log.Logger @@ -291,9 +290,9 @@ func (s *Service) getDataSourceFromQuery(ctx context.Context, user *models.Signe return nil, NewErrBadQuery("missing data source ID/UID") } -func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(map[string][]byte) map[string]string { - return func(m map[string][]byte) map[string]string { - decryptedJsonData, err := s.secretsService.DecryptJsonData(ctx, m) +func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string { + return func(ds *models.DataSource) map[string]string { + decryptedJsonData, err := s.dataSourceService.DecryptedValues(ctx, ds) if err != nil { s.log.Error("Failed to decrypt secure json data", "error", err) } diff --git a/pkg/services/query/query_test.go b/pkg/services/query/query_test.go index 3b0193302f6..b5e05ed5e35 100644 --- a/pkg/services/query/query_test.go +++ b/pkg/services/query/query_test.go @@ -2,6 +2,7 @@ package query_test import ( "context" + "encoding/json" "net/http" "testing" @@ -12,18 +13,29 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" + datasources "github.com/grafana/grafana/pkg/services/datasources/service" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/query" - "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/secrets/fakes" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" + secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" + "github.com/stretchr/testify/require" ) func TestQueryData(t *testing.T) { t.Run("it attaches custom headers to the request", func(t *testing.T) { - tc := setup() + tc := setup(t) tc.dataSourceCache.ds.JsonData = simplejson.NewFromAny(map[string]interface{}{"httpHeaderName1": "foo", "httpHeaderName2": "bar"}) - tc.secretService.decryptedJson = map[string]string{"httpHeaderValue1": "test-header", "httpHeaderValue2": "test-header2"} - _, err := tc.queryService.QueryData(context.Background(), nil, true, metricRequest(), false) + secureJsonData, err := json.Marshal(map[string]string{"httpHeaderValue1": "test-header", "httpHeaderValue2": "test-header2"}) + require.NoError(t, err) + + err = tc.secretStore.Set(context.Background(), tc.dataSourceCache.ds.OrgId, tc.dataSourceCache.ds.Name, "datasource", string(secureJsonData)) + require.NoError(t, err) + + _, err = tc.queryService.QueryData(context.Background(), nil, true, metricRequest(), false) require.Nil(t, err) require.Equal(t, map[string]string{"foo": "test-header", "bar": "test-header2"}, tc.pluginContext.req.Headers) @@ -36,7 +48,7 @@ func TestQueryData(t *testing.T) { } token = token.WithExtra(map[string]interface{}{"id_token": "id-token"}) - tc := setup() + tc := setup(t) tc.oauthTokenService.passThruEnabled = true tc.oauthTokenService.token = token @@ -51,26 +63,29 @@ func TestQueryData(t *testing.T) { }) } -func setup() *testContext { +func setup(t *testing.T) *testContext { pc := &fakePluginClient{} - sc := &fakeSecretsService{} dc := &fakeDataSourceCache{ds: &models.DataSource{}} tc := &fakeOAuthTokenService{} rv := &fakePluginRequestValidator{} + ss := kvstore.SetupTestService(t) + ssvc := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) + ds := datasources.ProvideService(nil, ssvc, ss, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + return &testContext{ pluginContext: pc, - secretService: sc, + secretStore: ss, dataSourceCache: dc, oauthTokenService: tc, pluginRequestValidator: rv, - queryService: query.ProvideService(nil, dc, nil, rv, sc, pc, tc), + queryService: query.ProvideService(nil, dc, nil, rv, ds, pc, tc), } } type testContext struct { pluginContext *fakePluginClient - secretService *fakeSecretsService + secretStore kvstore.SecretsKVStore dataSourceCache *fakeDataSourceCache oauthTokenService *fakeOAuthTokenService pluginRequestValidator *fakePluginRequestValidator @@ -108,16 +123,6 @@ func (ts *fakeOAuthTokenService) IsOAuthPassThruEnabled(*models.DataSource) bool return ts.passThruEnabled } -type fakeSecretsService struct { - secrets.Service - - decryptedJson map[string]string -} - -func (s *fakeSecretsService) DecryptJsonData(ctx context.Context, sjd map[string][]byte) (map[string]string, error) { - return s.decryptedJson, nil -} - type fakeDataSourceCache struct { ds *models.DataSource } diff --git a/pkg/services/secrets/kvstore/helpers.go b/pkg/services/secrets/kvstore/helpers.go new file mode 100644 index 00000000000..c00923ca8ea --- /dev/null +++ b/pkg/services/secrets/kvstore/helpers.go @@ -0,0 +1,29 @@ +package kvstore + +import ( + "testing" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/secrets/database" + "github.com/grafana/grafana/pkg/services/secrets/manager" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func SetupTestService(t *testing.T) SecretsKVStore { + t.Helper() + + sqlStore := sqlstore.InitTestDB(t) + store := database.ProvideSecretsStore(sqlstore.InitTestDB(t)) + secretsService := manager.SetupTestService(t, store) + + kv := &secretsKVStoreSQL{ + sqlStore: sqlStore, + log: log.New("secrets.kvstore"), + secretsService: secretsService, + decryptionCache: decryptionCache{ + cache: make(map[int64]cachedDecrypted), + }, + } + + return kv +} diff --git a/pkg/services/secrets/kvstore/kvstore.go b/pkg/services/secrets/kvstore/kvstore.go new file mode 100644 index 00000000000..b438aec3c41 --- /dev/null +++ b/pkg/services/secrets/kvstore/kvstore.go @@ -0,0 +1,77 @@ +package kvstore + +import ( + "context" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +const ( + // Wildcard to query all organizations + AllOrganizations = -1 +) + +func ProvideService(sqlStore sqlstore.Store, secretsService secrets.Service) SecretsKVStore { + return &secretsKVStoreSQL{ + sqlStore: sqlStore, + secretsService: secretsService, + log: log.New("secrets.kvstore"), + decryptionCache: decryptionCache{ + cache: make(map[int64]cachedDecrypted), + }, + } +} + +// SecretsKVStore is an interface for k/v store. +type SecretsKVStore interface { + Get(ctx context.Context, orgId int64, namespace string, typ string) (string, bool, error) + Set(ctx context.Context, orgId int64, namespace string, typ string, value string) error + Del(ctx context.Context, orgId int64, namespace string, typ string) error + Keys(ctx context.Context, orgId int64, namespace string, typ string) ([]Key, error) + Rename(ctx context.Context, orgId int64, namespace string, typ string, newNamespace string) error +} + +// WithType returns a kvstore wrapper with fixed orgId and type. +func With(kv SecretsKVStore, orgId int64, namespace string, typ string) *FixedKVStore { + return &FixedKVStore{ + kvStore: kv, + OrgId: orgId, + Namespace: namespace, + Type: typ, + } +} + +// FixedKVStore is a SecretsKVStore wrapper with fixed orgId, namespace and type. +type FixedKVStore struct { + kvStore SecretsKVStore + OrgId int64 + Namespace string + Type string +} + +func (kv *FixedKVStore) Get(ctx context.Context) (string, bool, error) { + return kv.kvStore.Get(ctx, kv.OrgId, kv.Namespace, kv.Type) +} + +func (kv *FixedKVStore) Set(ctx context.Context, value string) error { + return kv.kvStore.Set(ctx, kv.OrgId, kv.Namespace, kv.Type, value) +} + +func (kv *FixedKVStore) Del(ctx context.Context) error { + return kv.kvStore.Del(ctx, kv.OrgId, kv.Namespace, kv.Type) +} + +func (kv *FixedKVStore) Keys(ctx context.Context) ([]Key, error) { + return kv.kvStore.Keys(ctx, kv.OrgId, kv.Namespace, kv.Type) +} + +func (kv *FixedKVStore) Rename(ctx context.Context, newNamespace string) error { + err := kv.kvStore.Rename(ctx, kv.OrgId, kv.Namespace, kv.Type, newNamespace) + if err != nil { + return err + } + kv.Namespace = newNamespace + return nil +} diff --git a/pkg/services/secrets/kvstore/kvstore_test.go b/pkg/services/secrets/kvstore/kvstore_test.go new file mode 100644 index 00000000000..9b6bff4a371 --- /dev/null +++ b/pkg/services/secrets/kvstore/kvstore_test.go @@ -0,0 +1,226 @@ +package kvstore + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type TestCase struct { + OrgId int64 + Namespace string + Type string + Revision int64 +} + +func (t *TestCase) Value() string { + return fmt.Sprintf("%d:%s:%s:%d", t.OrgId, t.Namespace, t.Type, t.Revision) +} + +func TestKVStore(t *testing.T) { + kv := SetupTestService(t) + + ctx := context.Background() + + testCases := []*TestCase{ + { + OrgId: 0, + Namespace: "namespace1", + Type: "testing1", + }, + { + OrgId: 0, + Namespace: "namespace2", + Type: "testing2", + }, + { + OrgId: 1, + Namespace: "namespace1", + Type: "testing1", + }, + { + OrgId: 1, + Namespace: "namespace3", + Type: "testing3", + }, + } + + for _, tc := range testCases { + err := kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value()) + require.NoError(t, err) + } + + t.Run("get existing keys", func(t *testing.T) { + for _, tc := range testCases { + value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, tc.Value(), value) + } + }) + + t.Run("get nonexistent keys", func(t *testing.T) { + tcs := []*TestCase{ + { + OrgId: 0, + Namespace: "namespace3", + Type: "testing3", + }, + { + OrgId: 1, + Namespace: "namespace2", + Type: "testing2", + }, + { + OrgId: 2, + Namespace: "namespace1", + Type: "testing1", + }, + } + + for _, tc := range tcs { + value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.Nil(t, err) + require.False(t, ok) + require.Equal(t, "", value) + } + }) + + t.Run("modify existing key", func(t *testing.T) { + tc := testCases[0] + + value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, tc.Value(), value) + + tc.Revision += 1 + + err = kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value()) + require.NoError(t, err) + + value, ok, err = kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, tc.Value(), value) + }) + + t.Run("use fixed client", func(t *testing.T) { + tc := testCases[0] + + client := With(kv, tc.OrgId, tc.Namespace, tc.Type) + fmt.Println(client.Namespace, client.OrgId, client.Type) + + value, ok, err := client.Get(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, tc.Value(), value) + + tc.Revision += 1 + + err = client.Set(ctx, tc.Value()) + require.NoError(t, err) + + value, ok, err = client.Get(ctx) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, tc.Value(), value) + }) + + t.Run("deleting keys", func(t *testing.T) { + var stillHasKeys bool + for _, tc := range testCases { + if _, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type); err == nil && ok { + stillHasKeys = true + break + } + } + require.True(t, stillHasKeys, + "we are going to test key deletion, but there are no keys to delete in the database") + for _, tc := range testCases { + err := kv.Del(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.NoError(t, err) + } + for _, tc := range testCases { + _, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.NoError(t, err) + require.False(t, ok, "all keys should be deleted at this point") + } + }) + + t.Run("listing existing keys", func(t *testing.T) { + kv := SetupTestService(t) + + ctx := context.Background() + + namespace, typ := "listtest", "listtest" + + testCases := []*TestCase{ + { + OrgId: 1, + Type: typ, + Namespace: namespace, + }, + { + OrgId: 2, + Type: typ, + Namespace: namespace, + }, + { + OrgId: 3, + Type: typ, + Namespace: namespace, + }, + { + OrgId: 4, + Type: typ, + Namespace: namespace, + }, + { + OrgId: 1, + Type: typ, + Namespace: "other_key", + }, + { + OrgId: 4, + Type: typ, + Namespace: "another_one", + }, + } + + for _, tc := range testCases { + err := kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value()) + require.NoError(t, err) + } + + keys, err := kv.Keys(ctx, AllOrganizations, namespace, typ) + + require.NoError(t, err) + require.Len(t, keys, 4) + + found := 0 + + for _, key := range keys { + for _, tc := range testCases { + if key.OrgId == tc.OrgId && key.Namespace == tc.Namespace && key.Type == tc.Type { + found++ + break + } + } + } + + require.Equal(t, 4, found, "querying for all orgs should return 4 records") + + keys, err = kv.Keys(ctx, 1, namespace, typ) + + require.NoError(t, err) + require.Len(t, keys, 1, "querying for a specific org should return 1 record") + + keys, err = kv.Keys(ctx, AllOrganizations, "not_existing_namespace", "not_existing_type") + require.NoError(t, err, "querying a not existing namespace should not throw an error") + require.Len(t, keys, 0, "querying a not existing namespace should return an empty slice") + }) +} diff --git a/pkg/services/secrets/kvstore/model.go b/pkg/services/secrets/kvstore/model.go new file mode 100644 index 00000000000..6643de3a1a1 --- /dev/null +++ b/pkg/services/secrets/kvstore/model.go @@ -0,0 +1,31 @@ +package kvstore + +import ( + "time" +) + +// Item stored in k/v store. +type Item struct { + Id int64 + OrgId *int64 + Namespace *string + Type *string + Value string + + Created time.Time + Updated time.Time +} + +func (i *Item) TableName() string { + return "secrets" +} + +type Key struct { + OrgId int64 + Namespace string + Type string +} + +func (i *Key) TableName() string { + return "secrets" +} diff --git a/pkg/services/secrets/kvstore/sql.go b/pkg/services/secrets/kvstore/sql.go new file mode 100644 index 00000000000..08b1c9fe257 --- /dev/null +++ b/pkg/services/secrets/kvstore/sql.go @@ -0,0 +1,220 @@ +package kvstore + +import ( + "context" + "encoding/base64" + "sync" + "time" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +// secretsKVStoreSQL provides a key/value store backed by the Grafana database +type secretsKVStoreSQL struct { + log log.Logger + sqlStore sqlstore.Store + secretsService secrets.Service + decryptionCache decryptionCache +} + +type decryptionCache struct { + cache map[int64]cachedDecrypted + sync.Mutex +} + +type cachedDecrypted struct { + updated time.Time + value string +} + +var b64 = base64.RawStdEncoding + +// Get an item from the store +func (kv *secretsKVStoreSQL) Get(ctx context.Context, orgId int64, namespace string, typ string) (string, bool, error) { + item := Item{ + OrgId: &orgId, + Namespace: &namespace, + Type: &typ, + } + var isFound bool + var decryptedValue []byte + + err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error { + has, err := dbSession.Get(&item) + if err != nil { + kv.log.Debug("error getting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return err + } + if !has { + kv.log.Debug("secret value not found", "orgId", orgId, "type", typ, "namespace", namespace) + return nil + } + isFound = true + kv.log.Debug("got secret value", "orgId", orgId, "type", typ, "namespace", namespace) + return nil + }) + + if err == nil && isFound { + kv.decryptionCache.Lock() + defer kv.decryptionCache.Unlock() + + if cache, present := kv.decryptionCache.cache[item.Id]; present && item.Updated.Equal(cache.updated) { + return cache.value, isFound, err + } + + decodedValue, err := b64.DecodeString(item.Value) + if err != nil { + kv.log.Debug("error decoding secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return string(decryptedValue), isFound, err + } + + decryptedValue, err = kv.secretsService.Decrypt(ctx, decodedValue) + if err != nil { + kv.log.Debug("error decrypting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return string(decryptedValue), isFound, err + } + + kv.decryptionCache.cache[item.Id] = cachedDecrypted{ + updated: item.Updated, + value: string(decryptedValue), + } + } + + return string(decryptedValue), isFound, err +} + +// Set an item in the store +func (kv *secretsKVStoreSQL) Set(ctx context.Context, orgId int64, namespace string, typ string, value string) error { + encryptedValue, err := kv.secretsService.Encrypt(ctx, []byte(value), secrets.WithoutScope()) + if err != nil { + kv.log.Debug("error encrypting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return err + } + encodedValue := b64.EncodeToString(encryptedValue) + return kv.sqlStore.WithTransactionalDbSession(ctx, func(dbSession *sqlstore.DBSession) error { + item := Item{ + OrgId: &orgId, + Namespace: &namespace, + Type: &typ, + } + + has, err := dbSession.Get(&item) + if err != nil { + kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return err + } + + if has && item.Value == encodedValue { + kv.log.Debug("secret value not changed", "orgId", orgId, "type", typ, "namespace", namespace) + return nil + } + + item.Value = encodedValue + item.Updated = time.Now() + + if has { + // if item already exists we update it + _, err = dbSession.ID(item.Id).Update(&item) + if err != nil { + kv.log.Debug("error updating secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + } else { + kv.decryptionCache.cache[item.Id] = cachedDecrypted{ + updated: item.Updated, + value: value, + } + kv.log.Debug("secret value updated", "orgId", orgId, "type", typ, "namespace", namespace) + } + return err + } + + // if item doesn't exist we create it + item.Created = item.Updated + _, err = dbSession.Insert(&item) + if err != nil { + kv.log.Debug("error inserting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + } else { + kv.log.Debug("secret value inserted", "orgId", orgId, "type", typ, "namespace", namespace) + } + return err + }) +} + +// Del deletes an item from the store. +func (kv *secretsKVStoreSQL) Del(ctx context.Context, orgId int64, namespace string, typ string) error { + err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error { + item := Item{ + OrgId: &orgId, + Namespace: &namespace, + Type: &typ, + } + + has, err := dbSession.Get(&item) + if err != nil { + kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return err + } + + if has { + // if item exists we delete it + _, err = dbSession.ID(item.Id).Delete(&item) + if err != nil { + kv.log.Debug("error deleting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + } else { + delete(kv.decryptionCache.cache, item.Id) + kv.log.Debug("secret value deleted", "orgId", orgId, "type", typ, "namespace", namespace) + } + return err + } + return nil + }) + return err +} + +// Keys get all keys for a given namespace. To query for all +// organizations the constant 'kvstore.AllOrganizations' can be passed as orgId. +func (kv *secretsKVStoreSQL) Keys(ctx context.Context, orgId int64, namespace string, typ string) ([]Key, error) { + var keys []Key + err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error { + query := dbSession.Where("namespace = ?", namespace).And("type = ?", typ) + if orgId != AllOrganizations { + query.And("org_id = ?", orgId) + } + return query.Find(&keys) + }) + return keys, err +} + +// Rename an item in the store +func (kv *secretsKVStoreSQL) Rename(ctx context.Context, orgId int64, namespace string, typ string, newNamespace string) error { + return kv.sqlStore.WithTransactionalDbSession(ctx, func(dbSession *sqlstore.DBSession) error { + item := Item{ + OrgId: &orgId, + Namespace: &namespace, + Type: &typ, + } + + has, err := dbSession.Get(&item) + if err != nil { + kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return err + } + + item.Namespace = &newNamespace + item.Updated = time.Now() + + if has { + // if item already exists we update it + _, err = dbSession.ID(item.Id).Update(&item) + if err != nil { + kv.log.Debug("error updating secret namespace", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + } else { + kv.log.Debug("secret namespace updated", "orgId", orgId, "type", typ, "namespace", namespace) + } + return err + } + + return err + }) +} diff --git a/pkg/services/serviceaccounts/api/api_test.go b/pkg/services/serviceaccounts/api/api_test.go index c017b3dee0d..b60e5361da7 100644 --- a/pkg/services/serviceaccounts/api/api_test.go +++ b/pkg/services/serviceaccounts/api/api_test.go @@ -42,7 +42,7 @@ func TestServiceAccountsAPI_CreateServiceAccount(t *testing.T) { }() orgCmd := &models.CreateOrgCommand{Name: "Some Test Org"} - err := sqlstore.CreateOrg(context.Background(), orgCmd) + err := store.CreateOrg(context.Background(), orgCmd) require.Nil(t, err) type testCreateSATestCase struct { diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index ee780c53a44..24aa1a6aacd 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -479,7 +479,7 @@ func (ss *SQLStore) UpdateAlertNotificationWithUid(ctx context.Context, cmd *mod } func (ss *SQLStore) SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *models.SetAlertNotificationStateToCompleteCommand) error { - return inTransactionCtx(ctx, func(sess *DBSession) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { version := cmd.Version var current models.AlertNotificationState if _, err := sess.ID(cmd.Id).Get(¤t); err != nil { @@ -544,7 +544,7 @@ func (ss *SQLStore) SetAlertNotificationStateToPendingCommand(ctx context.Contex } func (ss *SQLStore) GetOrCreateAlertNotificationState(ctx context.Context, cmd *models.GetOrCreateNotificationStateQuery) error { - return inTransactionCtx(ctx, func(sess *DBSession) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { nj := &models.AlertNotificationState{} exist, err := getAlertNotificationState(ctx, sess, cmd, nj) diff --git a/pkg/services/sqlstore/annotation.go b/pkg/services/sqlstore/annotation.go index 83f3e0f319f..05da314540d 100644 --- a/pkg/services/sqlstore/annotation.go +++ b/pkg/services/sqlstore/annotation.go @@ -42,7 +42,7 @@ func NewSQLAnnotationRepo(sql *SQLStore) SQLAnnotationRepo { } func (r *SQLAnnotationRepo) Save(item *annotations.Item) error { - return inTransaction(func(sess *DBSession) error { + return r.sql.WithTransactionalDbSession(context.Background(), func(sess *DBSession) error { tags := models.ParseTagPairs(item.Tags) item.Tags = models.JoinTagPairs(tags) item.Created = timeNow().UnixNano() / int64(time.Millisecond) diff --git a/pkg/services/sqlstore/annotation_cleanup.go b/pkg/services/sqlstore/annotation_cleanup.go index 0f9a181777b..24c06071da5 100644 --- a/pkg/services/sqlstore/annotation_cleanup.go +++ b/pkg/services/sqlstore/annotation_cleanup.go @@ -13,6 +13,7 @@ import ( type AnnotationCleanupService struct { batchSize int64 log log.Logger + sqlstore *SQLStore } const ( @@ -92,7 +93,7 @@ func (acs *AnnotationCleanupService) executeUntilDoneOrCancelled(ctx context.Con return totalAffected, ctx.Err() default: var affected int64 - err := withDbSession(ctx, x, func(session *DBSession) error { + err := withDbSession(ctx, acs.sqlstore.engine, func(session *DBSession) error { res, err := session.Exec(sql) if err != nil { return err diff --git a/pkg/services/sqlstore/annotation_cleanup_test.go b/pkg/services/sqlstore/annotation_cleanup_test.go index b98792284c1..e3e0e4efa3d 100644 --- a/pkg/services/sqlstore/annotation_cleanup_test.go +++ b/pkg/services/sqlstore/annotation_cleanup_test.go @@ -87,7 +87,7 @@ func TestAnnotationCleanUp(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - cleaner := &AnnotationCleanupService{batchSize: 1, log: log.New("test-logger")} + cleaner := &AnnotationCleanupService{batchSize: 1, log: log.New("test-logger"), sqlstore: fakeSQL} affectedAnnotations, affectedAnnotationTags, err := cleaner.CleanAnnotations(context.Background(), test.cfg) require.NoError(t, err) @@ -142,7 +142,7 @@ func TestOldAnnotationsAreDeletedFirst(t *testing.T) { require.NoError(t, err, "cannot insert annotation") // run the clean up task to keep one annotation. - cleaner := &AnnotationCleanupService{batchSize: 1, log: log.New("test-logger")} + cleaner := &AnnotationCleanupService{batchSize: 1, log: log.New("test-logger"), sqlstore: fakeSQL} _, err = cleaner.cleanAnnotations(context.Background(), setting.AnnotationCleanupSettings{MaxCount: 1}, alertAnnotationType) require.NoError(t, err) diff --git a/pkg/services/sqlstore/migrations/secrets_mig.go b/pkg/services/sqlstore/migrations/secrets_mig.go index 3c2012a999f..d6bc6f02c90 100644 --- a/pkg/services/sqlstore/migrations/secrets_mig.go +++ b/pkg/services/sqlstore/migrations/secrets_mig.go @@ -18,4 +18,24 @@ func addSecretsMigration(mg *migrator.Migrator) { } mg.AddMigration("create data_keys table", migrator.NewAddTableMigration(dataKeysV1)) + + secretsV1 := migrator.Table{ + Name: "secrets", + Columns: []*migrator.Column{ + {Name: "id", Type: migrator.DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "org_id", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "namespace", Type: migrator.DB_NVarchar, Length: 255, Nullable: false}, + {Name: "type", Type: migrator.DB_NVarchar, Length: 255, Nullable: false}, + {Name: "value", Type: migrator.DB_Text, Nullable: true}, + {Name: "created", Type: migrator.DB_DateTime, Nullable: false}, + {Name: "updated", Type: migrator.DB_DateTime, Nullable: false}, + }, + Indices: []*migrator.Index{ + {Cols: []string{"org_id"}}, + {Cols: []string{"org_id", "namespace"}}, + {Cols: []string{"org_id", "namespace", "type"}, Type: migrator.UniqueIndex}, + }, + } + + mg.AddMigration("create secrets table", migrator.NewAddTableMigration(secretsV1)) } diff --git a/pkg/services/sqlstore/migrations/ualert/channel.go b/pkg/services/sqlstore/migrations/ualert/channel.go index 81c078c550e..27e0317c812 100644 --- a/pkg/services/sqlstore/migrations/ualert/channel.go +++ b/pkg/services/sqlstore/migrations/ualert/channel.go @@ -5,12 +5,11 @@ import ( "encoding/json" "errors" "fmt" - "sort" - "strings" + + "github.com/prometheus/alertmanager/pkg/labels" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/util" - "github.com/prometheus/alertmanager/pkg/labels" ) type notificationChannel struct { @@ -26,11 +25,88 @@ type notificationChannel struct { } // channelsPerOrg maps notification channels per organisation -type channelsPerOrg map[int64]map[interface{}]*notificationChannel +type channelsPerOrg map[int64][]*notificationChannel // channelMap maps notification channels per organisation type defaultChannelsPerOrg map[int64][]*notificationChannel +// uidOrID for both uid and ID, primarily used for mapping legacy channel to migrated receiver. +type uidOrID interface{} + +// setupAlertmanagerConfigs creates Alertmanager configs with migrated receivers and routes. +func (m *migration) setupAlertmanagerConfigs(rulesPerOrg map[int64]map[string]dashAlert) (amConfigsPerOrg, error) { + // allChannels: channelUID -> channelConfig + allChannelsPerOrg, defaultChannelsPerOrg, err := m.getNotificationChannelMap() + if err != nil { + return nil, fmt.Errorf("failed to load notification channels: %w", err) + } + + amConfigPerOrg := make(amConfigsPerOrg, len(allChannelsPerOrg)) + for orgID, channels := range allChannelsPerOrg { + amConfig := &PostableUserConfig{ + AlertmanagerConfig: PostableApiAlertingConfig{ + Receivers: make([]*PostableApiReceiver, 0), + }, + } + amConfigPerOrg[orgID] = amConfig + + // Create all newly migrated receivers from legacy notification channels. + receiversMap, receivers, err := m.createReceivers(channels) + if err != nil { + return nil, fmt.Errorf("failed to create receiver in orgId %d: %w", orgID, err) + } + + // No need to create an Alertmanager configuration if there are no receivers left that aren't obsolete. + if len(receivers) == 0 { + m.mg.Logger.Warn("no available receivers", "orgId", orgID) + continue + } + + amConfig.AlertmanagerConfig.Receivers = receivers + + defaultReceivers := make(map[string]struct{}) + defaultChannels, ok := defaultChannelsPerOrg[orgID] + if ok { + // If the organization has default channels build a map of default receivers, used to create alert-specific routes later. + for _, c := range defaultChannels { + defaultReceivers[c.Name] = struct{}{} + } + } + defaultReceiver, defaultRoute, err := m.createDefaultRouteAndReceiver(defaultChannels) + if err != nil { + return nil, fmt.Errorf("failed to create default route & receiver in orgId %d: %w", orgID, err) + } + amConfig.AlertmanagerConfig.Route = defaultRoute + if defaultReceiver != nil { + amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, defaultReceiver) + } + + // Create routes + if rules, ok := rulesPerOrg[orgID]; ok { + for ruleUid, da := range rules { + route, err := m.createRouteForAlert(ruleUid, da, receiversMap, defaultReceivers) + if err != nil { + return nil, fmt.Errorf("failed to create route for alert %s in orgId %d: %w", da.Name, orgID, err) + } + + if route != nil { + amConfigPerOrg[da.OrgId].AlertmanagerConfig.Route.Routes = append(amConfigPerOrg[da.OrgId].AlertmanagerConfig.Route.Routes, route) + } + } + } + + // Validate the alertmanager configuration produced, this gives a chance to catch bad configuration at migration time. + // Validation between legacy and unified alerting can be different (e.g. due to bug fixes) so this would fail the migration in that case. + if err := m.validateAlertmanagerConfig(orgID, amConfig); err != nil { + return nil, fmt.Errorf("failed to validate AlertmanagerConfig in orgId %d: %w", orgID, err) + } + } + + return amConfigPerOrg, nil +} + +// getNotificationChannelMap returns a map of all channelUIDs to channel config as well as a separate map for just those channels that are default. +// For any given Organization, all channels in defaultChannelsPerOrg should also exist in channelsPerOrg. func (m *migration) getNotificationChannelMap() (channelsPerOrg, defaultChannelsPerOrg, error) { q := ` SELECT id, @@ -58,15 +134,13 @@ func (m *migration) getNotificationChannelMap() (channelsPerOrg, defaultChannels allChannelsMap := make(channelsPerOrg) defaultChannelsMap := make(defaultChannelsPerOrg) for i, c := range allChannels { - if _, ok := allChannelsMap[c.OrgID]; !ok { // new seen org - allChannelsMap[c.OrgID] = make(map[interface{}]*notificationChannel) - } - if c.Uid != "" { - allChannelsMap[c.OrgID][c.Uid] = &allChannels[i] - } - if c.ID != 0 { - allChannelsMap[c.OrgID][c.ID] = &allChannels[i] + if c.Type == "hipchat" || c.Type == "sensu" { + m.mg.Logger.Error("alert migration error: discontinued notification channel found", "type", c.Type, "name", c.Name, "uid", c.Uid) + continue } + + allChannelsMap[c.OrgID] = append(allChannelsMap[c.OrgID], &allChannels[i]) + if c.IsDefault { defaultChannelsMap[c.OrgID] = append(defaultChannelsMap[c.OrgID], &allChannels[i]) } @@ -75,257 +149,200 @@ func (m *migration) getNotificationChannelMap() (channelsPerOrg, defaultChannels return allChannelsMap, defaultChannelsMap, nil } -func (m *migration) updateReceiverAndRoute(allChannels channelsPerOrg, defaultChannels defaultChannelsPerOrg, da dashAlert, rule *alertRule, amConfig *PostableUserConfig) error { - // Create receiver and route for this rule. - if allChannels == nil { - return nil - } - - channelIDs := extractChannelIDs(da) - if len(channelIDs) == 0 { - // If there are no channels associated, we skip adding any routes, - // receivers or labels to rules so that it goes through the default - // route. - return nil - } - - recv, route, err := m.makeReceiverAndRoute(rule.UID, rule.OrgID, channelIDs, defaultChannels[rule.OrgID], allChannels[rule.OrgID]) +// Create a notifier (PostableGrafanaReceiver) from a legacy notification channel +func (m *migration) createNotifier(c *notificationChannel) (*PostableGrafanaReceiver, error) { + uid, err := m.generateChannelUID() if err != nil { - return err + return nil, err } - if recv != nil { - amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv) - } - if route != nil { - amConfig.AlertmanagerConfig.Route.Routes = append(amConfig.AlertmanagerConfig.Route.Routes, route) + settings, secureSettings, err := migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings) + if err != nil { + return nil, err } - return nil + return &PostableGrafanaReceiver{ + UID: uid, + Name: c.Name, + Type: c.Type, + DisableResolveMessage: c.DisableResolveMessage, + Settings: settings, + SecureSettings: secureSettings, + }, nil } -func (m *migration) makeReceiverAndRoute(ruleUid string, orgID int64, channelUids []interface{}, defaultChannels []*notificationChannel, allChannels map[interface{}]*notificationChannel) (*PostableApiReceiver, *Route, error) { - portedChannels := []*PostableGrafanaReceiver{} - var receiver *PostableApiReceiver - - addChannel := func(c *notificationChannel) error { - if c.Type == "hipchat" || c.Type == "sensu" { - m.mg.Logger.Error("alert migration error: discontinued notification channel found", "type", c.Type, "name", c.Name, "uid", c.Uid) - return nil - } - - uid, ok := m.generateChannelUID() - if !ok { - return errors.New("failed to generate UID for notification channel") - } - - if _, ok := m.migratedChannelsPerOrg[orgID]; !ok { - m.migratedChannelsPerOrg[orgID] = make(map[*notificationChannel]struct{}) - } - m.migratedChannelsPerOrg[orgID][c] = struct{}{} - settings, decryptedSecureSettings, err := migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings) +// Create one receiver for every unique notification channel. +func (m *migration) createReceivers(allChannels []*notificationChannel) (map[uidOrID]*PostableApiReceiver, []*PostableApiReceiver, error) { + var receivers []*PostableApiReceiver + receiversMap := make(map[uidOrID]*PostableApiReceiver) + for _, c := range allChannels { + notifier, err := m.createNotifier(c) if err != nil { - return err + return nil, nil, err } - portedChannels = append(portedChannels, &PostableGrafanaReceiver{ - UID: uid, - Name: c.Name, - Type: c.Type, - DisableResolveMessage: c.DisableResolveMessage, - Settings: settings, - SecureSettings: decryptedSecureSettings, - }) - - return nil - } - - // Remove obsolete notification channels. - filteredChannelUids := make(map[interface{}]struct{}) - for _, uid := range channelUids { - c, ok := allChannels[uid] - if ok { - // always store the channel UID to prevent duplicates - filteredChannelUids[c.Uid] = struct{}{} - } else { - m.mg.Logger.Warn("ignoring obsolete notification channel", "uid", uid) + recv := &PostableApiReceiver{ + Name: c.Name, // Channel name is unique within an Org. + GrafanaManagedReceivers: []*PostableGrafanaReceiver{notifier}, } - } - // Add default channels that are not obsolete. - for _, c := range defaultChannels { - id := interface{}(c.Uid) - if c.Uid == "" { - id = c.ID + + receivers = append(receivers, recv) + + // Store receivers for creating routes from alert rules later. + if c.Uid != "" { + receiversMap[c.Uid] = recv } - c, ok := allChannels[id] - if ok { - // always store the channel UID to prevent duplicates - filteredChannelUids[c.Uid] = struct{}{} + if c.ID != 0 { + // In certain circumstances, the alert rule uses ID instead of uid. So, we add this to be able to lookup by ID in case. + receiversMap[c.ID] = recv } } - if len(filteredChannelUids) == 0 && ruleUid != "default_route" { - // We use the default route instead. No need to add additional route. - return nil, nil, nil - } + return receiversMap, receivers, nil +} - chanKey, err := makeKeyForChannelGroup(filteredChannelUids) - if err != nil { - return nil, nil, err - } +// Create the root-level route with the default receiver. If no new receiver is created specifically for the root-level route, the returned receiver will be nil. +func (m *migration) createDefaultRouteAndReceiver(defaultChannels []*notificationChannel) (*PostableApiReceiver, *Route, error) { + var defaultReceiver *PostableApiReceiver - var receiverName string - - if _, ok := m.portedChannelGroupsPerOrg[orgID]; !ok { - m.portedChannelGroupsPerOrg[orgID] = make(map[string]string) - } - if rn, ok := m.portedChannelGroupsPerOrg[orgID][chanKey]; ok { - // We have ported these exact set of channels already. Re-use it. - receiverName = rn - if receiverName == "autogen-contact-point-default" { - // We don't need to create new routes if it's the default contact point. - return nil, nil, nil + defaultReceiverName := "autogen-contact-point-default" + if len(defaultChannels) != 1 { + // If there are zero or more than one default channels we create a separate contact group that is used only in the root policy. This is to simplify the migrated notification policy structure. + // If we ever allow more than one receiver per route this won't be necessary. + defaultReceiver = &PostableApiReceiver{ + Name: defaultReceiverName, + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, } - } else { - for n := range filteredChannelUids { - if err := addChannel(allChannels[n]); err != nil { + + for _, c := range defaultChannels { + // Need to create a new notifier to prevent uid conflict. + defaultNotifier, err := m.createNotifier(c) + if err != nil { return nil, nil, err } - } - if ruleUid == "default_route" { - receiverName = "autogen-contact-point-default" - } else { - m.lastReceiverID++ - receiverName = fmt.Sprintf("autogen-contact-point-%d", m.lastReceiverID) - } - - m.portedChannelGroupsPerOrg[orgID][chanKey] = receiverName - receiver = &PostableApiReceiver{ - Name: receiverName, - GrafanaManagedReceivers: portedChannels, + defaultReceiver.GrafanaManagedReceivers = append(defaultReceiver.GrafanaManagedReceivers, defaultNotifier) } + } else { + // If there is only a single default channel, we don't need a separate receiver to hold it. We can reuse the existing receiver for that single notifier. + defaultReceiverName = defaultChannels[0].Name } - n, v := getLabelForRouteMatching(ruleUid) + defaultRoute := &Route{ + Receiver: defaultReceiverName, + Routes: make([]*Route, 0), + } + + return defaultReceiver, defaultRoute, nil +} + +// Wrapper to select receivers for given alert rules based on associated notification channels and then create the migrated route. +func (m *migration) createRouteForAlert(ruleUID string, da dashAlert, receivers map[uidOrID]*PostableApiReceiver, defaultReceivers map[string]struct{}) (*Route, error) { + // Create route(s) for alert + filteredReceiverNames := m.filterReceiversForAlert(da, receivers, defaultReceivers) + + if len(filteredReceiverNames) != 0 { + // Only create a route if there are specific receivers, otherwise it defaults to the root-level route. + route, err := createRoute(ruleUID, filteredReceiverNames) + if err != nil { + return nil, err + } + + return route, nil + } + + return nil, nil +} + +// Create route(s) for the given alert ruleUID and receivers. +// If the alert had a single channel, it will now have a single route/policy. If the alert had multiple channels, it will now have multiple nested routes/policies. +func createRoute(ruleUID string, filteredReceiverNames map[string]interface{}) (*Route, error) { + n, v := getLabelForRouteMatching(ruleUID) mat, err := labels.NewMatcher(labels.MatchEqual, n, v) if err != nil { - return nil, nil, err - } - route := &Route{ - Receiver: receiverName, - Matchers: Matchers{mat}, + return nil, err } - return receiver, route, nil -} - -// makeKeyForChannelGroup generates a unique for this group of channels UIDs. -func makeKeyForChannelGroup(channelUids map[interface{}]struct{}) (string, error) { - uids := make([]string, 0, len(channelUids)) - for u := range channelUids { - switch uid := u.(type) { - case string: - uids = append(uids, uid) - case int, int32, int64: - uids = append(uids, fmt.Sprintf("%d", uid)) - default: - // Should never happen. - return "", fmt.Errorf("unknown channel UID type: %T", u) - } - } - - sort.Strings(uids) - return strings.Join(uids, "::sep::"), nil -} - -// addDefaultChannels should be called before adding any other routes. -func (m *migration) addDefaultChannels(amConfigsPerOrg amConfigsPerOrg, allChannels channelsPerOrg, defaultChannels defaultChannelsPerOrg) error { - for orgID := range allChannels { - if _, ok := amConfigsPerOrg[orgID]; !ok { - amConfigsPerOrg[orgID] = &PostableUserConfig{ - AlertmanagerConfig: PostableApiAlertingConfig{ - Receivers: make([]*PostableApiReceiver, 0), - Route: &Route{ - Routes: make([]*Route, 0), - }, - }, + var route *Route + if len(filteredReceiverNames) == 1 { + for name := range filteredReceiverNames { + route = &Route{ + Receiver: name, + Matchers: Matchers{mat}, } } - // Default route and receiver. - recv, route, err := m.makeReceiverAndRoute("default_route", orgID, nil, defaultChannels[orgID], allChannels[orgID]) - if err != nil { - // if one fails it will fail the migration - return err + } else { + nestedRoutes := []*Route{} + for name := range filteredReceiverNames { + r := &Route{ + Receiver: name, + Matchers: Matchers{mat}, + Continue: true, + } + nestedRoutes = append(nestedRoutes, r) } - if recv != nil { - amConfigsPerOrg[orgID].AlertmanagerConfig.Receivers = append(amConfigsPerOrg[orgID].AlertmanagerConfig.Receivers, recv) - } - if route != nil { - route.Matchers = nil // Don't need matchers for root route. - amConfigsPerOrg[orgID].AlertmanagerConfig.Route = route + route = &Route{ + Matchers: Matchers{mat}, + Routes: nestedRoutes, } } - return nil + + return route, nil } -func (m *migration) addUnmigratedChannels(orgID int64, amConfigs *PostableUserConfig, allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel) error { - // Unmigrated channels. - portedChannels := []*PostableGrafanaReceiver{} - receiver := &PostableApiReceiver{ - Name: "autogen-unlinked-channel-recv", +// Filter receivers to select those that were associated to the given rule as channels. +func (m *migration) filterReceiversForAlert(da dashAlert, receivers map[uidOrID]*PostableApiReceiver, defaultReceivers map[string]struct{}) map[string]interface{} { + channelIDs := extractChannelIDs(da) + if len(channelIDs) == 0 { + // If there are no channels associated, we use the default route. + return nil } - for _, c := range allChannels { - if _, ok := m.migratedChannelsPerOrg[orgID]; !ok { - m.migratedChannelsPerOrg[orgID] = make(map[*notificationChannel]struct{}) - } - _, ok := m.migratedChannelsPerOrg[orgID][c] + + // Filter receiver names. + filteredReceiverNames := make(map[string]interface{}) + for _, uidOrId := range channelIDs { + recv, ok := receivers[uidOrId] if ok { - continue + filteredReceiverNames[recv.Name] = struct{}{} // Deduplicate on contact point name. + } else { + m.mg.Logger.Warn("alert linked to obsolete notification channel, ignoring", "alert", da.Name, "uid", uidOrId) } - if c.Type == "hipchat" || c.Type == "sensu" { - m.mg.Logger.Error("alert migration error: discontinued notification channel found", "type", c.Type, "name", c.Name, "uid", c.Uid) - continue - } - - uid, ok := m.generateChannelUID() - if !ok { - return errors.New("failed to generate UID for notification channel") - } - - m.migratedChannelsPerOrg[orgID][c] = struct{}{} - settings, decryptedSecureSettings, err := migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings) - if err != nil { - return err - } - portedChannels = append(portedChannels, &PostableGrafanaReceiver{ - UID: uid, - Name: c.Name, - Type: c.Type, - DisableResolveMessage: c.DisableResolveMessage, - Settings: settings, - SecureSettings: decryptedSecureSettings, - }) - } - receiver.GrafanaManagedReceivers = portedChannels - if len(portedChannels) > 0 { - amConfigs.AlertmanagerConfig.Receivers = append(amConfigs.AlertmanagerConfig.Receivers, receiver) } - return nil + coveredByDefault := func(names map[string]interface{}) bool { + // Check if all receivers are also default ones and if so, just use the default route. + for n := range names { + if _, ok := defaultReceivers[n]; !ok { + return false + } + } + return true + } + + if len(filteredReceiverNames) == 0 || coveredByDefault(filteredReceiverNames) { + // Use the default route instead. + return nil + } + + // Add default receivers alongside rule-specific ones. + for n := range defaultReceivers { + filteredReceiverNames[n] = struct{}{} + } + + return filteredReceiverNames } -func (m *migration) generateChannelUID() (string, bool) { +func (m *migration) generateChannelUID() (string, error) { for i := 0; i < 5; i++ { gen := util.GenerateShortUID() if _, ok := m.seenChannelUIDs[gen]; !ok { m.seenChannelUIDs[gen] = struct{}{} - return gen, true + return gen, nil } } - return "", false + return "", errors.New("failed to generate UID for notification channel") } // Some settings were migrated from settings to secure settings in between. @@ -354,7 +371,7 @@ func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json, keys = []string{"api_secret"} } - decryptedSecureSettings := secureSettings.Decrypt() + newSecureSettings := secureSettings.Decrypt() cloneSettings := simplejson.New() settingsMap, err := settings.Map() if err != nil { @@ -364,25 +381,30 @@ func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json, cloneSettings.Set(k, v) } for _, k := range keys { - if v, ok := decryptedSecureSettings[k]; ok && v != "" { + if v, ok := newSecureSettings[k]; ok && v != "" { continue } sv := cloneSettings.Get(k).MustString() if sv != "" { - decryptedSecureSettings[k] = sv + newSecureSettings[k] = sv cloneSettings.Del(k) } } - return cloneSettings, decryptedSecureSettings, nil + encryptedData := GetEncryptedJsonData(newSecureSettings) + for k, v := range encryptedData { + newSecureSettings[k] = base64.StdEncoding.EncodeToString(v) + } + + return cloneSettings, newSecureSettings, nil } func getLabelForRouteMatching(ruleUID string) (string, string) { return "rule_uid", ruleUID } -func extractChannelIDs(d dashAlert) (channelUids []interface{}) { +func extractChannelIDs(d dashAlert) (channelUids []uidOrID) { // Extracting channel UID/ID. for _, ui := range d.ParsedSettings.Notifications { if ui.UID != "" { @@ -409,18 +431,6 @@ type PostableUserConfig struct { type amConfigsPerOrg = map[int64]*PostableUserConfig -func (c *PostableUserConfig) EncryptSecureSettings() error { - for _, r := range c.AlertmanagerConfig.Receivers { - for _, gr := range r.GrafanaManagedReceivers { - encryptedData := GetEncryptedJsonData(gr.SecureSettings) - for k, v := range encryptedData { - gr.SecureSettings[k] = base64.StdEncoding.EncodeToString(v) - } - } - } - return nil -} - type PostableApiAlertingConfig struct { Route *Route `yaml:"route,omitempty" json:"route,omitempty"` Templates []string `yaml:"templates" json:"templates"` @@ -431,6 +441,7 @@ type Route struct { Receiver string `yaml:"receiver,omitempty" json:"receiver,omitempty"` Matchers Matchers `yaml:"matchers,omitempty" json:"matchers,omitempty"` Routes []*Route `yaml:"routes,omitempty" json:"routes,omitempty"` + Continue bool `yaml:"continue,omitempty" json:"continue,omitempty"` } type Matchers labels.Matchers diff --git a/pkg/services/sqlstore/migrations/ualert/channel_test.go b/pkg/services/sqlstore/migrations/ualert/channel_test.go new file mode 100644 index 00000000000..688b3d62eb1 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/channel_test.go @@ -0,0 +1,349 @@ +package ualert + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/components/simplejson" +) + +func TestFilterReceiversForAlert(t *testing.T) { + tc := []struct { + name string + da dashAlert + receivers map[uidOrID]*PostableApiReceiver + defaultReceivers map[string]struct{} + expected map[string]interface{} + }{ + { + name: "when an alert has multiple channels, each should filter for the correct receiver", + da: dashAlert{ + ParsedSettings: &dashAlertSettings{ + Notifications: []dashAlertNot{{UID: "uid1"}, {UID: "uid2"}}, + }, + }, + receivers: map[uidOrID]*PostableApiReceiver{ + "uid1": { + Name: "recv1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid2": { + Name: "recv2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid3": { + Name: "recv3", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + }, + defaultReceivers: map[string]struct{}{}, + expected: map[string]interface{}{ + "recv1": struct{}{}, + "recv2": struct{}{}, + }, + }, + { + name: "when default receivers exist, they should be added to an alert's filtered receivers", + da: dashAlert{ + ParsedSettings: &dashAlertSettings{ + Notifications: []dashAlertNot{{UID: "uid1"}}, + }, + }, + receivers: map[uidOrID]*PostableApiReceiver{ + "uid1": { + Name: "recv1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid2": { + Name: "recv2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid3": { + Name: "recv3", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + }, + defaultReceivers: map[string]struct{}{ + "recv2": {}, + }, + expected: map[string]interface{}{ + "recv1": struct{}{}, // From alert + "recv2": struct{}{}, // From default + }, + }, + { + name: "when an alert has a channels associated by ID instead of UID, it should be included", + da: dashAlert{ + ParsedSettings: &dashAlertSettings{ + Notifications: []dashAlertNot{{ID: int64(42)}}, + }, + }, + receivers: map[uidOrID]*PostableApiReceiver{ + int64(42): { + Name: "recv1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + }, + defaultReceivers: map[string]struct{}{}, + expected: map[string]interface{}{ + "recv1": struct{}{}, + }, + }, + { + name: "when an alert's receivers are covered by the defaults, return nil to use default receiver downstream", + da: dashAlert{ + ParsedSettings: &dashAlertSettings{ + Notifications: []dashAlertNot{{UID: "uid1"}}, + }, + }, + receivers: map[uidOrID]*PostableApiReceiver{ + "uid1": { + Name: "recv1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid2": { + Name: "recv2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid3": { + Name: "recv3", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + }, + defaultReceivers: map[string]struct{}{ + "recv1": {}, + "recv2": {}, + }, + expected: nil, // recv1 is already a default + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + m := newTestMigration(t) + res := m.filterReceiversForAlert(tt.da, tt.receivers, tt.defaultReceivers) + + require.Equal(t, tt.expected, res) + }) + } +} + +func TestCreateRoute(t *testing.T) { + tc := []struct { + name string + ruleUID string + filteredReceiverNames map[string]interface{} + expected *Route + expErr error + }{ + { + name: "when a single receiver is passed in, the route should be simple and not nested", + ruleUID: "r_uid1", + filteredReceiverNames: map[string]interface{}{ + "recv1": struct{}{}, + }, + expected: &Route{ + Receiver: "recv1", + Matchers: Matchers{{Type: 0, Name: "rule_uid", Value: "r_uid1"}}, + Routes: nil, + Continue: false, + }, + }, + { + name: "when multiple receivers are passed in, the route should be nested with continue=true", + ruleUID: "r_uid1", + filteredReceiverNames: map[string]interface{}{ + "recv1": struct{}{}, + "recv2": struct{}{}, + }, + expected: &Route{ + Receiver: "", + Matchers: Matchers{{Type: 0, Name: "rule_uid", Value: "r_uid1"}}, + Routes: []*Route{ + { + Receiver: "recv1", + Matchers: Matchers{{Type: 0, Name: "rule_uid", Value: "r_uid1"}}, + Routes: nil, + Continue: true, + }, + { + Receiver: "recv2", + Matchers: Matchers{{Type: 0, Name: "rule_uid", Value: "r_uid1"}}, + Routes: nil, + Continue: true, + }, + }, + Continue: false, + }, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + res, err := createRoute(tt.ruleUID, tt.filteredReceiverNames) + if tt.expErr != nil { + require.Error(t, err) + require.EqualError(t, err, tt.expErr.Error()) + return + } + + require.NoError(t, err) + + // Compare route slice separately since order is not guaranteed + expRoutes := tt.expected.Routes + tt.expected.Routes = nil + actRoutes := res.Routes + res.Routes = nil + + require.Equal(t, tt.expected, res) + require.ElementsMatch(t, expRoutes, actRoutes) + }) + } +} + +func createNotChannel(t *testing.T, uid string, id int64, name string) *notificationChannel { + t.Helper() + return ¬ificationChannel{Uid: uid, ID: id, Name: name, Settings: simplejson.New()} +} + +func TestCreateReceivers(t *testing.T) { + tc := []struct { + name string + allChannels []*notificationChannel + defaultChannels []*notificationChannel + expRecvMap map[uidOrID]*PostableApiReceiver + expRecv []*PostableApiReceiver + expErr error + }{ + { + name: "when given notification channels migrate them to receivers", + allChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name1"), createNotChannel(t, "uid2", int64(2), "name2")}, + expRecvMap: map[uidOrID]*PostableApiReceiver{ + "uid1": { + Name: "name1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}}, + }, + "uid2": { + Name: "name2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name2"}}, + }, + int64(1): { + Name: "name1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}}, + }, + int64(2): { + Name: "name2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name2"}}, + }, + }, + expRecv: []*PostableApiReceiver{ + { + Name: "name1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}}, + }, + { + Name: "name2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name2"}}, + }, + }, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + m := newTestMigration(t) + recvMap, recvs, err := m.createReceivers(tt.allChannels) + if tt.expErr != nil { + require.Error(t, err) + require.EqualError(t, err, tt.expErr.Error()) + return + } + + require.NoError(t, err) + + // We ignore certain fields for the purposes of this test + for _, recv := range recvs { + for _, not := range recv.GrafanaManagedReceivers { + not.UID = "" + not.Settings = nil + not.SecureSettings = nil + } + } + + require.Equal(t, tt.expRecvMap, recvMap) + require.ElementsMatch(t, tt.expRecv, recvs) + }) + } +} + +func TestCreateDefaultRouteAndReceiver(t *testing.T) { + tc := []struct { + name string + amConfig *PostableUserConfig + defaultChannels []*notificationChannel + expRecv *PostableApiReceiver + expRoute *Route + expErr error + }{ + { + name: "when given multiple default notification channels migrate them to a single receiver", + defaultChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name1"), createNotChannel(t, "uid2", int64(2), "name2")}, + expRecv: &PostableApiReceiver{ + Name: "autogen-contact-point-default", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}, {Name: "name2"}}, + }, + expRoute: &Route{ + Receiver: "autogen-contact-point-default", + Routes: make([]*Route, 0), + }, + }, + { + name: "when given no default notification channels create a single empty receiver for default", + defaultChannels: []*notificationChannel{}, + expRecv: &PostableApiReceiver{ + Name: "autogen-contact-point-default", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + expRoute: &Route{ + Receiver: "autogen-contact-point-default", + Routes: make([]*Route, 0), + }, + }, + { + name: "when given a single default notification channels don't create a new default receiver", + defaultChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name1")}, + expRecv: nil, + expRoute: &Route{ + Receiver: "name1", + Routes: make([]*Route, 0), + }, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + m := newTestMigration(t) + recv, route, err := m.createDefaultRouteAndReceiver(tt.defaultChannels) + if tt.expErr != nil { + require.Error(t, err) + require.EqualError(t, err, tt.expErr.Error()) + return + } + + require.NoError(t, err) + + // We ignore certain fields for the purposes of this test + if recv != nil { + for _, not := range recv.GrafanaManagedReceivers { + not.UID = "" + not.Settings = nil + not.SecureSettings = nil + } + } + + require.Equal(t, tt.expRecv, recv) + require.Equal(t, tt.expRoute, route) + }) + } +} diff --git a/pkg/services/sqlstore/migrations/ualert/migration_test.go b/pkg/services/sqlstore/migrations/ualert/migration_test.go index 8f7de0ef342..0061d9f5871 100644 --- a/pkg/services/sqlstore/migrations/ualert/migration_test.go +++ b/pkg/services/sqlstore/migrations/ualert/migration_test.go @@ -137,15 +137,18 @@ func TestDashAlertMigration(t *testing.T) { Route: &ualert.Route{ Receiver: "autogen-contact-point-default", Routes: []*ualert.Route{ - {Receiver: "autogen-contact-point-1", Matchers: newMatchers(labels.MatchEqual, "alert_name", "alert1")}, // These Matchers are temporary and will be replaced below with generated rule_uid. - {Receiver: "autogen-contact-point-2", Matchers: newMatchers(labels.MatchEqual, "alert_name", "alert2")}, - {Receiver: "autogen-contact-point-3", Matchers: newMatchers(labels.MatchEqual, "alert_name", "alert3")}, + {Receiver: "notifier1", Matchers: createAlertNameMatchers("alert1")}, // These Matchers are temporary and will be replaced below with generated rule_uid. + {Matchers: createAlertNameMatchers("alert2"), Routes: []*ualert.Route{ + {Receiver: "notifier2", Matchers: createAlertNameMatchers("alert2"), Continue: true}, + {Receiver: "notifier3", Matchers: createAlertNameMatchers("alert2"), Continue: true}, + }}, + {Receiver: "notifier3", Matchers: createAlertNameMatchers("alert3")}, }, }, Receivers: []*ualert.PostableApiReceiver{ - {Name: "autogen-contact-point-1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, // email - {Name: "autogen-contact-point-2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}, {Name: "notifier3", Type: "opsgenie"}}}, // slack+opsgenie - {Name: "autogen-contact-point-3", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}, // opsgenie + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, + {Name: "notifier3", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}, {Name: "autogen-contact-point-default"}, // empty default }, }, @@ -153,29 +156,34 @@ func TestDashAlertMigration(t *testing.T) { int64(2): { AlertmanagerConfig: ualert.PostableApiAlertingConfig{ Route: &ualert.Route{ - Receiver: "autogen-contact-point-default", + Receiver: "notifier6", Routes: []*ualert.Route{ - {Receiver: "autogen-contact-point-4", Matchers: newMatchers(labels.MatchEqual, "alert_name", "alert4")}, - {Receiver: "autogen-contact-point-5", Matchers: newMatchers(labels.MatchEqual, "alert_name", "alert5")}, + {Matchers: createAlertNameMatchers("alert4"), Routes: []*ualert.Route{ + {Receiver: "notifier4", Matchers: createAlertNameMatchers("alert4"), Continue: true}, + {Receiver: "notifier6", Matchers: createAlertNameMatchers("alert4"), Continue: true}, + }}, + {Matchers: createAlertNameMatchers("alert5"), Routes: []*ualert.Route{ + {Receiver: "notifier4", Matchers: createAlertNameMatchers("alert5"), Continue: true}, + {Receiver: "notifier5", Matchers: createAlertNameMatchers("alert5"), Continue: true}, + {Receiver: "notifier6", Matchers: createAlertNameMatchers("alert5"), Continue: true}, + }}, }, }, Receivers: []*ualert.PostableApiReceiver{ - {Name: "autogen-contact-point-4", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier4", Type: "email"}, {Name: "notifier6", Type: "opsgenie"}}}, // email - {Name: "autogen-contact-point-5", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier4", Type: "email"}, {Name: "notifier5", Type: "slack"}, {Name: "notifier6", Type: "opsgenie"}}}, // email+slack+opsgenie - {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier6", Type: "opsgenie"}}}, // empty default + {Name: "notifier4", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier4", Type: "email"}}}, + {Name: "notifier5", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier5", Type: "slack"}}}, + {Name: "notifier6", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier6", Type: "opsgenie"}}}, // empty default }, }, }, }, }, { - name: "when default channel, add to autogen-contact-point-default", + name: "when no default channel, create empty autogen-contact-point-default", legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), // default - }, - alerts: []*models.Alert{ - createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1"}), + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), }, + alerts: []*models.Alert{}, expected: map[int64]*ualert.PostableUserConfig{ int64(1): { AlertmanagerConfig: ualert.PostableApiAlertingConfig{ @@ -183,7 +191,49 @@ func TestDashAlertMigration(t *testing.T) { Receiver: "autogen-contact-point-default", }, Receivers: []*ualert.PostableApiReceiver{ - {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "autogen-contact-point-default"}, + }, + }, + }, + }, + }, + { + name: "when single default channel, don't create autogen-contact-point-default", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), + }, + alerts: []*models.Alert{}, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "notifier1", + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + }, + }, + }, + }, + }, + { + name: "when multiple default channels, add them to autogen-contact-point-default as well", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), + createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, true), + }, + alerts: []*models.Alert{}, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "autogen-contact-point-default", + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, + {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}, }, }, }, @@ -196,22 +246,18 @@ func TestDashAlertMigration(t *testing.T) { createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, true), // default }, - alerts: []*models.Alert{ - createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier2"}), // + notifier1, notifier3 - }, + alerts: []*models.Alert{}, expected: map[int64]*ualert.PostableUserConfig{ int64(1): { AlertmanagerConfig: ualert.PostableApiAlertingConfig{ Route: &ualert.Route{ Receiver: "autogen-contact-point-default", - Routes: []*ualert.Route{ - {Receiver: "autogen-contact-point-1", Matchers: newMatchers(labels.MatchEqual, "alert_name", "alert1")}, - }, }, Receivers: []*ualert.PostableApiReceiver{ - {Name: "autogen-contact-point-1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}, {Name: "notifier3", Type: "opsgenie"}}}, - {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier3", Type: "opsgenie"}}}, - }, + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, + {Name: "notifier3", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}, + {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier3", Type: "opsgenie"}}}}, }, }, }, @@ -233,6 +279,8 @@ func TestDashAlertMigration(t *testing.T) { Receiver: "autogen-contact-point-default", }, Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}, }, }, @@ -240,13 +288,13 @@ func TestDashAlertMigration(t *testing.T) { }, }, { - name: "when alerts share all channels, only create one receiver for all of them", + name: "when alerts share channels, only create one receiver per legacy channel", legacyChannels: []*models.AlertNotification{ createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), }, alerts: []*models.Alert{ - createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1", "notifier2"}), + createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1"}), createAlert(t, int64(1), int64(1), int64(1), "alert2", []string{"notifier1", "notifier2"}), }, expected: map[int64]*ualert.PostableUserConfig{ @@ -255,12 +303,16 @@ func TestDashAlertMigration(t *testing.T) { Route: &ualert.Route{ Receiver: "autogen-contact-point-default", Routes: []*ualert.Route{ - {Receiver: "autogen-contact-point-1", Matchers: newMatchers(labels.MatchEqual, "alert_name", "alert1")}, - {Receiver: "autogen-contact-point-1", Matchers: newMatchers(labels.MatchEqual, "alert_name", "alert2")}, + {Receiver: "notifier1", Matchers: createAlertNameMatchers("alert1")}, + {Matchers: createAlertNameMatchers("alert2"), Routes: []*ualert.Route{ + {Receiver: "notifier1", Matchers: createAlertNameMatchers("alert2"), Continue: true}, + {Receiver: "notifier2", Matchers: createAlertNameMatchers("alert2"), Continue: true}, + }}, }, }, Receivers: []*ualert.PostableApiReceiver{ - {Name: "autogen-contact-point-1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}, + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, {Name: "autogen-contact-point-default"}, }, }, @@ -268,35 +320,9 @@ func TestDashAlertMigration(t *testing.T) { }, }, { - name: "when channel not linked to any alerts, migrate it to autogen-unlinked-channel-recv", + name: "when channel not linked to any alerts, still create a receiver for it", legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), // default - createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, true), // default - createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, false), // unlinked - }, - alerts: []*models.Alert{ - createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1"}), - createAlert(t, int64(1), int64(2), int64(3), "alert3", []string{}), - }, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "autogen-contact-point-default", - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}, - {Name: "autogen-unlinked-channel-recv", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}, - }, - }, - }, - }, - }, - { - name: "when unsupported channels, do not migrate them", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "hipchat", "", false), - createAlertNotification(t, int64(1), "notifier2", "sensu", "", false), + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), }, alerts: []*models.Alert{}, expected: map[int64]*ualert.PostableUserConfig{ @@ -306,6 +332,29 @@ func TestDashAlertMigration(t *testing.T) { Receiver: "autogen-contact-point-default", }, Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "autogen-contact-point-default"}, + }, + }, + }, + }, + }, + { + name: "when unsupported channels, do not migrate them", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + createAlertNotification(t, int64(1), "notifier2", "hipchat", "", false), + createAlertNotification(t, int64(1), "notifier3", "sensu", "", false), + }, + alerts: []*models.Alert{}, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "autogen-contact-point-default", + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, {Name: "autogen-contact-point-default"}, }, }, @@ -327,11 +376,11 @@ func TestDashAlertMigration(t *testing.T) { Route: &ualert.Route{ Receiver: "autogen-contact-point-default", Routes: []*ualert.Route{ - {Receiver: "autogen-contact-point-1", Matchers: newMatchers(labels.MatchEqual, "alert_name", "alert1")}, + {Receiver: "notifier1", Matchers: createAlertNameMatchers("alert1")}, }, }, Receivers: []*ualert.PostableApiReceiver{ - {Name: "autogen-contact-point-1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, // no sensu + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, {Name: "autogen-contact-point-default"}, }, }, @@ -592,8 +641,8 @@ func boolPointer(b bool) *bool { return &b } -// newMatchers creates a new ualert.Matchers given MatchType, name, and value. -func newMatchers(t labels.MatchType, n, v string) ualert.Matchers { - matcher, _ := labels.NewMatcher(t, n, v) +// createAlertNameMatchers creates a temporary alert_name Matchers that will be replaced during runtime with the generated rule_uid. +func createAlertNameMatchers(alertName string) ualert.Matchers { + matcher, _ := labels.NewMatcher(labels.MatchEqual, "alert_name", alertName) return ualert.Matchers(labels.Matchers{matcher}) } diff --git a/pkg/services/sqlstore/migrations/ualert/testing.go b/pkg/services/sqlstore/migrations/ualert/testing.go index af99d06e665..d238a7ce4f5 100644 --- a/pkg/services/sqlstore/migrations/ualert/testing.go +++ b/pkg/services/sqlstore/migrations/ualert/testing.go @@ -16,8 +16,6 @@ func newTestMigration(t *testing.T) *migration { Logger: log.New("test"), }, - migratedChannelsPerOrg: make(map[int64]map[*notificationChannel]struct{}), - portedChannelGroupsPerOrg: make(map[int64]map[string]string), - seenChannelUIDs: make(map[string]struct{}), + seenChannelUIDs: make(map[string]struct{}), } } diff --git a/pkg/services/sqlstore/migrations/ualert/ualert.go b/pkg/services/sqlstore/migrations/ualert/ualert.go index 8078478d0dc..7c91f53aaf2 100644 --- a/pkg/services/sqlstore/migrations/ualert/ualert.go +++ b/pkg/services/sqlstore/migrations/ualert/ualert.go @@ -69,10 +69,8 @@ func AddDashAlertMigration(mg *migrator.Migrator) { mg.Logger.Error("alert migration error: could not clear alert migration for removing data", "error", err) } mg.AddMigration(migTitle, &migration{ - seenChannelUIDs: make(map[string]struct{}), - migratedChannelsPerOrg: make(map[int64]map[*notificationChannel]struct{}), - portedChannelGroupsPerOrg: make(map[int64]map[string]string), - silences: make(map[int64][]*pb.MeshSilence), + seenChannelUIDs: make(map[string]struct{}), + silences: make(map[int64][]*pb.MeshSilence), }) case !mg.Cfg.UnifiedAlerting.IsEnabled() && migrationRun: // Remove the migration entry that creates unified alerting data. This is so when the feature @@ -213,11 +211,8 @@ type migration struct { sess *xorm.Session mg *migrator.Migrator - seenChannelUIDs map[string]struct{} - migratedChannelsPerOrg map[int64]map[*notificationChannel]struct{} - silences map[int64][]*pb.MeshSilence - portedChannelGroupsPerOrg map[int64]map[string]string // Org -> Channel group key -> receiver name. - lastReceiverID int // For the auto generated receivers. + seenChannelUIDs map[string]struct{} + silences map[int64][]*pb.MeshSilence } func (m *migration) SQL(dialect migrator.Dialect) string { @@ -247,21 +242,12 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { return err } - // allChannels: channelUID -> channelConfig - allChannelsPerOrg, defaultChannelsPerOrg, err := m.getNotificationChannelMap() - if err != nil { - return err - } - - amConfigPerOrg := make(amConfigsPerOrg, len(allChannelsPerOrg)) - err = m.addDefaultChannels(amConfigPerOrg, allChannelsPerOrg, defaultChannelsPerOrg) - if err != nil { - return err - } - // cache for folders created for dashboards that have custom permissions folderCache := make(map[string]*dashboard) + // Store of newly created rules to later create routes + rulesPerOrg := make(map[int64]map[string]dashAlert) + for _, da := range dashAlerts { newCond, err := transConditions(*da.ParsedSettings, da.OrgId, dsIDMap) if err != nil { @@ -358,11 +344,15 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { return err } - if _, ok := amConfigPerOrg[rule.OrgID]; !ok { - m.mg.Logger.Info("no configuration found", "org", rule.OrgID) + if _, ok := rulesPerOrg[rule.OrgID]; !ok { + rulesPerOrg[rule.OrgID] = make(map[string]dashAlert) + } + if _, ok := rulesPerOrg[rule.OrgID][rule.UID]; !ok { + rulesPerOrg[rule.OrgID][rule.UID] = da } else { - if err := m.updateReceiverAndRoute(allChannelsPerOrg, defaultChannelsPerOrg, da, rule, amConfigPerOrg[rule.OrgID]); err != nil { - return err + return MigrationError{ + Err: fmt.Errorf("duplicate generated rule UID"), + AlertId: da.Id, } } @@ -392,39 +382,22 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { } } - for orgID, amConfig := range amConfigPerOrg { - // Create a separate receiver for all the unmigrated channels. - err = m.addUnmigratedChannels(orgID, amConfig, allChannelsPerOrg[orgID], defaultChannelsPerOrg[orgID]) - if err != nil { - return err - } - - // No channels, hence don't require Alertmanager config - skip it. - if len(allChannelsPerOrg[orgID]) == 0 { - m.mg.Logger.Info("alert migration: no notification channel found, skipping Alertmanager config") - continue - } - - // Encrypt the secure settings before we continue. - if err := amConfig.EncryptSecureSettings(); err != nil { - return err - } - - // Validate the alertmanager configuration produced, this gives a chance to catch bad configuration at migration time. - // Validation between legacy and unified alerting can be different (e.g. due to bug fixes) so this would fail the migration in that case. - if err := m.validateAlertmanagerConfig(orgID, amConfig); err != nil { - return err - } - - if err := m.writeAlertmanagerConfig(orgID, amConfig); err != nil { - return err - } - + for orgID := range rulesPerOrg { if err := m.writeSilencesFile(orgID); err != nil { m.mg.Logger.Error("alert migration error: failed to write silence file", "err", err) } } + amConfigPerOrg, err := m.setupAlertmanagerConfigs(rulesPerOrg) + if err != nil { + return err + } + for orgID, amConfig := range amConfigPerOrg { + if err := m.writeAlertmanagerConfig(orgID, amConfig); err != nil { + return err + } + } + return nil } diff --git a/pkg/services/sqlstore/migrations/ualert/ualert_test.go b/pkg/services/sqlstore/migrations/ualert/ualert_test.go index e5bdda6fc07..75abe7369fa 100644 --- a/pkg/services/sqlstore/migrations/ualert/ualert_test.go +++ b/pkg/services/sqlstore/migrations/ualert/ualert_test.go @@ -1,11 +1,14 @@ package ualert import ( + "encoding/base64" "encoding/json" "fmt" "sort" "testing" + "github.com/prometheus/alertmanager/pkg/labels" + "github.com/stretchr/testify/require" "xorm.io/xorm" "github.com/grafana/grafana/pkg/components/simplejson" @@ -13,9 +16,6 @@ import ( "github.com/grafana/grafana/pkg/services/sqlstore/sqlutil" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" - "github.com/prometheus/alertmanager/pkg/labels" - - "github.com/stretchr/testify/require" ) var MigTitle = migTitle @@ -208,6 +208,18 @@ func configFromReceivers(t *testing.T, receivers []*PostableGrafanaReceiver) *Po } } +func (c *PostableUserConfig) EncryptSecureSettings() error { + for _, r := range c.AlertmanagerConfig.Receivers { + for _, gr := range r.GrafanaManagedReceivers { + encryptedData := GetEncryptedJsonData(gr.SecureSettings) + for k, v := range encryptedData { + gr.SecureSettings[k] = base64.StdEncoding.EncodeToString(v) + } + } + } + return nil +} + const invalidUri = "�6�M��)uk譹1(�h`$�o�N>mĕ����cS2�dh![ę� ���`csB�!��OSxP�{�" func Test_getAlertFolderNameFromDashboard(t *testing.T) { diff --git a/pkg/services/sqlstore/mockstore/mockstore.go b/pkg/services/sqlstore/mockstore/mockstore.go index f8b7deac7bf..6a1c9a56651 100644 --- a/pkg/services/sqlstore/mockstore/mockstore.go +++ b/pkg/services/sqlstore/mockstore/mockstore.go @@ -117,6 +117,9 @@ func (m *SQLStoreMock) GetOrgByNameHandler(ctx context.Context, query *models.Ge func (m *SQLStoreMock) CreateOrgWithMember(name string, userID int64) (models.Org, error) { return *m.ExpectedOrg, nil } +func (m *SQLStoreMock) CreateOrg(ctx context.Context, cmd *models.CreateOrgCommand) error { + return m.ExpectedError +} func (m *SQLStoreMock) UpdateOrg(ctx context.Context, cmd *models.UpdateOrgCommand) error { return m.ExpectedError diff --git a/pkg/services/sqlstore/org.go b/pkg/services/sqlstore/org.go index 1cdd72a938a..af65873fb47 100644 --- a/pkg/services/sqlstore/org.go +++ b/pkg/services/sqlstore/org.go @@ -149,8 +149,8 @@ func (ss *SQLStore) CreateOrgWithMember(name string, userID int64) (models.Org, return createOrg(name, userID, ss.engine) } -func CreateOrg(ctx context.Context, cmd *models.CreateOrgCommand) error { - org, err := createOrg(cmd.Name, cmd.UserId, x) +func (ss *SQLStore) CreateOrg(ctx context.Context, cmd *models.CreateOrgCommand) error { + org, err := createOrg(cmd.Name, cmd.UserId, ss.engine) if err != nil { return err } diff --git a/pkg/services/sqlstore/org_test.go b/pkg/services/sqlstore/org_test.go index a86b268776e..ea002c51de5 100644 --- a/pkg/services/sqlstore/org_test.go +++ b/pkg/services/sqlstore/org_test.go @@ -27,7 +27,7 @@ func TestAccountDataAccess(t *testing.T) { for i := 1; i < 4; i++ { cmd = &models.CreateOrgCommand{Name: fmt.Sprint("Org #", i)} - err = CreateOrg(context.Background(), cmd) + err = sqlStore.CreateOrg(context.Background(), cmd) require.NoError(t, err) ids = append(ids, cmd.Result.Id) @@ -44,7 +44,7 @@ func TestAccountDataAccess(t *testing.T) { sqlStore = InitTestDB(t) for i := 1; i < 4; i++ { cmd := &models.CreateOrgCommand{Name: fmt.Sprint("Org #", i)} - err := CreateOrg(context.Background(), cmd) + err := sqlStore.CreateOrg(context.Background(), cmd) require.NoError(t, err) } diff --git a/pkg/services/sqlstore/quota_test.go b/pkg/services/sqlstore/quota_test.go index 8358ce2baa2..f6c68284555 100644 --- a/pkg/services/sqlstore/quota_test.go +++ b/pkg/services/sqlstore/quota_test.go @@ -49,7 +49,7 @@ func TestQuotaCommandsAndQueries(t *testing.T) { UserId: 1, } - err := CreateOrg(context.Background(), &userCmd) + err := sqlStore.CreateOrg(context.Background(), &userCmd) require.NoError(t, err) orgId = userCmd.Result.Id diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index 7baa450cb99..f4833d90235 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -32,7 +32,6 @@ import ( ) var ( - x *xorm.Engine dialect migrator.Dialect sqlog log.Logger = log.New("sqlstore") @@ -101,13 +100,11 @@ func newSQLStore(cfg *setting.Cfg, cacheService *localcache.CacheService, engine ss.Dialect = migrator.NewDialect(ss.engine) - // temporarily still set global var - x = ss.engine dialect = ss.Dialect // Init repo instances annotations.SetRepository(&SQLAnnotationRepo{sql: ss}) - annotations.SetAnnotationCleaner(&AnnotationCleanupService{batchSize: ss.Cfg.AnnotationCleanupJobBatchSize, log: log.New("annotationcleaner")}) + annotations.SetAnnotationCleaner(&AnnotationCleanupService{batchSize: ss.Cfg.AnnotationCleanupJobBatchSize, log: log.New("annotationcleaner"), sqlstore: ss}) // if err := ss.Reset(); err != nil { // return nil, err diff --git a/pkg/services/sqlstore/store.go b/pkg/services/sqlstore/store.go index 8871a68d8fb..1de411c4bc0 100644 --- a/pkg/services/sqlstore/store.go +++ b/pkg/services/sqlstore/store.go @@ -19,6 +19,7 @@ type Store interface { HasEditPermissionInFolders(ctx context.Context, query *models.HasEditPermissionInFoldersQuery) error SearchDashboardSnapshots(ctx context.Context, query *models.GetDashboardSnapshotsQuery) error GetOrgByName(name string) (*models.Org, error) + CreateOrg(ctx context.Context, cmd *models.CreateOrgCommand) error CreateOrgWithMember(name string, userID int64) (models.Org, error) UpdateOrg(ctx context.Context, cmd *models.UpdateOrgCommand) error UpdateOrgAddress(ctx context.Context, cmd *models.UpdateOrgAddressCommand) error diff --git a/pkg/services/sqlstore/team.go b/pkg/services/sqlstore/team.go index d78c1aa873e..d8325604d7f 100644 --- a/pkg/services/sqlstore/team.go +++ b/pkg/services/sqlstore/team.go @@ -351,7 +351,7 @@ func getTeamMember(sess *DBSession, orgId int64, teamId int64, userId int64) (mo // UpdateTeamMember updates a team member func (ss *SQLStore) UpdateTeamMember(ctx context.Context, cmd *models.UpdateTeamMemberCommand) error { - return inTransaction(func(sess *DBSession) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { return updateTeamMember(sess, cmd.OrgId, cmd.TeamId, cmd.UserId, cmd.Permission) }) } @@ -437,7 +437,7 @@ func updateTeamMember(sess *DBSession, orgID, teamID, userID int64, permission m // RemoveTeamMember removes a member from a team func (ss *SQLStore) RemoveTeamMember(ctx context.Context, cmd *models.RemoveTeamMemberCommand) error { - return inTransaction(func(sess *DBSession) error { + return ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { return removeTeamMember(sess, cmd) }) } diff --git a/pkg/services/sqlstore/transactions.go b/pkg/services/sqlstore/transactions.go index 7accf1a844c..341427527a7 100644 --- a/pkg/services/sqlstore/transactions.go +++ b/pkg/services/sqlstore/transactions.go @@ -32,8 +32,8 @@ func (ss *SQLStore) inTransactionWithRetry(ctx context.Context, fn func(ctx cont }, retry) } -func inTransactionWithRetry(callback DBTransactionFunc, retry int) error { - return inTransactionWithRetryCtx(context.Background(), x, callback, retry) +func inTransactionWithRetry(callback DBTransactionFunc, engine *xorm.Engine, retry int) error { + return inTransactionWithRetryCtx(context.Background(), engine, callback, retry) } func inTransactionWithRetryCtx(ctx context.Context, engine *xorm.Engine, callback DBTransactionFunc, retry int) error { @@ -68,7 +68,7 @@ func inTransactionWithRetryCtx(ctx context.Context, engine *xorm.Engine, callbac time.Sleep(time.Millisecond * time.Duration(10)) sqlog.Info("Database locked, sleeping then retrying", "error", err, "retry", retry) - return inTransactionWithRetry(callback, retry+1) + return inTransactionWithRetry(callback, engine, retry+1) } if err != nil { @@ -91,11 +91,3 @@ func inTransactionWithRetryCtx(ctx context.Context, engine *xorm.Engine, callbac return nil } - -func inTransaction(callback DBTransactionFunc) error { - return inTransactionWithRetry(callback, 0) -} - -func inTransactionCtx(ctx context.Context, callback DBTransactionFunc) error { - return inTransactionWithRetryCtx(ctx, x, callback, 0) -} diff --git a/pkg/services/sqlstore/user_test.go b/pkg/services/sqlstore/user_test.go index f91e70134ff..bd971fa6f66 100644 --- a/pkg/services/sqlstore/user_test.go +++ b/pkg/services/sqlstore/user_test.go @@ -80,7 +80,7 @@ func TestUserDataAccess(t *testing.T) { }() orgCmd := &models.CreateOrgCommand{Name: "Some Test Org"} - err := CreateOrg(context.Background(), orgCmd) + err := ss.CreateOrg(context.Background(), orgCmd) require.Nil(t, err) cmd := models.CreateUserCommand{ diff --git a/pkg/services/thumbs/crawler.go b/pkg/services/thumbs/crawler.go index c641455728c..7e65a3590cf 100644 --- a/pkg/services/thumbs/crawler.go +++ b/pkg/services/thumbs/crawler.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/grafana/grafana/pkg/setting" "golang.org/x/sync/errgroup" "github.com/grafana/grafana/pkg/infra/log" @@ -19,8 +20,10 @@ import ( ) type simpleCrawler struct { - renderService rendering.Service - threadCount int + renderService rendering.Service + threadCount int + concurrentLimit int + renderingTimeout time.Duration glive *live.GrafanaLive thumbnailRepo thumbnailRepo @@ -36,13 +39,17 @@ type simpleCrawler struct { renderingSessionByOrgId map[int64]rendering.Session } -func newSimpleCrawler(renderService rendering.Service, gl *live.GrafanaLive, repo thumbnailRepo) dashRenderer { +func newSimpleCrawler(renderService rendering.Service, gl *live.GrafanaLive, repo thumbnailRepo, cfg *setting.Cfg, settings setting.DashboardPreviewsSettings) dashRenderer { + threadCount := int(settings.CrawlThreadCount) c := &simpleCrawler{ - renderService: renderService, - threadCount: 6, - glive: gl, - thumbnailRepo: repo, - log: log.New("thumbnails_crawler"), + // temporarily increases the concurrentLimit from the 'cfg.RendererConcurrentRequestLimit' to 'cfg.RendererConcurrentRequestLimit + crawlerThreadCount' + concurrentLimit: cfg.RendererConcurrentRequestLimit + threadCount, + renderingTimeout: settings.RenderingTimeout, + renderService: renderService, + threadCount: threadCount, + glive: gl, + thumbnailRepo: repo, + log: log.New("thumbnails_crawler"), status: crawlStatus{ State: initializing, Complete: 0, @@ -154,11 +161,11 @@ func (r *simpleCrawler) Run(ctx context.Context, auth CrawlerAuth, mode CrawlerM r.auth = auth r.opts = rendering.Opts{ TimeoutOpts: rendering.TimeoutOpts{ - Timeout: 20 * time.Second, + Timeout: r.renderingTimeout, RequestTimeoutMultiplier: 3, }, Theme: theme, - ConcurrentLimit: 10, + ConcurrentLimit: r.concurrentLimit, } r.renderingSessionByOrgId = make(map[int64]rendering.Session) diff --git a/pkg/services/thumbs/service.go b/pkg/services/thumbs/service.go index 45cba2828aa..72e0ccb450f 100644 --- a/pkg/services/thumbs/service.go +++ b/pkg/services/thumbs/service.go @@ -51,6 +51,8 @@ type thumbService struct { crawlLockServiceActionName string log log.Logger usageStatsService usagestats.Service + canRunCrawler bool + settings setting.DashboardPreviewsSettings } type crawlerScheduleOptions struct { @@ -67,30 +69,34 @@ func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockS if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) { return &dummyService{} } - logger := log.New("thumbnails_service") + logger := log.New("previews_service") + logger.Info("initialized thumb", "settings", cfg.DashboardPreviews) thumbnailRepo := newThumbnailRepo(store) + canRunCrawler := true crawlerAuth, err := authSetupService.Setup(context.Background()) if err != nil { - logger.Error("Failed to setup auth for the dashboard previews crawler", "err", err) - return &dummyService{} + logger.Error("failed to setup auth for the dashboard previews crawler", "err", err) + canRunCrawler = false } t := &thumbService{ usageStatsService: usageStatsService, renderingService: renderService, - renderer: newSimpleCrawler(renderService, gl, thumbnailRepo), + renderer: newSimpleCrawler(renderService, gl, thumbnailRepo, cfg, cfg.DashboardPreviews), thumbnailRepo: thumbnailRepo, store: store, features: features, lockService: lockService, crawlLockServiceActionName: "dashboard-crawler", log: logger, + canRunCrawler: canRunCrawler, + settings: cfg.DashboardPreviews, scheduleOptions: crawlerScheduleOptions{ - tickerInterval: time.Hour, - crawlInterval: time.Hour * 12, - maxCrawlDuration: time.Hour, + tickerInterval: 5 * time.Minute, + crawlInterval: cfg.DashboardPreviews.SchedulerInterval, + maxCrawlDuration: cfg.DashboardPreviews.MaxCrawlDuration, crawlerMode: CrawlerModeThumbs, thumbnailKind: models.ThumbnailKindDefault, themes: []models.Theme{models.ThemeDark, models.ThemeLight}, @@ -401,6 +407,10 @@ func (hs *thumbService) getDashboardId(c *models.ReqContext, uid string) (int64, } func (hs *thumbService) runOnDemandCrawl(parentCtx context.Context, theme models.Theme, mode CrawlerMode, kind models.ThumbnailKind, authOpts rendering.AuthOpts) { + if !hs.canRunCrawler { + return + } + crawlerCtx, cancel := context.WithTimeout(parentCtx, hs.scheduleOptions.maxCrawlDuration) defer cancel() @@ -435,7 +445,7 @@ func (hs *thumbService) runScheduledCrawl(parentCtx context.Context) { } func (hs *thumbService) Run(ctx context.Context) error { - if !hs.features.IsEnabled(featuremgmt.FlagDashboardPreviewsScheduler) { + if !hs.canRunCrawler { return nil } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 58d695f41fc..43974b76986 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -439,6 +439,8 @@ type Cfg struct { // Query history QueryHistoryEnabled bool + + DashboardPreviews DashboardPreviewsSettings } type CommandLineArgs struct { @@ -1001,6 +1003,8 @@ func (cfg *Cfg) Load(args CommandLineArgs) error { cfg.readDataSourcesSettings() + cfg.DashboardPreviews = readDashboardPreviewsSettings(iniFile) + if VerifyEmailEnabled && !cfg.Smtp.Enabled { cfg.Logger.Warn("require_email_validation is enabled but smtp is disabled") } diff --git a/pkg/setting/setting_dashboard_previews.go b/pkg/setting/setting_dashboard_previews.go new file mode 100644 index 00000000000..f73e046e99f --- /dev/null +++ b/pkg/setting/setting_dashboard_previews.go @@ -0,0 +1,31 @@ +package setting + +import ( + "time" + + "gopkg.in/ini.v1" +) + +type DashboardPreviewsSettings struct { + SchedulerInterval time.Duration + MaxCrawlDuration time.Duration + RenderingTimeout time.Duration + CrawlThreadCount uint32 +} + +func readDashboardPreviewsSettings(iniFile *ini.File) DashboardPreviewsSettings { + maxThreadCount := uint32(20) + + s := DashboardPreviewsSettings{} + + previewsCrawlerSection := iniFile.Section("dashboard_previews.crawler") + s.CrawlThreadCount = uint32(previewsCrawlerSection.Key("thread_count").MustUint(6)) + if s.CrawlThreadCount > maxThreadCount { + s.CrawlThreadCount = maxThreadCount + } + + s.SchedulerInterval = previewsCrawlerSection.Key("scheduler_interval").MustDuration(12 * time.Hour) + s.MaxCrawlDuration = previewsCrawlerSection.Key("max_crawl_duration").MustDuration(1 * time.Hour) + s.RenderingTimeout = previewsCrawlerSection.Key("rendering_timeout").MustDuration(20 * time.Second) + return s +} diff --git a/pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource.go b/pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource.go index 84279f4de67..0b9c2f5f0d3 100644 --- a/pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource.go +++ b/pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "path" + "regexp" "sort" "strings" "time" @@ -32,7 +33,8 @@ type AzureMonitorDatasource struct { var ( // Used to convert the aggregation value to the Azure enum for deep linking - aggregationTypeMap = map[string]int{"None": 0, "Total": 1, "Minimum": 2, "Maximum": 3, "Average": 4, "Count": 7} + aggregationTypeMap = map[string]int{"None": 0, "Total": 1, "Minimum": 2, "Maximum": 3, "Average": 4, "Count": 7} + resourceNameLandmark = regexp.MustCompile(`(?i)(/(?P[\w-\.]+)/providers/Microsoft\.Insights/metrics)`) ) const azureMonitorAPIVersion = "2018-01-01" @@ -74,12 +76,6 @@ func (e *AzureMonitorDatasource) buildQueries(queries []backend.DataQuery, dsInf azJSONModel := queryJSONModel.AzureMonitor - urlComponents := map[string]string{} - urlComponents["subscription"] = queryJSONModel.Subscription - urlComponents["resourceGroup"] = azJSONModel.ResourceGroup - urlComponents["metricDefinition"] = azJSONModel.MetricDefinition - urlComponents["resourceName"] = azJSONModel.ResourceName - ub := urlBuilder{ ResourceURI: azJSONModel.ResourceURI, // Legacy, used to reconstruct resource URI if it's not present @@ -91,6 +87,19 @@ func (e *AzureMonitorDatasource) buildQueries(queries []backend.DataQuery, dsInf } azureURL := ub.BuildMetricsURL() + resourceName := azJSONModel.ResourceName + if resourceName == "" { + resourceName = extractResourceNameFromMetricsURL(azureURL) + } + + urlComponents := map[string]string{} + urlComponents["resourceURI"] = azJSONModel.ResourceURI + // Legacy fields used for constructing a deep link to display the query in Azure Portal. + urlComponents["subscription"] = queryJSONModel.Subscription + urlComponents["resourceGroup"] = azJSONModel.ResourceGroup + urlComponents["metricDefinition"] = azJSONModel.MetricDefinition + urlComponents["resourceName"] = resourceName + alias := azJSONModel.Alias timeGrain := azJSONModel.TimeGrain @@ -338,12 +347,18 @@ func getQueryUrl(query *types.AzureMonitorQuery, azurePortalUrl string) (string, } escapedTime := url.QueryEscape(string(timespan)) - id := fmt.Sprintf("/subscriptions/%v/resourceGroups/%v/providers/%v/%v", - query.UrlComponents["subscription"], - query.UrlComponents["resourceGroup"], - query.UrlComponents["metricDefinition"], - query.UrlComponents["resourceName"], - ) + id := query.UrlComponents["resourceURI"] + + if id == "" { + ub := urlBuilder{ + Subscription: query.UrlComponents["subscription"], + ResourceGroup: query.UrlComponents["resourceGroup"], + MetricDefinition: query.UrlComponents["metricDefinition"], + ResourceName: query.UrlComponents["resourceName"], + } + id = ub.buildResourceURIFromLegacyQuery() + } + chartDef, err := json.Marshal(map[string]interface{}{ "v2charts": []interface{}{ map[string]interface{}{ @@ -368,6 +383,10 @@ func getQueryUrl(query *types.AzureMonitorQuery, azurePortalUrl string) (string, return "", err } escapedChart := url.QueryEscape(string(chartDef)) + // Azure Portal will timeout if the chart definition includes a space character encoded as '+'. + // url.QueryEscape encodes spaces as '+'. + // Note: this will not encode '+' literals as those are already encoded as '%2B' by url.QueryEscape + escapedChart = strings.ReplaceAll(escapedChart, "+", "%20") return fmt.Sprintf("%s/#blade/Microsoft_Azure_MonitoringMetrics/Metrics.ReactView/Referer/MetricsExplorer/TimeContext/%s/ChartDefinition/%s", azurePortalUrl, escapedTime, escapedChart), nil } @@ -463,3 +482,20 @@ func toGrafanaUnit(unit string) string { // 1. Do not have a corresponding unit in Grafana's current list. // 2. Do not have the unit listed in any of Azure Monitor's supported metrics anyways. } + +func extractResourceNameFromMetricsURL(url string) string { + matches := resourceNameLandmark.FindStringSubmatch(url) + resourceName := "" + + if matches == nil { + return resourceName + } + + for i, name := range resourceNameLandmark.SubexpNames() { + if name == "resourceName" { + resourceName = matches[i] + } + } + + return resourceName +} diff --git a/pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource_test.go b/pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource_test.go index 89faac286c0..912def85bb9 100644 --- a/pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource_test.go +++ b/pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource_test.go @@ -38,19 +38,22 @@ func TestAzureMonitorBuildQueries(t *testing.T) { azureMonitorVariedProperties map[string]interface{} azureMonitorQueryTarget string expectedInterval string + resourceURI string queryInterval time.Duration }{ { name: "Parse queries from frontend and build AzureMonitor API queries", azureMonitorVariedProperties: map[string]interface{}{ - "timeGrain": "PT1M", - "top": "10", + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", + "timeGrain": "PT1M", + "top": "10", }, + resourceURI: "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", expectedInterval: "PT1M", azureMonitorQueryTarget: "aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines×pan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z", }, { - name: "time grain set to auto", + name: "legacy query without resourceURI and time grain set to auto", azureMonitorVariedProperties: map[string]interface{}{ "timeGrain": "auto", "top": "10", @@ -60,7 +63,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) { azureMonitorQueryTarget: "aggregation=Average&api-version=2018-01-01&interval=PT15M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines×pan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z", }, { - name: "time grain set to auto", + name: "legacy query without resourceURI and time grain set to auto", azureMonitorVariedProperties: map[string]interface{}{ "timeGrain": "auto", "allowedTimeGrainsMs": []int64{60000, 300000}, @@ -71,7 +74,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) { azureMonitorQueryTarget: "aggregation=Average&api-version=2018-01-01&interval=PT5M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines×pan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z", }, { - name: "has a dimension filter", + name: "legacy query without resourceURI and has a dimension filter", azureMonitorVariedProperties: map[string]interface{}{ "timeGrain": "PT1M", "dimension": "blob", @@ -83,7 +86,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) { azureMonitorQueryTarget: "%24filter=blob+eq+%27%2A%27&aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines×pan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z&top=30", }, { - name: "has a dimension filter and none Dimension", + name: "legacy query without resourceURI and has a dimension filter and none Dimension", azureMonitorVariedProperties: map[string]interface{}{ "timeGrain": "PT1M", "dimension": "None", @@ -95,7 +98,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) { azureMonitorQueryTarget: "aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines×pan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z", }, { - name: "has dimensionFilter*s* property with one dimension", + name: "legacy query without resourceURI and has dimensionFilter*s* property with one dimension", azureMonitorVariedProperties: map[string]interface{}{ "timeGrain": "PT1M", "dimensionFilters": []types.AzureMonitorDimensionFilter{{Dimension: "blob", Operator: "eq", Filter: "*"}}, @@ -106,7 +109,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) { azureMonitorQueryTarget: "%24filter=blob+eq+%27%2A%27&aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines×pan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z&top=30", }, { - name: "has dimensionFilter*s* property with two dimensions", + name: "legacy query without resourceURI and has dimensionFilter*s* property with two dimensions", azureMonitorVariedProperties: map[string]interface{}{ "timeGrain": "PT1M", "dimensionFilters": []types.AzureMonitorDimensionFilter{{Dimension: "blob", Operator: "eq", Filter: "*"}, {Dimension: "tier", Operator: "eq", Filter: "*"}}, @@ -117,7 +120,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) { azureMonitorQueryTarget: "%24filter=blob+eq+%27%2A%27+and+tier+eq+%27%2A%27&aggregation=Average&api-version=2018-01-01&interval=PT1M&metricnames=Percentage+CPU&metricnamespace=Microsoft.Compute-virtualMachines×pan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z&top=30", }, { - name: "has a dimension filter without specifying a top", + name: "legacy query without resourceURI and has a dimension filter without specifying a top", azureMonitorVariedProperties: map[string]interface{}{ "timeGrain": "PT1M", "dimension": "blob", @@ -165,6 +168,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) { azureMonitorQuery := &types.AzureMonitorQuery{ URL: "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana/providers/microsoft.insights/metrics", UrlComponents: map[string]string{ + "resourceURI": tt.resourceURI, "metricDefinition": "Microsoft.Compute/virtualMachines", "resourceGroup": "grafanastaging", "resourceName": "grafana", @@ -188,7 +192,7 @@ func TestAzureMonitorBuildQueries(t *testing.T) { expected := `http://ds/#blade/Microsoft_Azure_MonitoringMetrics/Metrics.ReactView/Referer/MetricsExplorer/` + `TimeContext/%7B%22absolute%22%3A%7B%22startTime%22%3A%222018-03-15T13%3A00%3A00Z%22%2C%22endTime%22%3A%222018-03-15T13%3A34%3A00Z%22%7D%7D/` + `ChartDefinition/%7B%22v2charts%22%3A%5B%7B%22metrics%22%3A%5B%7B%22resourceMetadata%22%3A%7B%22id%22%3A%22%2Fsubscriptions%2F12345678-aaaa-bbbb-cccc-123456789abc%2FresourceGroups%2Fgrafanastaging%2Fproviders%2FMicrosoft.Compute%2FvirtualMachines%2Fgrafana%22%7D%2C` + - `%22name%22%3A%22Percentage+CPU%22%2C%22aggregationType%22%3A4%2C%22namespace%22%3A%22Microsoft.Compute-virtualMachines%22%2C%22metricVisualization%22%3A%7B%22displayName%22%3A%22Percentage+CPU%22%2C%22resourceDisplayName%22%3A%22grafana%22%7D%7D%5D%7D%5D%7D` + `%22name%22%3A%22Percentage%20CPU%22%2C%22aggregationType%22%3A4%2C%22namespace%22%3A%22Microsoft.Compute-virtualMachines%22%2C%22metricVisualization%22%3A%7B%22displayName%22%3A%22Percentage%20CPU%22%2C%22resourceDisplayName%22%3A%22grafana%22%7D%7D%5D%7D%5D%7D` actual, err := getQueryUrl(queries[0], "http://ds") require.NoError(t, err) require.Equal(t, expected, actual) @@ -214,19 +218,19 @@ func makeTestDataLink(url string) data.DataLink { func TestAzureMonitorParseResponse(t *testing.T) { // datalinks for the test frames averageLink := makeTestDataLink(`http://ds/#blade/Microsoft_Azure_MonitoringMetrics/Metrics.ReactView/Referer/MetricsExplorer/TimeContext/%7B%22absolute%22%3A%7B%22startTime%22%3A%220001-01-01T00%3A00%3A00Z%22%2C%22endTime%22%3A%220001-01-01T00%3A00%3A00Z%22%7D%7D/` + - `ChartDefinition/%7B%22v2charts%22%3A%5B%7B%22metrics%22%3A%5B%7B%22resourceMetadata%22%3A%7B%22id%22%3A%22%2Fsubscriptions%2F%2FresourceGroups%2F%2Fproviders%2F%2Fgrafana%22%7D%2C%22name%22%3A%22%22%2C%22aggregationType%22%3A4%2C%22namespace%22%3A%22%22%2C` + + `ChartDefinition/%7B%22v2charts%22%3A%5B%7B%22metrics%22%3A%5B%7B%22resourceMetadata%22%3A%7B%22id%22%3A%22%2Fsubscriptions%2F12345678-aaaa-bbbb-cccc-123456789abc%2FresourceGroups%2Fgrafanastaging%2Fproviders%2FMicrosoft.Compute%2FvirtualMachines%2Fgrafana%22%7D%2C%22name%22%3A%22%22%2C%22aggregationType%22%3A4%2C%22namespace%22%3A%22%22%2C` + `%22metricVisualization%22%3A%7B%22displayName%22%3A%22%22%2C%22resourceDisplayName%22%3A%22grafana%22%7D%7D%5D%7D%5D%7D`) totalLink := makeTestDataLink(`http://ds/#blade/Microsoft_Azure_MonitoringMetrics/Metrics.ReactView/Referer/MetricsExplorer/TimeContext/%7B%22absolute%22%3A%7B%22startTime%22%3A%220001-01-01T00%3A00%3A00Z%22%2C%22endTime%22%3A%220001-01-01T00%3A00%3A00Z%22%7D%7D/` + - `ChartDefinition/%7B%22v2charts%22%3A%5B%7B%22metrics%22%3A%5B%7B%22resourceMetadata%22%3A%7B%22id%22%3A%22%2Fsubscriptions%2F%2FresourceGroups%2F%2Fproviders%2F%2Fgrafana%22%7D%2C%22name%22%3A%22%22%2C%22aggregationType%22%3A1%2C%22namespace%22%3A%22%22%2C` + + `ChartDefinition/%7B%22v2charts%22%3A%5B%7B%22metrics%22%3A%5B%7B%22resourceMetadata%22%3A%7B%22id%22%3A%22%2Fsubscriptions%2F12345678-aaaa-bbbb-cccc-123456789abc%2FresourceGroups%2Fgrafanastaging%2Fproviders%2FMicrosoft.Compute%2FvirtualMachines%2Fgrafana%22%7D%2C%22name%22%3A%22%22%2C%22aggregationType%22%3A1%2C%22namespace%22%3A%22%22%2C` + `%22metricVisualization%22%3A%7B%22displayName%22%3A%22%22%2C%22resourceDisplayName%22%3A%22grafana%22%7D%7D%5D%7D%5D%7D`) maxLink := makeTestDataLink(`http://ds/#blade/Microsoft_Azure_MonitoringMetrics/Metrics.ReactView/Referer/MetricsExplorer/TimeContext/%7B%22absolute%22%3A%7B%22startTime%22%3A%220001-01-01T00%3A00%3A00Z%22%2C%22endTime%22%3A%220001-01-01T00%3A00%3A00Z%22%7D%7D/` + - `ChartDefinition/%7B%22v2charts%22%3A%5B%7B%22metrics%22%3A%5B%7B%22resourceMetadata%22%3A%7B%22id%22%3A%22%2Fsubscriptions%2F%2FresourceGroups%2F%2Fproviders%2F%2Fgrafana%22%7D%2C%22name%22%3A%22%22%2C%22aggregationType%22%3A3%2C%22namespace%22%3A%22%22%2C` + + `ChartDefinition/%7B%22v2charts%22%3A%5B%7B%22metrics%22%3A%5B%7B%22resourceMetadata%22%3A%7B%22id%22%3A%22%2Fsubscriptions%2F12345678-aaaa-bbbb-cccc-123456789abc%2FresourceGroups%2Fgrafanastaging%2Fproviders%2FMicrosoft.Compute%2FvirtualMachines%2Fgrafana%22%7D%2C%22name%22%3A%22%22%2C%22aggregationType%22%3A3%2C%22namespace%22%3A%22%22%2C` + `%22metricVisualization%22%3A%7B%22displayName%22%3A%22%22%2C%22resourceDisplayName%22%3A%22grafana%22%7D%7D%5D%7D%5D%7D`) minLink := makeTestDataLink(`http://ds/#blade/Microsoft_Azure_MonitoringMetrics/Metrics.ReactView/Referer/MetricsExplorer/TimeContext/%7B%22absolute%22%3A%7B%22startTime%22%3A%220001-01-01T00%3A00%3A00Z%22%2C%22endTime%22%3A%220001-01-01T00%3A00%3A00Z%22%7D%7D/` + - `ChartDefinition/%7B%22v2charts%22%3A%5B%7B%22metrics%22%3A%5B%7B%22resourceMetadata%22%3A%7B%22id%22%3A%22%2Fsubscriptions%2F%2FresourceGroups%2F%2Fproviders%2F%2Fgrafana%22%7D%2C%22name%22%3A%22%22%2C%22aggregationType%22%3A2%2C%22namespace%22%3A%22%22%2C` + + `ChartDefinition/%7B%22v2charts%22%3A%5B%7B%22metrics%22%3A%5B%7B%22resourceMetadata%22%3A%7B%22id%22%3A%22%2Fsubscriptions%2F12345678-aaaa-bbbb-cccc-123456789abc%2FresourceGroups%2Fgrafanastaging%2Fproviders%2FMicrosoft.Compute%2FvirtualMachines%2Fgrafana%22%7D%2C%22name%22%3A%22%22%2C%22aggregationType%22%3A2%2C%22namespace%22%3A%22%22%2C` + `%22metricVisualization%22%3A%7B%22displayName%22%3A%22%22%2C%22resourceDisplayName%22%3A%22grafana%22%7D%7D%5D%7D%5D%7D`) countLink := makeTestDataLink(`http://ds/#blade/Microsoft_Azure_MonitoringMetrics/Metrics.ReactView/Referer/MetricsExplorer/TimeContext/%7B%22absolute%22%3A%7B%22startTime%22%3A%220001-01-01T00%3A00%3A00Z%22%2C%22endTime%22%3A%220001-01-01T00%3A00%3A00Z%22%7D%7D/` + - `ChartDefinition/%7B%22v2charts%22%3A%5B%7B%22metrics%22%3A%5B%7B%22resourceMetadata%22%3A%7B%22id%22%3A%22%2Fsubscriptions%2F%2FresourceGroups%2F%2Fproviders%2F%2Fgrafana%22%7D%2C%22name%22%3A%22%22%2C%22aggregationType%22%3A7%2C%22namespace%22%3A%22%22%2C` + + `ChartDefinition/%7B%22v2charts%22%3A%5B%7B%22metrics%22%3A%5B%7B%22resourceMetadata%22%3A%7B%22id%22%3A%22%2Fsubscriptions%2F12345678-aaaa-bbbb-cccc-123456789abc%2FresourceGroups%2Fgrafanastaging%2Fproviders%2FMicrosoft.Compute%2FvirtualMachines%2Fgrafana%22%7D%2C%22name%22%3A%22%22%2C%22aggregationType%22%3A7%2C%22namespace%22%3A%22%22%2C` + `%22metricVisualization%22%3A%7B%22displayName%22%3A%22%22%2C%22resourceDisplayName%22%3A%22grafana%22%7D%7D%5D%7D%5D%7D`) tests := []struct { @@ -242,6 +246,7 @@ func TestAzureMonitorParseResponse(t *testing.T) { mockQuery: &types.AzureMonitorQuery{ UrlComponents: map[string]string{ "resourceName": "grafana", + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", }, Params: url.Values{ "aggregation": {"Average"}, @@ -263,6 +268,7 @@ func TestAzureMonitorParseResponse(t *testing.T) { mockQuery: &types.AzureMonitorQuery{ UrlComponents: map[string]string{ "resourceName": "grafana", + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", }, Params: url.Values{ "aggregation": {"Total"}, @@ -284,6 +290,7 @@ func TestAzureMonitorParseResponse(t *testing.T) { mockQuery: &types.AzureMonitorQuery{ UrlComponents: map[string]string{ "resourceName": "grafana", + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", }, Params: url.Values{ "aggregation": {"Maximum"}, @@ -305,6 +312,7 @@ func TestAzureMonitorParseResponse(t *testing.T) { mockQuery: &types.AzureMonitorQuery{ UrlComponents: map[string]string{ "resourceName": "grafana", + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", }, Params: url.Values{ "aggregation": {"Minimum"}, @@ -326,6 +334,7 @@ func TestAzureMonitorParseResponse(t *testing.T) { mockQuery: &types.AzureMonitorQuery{ UrlComponents: map[string]string{ "resourceName": "grafana", + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", }, Params: url.Values{ "aggregation": {"Count"}, @@ -347,6 +356,7 @@ func TestAzureMonitorParseResponse(t *testing.T) { mockQuery: &types.AzureMonitorQuery{ UrlComponents: map[string]string{ "resourceName": "grafana", + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", }, Params: url.Values{ "aggregation": {"Average"}, @@ -382,6 +392,7 @@ func TestAzureMonitorParseResponse(t *testing.T) { Alias: "custom {{resourcegroup}} {{namespace}} {{resourceName}} {{metric}}", UrlComponents: map[string]string{ "resourceName": "grafana", + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", }, Params: url.Values{ "aggregation": {"Total"}, @@ -404,6 +415,7 @@ func TestAzureMonitorParseResponse(t *testing.T) { Alias: "{{dimensionname}}={{DimensionValue}}", UrlComponents: map[string]string{ "resourceName": "grafana", + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", }, Params: url.Values{ "aggregation": {"Average"}, @@ -441,6 +453,7 @@ func TestAzureMonitorParseResponse(t *testing.T) { Alias: "{{resourcegroup}} {Blob Type={{blobtype}}, Tier={{Tier}}}", UrlComponents: map[string]string{ "resourceName": "grafana", + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", }, Params: url.Values{ "aggregation": {"Average"}, @@ -479,6 +492,7 @@ func TestAzureMonitorParseResponse(t *testing.T) { Alias: "custom", UrlComponents: map[string]string{ "resourceName": "grafana", + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", }, Params: url.Values{ "aggregation": {"Average"}, @@ -494,6 +508,57 @@ func TestAzureMonitorParseResponse(t *testing.T) { }).SetConfig(&data.FieldConfig{DisplayName: "custom", Links: []data.DataLink{averageLink}})), }, }, + { + name: "with legacy azure monitor query properties and without a resource uri", + responseFile: "2-azure-monitor-response-total.json", + mockQuery: &types.AzureMonitorQuery{ + Alias: "custom {{resourcegroup}} {{namespace}} {{resourceName}} {{metric}}", + UrlComponents: map[string]string{ + "subscription": "12345678-aaaa-bbbb-cccc-123456789abc", + "resourceGroup": "grafanastaging", + "metricDefinition": "Microsoft.Compute/virtualMachines", + "resourceName": "grafana", + }, + Params: url.Values{ + "aggregation": {"Total"}, + }, + }, + expectedFrames: data.Frames{ + data.NewFrame("", + data.NewField("Time", nil, + makeDates(time.Date(2019, 2, 9, 13, 29, 0, 0, time.UTC), 5, time.Minute), + ).SetConfig(&data.FieldConfig{Links: []data.DataLink{totalLink}}), + data.NewField("Percentage CPU", nil, []*float64{ + ptr.Float64(8.26), ptr.Float64(8.7), ptr.Float64(14.82), ptr.Float64(10.07), ptr.Float64(8.52), + }).SetConfig(&data.FieldConfig{Unit: "percent", DisplayName: "custom grafanastaging Microsoft.Compute/virtualMachines grafana Percentage CPU", Links: []data.DataLink{totalLink}})), + }, + }, + { + name: "with legacy azure monitor query properties and with a resource uri it should use the resource uri", + responseFile: "2-azure-monitor-response-total.json", + mockQuery: &types.AzureMonitorQuery{ + Alias: "custom {{resourcegroup}} {{namespace}} {{resourceName}} {{metric}}", + UrlComponents: map[string]string{ + "resourceURI": "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/grafana", + "subscription": "12345678-aaaa-bbbb-cccc-123456789abc-nope", + "resourceGroup": "grafanastaging-nope", + "metricDefinition": "Microsoft.Compute/virtualMachines-nope", + "resourceName": "grafana", + }, + Params: url.Values{ + "aggregation": {"Total"}, + }, + }, + expectedFrames: data.Frames{ + data.NewFrame("", + data.NewField("Time", nil, + makeDates(time.Date(2019, 2, 9, 13, 29, 0, 0, time.UTC), 5, time.Minute), + ).SetConfig(&data.FieldConfig{Links: []data.DataLink{totalLink}}), + data.NewField("Percentage CPU", nil, []*float64{ + ptr.Float64(8.26), ptr.Float64(8.7), ptr.Float64(14.82), ptr.Float64(10.07), ptr.Float64(8.52), + }).SetConfig(&data.FieldConfig{Unit: "percent", DisplayName: "custom grafanastaging Microsoft.Compute/virtualMachines grafana Percentage CPU", Links: []data.DataLink{totalLink}})), + }, + }, } datasource := &AzureMonitorDatasource{} @@ -609,3 +674,21 @@ func TestAzureMonitorCreateRequest(t *testing.T) { }) } } + +func TestExtractResourceNameFromMetricsURL(t *testing.T) { + t.Run("it should extract the resourceName from a well-formed Metrics URL", func(t *testing.T) { + url := "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/Grafana-Test.VM/providers/microsoft.insights/metrics" + expected := "Grafana-Test.VM" + require.Equal(t, expected, extractResourceNameFromMetricsURL((url))) + }) + t.Run("it should extract the resourceName from a well-formed Metrics URL in a case insensitive manner", func(t *testing.T) { + url := "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/Grafana-Test.VM/pRoViDeRs/MiCrOsOfT.iNsIgHtS/mEtRiCs" + expected := "Grafana-Test.VM" + require.Equal(t, expected, extractResourceNameFromMetricsURL((url))) + }) + t.Run("it should return an empty string if no match is found", func(t *testing.T) { + url := "/subscriptions/12345678-aaaa-bbbb-cccc-123456789abc/resourceGroups/grafanastaging/providers/Microsoft.Compute/virtualMachines/Grafana-Test.VM/providers/microsoft.insights/nope-this-part-does-not-match" + expected := "" + require.Equal(t, expected, extractResourceNameFromMetricsURL((url))) + }) +} diff --git a/pkg/tsdb/azuremonitor/metrics/url-builder.go b/pkg/tsdb/azuremonitor/metrics/url-builder.go index 8c3fad8fd37..432e80a98aa 100644 --- a/pkg/tsdb/azuremonitor/metrics/url-builder.go +++ b/pkg/tsdb/azuremonitor/metrics/url-builder.go @@ -18,7 +18,7 @@ type urlBuilder struct { ResourceName string } -func (params *urlBuilder) buildMetricsURLFromLegacyQuery() string { +func (params *urlBuilder) buildResourceURIFromLegacyQuery() string { subscription := params.Subscription if params.Subscription == "" { @@ -54,7 +54,7 @@ func (params *urlBuilder) BuildMetricsURL() string { // Prior to Grafana 9, we had a legacy query object rather than a resourceURI, so we manually create the resource URI if resourceURI == "" { - resourceURI = params.buildMetricsURLFromLegacyQuery() + resourceURI = params.buildResourceURIFromLegacyQuery() } return fmt.Sprintf("%s/providers/microsoft.insights/metrics", resourceURI) diff --git a/pkg/tsdb/legacydata/service/service.go b/pkg/tsdb/legacydata/service/service.go index b946d255252..5c7f5ac5b7a 100644 --- a/pkg/tsdb/legacydata/service/service.go +++ b/pkg/tsdb/legacydata/service/service.go @@ -40,6 +40,11 @@ func (h *Service) HandleRequest(ctx context.Context, ds *models.DataSource, quer return legacydata.DataResponse{}, err } + decryptedValues, err := h.dataSourcesService.DecryptedValues(ctx, ds) + if err != nil { + return legacydata.DataResponse{}, err + } + instanceSettings := &backend.DataSourceInstanceSettings{ ID: ds.Id, Name: ds.Name, @@ -49,7 +54,7 @@ func (h *Service) HandleRequest(ctx context.Context, ds *models.DataSource, quer BasicAuthEnabled: ds.BasicAuth, BasicAuthUser: ds.BasicAuthUser, JSONData: jsonDataBytes, - DecryptedSecureJSONData: h.dataSourcesService.DecryptedValues(ds), + DecryptedSecureJSONData: decryptedValues, Updated: ds.Updated, UID: ds.Uid, } diff --git a/pkg/tsdb/legacydata/service/service_test.go b/pkg/tsdb/legacydata/service/service_test.go index 7e895da093a..33bef6cf3b5 100644 --- a/pkg/tsdb/legacydata/service/service_test.go +++ b/pkg/tsdb/legacydata/service/service_test.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/oauthtoken" "github.com/grafana/grafana/pkg/services/secrets/fakes" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/legacydata" @@ -38,8 +39,9 @@ func TestHandleRequest(t *testing.T) { actualReq = req return backend.NewQueryDataResponse(), nil } + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) s := ProvideService(client, nil, dsService) ds := &models.DataSource{Id: 12, Type: "unregisteredType", JsonData: simplejson.New()} diff --git a/pkg/tsdb/loki/api.go b/pkg/tsdb/loki/api.go index 87e340dfc8c..d1d1eb6f813 100644 --- a/pkg/tsdb/loki/api.go +++ b/pkg/tsdb/loki/api.go @@ -25,7 +25,7 @@ func newLokiAPI(client *http.Client, url string, log log.Logger) *LokiAPI { return &LokiAPI{client: client, url: url, log: log} } -func makeRequest(ctx context.Context, lokiDsUrl string, query lokiQuery) (*http.Request, error) { +func makeDataRequest(ctx context.Context, lokiDsUrl string, query lokiQuery) (*http.Request, error) { qs := url.Values{} qs.Set("query", query.Expr) @@ -135,8 +135,8 @@ func makeLokiError(body io.ReadCloser) error { return fmt.Errorf("%v", errorMessage) } -func (api *LokiAPI) Query(ctx context.Context, query lokiQuery) (*loghttp.QueryResponse, error) { - req, err := makeRequest(ctx, api.url, query) +func (api *LokiAPI) DataQuery(ctx context.Context, query lokiQuery) (*loghttp.QueryResponse, error) { + req, err := makeDataRequest(ctx, api.url, query) if err != nil { return nil, err } @@ -164,3 +164,41 @@ func (api *LokiAPI) Query(ctx context.Context, query lokiQuery) (*loghttp.QueryR return &response, nil } + +func makeRawRequest(ctx context.Context, lokiDsUrl string, resourceURL string) (*http.Request, error) { + lokiUrl, err := url.Parse(lokiDsUrl) + if err != nil { + return nil, err + } + + url, err := lokiUrl.Parse(resourceURL) + if err != nil { + return nil, err + } + + return http.NewRequestWithContext(ctx, "GET", url.String(), nil) +} + +func (api *LokiAPI) RawQuery(ctx context.Context, resourceURL string) ([]byte, error) { + req, err := makeRawRequest(ctx, api.url, resourceURL) + if err != nil { + return nil, err + } + + resp, err := api.client.Do(req) + if err != nil { + return nil, err + } + + defer func() { + if err := resp.Body.Close(); err != nil { + api.log.Warn("Failed to close response body", "err", err) + } + }() + + if resp.StatusCode/100 != 2 { + return nil, makeLokiError(resp.Body) + } + + return io.ReadAll(resp.Body) +} diff --git a/pkg/tsdb/loki/loki.go b/pkg/tsdb/loki/loki.go index e2112d33623..67d5b8b6c8d 100644 --- a/pkg/tsdb/loki/loki.go +++ b/pkg/tsdb/loki/loki.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "regexp" + "strings" "sync" "github.com/grafana/grafana-plugin-sdk-go/backend" @@ -25,8 +26,9 @@ type Service struct { } var ( - _ backend.QueryDataHandler = (*Service)(nil) - _ backend.StreamHandler = (*Service)(nil) + _ backend.QueryDataHandler = (*Service)(nil) + _ backend.StreamHandler = (*Service)(nil) + _ backend.CallResourceHandler = (*Service)(nil) ) func ProvideService(httpClientProvider httpclient.Provider, tracer tracing.Tracer) *Service { @@ -89,6 +91,40 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst } } +func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { + url := req.URL + + // a very basic is-this-url-valid check + if req.Method != "GET" { + return fmt.Errorf("invalid resource method: %s", req.Method) + } + if (!strings.HasPrefix(url, "/loki/api/v1/labels?")) && + (!strings.HasPrefix(url, "/loki/api/v1/label/")) && // the `/label/$label_name/values` form + (!strings.HasPrefix(url, "/loki/api/v1/series?")) { + return fmt.Errorf("invalid resource URL: %s", url) + } + + dsInfo, err := s.getDSInfo(req.PluginContext) + if err != nil { + return err + } + + api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, s.plog) + bytes, err := api.RawQuery(ctx, url) + + if err != nil { + return err + } + + return sender.Send(&backend.CallResourceResponse{ + Status: http.StatusOK, + Headers: map[string][]string{ + "content-type": {"application/json"}, + }, + Body: bytes, + }) +} + func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { result := backend.NewQueryDataResponse() @@ -129,7 +165,7 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) // we extracted this part of the functionality to make it easy to unit-test it func runQuery(ctx context.Context, api *LokiAPI, query *lokiQuery) (data.Frames, error) { - value, err := api.Query(ctx, *query) + value, err := api.DataQuery(ctx, *query) if err != nil { return data.Frames{}, err } diff --git a/public/app/angular/angular_wrappers.ts b/public/app/angular/angular_wrappers.ts index 6827c913d8d..11d77c90a1d 100644 --- a/public/app/angular/angular_wrappers.ts +++ b/public/app/angular/angular_wrappers.ts @@ -15,8 +15,6 @@ import { FolderPicker } from 'app/core/components/Select/FolderPicker'; import { TimePickerSettings } from 'app/features/dashboard/components/DashboardSettings/TimePickerSettings'; import { AnnotationQueryEditor as CloudMonitoringAnnotationQueryEditor } from 'app/plugins/datasource/cloud-monitoring/components/AnnotationQueryEditor'; import { QueryEditor as CloudMonitoringQueryEditor } from 'app/plugins/datasource/cloud-monitoring/components/QueryEditor'; -import { AnnotationQueryEditor as CloudWatchAnnotationQueryEditor } from 'app/plugins/datasource/cloudwatch/components/AnnotationQueryEditor'; -import QueryEditor from 'app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/QueryEditor'; import EmptyListCTA from '../core/components/EmptyListCTA/EmptyListCTA'; import { Footer } from '../core/components/Footer/Footer'; @@ -125,11 +123,7 @@ export function registerAngularDirectives() { ['datasource', { watchDepth: 'reference' }], ['templateSrv', { watchDepth: 'reference' }], ]); - react2AngularDirective('cloudwatchAnnotationQueryEditor', CloudWatchAnnotationQueryEditor, [ - 'query', - 'onChange', - ['datasource', { watchDepth: 'reference' }], - ]); + react2AngularDirective('secretFormField', SecretFormField, [ 'value', 'isConfigured', @@ -198,12 +192,6 @@ export function registerAngularDirectives() { ['onHideTimePickerChange', { watchDepth: 'reference', wrapApply: true }], ]); - react2AngularDirective('azureMonitorQueryEditor', QueryEditor, [ - 'query', - ['datasource', { watchDepth: 'reference' }], - 'onChange', - ]); - react2AngularDirective('clipboardButton', ClipboardButton, [ ['getText', { watchDepth: 'reference', wrapApply: true }], ]); diff --git a/public/app/core/components/AppNotifications/AppNotificationItem.tsx b/public/app/core/components/AppNotifications/AppNotificationItem.tsx index 3ec1a2b59a2..a02b1e2edb6 100644 --- a/public/app/core/components/AppNotifications/AppNotificationItem.tsx +++ b/public/app/core/components/AppNotifications/AppNotificationItem.tsx @@ -3,6 +3,7 @@ import React from 'react'; import { useEffectOnce } from 'react-use'; import { GrafanaTheme2 } from '@grafana/data'; +import { config } from '@grafana/runtime'; import { Alert, useStyles2 } from '@grafana/ui'; import { AppNotification, timeoutMap } from 'app/types'; @@ -20,6 +21,8 @@ export default function AppNotificationItem({ appNotification, onClearNotificati }, timeoutMap[appNotification.severity]); }); + const showTraceId = config.featureToggles.tracing && appNotification.traceId; + return (
{appNotification.component || appNotification.text} - {appNotification.traceId && Trace ID: {appNotification.traceId}} + {showTraceId && Trace ID: {appNotification.traceId}}
); diff --git a/public/app/core/components/AppNotifications/StoredNotificationItem.tsx b/public/app/core/components/AppNotifications/StoredNotificationItem.tsx index d6fe2992b5a..57b612db7ae 100644 --- a/public/app/core/components/AppNotifications/StoredNotificationItem.tsx +++ b/public/app/core/components/AppNotifications/StoredNotificationItem.tsx @@ -3,6 +3,7 @@ import { formatDistanceToNow } from 'date-fns'; import React, { ReactNode } from 'react'; import { GrafanaTheme2 } from '@grafana/data'; +import { config } from '@grafana/runtime'; import { Icon, IconButton, IconName, useTheme2 } from '@grafana/ui'; import { getIconFromSeverity } from '@grafana/ui/src/components/Alert/Alert'; @@ -27,6 +28,7 @@ export const StoredNotificationItem = ({ }: Props) => { const theme = useTheme2(); const styles = getStyles(theme, severity); + const showTraceId = config.featureToggles.tracing && traceId; return (
@@ -35,7 +37,7 @@ export const StoredNotificationItem = ({
{title}
{children}
- {traceId && `Trace ID: ${traceId}`} + {showTraceId && `Trace ID: ${traceId}`}
diff --git a/public/app/core/components/NavBar/Next/NavBarNext.tsx b/public/app/core/components/NavBar/Next/NavBarNext.tsx index 1b97427d1a4..a7a3b60d5dd 100644 --- a/public/app/core/components/NavBar/Next/NavBarNext.tsx +++ b/public/app/core/components/NavBar/Next/NavBarNext.tsx @@ -96,7 +96,6 @@ export const NavBarNext = React.memo(() => {
    toggleTheme(true)); } - private globalEsc() { + globalEsc() { const anyDoc = document as any; const activeElement = anyDoc.activeElement; diff --git a/public/app/features/admin/ExportStartButton.tsx b/public/app/features/admin/ExportStartButton.tsx new file mode 100644 index 00000000000..f94d4f5b081 --- /dev/null +++ b/public/app/features/admin/ExportStartButton.tsx @@ -0,0 +1,62 @@ +import { css } from '@emotion/css'; +import React, { useState } from 'react'; + +import { GrafanaTheme2 } from '@grafana/data'; +import { getBackendSrv } from '@grafana/runtime'; +import { Button, CodeEditor, Modal, useTheme2 } from '@grafana/ui'; + +export const ExportStartButton = () => { + const styles = getStyles(useTheme2()); + const [open, setOpen] = useState(false); + const [body, setBody] = useState({ + format: 'git', + git: {}, + }); + const onDismiss = () => setOpen(false); + const doStart = () => { + getBackendSrv() + .post('/api/admin/export', body) + .then((v) => { + console.log('GOT', v); + onDismiss(); + }); + }; + + return ( + <> + +
    + { + setBody(JSON.parse(text)); // force JSON? + }} + /> +
    + + + + +     + + + + ); +}; + +const getStyles = (theme: GrafanaTheme2) => { + return { + wrap: css` + border: 2px solid #111; + `, + }; +}; diff --git a/public/app/features/admin/ExportStatus.tsx b/public/app/features/admin/ExportStatus.tsx new file mode 100644 index 00000000000..6b638b8f358 --- /dev/null +++ b/public/app/features/admin/ExportStatus.tsx @@ -0,0 +1,82 @@ +import { css } from '@emotion/css'; +import React, { useEffect, useState } from 'react'; + +import { GrafanaTheme2, isLiveChannelMessageEvent, isLiveChannelStatusEvent, LiveChannelScope } from '@grafana/data'; +import { getBackendSrv, getGrafanaLiveSrv } from '@grafana/runtime'; +import { Button, useTheme2 } from '@grafana/ui'; + +import { ExportStartButton } from './ExportStartButton'; + +interface ExportStatusMessage { + running: boolean; + target: string; + started: number; + finished: number; + update: number; + count: number; + current: number; + last: string; + status: string; +} + +export const ExportStatus = () => { + const styles = getStyles(useTheme2()); + const [status, setStatus] = useState(); + + useEffect(() => { + const subscription = getGrafanaLiveSrv() + .getStream({ + scope: LiveChannelScope.Grafana, + namespace: 'broadcast', + path: 'export', + }) + .subscribe({ + next: (evt) => { + if (isLiveChannelMessageEvent(evt)) { + setStatus(evt.message); + } else if (isLiveChannelStatusEvent(evt)) { + setStatus(evt.message); + } + }, + }); + return () => { + subscription.unsubscribe(); + }; + }, []); + + if (!status) { + return ( +
    + +
    + ); + } + + return ( +
    + 
    {JSON.stringify(status, null, 2)}
    + {Boolean(!status.running) && } + {Boolean(status.running) && ( + + )} +
    + ); +}; + +const getStyles = (theme: GrafanaTheme2) => { + return { + wrap: css` + border: 4px solid red; + `, + running: css` + border: 4px solid green; + `, + }; +}; diff --git a/public/app/features/admin/ServerStats.tsx b/public/app/features/admin/ServerStats.tsx index 672d1ba2746..1b3cf7799d6 100644 --- a/public/app/features/admin/ServerStats.tsx +++ b/public/app/features/admin/ServerStats.tsx @@ -10,6 +10,7 @@ import { contextSrv } from '../../core/services/context_srv'; import { Loader } from '../plugins/admin/components/Loader'; import { CrawlerStatus } from './CrawlerStatus'; +import { ExportStatus } from './ExportStatus'; import { getServerStats, ServerStat } from './state/apis'; export const ServerStats = () => { @@ -98,6 +99,7 @@ export const ServerStats = () => { )} {config.featureToggles.dashboardPreviews && config.featureToggles.dashboardPreviewsAdmin && } + {config.featureToggles.export && } ); }; diff --git a/public/app/features/alerting/unified/RuleList.test.tsx b/public/app/features/alerting/unified/RuleList.test.tsx index 2b3bcb17330..e802637720c 100644 --- a/public/app/features/alerting/unified/RuleList.test.tsx +++ b/public/app/features/alerting/unified/RuleList.test.tsx @@ -8,20 +8,23 @@ import { byLabelText, byRole, byTestId, byText } from 'testing-library-selector' import { locationService, setDataSourceSrv } from '@grafana/runtime'; import { configureStore } from 'app/store/configureStore'; +import { AccessControlAction } from 'app/types'; import { PromAlertingRuleState, PromApplication } from 'app/types/unified-alerting-dto'; import RuleList from './RuleList'; import { fetchBuildInfo } from './api/buildInfo'; import { fetchRules } from './api/prometheus'; -import { fetchRulerRules, deleteRulerRulesGroup, deleteNamespace, setRulerRuleGroup } from './api/ruler'; +import { deleteNamespace, deleteRulerRulesGroup, fetchRulerRules, setRulerRuleGroup } from './api/ruler'; import { + enableRBAC, + grantUserPermissions, mockDataSource, + MockDataSourceSrv, mockPromAlert, mockPromAlertingRule, mockPromRecordingRule, mockPromRuleGroup, mockPromRuleNamespace, - MockDataSourceSrv, somePromRules, someRulerRules, } from './mocks'; @@ -101,6 +104,8 @@ const ui = { moreErrorsButton: byRole('button', { name: /more errors/ }), editCloudGroupIcon: byTestId('edit-group'), + newRuleButton: byRole('link', { name: 'New alert rule' }), + editGroupModal: { namespaceInput: byLabelText('Namespace'), ruleGroupInput: byLabelText('Rule group'), @@ -631,4 +636,105 @@ describe('RuleList', () => { ); }); }); + + describe('RBAC Enabled', () => { + describe('Grafana Managed Alerts', () => { + it('New alert button should be visible when the user has alert rule create and folder read permissions and no rules exists', async () => { + enableRBAC(); + + grantUserPermissions([ + AccessControlAction.FoldersRead, + AccessControlAction.AlertingRuleCreate, + AccessControlAction.AlertingRuleRead, + ]); + + mocks.getAllDataSourcesMock.mockReturnValue([]); + setDataSourceSrv(new MockDataSourceSrv({})); + mocks.api.fetchRules.mockResolvedValue([]); + mocks.api.fetchRulerRules.mockResolvedValue({}); + + renderRuleList(); + + await waitFor(() => expect(mocks.api.fetchRules).toHaveBeenCalledTimes(1)); + expect(ui.newRuleButton.get()).toBeInTheDocument(); + }); + + it('New alert button should be visible when the user has alert rule create and folder read permissions and rules already exists', async () => { + enableRBAC(); + + grantUserPermissions([ + AccessControlAction.FoldersRead, + AccessControlAction.AlertingRuleCreate, + AccessControlAction.AlertingRuleRead, + ]); + + mocks.getAllDataSourcesMock.mockReturnValue([]); + setDataSourceSrv(new MockDataSourceSrv({})); + mocks.api.fetchRules.mockResolvedValue(somePromRules('grafana')); + mocks.api.fetchRulerRules.mockResolvedValue(someRulerRules); + + renderRuleList(); + + await waitFor(() => expect(mocks.api.fetchRules).toHaveBeenCalledTimes(1)); + expect(ui.newRuleButton.get()).toBeInTheDocument(); + }); + }); + + describe('Cloud Alerts', () => { + it('New alert button should be visible when the user has the alert rule external write and datasource read permissions and no rules exists', async () => { + enableRBAC(); + + grantUserPermissions([ + // AccessControlAction.AlertingRuleRead, + AccessControlAction.DataSourcesRead, + AccessControlAction.AlertingRuleExternalRead, + AccessControlAction.AlertingRuleExternalWrite, + ]); + + mocks.getAllDataSourcesMock.mockReturnValue([dataSources.prom]); + setDataSourceSrv(new MockDataSourceSrv({ prom: dataSources.prom })); + mocks.api.fetchBuildInfo.mockResolvedValue({ + application: PromApplication.Cortex, + features: { + rulerApiEnabled: true, + }, + }); + + mocks.api.fetchRules.mockResolvedValue([]); + mocks.api.fetchRulerRules.mockResolvedValue({}); + + renderRuleList(); + + await waitFor(() => expect(mocks.api.fetchRules).toHaveBeenCalledTimes(1)); + expect(ui.newRuleButton.get()).toBeInTheDocument(); + }); + + it('New alert button should be visible when the user has the alert rule external write and data source read permissions and rules already exists', async () => { + enableRBAC(); + + grantUserPermissions([ + AccessControlAction.DataSourcesRead, + AccessControlAction.AlertingRuleExternalRead, + AccessControlAction.AlertingRuleExternalWrite, + ]); + + mocks.getAllDataSourcesMock.mockReturnValue([dataSources.prom]); + setDataSourceSrv(new MockDataSourceSrv({ prom: dataSources.prom })); + mocks.api.fetchBuildInfo.mockResolvedValue({ + application: PromApplication.Cortex, + features: { + rulerApiEnabled: true, + }, + }); + + mocks.api.fetchRules.mockResolvedValue(somePromRules('Cortex')); + mocks.api.fetchRulerRules.mockResolvedValue(someRulerRules); + + renderRuleList(); + + await waitFor(() => expect(mocks.api.fetchRules).toHaveBeenCalledTimes(1)); + expect(ui.newRuleButton.get()).toBeInTheDocument(); + }); + }); + }); }); diff --git a/public/app/features/alerting/unified/components/rule-editor/RuleFolderPicker.tsx b/public/app/features/alerting/unified/components/rule-editor/RuleFolderPicker.tsx index 1e65372a238..518348877aa 100644 --- a/public/app/features/alerting/unified/components/rule-editor/RuleFolderPicker.tsx +++ b/public/app/features/alerting/unified/components/rule-editor/RuleFolderPicker.tsx @@ -1,6 +1,7 @@ import React, { FC } from 'react'; import { FolderPicker, Props as FolderPickerProps } from 'app/core/components/Select/FolderPicker'; +import { PermissionLevelString } from 'app/types'; export interface Folder { title: string; @@ -12,5 +13,12 @@ export interface Props extends Omit = ({ value, ...props }) => ( - + ); diff --git a/public/app/features/alerting/unified/mocks.ts b/public/app/features/alerting/unified/mocks.ts index 0dcb67cca28..70c9070c8fd 100644 --- a/public/app/features/alerting/unified/mocks.ts +++ b/public/app/features/alerting/unified/mocks.ts @@ -6,7 +6,8 @@ import { DataSourceRef, ScopedVars, } from '@grafana/data'; -import { DataSourceSrv, GetDataSourceListFilters, config } from '@grafana/runtime'; +import { config, DataSourceSrv, GetDataSourceListFilters } from '@grafana/runtime'; +import { contextSrv } from 'app/core/services/context_srv'; import { DatasourceSrv } from 'app/features/plugins/datasource_srv'; import { AlertmanagerAlert, @@ -18,8 +19,8 @@ import { Silence, SilenceState, } from 'app/plugins/datasource/alertmanager/types'; -import { FolderDTO } from 'app/types'; -import { AlertingRule, Alert, RecordingRule, RuleGroup, RuleNamespace, CombinedRule } from 'app/types/unified-alerting'; +import { AccessControlAction, FolderDTO } from 'app/types'; +import { Alert, AlertingRule, CombinedRule, RecordingRule, RuleGroup, RuleNamespace } from 'app/types/unified-alerting'; import { GrafanaAlertStateDecision, GrafanaRuleDefinition, @@ -465,3 +466,13 @@ export const mockFolder = (partial?: Partial): FolderDTO => { ...partial, }; }; + +export const enableRBAC = () => { + jest.spyOn(contextSrv, 'accessControlEnabled').mockReturnValue(true); +}; + +export const grantUserPermissions = (permissions: AccessControlAction[]) => { + jest + .spyOn(contextSrv, 'hasPermission') + .mockImplementation((action) => permissions.includes(action as AccessControlAction)); +}; diff --git a/public/app/features/alerting/unified/state/actions.ts b/public/app/features/alerting/unified/state/actions.ts index 071aefa3377..51486d9859b 100644 --- a/public/app/features/alerting/unified/state/actions.ts +++ b/public/app/features/alerting/unified/state/actions.ts @@ -402,6 +402,9 @@ export const saveRuleFormAction = createAsyncThunk( const newLocation = `/alerting/${encodeURIComponent(stringifiedIdentifier)}/edit`; if (locationService.getLocation().pathname !== newLocation) { locationService.replace(newLocation); + } else { + // refresh the details of the current editable rule after saving + thunkAPI.dispatch(fetchEditableRuleAction(identifier)); } } })() diff --git a/public/app/features/alerting/unified/utils/access-control.ts b/public/app/features/alerting/unified/utils/access-control.ts index 0f03e12af10..9b736179118 100644 --- a/public/app/features/alerting/unified/utils/access-control.ts +++ b/public/app/features/alerting/unified/utils/access-control.ts @@ -108,9 +108,11 @@ export function evaluateAccess(actions: AccessControlAction[], fallBackUserRoles export function getRulesAccess() { return { canCreateGrafanaRules: - contextSrv.hasEditPermissionInFolders && + contextSrv.hasAccess(AccessControlAction.FoldersRead, contextSrv.isEditor) && contextSrv.hasAccess(rulesPermissions.create.grafana, contextSrv.isEditor), - canCreateCloudRules: contextSrv.hasAccess(rulesPermissions.create.external, contextSrv.isEditor), + canCreateCloudRules: + contextSrv.hasAccess(AccessControlAction.DataSourcesRead, contextSrv.isEditor) && + contextSrv.hasAccess(rulesPermissions.create.external, contextSrv.isEditor), canEditRules: (rulesSourceName: string) => contextSrv.hasAccess(getRulesPermissions(rulesSourceName).update, contextSrv.isEditor), }; diff --git a/public/app/features/alerting/unified/utils/rulerClient.ts b/public/app/features/alerting/unified/utils/rulerClient.ts index f8fd8ffab05..0398bd47872 100644 --- a/public/app/features/alerting/unified/utils/rulerClient.ts +++ b/public/app/features/alerting/unified/utils/rulerClient.ts @@ -188,29 +188,23 @@ export function getRulerClient(rulerConfig: RulerDataSourceConfig): RulerClient await setRulerRuleGroup(rulerConfig, namespace, payload); - return { uid: '', ruleSourceName: GRAFANA_RULES_SOURCE_NAME }; + return { uid: newRule.grafana_alert.uid ?? '', ruleSourceName: GRAFANA_RULES_SOURCE_NAME }; }; - // we can't move the rule in a single atomic operation so we have to - // 1. add the rule to the new group - // 2. remove the rule from the old one + // move the rule to another namespace / groupname const moveGrafanaRule = async ( namespace: string, group: { name: string; interval: string }, existingRule: RuleWithLocation, newRule: PostableRuleGrafanaRuleDTO ): Promise => { + // make sure our updated alert has the same UID as before + // that way the rule is automatically moved to the new namespace / group name + copyGrafanaUID(existingRule, newRule); + // add the new rule to the requested namespace and group const identifier = await addRuleToNamespaceAndGroup(namespace, group, newRule); - // remove the rule from the previous namespace and group - await deleteRule({ - ruleSourceName: existingRule.ruleSourceName, - namespace: existingRule.namespace, - group: existingRule.group, - rule: newRule as RulerGrafanaRuleDTO, - }); - return identifier; }; @@ -219,19 +213,13 @@ export function getRulerClient(rulerConfig: RulerDataSourceConfig): RulerClient newRule: PostableRuleGrafanaRuleDTO, interval: string ): Promise => { - // type guard to make sure we're working with a Grafana managed rule - if (!isGrafanaRulerRule(existingRule.rule)) { - throw new Error('The rule is not a Grafana managed rule'); - } - // make sure our updated alert has the same UID as before - const uid = existingRule.rule.grafana_alert.uid; - newRule.grafana_alert.uid = uid; + copyGrafanaUID(existingRule, newRule); // create the new array of rules we want to send to the group const newRules = existingRule.group.rules .filter((rule): rule is RulerGrafanaRuleDTO => isGrafanaRulerRule(rule)) - .filter((rule) => rule.grafana_alert.uid !== uid) + .filter((rule) => rule.grafana_alert.uid !== existingRule.rule.grafana_alert.uid) .concat(newRule as RulerGrafanaRuleDTO); await setRulerRuleGroup(rulerConfig, existingRule.namespace, { @@ -240,7 +228,7 @@ export function getRulerClient(rulerConfig: RulerDataSourceConfig): RulerClient rules: newRules, }); - return { uid: '', ruleSourceName: GRAFANA_RULES_SOURCE_NAME }; + return { uid: existingRule.rule.grafana_alert.uid, ruleSourceName: GRAFANA_RULES_SOURCE_NAME }; }; // Would be nice to somehow align checking of ruler type between different methods @@ -252,3 +240,17 @@ export function getRulerClient(rulerConfig: RulerDataSourceConfig): RulerClient saveGrafanaRule, }; } + +//copy the Grafana rule UID from the old rule to the new rule +function copyGrafanaUID( + oldRule: RuleWithLocation, + newRule: PostableRuleGrafanaRuleDTO +): asserts oldRule is RuleWithLocation { + // type guard to make sure we're working with a Grafana managed rule + if (!isGrafanaRulerRule(oldRule.rule)) { + throw new Error('The rule is not a Grafana managed rule'); + } + + const uid = oldRule.rule.grafana_alert.uid; + newRule.grafana_alert.uid = uid; +} diff --git a/public/app/features/commandPalette/CommandPalette.tsx b/public/app/features/commandPalette/CommandPalette.tsx index db0e978be14..fe17e292fb8 100644 --- a/public/app/features/commandPalette/CommandPalette.tsx +++ b/public/app/features/commandPalette/CommandPalette.tsx @@ -33,8 +33,7 @@ import getGlobalActions from './actions/global.static.actions'; export const CommandPalette = () => { const styles = useStyles2(getSearchStyles); const [actions, setActions] = useState([]); - const { notHidden, query, showing } = useKBar((state) => ({ - notHidden: state.visualState !== VisualState.hidden, + const { query, showing } = useKBar((state) => ({ showing: state.visualState === VisualState.showing, })); const isNotLogin = locationService.getLocation().pathname !== '/login'; @@ -45,12 +44,6 @@ export const CommandPalette = () => { }; }); - keybindingSrv.bind('esc', () => { - if (notHidden) { - query.setVisualState(VisualState.animatingOut); - } - }); - useEffect(() => { (async () => { if (isNotLogin) { @@ -65,7 +58,18 @@ export const CommandPalette = () => { useEffect(() => { if (showing) { reportInteraction('commandPalette_opened'); + + keybindingSrv.bindGlobal('esc', () => { + query.setVisualState(VisualState.animatingOut); + }); } + + return () => { + keybindingSrv.bindGlobal('esc', () => { + keybindingSrv.globalEsc(); + }); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps }, [showing]); useRegisterActions(actions, [actions]); diff --git a/public/app/features/explore/ExploreDrawer.tsx b/public/app/features/explore/ExploreDrawer.tsx index 8220dc748d9..10fa3489d5e 100644 --- a/public/app/features/explore/ExploreDrawer.tsx +++ b/public/app/features/explore/ExploreDrawer.tsx @@ -9,9 +9,9 @@ import { stylesFactory, useTheme2 } from '@grafana/ui'; // Types -const drawerSlide = keyframes` +const drawerSlide = (theme: GrafanaTheme2) => keyframes` 0% { - transform: translateY(400px); + transform: translateY(${theme.components.horizontalDrawer.defaultHeight}px); } 100% { @@ -32,7 +32,7 @@ const getStyles = stylesFactory((theme: GrafanaTheme2) => { `, drawerActive: css` opacity: 1; - animation: 0.5s ease-out ${drawerSlide}; + animation: 0.5s ease-out ${drawerSlide(theme)}; `, rzHandle: css` background: ${theme.colors.secondary.main}; @@ -66,7 +66,7 @@ export function ExploreDrawer(props: Props) { return ( {}}> + ); diff --git a/public/app/features/explore/RichHistory/RichHistoryContainer.tsx b/public/app/features/explore/RichHistory/RichHistoryContainer.tsx index 0824dae0aac..bacf5c7254e 100644 --- a/public/app/features/explore/RichHistory/RichHistoryContainer.tsx +++ b/public/app/features/explore/RichHistory/RichHistoryContainer.tsx @@ -2,6 +2,7 @@ import React, { useEffect, useState } from 'react'; import { connect, ConnectedProps } from 'react-redux'; +import { useTheme2 } from '@grafana/ui'; // Types import { ExploreItemState, StoreState } from 'app/types'; import { ExploreId } from 'app/types/explore'; @@ -54,7 +55,8 @@ interface OwnProps { export type Props = ConnectedProps & OwnProps; export function RichHistoryContainer(props: Props) { - const [height, setHeight] = useState(400); + const theme = useTheme2(); + const [height, setHeight] = useState(theme.components.horizontalDrawer.defaultHeight); const { richHistory, diff --git a/public/app/features/inspector/styles.ts b/public/app/features/inspector/styles.ts index 66aef0e653d..123ccb131a7 100644 --- a/public/app/features/inspector/styles.ts +++ b/public/app/features/inspector/styles.ts @@ -26,6 +26,7 @@ export const getPanelInspectorStyles = stylesFactory(() => { content: css` flex-grow: 1; height: 100%; + overflow: scroll; `, editor: css` font-family: monospace; diff --git a/public/app/features/query/components/QueryEditorRow.tsx b/public/app/features/query/components/QueryEditorRow.tsx index 4ed1185e9a2..f545f04c29b 100644 --- a/public/app/features/query/components/QueryEditorRow.tsx +++ b/public/app/features/query/components/QueryEditorRow.tsx @@ -416,6 +416,7 @@ export class QueryEditorRow extends PureComponent this.onClickExample(query)} + query={this.props.query} datasource={datasource} /> diff --git a/public/app/features/search/hooks/useShowDashboardPreviews.ts b/public/app/features/search/hooks/useShowDashboardPreviews.ts index ebddd8b607b..80e50f37e7b 100644 --- a/public/app/features/search/hooks/useShowDashboardPreviews.ts +++ b/public/app/features/search/hooks/useShowDashboardPreviews.ts @@ -6,7 +6,7 @@ import { PREVIEWS_LOCAL_STORAGE_KEY } from '../constants'; export const useShowDashboardPreviews = () => { const previewFeatureEnabled = Boolean(config.featureToggles.dashboardPreviews); - const [showPreviews, setShowPreviews] = useLocalStorage(PREVIEWS_LOCAL_STORAGE_KEY, previewFeatureEnabled); + const [showPreviews, setShowPreviews] = useLocalStorage(PREVIEWS_LOCAL_STORAGE_KEY, false); return { showPreviews: Boolean(showPreviews && previewFeatureEnabled), previewFeatureEnabled, setShowPreviews }; }; diff --git a/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts b/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts index 38587ea8cf6..5ce973cca66 100644 --- a/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts +++ b/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts @@ -53,7 +53,7 @@ export function setupMockedDataSource({ const fetchMock = jest.fn().mockReturnValue(of({ data })); setBackendSrv({ fetch: fetchMock } as any); - return { datasource, fetchMock }; + return { datasource, fetchMock, templateService }; } export const metricVariable: CustomVariableModel = { diff --git a/public/app/plugins/datasource/cloudwatch/components/AnnotationQueryEditor.tsx b/public/app/plugins/datasource/cloudwatch/components/AnnotationQueryEditor.tsx index 72bb082d99e..84660850dc7 100644 --- a/public/app/plugins/datasource/cloudwatch/components/AnnotationQueryEditor.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/AnnotationQueryEditor.tsx @@ -1,8 +1,8 @@ import React, { ChangeEvent } from 'react'; import { PanelData } from '@grafana/data'; -import { EditorField, EditorHeader, EditorRow, InlineSelect, Space } from '@grafana/experimental'; -import { Input, Switch } from '@grafana/ui'; +import { EditorField, EditorHeader, EditorRow, EditorSwitch, InlineSelect, Space } from '@grafana/experimental'; +import { Input } from '@grafana/ui'; import { CloudWatchDatasource } from '../datasource'; import { useRegions } from '../hooks'; @@ -52,7 +52,7 @@ export function AnnotationQueryEditor(props: React.PropsWithChildren) { /> - { onChange({ @@ -62,18 +62,16 @@ export function AnnotationQueryEditor(props: React.PropsWithChildren) { }} /> - + ) => onChange({ ...query, actionPrefix: event.target.value }) } /> - + ) => onChange({ ...query, alarmNamePrefix: event.target.value }) diff --git a/public/app/plugins/datasource/cloudwatch/components/LogsCheatSheet.tsx b/public/app/plugins/datasource/cloudwatch/components/LogsCheatSheet.tsx index 87aa2ae1df7..697720b3c35 100644 --- a/public/app/plugins/datasource/cloudwatch/components/LogsCheatSheet.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/LogsCheatSheet.tsx @@ -229,8 +229,15 @@ export default class LogsCheatSheet extends PureComponent<
    - this.onClickExample({ refId: 'A', expression: expr, queryMode: 'Logs', region: 'default', id: 'A' }) + onClick={() => + this.onClickExample({ + refId: this.props.query.refId ?? 'A', + expression: expr, + queryMode: 'Logs', + region: this.props.query.region, + id: this.props.query.refId ?? 'A', + logGroupNames: 'logGroupNames' in this.props.query ? this.props.query.logGroupNames : [], + }) } > 
    {renderHighlightedMarkup(expr, keyPrefix)}
    diff --git a/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx b/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx index 079b53d3010..db61ea5cd8d 100644 --- a/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx @@ -1,8 +1,8 @@ import React from 'react'; import { SelectableValue } from '@grafana/data'; -import { EditorField, EditorFieldGroup, EditorRow, EditorRows } from '@grafana/experimental'; -import { Select, Switch } from '@grafana/ui'; +import { EditorField, EditorFieldGroup, EditorRow, EditorRows, EditorSwitch } from '@grafana/experimental'; +import { Select } from '@grafana/ui'; import { Dimensions } from '..'; import { CloudWatchDatasource } from '../../datasource'; @@ -128,7 +128,7 @@ export function MetricStatEditor({ optional={true} tooltip="Only show metrics that exactly match all defined dimension names." > - { diff --git a/public/app/plugins/datasource/cloudwatch/components/MetricsQueryEditor.tsx b/public/app/plugins/datasource/cloudwatch/components/MetricsQueryEditor.tsx index 38f63c9b58f..9b33f26cce2 100644 --- a/public/app/plugins/datasource/cloudwatch/components/MetricsQueryEditor.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/MetricsQueryEditor.tsx @@ -147,6 +147,7 @@ export class MetricsQueryEditor extends PureComponent { width={26} optional tooltip="ID can be used to reference other queries in math expressions. The ID can include numbers, letters, and underscore, and must start with a lowercase letter." + invalid={!!query.id && !/^$|^[a-z][a-zA-Z0-9_]*$/.test(query.id)} > { this.onChange({ ...metricsQuery, id: event.target.value }) } type="text" - invalid={!!query.id && !/^$|^[a-z][a-zA-Z0-9_]*$/.test(query.id)} value={query.id} /> diff --git a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderSelectRow.tsx b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderSelectRow.tsx index 4caa43aa03d..6b1a6781980 100644 --- a/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderSelectRow.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/SQLBuilderEditor/SQLBuilderSelectRow.tsx @@ -1,8 +1,8 @@ import React, { useEffect, useMemo } from 'react'; import { SelectableValue, toOption } from '@grafana/data'; -import { EditorField, EditorFieldGroup } from '@grafana/experimental'; -import { Select, Switch } from '@grafana/ui'; +import { EditorField, EditorFieldGroup, EditorSwitch } from '@grafana/experimental'; +import { Select } from '@grafana/ui'; import { STATISTICS } from '../../cloudwatch-sql/language'; import { CloudWatchDatasource } from '../../datasource'; @@ -87,7 +87,7 @@ const SQLBuilderSelectRow: React.FC = ({ datasource, q - @@ -97,12 +97,11 @@ const SQLBuilderSelectRow: React.FC = ({ datasource, q {withSchemaEnabled && ( - + value && onQueryChange(setOrderBy(query, value))} @@ -46,14 +46,13 @@ const SQLOrderByGroup: React.FC = ({ query, onQueryCha onClick={() => onQueryChange(setSql(query, { orderBy: undefined }))} /> )} - + - + -
    -
-
- -
- -
-
-
-
-
- -
- -
-
-
-
-
-
- -
-
- -
-
- -
- -
-
- -
-
- -
-
- -
-
- -
- 
Annotation Query Format

-An annotation is an event that is overlaid on top of graphs. The query can have up to three columns per row, the datetime column is mandatory. Annotation rendering is expensive so it is important to limit the number of rows returned.
-
-- column with the datetime type.
-- column with alias: Text or text for the annotation text
-- column with alias: Tags or tags for annotation tags. This is should return a comma separated string of tags e.g. 'tag1,tag2'
-
-Macros:
-    - $__timeFilter() -> TimeGenerated ≥ datetime(2018-06-05T18:09:58.907Z) and TimeGenerated ≤ datetime(2018-06-05T20:09:58.907Z)
-    - $__timeFilter(datetimeColumn) ->  datetimeColumn  ≥ datetime(2018-06-05T18:09:58.907Z) and datetimeColumn ≤ datetime(2018-06-05T20:09:58.907Z)
-
-    Or build your own conditionals using these built-in variables which just return the values:
-    - $__timeFrom ->  datetime(2018-06-05T18:09:58.907Z)
-    - $__timeTo -> datetime(2018-06-05T20:09:58.907Z)
-    - $__interval -> 5m
-
-
-
diff --git a/public/app/plugins/datasource/jaeger/responseTransform.ts b/public/app/plugins/datasource/jaeger/responseTransform.ts index 4e6e8683aa7..07123123422 100644 --- a/public/app/plugins/datasource/jaeger/responseTransform.ts +++ b/public/app/plugins/datasource/jaeger/responseTransform.ts @@ -71,6 +71,7 @@ export function createTableFrame(data: TraceResponse[], instanceSettings: DataSo name: 'traceID', type: FieldType.string, config: { + unit: 'string', displayNameFromDS: 'Trace ID', links: [ { diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index 9897ae2cc15..9bae8fc4890 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -439,8 +439,13 @@ export class LokiDatasource } async metadataRequest(url: string, params?: Record) { - const res = await lastValueFrom(this._request(url, params, { hideFromInspector: true })); - return res.data.data || res.data.values || []; + if (config.featureToggles.lokiBackendMode) { + const res = await this.getResource(url, params); + return res.data || []; + } else { + const res = await lastValueFrom(this._request(url, params, { hideFromInspector: true })); + return res.data.data || []; + } } async metricFindQuery(query: string) { @@ -474,7 +479,7 @@ export class LokiDatasource } async labelNamesQuery() { - const url = `${LOKI_ENDPOINT}/label`; + const url = `${LOKI_ENDPOINT}/labels`; const params = this.getTimeRangeParams(); const result = await this.metadataRequest(url, params); return result.map((value: string) => ({ text: value })); diff --git a/public/app/plugins/datasource/loki/language_provider.test.ts b/public/app/plugins/datasource/loki/language_provider.test.ts index 168d696855e..194f388ef00 100644 --- a/public/app/plugins/datasource/loki/language_provider.test.ts +++ b/public/app/plugins/datasource/loki/language_provider.test.ts @@ -259,7 +259,7 @@ describe('Request URL', () => { const instance = new LanguageProvider(datasourceWithLabels); instance.fetchLabels(); - const expectedUrl = '/loki/api/v1/label'; + const expectedUrl = '/loki/api/v1/labels'; expect(datasourceSpy).toHaveBeenCalledWith(expectedUrl, rangeParams); }); }); diff --git a/public/app/plugins/datasource/loki/language_provider.ts b/public/app/plugins/datasource/loki/language_provider.ts index 330df0562a2..9b5aeba208d 100644 --- a/public/app/plugins/datasource/loki/language_provider.ts +++ b/public/app/plugins/datasource/loki/language_provider.ts @@ -364,7 +364,7 @@ export default class LokiLanguageProvider extends LanguageProvider { * Fetches all label keys */ async fetchLabels(): Promise { - const url = '/loki/api/v1/label'; + const url = '/loki/api/v1/labels'; const timeRange = this.datasource.getTimeRangeParams(); this.labelFetchTs = Date.now().valueOf(); diff --git a/public/app/plugins/datasource/loki/mocks.ts b/public/app/plugins/datasource/loki/mocks.ts index 9d5e7b0ca44..4fb4b50dbf3 100644 --- a/public/app/plugins/datasource/loki/mocks.ts +++ b/public/app/plugins/datasource/loki/mocks.ts @@ -21,7 +21,7 @@ export function makeMockLokiDatasource(labelsAndValues: Labels, series?: SeriesF const lokiLabelsAndValuesEndpointRegex = /^\/loki\/api\/v1\/label\/(\w*)\/values/; const lokiSeriesEndpointRegex = /^\/loki\/api\/v1\/series/; - const lokiLabelsEndpoint = `${LOKI_ENDPOINT}/label`; + const lokiLabelsEndpoint = `${LOKI_ENDPOINT}/labels`; const rangeMock = { start: 1560153109000, end: 1560163909000, diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilderOptions.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilderOptions.tsx index f4a6b0f141c..f2c395e10c1 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilderOptions.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilderOptions.tsx @@ -1,8 +1,8 @@ import React, { SyntheticEvent } from 'react'; import { CoreApp, SelectableValue } from '@grafana/data'; -import { EditorRow, EditorField } from '@grafana/experimental'; -import { RadioButtonGroup, Select, Switch } from '@grafana/ui'; +import { EditorRow, EditorField, EditorSwitch } from '@grafana/experimental'; +import { RadioButtonGroup, Select } from '@grafana/ui'; import { getQueryTypeChangeHandler, getQueryTypeOptions } from '../../components/PromExploreExtraField'; import { FORMAT_OPTIONS, INTERVAL_FACTOR_OPTIONS } from '../../components/PromQueryEditor'; @@ -82,7 +82,7 @@ export const PromQueryBuilderOptions = React.memo(({ query, app, onChange {shouldShowExemplarSwitch(query, app) && ( - + )} {query.intervalFactor && query.intervalFactor > 1 && ( diff --git a/public/app/plugins/datasource/tempo/resultTransformer.test.ts b/public/app/plugins/datasource/tempo/resultTransformer.test.ts index b6e25519c8a..f5a1741f106 100644 --- a/public/app/plugins/datasource/tempo/resultTransformer.test.ts +++ b/public/app/plugins/datasource/tempo/resultTransformer.test.ts @@ -102,6 +102,9 @@ describe('createTableFrameFromSearch()', () => { expect(frame.fields[0].name).toBe('traceID'); expect(frame.fields[0].values.get(0)).toBe('e641dcac1c3a0565'); + // TraceID must have unit = 'string' to prevent the ID from rendering as Infinity + expect(frame.fields[0].config.unit).toBe('string'); + expect(frame.fields[1].name).toBe('traceName'); expect(frame.fields[1].values.get(0)).toBe('c10d7ca4e3a00354 '); diff --git a/public/app/plugins/datasource/tempo/resultTransformer.ts b/public/app/plugins/datasource/tempo/resultTransformer.ts index 20a1419dd6f..74e385f1e92 100644 --- a/public/app/plugins/datasource/tempo/resultTransformer.ts +++ b/public/app/plugins/datasource/tempo/resultTransformer.ts @@ -600,6 +600,7 @@ export function createTableFrameFromSearch(data: SearchResponse[], instanceSetti name: 'traceID', type: FieldType.string, config: { + unit: 'string', displayNameFromDS: 'Trace ID', links: [ { diff --git a/public/app/types/unified-alerting.ts b/public/app/types/unified-alerting.ts index 5c98037378e..09df8ca9589 100644 --- a/public/app/types/unified-alerting.ts +++ b/public/app/types/unified-alerting.ts @@ -97,11 +97,11 @@ export interface CombinedRuleNamespace { groups: CombinedRuleGroup[]; } -export interface RuleWithLocation { +export interface RuleWithLocation { ruleSourceName: string; namespace: string; group: RulerRuleGroupDTO; - rule: RulerRuleDTO; + rule: T; } export interface PromRuleWithLocation { diff --git a/scripts/drone/pipelines/main.star b/scripts/drone/pipelines/main.star index e86e337593f..1c1aa5dedca 100644 --- a/scripts/drone/pipelines/main.star +++ b/scripts/drone/pipelines/main.star @@ -238,19 +238,19 @@ def main_pipelines(edition): steps=[download_grabpl_step(), identify_runner_step(),] + integration_test_steps, volumes=volumes, ), pipeline( - name='windows-main', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), + name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), steps=[identify_runner_step('windows')] + windows_steps, depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows', ), notify_pipeline( name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger, template=drone_change_template, secret='drone-changes-webhook', ), pipeline( - name='publish-main', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), + name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), steps=[download_grabpl_step(), identify_runner_step(),] + store_steps, - depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main', ], + depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ], ), notify_pipeline( - name='notify-main', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']), - depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'windows-main', 'publish-main'], + name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']), + depends_on=['main-test', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', 'main-publish'], template=failure_template, secret='slack_webhook' )] diff --git a/yarn.lock b/yarn.lock index 89c0142448e..396815c78c8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4135,9 +4135,9 @@ __metadata: languageName: node linkType: hard -"@grafana/experimental@npm:^0.0.2-canary.25": - version: 0.0.2-canary.25 - resolution: "@grafana/experimental@npm:0.0.2-canary.25" +"@grafana/experimental@npm:^0.0.2-canary.30": + version: 0.0.2-canary.30 + resolution: "@grafana/experimental@npm:0.0.2-canary.30" dependencies: "@types/uuid": ^8.3.3 uuid: ^8.3.2 @@ -4145,7 +4145,7 @@ __metadata: "@emotion/css": 11.1.3 react: 17.0.1 react-select: 5.2.1 - checksum: 20532d6a1ff1bb7a98db71728bc34474f87663431fef349a53f4673d12b6356336b9250bc85028693e7bbfeec8e468d9910837f4062cb49239d581dc974c55c9 + checksum: b5b453b9372cde8f89021c50ae1191a2506ebbb069ad6331d22a5c7267ecd8c0db7f9f5fdd9cfab49fafce4e0b6809609e67449b78fe3ccc63cedfeceb64b911 languageName: node linkType: hard @@ -20530,7 +20530,7 @@ __metadata: "@grafana/e2e": "workspace:*" "@grafana/e2e-selectors": "workspace:*" "@grafana/eslint-config": 3.0.0 - "@grafana/experimental": ^0.0.2-canary.25 + "@grafana/experimental": ^0.0.2-canary.30 "@grafana/google-sdk": 0.0.3 "@grafana/lezer-logql": ^0.0.11 "@grafana/runtime": "workspace:*"