Merge remote-tracking branch 'origin/main' into grafana-git-ui-sync

2025-02-12 13:20:32 +03:00
commit f07688a9f9
@@ -148,8 +148,6 @@ You can toggle between the two options. Once you have created an alert rule, the

 Switching from advanced to default may result in queries and expressions that cannot be converted. In this case, a warning message asks if you want to continue to reset to default settings.

-Default and advanced options are enabled by default for Grafana Cloud users and this feature is being rolled out progressively. OSS users can enable them via the [`alertingQueryAndExpressionsStepMode` feature toggle](/setup-grafana/configure-grafana/feature-toggles/).
-
 {{< docs/shared lookup="alerts/configure-alert-rule-name.md" source="grafana" version="<GRAFANA_VERSION>" >}}

 ## Define query and condition
@@ -61,7 +61,7 @@ refs:

 # Configure silences

-Silences stop notifications from getting created and last for only a specified window of time. Use them to temporarily prevent alert notifications, such as during incident response or a maintenance window.
+Silences stop notifications from being created for a specified time window but do not interrupt alert evaluation. Use them to temporarily prevent alert notifications, such as during incident response or a maintenance window.

 {{< admonition type="note" >}}
 Silences are assigned to a [specific Alertmanager](ref:alertmanager-architecture) and only suppress notifications for alerts managed by that Alertmanager.
@@ -86,12 +86,7 @@ refs:
      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/manage-contact-points/integrations/configure-mqtt/
    - pattern: /docs/grafana-cloud/
      destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/manage-contact-points/integrations/configure-mqtt/
-  alertmanager-architecture:
-    - pattern: /docs/grafana/
-      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/#alertmanager-architecture
-    - pattern: /docs/grafana-cloud/
-      destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/#alertmanager-architecture
-  external-alertmanager:
+  configure-alertmanager:
    - pattern: /docs/grafana/
      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/set-up/configure-alertmanager/
    - pattern: /docs/grafana-cloud/
@@ -101,13 +96,23 @@ refs:
      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/manage-notification-templates/
    - pattern: /docs/grafana-cloud/
      destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/template-notifications/manage-notification-templates/
+  configure-grafana-alerts:
+    - pattern: /docs/grafana/
+      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-grafana-managed-rule/
+    - pattern: /docs/grafana-cloud/
+      destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-grafana-managed-rule/
+  configure-contact-points:
+    - pattern: /docs/grafana/
+      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/manage-contact-points/
+    - pattern: /docs/grafana-cloud/
+      destination: /docs/grafana-cloud/alerting-and-irm/alerting/configure-notifications/manage-contact-points/
 ---

 # Configure contact points

 Use contact points to specify where to receive alert notifications. Contact points contain the configuration for sending alert notifications, including destinations like email, Slack, OnCall, webhooks, and their notification messages.

-A contact point can have one or multiple destinations, known as [contact point integrations](#list-of-supported-integrations). Alert notifications are sent to each integration within the chosen contact point.
+A contact point can have one or multiple destinations, known as [contact point integrations](#supported-contact-point-integrations). Alert notifications are sent to each integration within the chosen contact point.

 On the **Contact Points** tab, you can:

@@ -119,9 +124,42 @@ On the **Contact Points** tab, you can:
 - Delete contact points. Note that you cannot delete contact points that are in use by a notification policy. To proceed, either delete the notification policy or update it to use another contact point.

 {{% admonition type="note" %}}
-Contact points are assigned to a [specific Alertmanager](ref:alertmanager-architecture) and cannot be used by notification policies in other Alertmanagers.
+Contact points are assigned to a [specific Alertmanager](ref:configure-alertmanager) and cannot be used by notification policies in other Alertmanagers.
 {{% /admonition %}}

+## Supported contact point integrations
+
+Each contact point integration has its own configuration options and setup process. The following list shows the contact point integrations supported by Grafana.
+
+{{< column-list >}}
+
+- Alertmanager
+- [AWS SNS](ref:sns)
+- Cisco Webex Teams
+- DingDing
+- [Discord](ref:discord)
+- [Email](ref:email)
+- [Google Chat](ref:gchat)
+- [Grafana Oncall](ref:oncall)
+- Kafka REST Proxy
+- Line
+- [Microsoft Teams](ref:teams)
+- [MQTT](ref:mqtt)
+- [Opsgenie](ref:opsgenie)
+- [Pagerduty](ref:pagerduty)
+- Pushover
+- Sensu Go
+- [Slack](ref:slack)
+- [Telegram](ref:telegram)
+- Threema Gateway
+- VictorOps
+- [Webhook](ref:webhook)
+- WeCom
+
+{{< /column-list >}}
+
+Some of the integrations above are not supported by Prometheus Alertmanager. For the list of supported integrations, refer to the [Prometheus Alertmanager receiver settings](https://prometheus.io/docs/alerting/latest/configuration/#receiver-integration-settings).
+
 ## Add a contact point

 Complete the following steps to add a contact point.
@@ -132,7 +170,7 @@ Complete the following steps to add a contact point.
 1. On the **Contact Points** tab, click **+ Add contact point**.
 1. Enter a descriptive name for the contact point.
 1. From **Integration**, select a type and fill out mandatory fields. For example, if you choose email, enter the email addresses. Or if you choose Slack, enter the Slack channel and users who should be contacted.
-1. Some contact point integrations, like email or Webhook, have optional settings. In **Optional settings**, specify additional settings for the selected contact point integration.
+1. Some [contact point integrations](#supported-contact-point-integrations), like email or Webhook, have optional settings. In **Optional settings**, specify additional settings for the selected contact point integration.
 1. In Notification settings, optionally select **Disable resolved message** if you do not want to be notified when an alert resolves.
 1. Save your changes.

@@ -148,6 +186,16 @@ To add another integration to a contact point, complete the following steps.
   - In **Optional settings**, specify additional settings for the selected contact point integration.
 1. Save your changes.

+## Customize notification messages
+
+In contact points, you can also customize notification messages. For example, when setting up an email contact point integration, click **Message** or **Subject** to modify it.
+
+By default, notification messages include common alert details, which are usually sufficient for most cases.
+
+If necessary, you can customize the content and format of notification messages. You can create a custom notification template, which can then be applied to one or more contact points.
+
+On the **Notification templates** tab, you can view, edit, copy or delete notification templates. Refer to [manage notification templates](ref:manage-notification-templates) for instructions on selecting or creating a template for a contact point.
+
 ## Test a contact point

 Testing a contact point is only available for Grafana Alertmanager. Complete the following steps to test a contact point.
@@ -159,45 +207,9 @@ Testing a contact point is only available for Grafana Alertmanager. Complete the
 1. Choose whether to send a predefined test notification or choose custom to add your own custom annotations and labels to include in the notification.
 1. Click **Send test notification** to fire the alert.

-## Customize notification messages
+## Enable notifications for a contact point

-In contact points, you can also customize notification messages. For example, when setting up an email contact point integration, click **Message** or **Subject** to modify it.
+After creating a contact point, you can enable it to receive alert notifications using one of the following methods:

-By default, notification messages include common alert details, which are usually sufficient for most cases.
-
-If necessary, you can customize the content and format of notification messages. You can create a custom notification template, which can then be applied to one or more contact points.
-
-On the **Notification templates** tab, you can view, edit, copy or delete notification templates. Refer to [manage notification templates](ref:manage-notification-templates) for instructions on selecting or creating a template for a contact point.
-
-## List of supported integrations
-
-Each contact point integration has its own configuration options and setup process. In most cases, this involves providing an API key or a Webhook URL.
-
-The following table lists the contact point integrations supported by Grafana.
-
-| Name                         | Type                      |
-| ---------------------------- | ------------------------- |
-| Alertmanager                 | `prometheus-alertmanager` |
-| [Amazon SNS](ref:sns)        | `sns`                     |
-| Cisco Webex Teams            | `webex`                   |
-| DingDing                     | `dingding`                |
-| [Discord](ref:discord)       | `discord`                 |
-| [Email](ref:email)           | `email`                   |
-| [Google Chat](ref:gchat)     | `googlechat`              |
-| [Grafana Oncall](ref:oncall) | `oncall`                  |
-| Kafka REST Proxy             | `kafka`                   |
-| Line                         | `line`                    |
-| [Microsoft Teams](ref:teams) | `teams`                   |
-| [MQTT](ref:mqtt)             | `mqtt`                    |
-| [Opsgenie](ref:opsgenie)     | `opsgenie`                |
-| [Pagerduty](ref:pagerduty)   | `pagerduty`               |
-| Pushover                     | `pushover`                |
-| Sensu Go                     | `sensugo`                 |
-| [Slack](ref:slack)           | `slack`                   |
-| [Telegram](ref:telegram)     | `telegram`                |
-| Threema Gateway              | `threema`                 |
-| VictorOps                    | `victorops`               |
-| [Webhook](ref:webhook)       | `webhook`                 |
-| WeCom                        | `wecom`                   |
-
-Some of these integrations are not compatible with [external Alertmanagers](ref:external-alertmanager). For the list of Prometheus Alertmanager integrations, refer to the [Prometheus Alertmanager receiver settings](https://prometheus.io/docs/alerting/latest/configuration/#receiver-integration-settings).
+- **Assign it to alert rules** – Select the contact point in the [notifications options for Grafana-managed alert rules](ref:configure-grafana-alerts) to directly associate it with specific alerts.
+- **Assign it to notification policies** – Add the contact point to one or more [notification policies](ref:configure-contact-points), which manage the alert notifications you want the contact point to receive.
@@ -39,7 +39,9 @@ refs:

 # Configure mute timings

-A mute timing is a recurring interval of time when no new notifications for a policy are generated or sent. Use them to prevent alerts from firing a specific and reoccurring period, for example, a regular maintenance period or weekends.
+A mute timing is a recurring interval that stops notifications for one or multiple notification policies during a specified period. It suppresses notifications but does not interrupt alert evaluation.
+
+Use mute timings to temporarily pause notifications for a specific recurring period, such as a regular maintenance window or weekends.

 {{< admonition type="note" >}}
 Mute timings are assigned to a [specific Alertmanager](ref:alertmanager-architecture) and only suppress notifications for alerts managed by that Alertmanager.
@@ -132,7 +132,7 @@ Each notification policy decides where to send the alert (contact point) and whe

 ### Silences and mute timings

-[Silences](ref:silences) and [mute timings](ref:mute-timings) allow you to pause notifications for specific alerts or even entire notification policies. Use a silence to pause notifications on an ad-hoc basis, such as during a maintenance window; and use mute timings to pause notifications at regular intervals, such as evenings and weekends.
+[Silences](ref:silences) and [mute timings](ref:mute-timings) allow you to pause notifications without interrupting alert rule evaluation. Use a silence to pause notifications on a one-time basis, such as during a maintenance window; and use mute timings to pause notifications at regular intervals, such as evenings and weekends.

 ### Architecture

@@ -25,9 +25,17 @@ refs:
      destination: /docs/grafana-cloud/connect-externally-hosted/data-sources/prometheus/configure-prometheus-data-source/#alerting
  queries-and-conditions:
    - pattern: /docs/grafana/
-      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/
+      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/#data-source-queries
    - pattern: /docs/grafana-cloud/
-      destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/
+      destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/#data-source-queries
+  alert-condition:
+    - pattern: /docs/grafana/
+      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/#alert-condition
+    - pattern: /docs/grafana-cloud/
+      destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/#alert-condition
+  recorded-queries:
+    - pattern: /docs/
+      destination: /docs/grafana/<GRAFANA_VERSION>/administration/recorded-queries/
  notification-images:
    - pattern: /docs/grafana/
      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/configure-notifications/template-notifications/images-in-notifications/
@@ -45,14 +53,9 @@ refs:
      destination: /docs/grafana-cloud/alerting-and-irm/alerting/alerting-rules/create-recording-rules/
  expression-queries:
    - pattern: /docs/grafana/
-      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/#expression-queries
+      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/#advanced-options-expressions
    - pattern: /docs/grafana-cloud/
-      destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/#expression-queries
-  alert-condition:
-    - pattern: /docs/grafana/
-      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/queries-conditions/#alert-condition
-    - pattern: /docs/grafana-cloud/
-      destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/#alert-condition
+      destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/queries-conditions/#advanced-options-expressions
  alert-rule-evaluation:
    - pattern: /docs/grafana/
      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/rule-evaluation/
@@ -64,8 +67,8 @@ refs:

 An alert rule is a set of evaluation criteria for when an alert rule should fire. An alert rule consists of:

- Queries and expressions that select the data set to evaluate.
- A condition (the threshold) that the query must meet or exceed to trigger the alert instance.
+- [Queries](ref:queries-and-conditions) that select the dataset to evaluate.
+- An [alert condition](ref:alert-condition) (the threshold) that the query must meet or exceed to trigger the alert instance.
 - An interval that specifies the frequency of [alert rule evaluation](ref:alert-rule-evaluation) and a duration indicating how long the condition must be met to trigger the alert instance.
 - Other customizable options, for example, setting what should happen in the absence of data, notification messages, and more.

@@ -17,21 +17,16 @@ labels:
 title: Queries and conditions
 weight: 104
 refs:
-  data-sources:
-    - pattern: /docs/grafana/
-      destination: /docs/grafana/<GRAFANA_VERSION>/datasources/
-    - pattern: /docs/grafana-cloud/
-      destination: /docs/grafana-cloud/connect-externally-hosted/data-sources/
  data-source-alerting:
    - pattern: /docs/grafana/
      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rules/#supported-data-sources
    - pattern: /docs/grafana-cloud/
      destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rules/#supported-data-sources
-  alert-rule-evaluation:
+  state-and-health:
    - pattern: /docs/grafana/
-      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/alert-rule-evaluation/
+      destination: /docs/grafana/<GRAFANA_VERSION>/alerting/fundamentals/state-and-health/
    - pattern: /docs/grafana-cloud/
-      destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/alert-rule-evaluation/
+      destination: /docs/grafana-cloud/alerting-and-irm/alerting/fundamentals/state-and-health/
  query-transform-data:
    - pattern: /docs/grafana/
      destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/query-transform-data/
@@ -41,71 +36,116 @@ refs:

 # Queries and conditions

-In Grafana, queries fetch and transform data from [data sources,](ref:data-sources) which include databases like MySQL or PostgreSQL, time series databases like Prometheus or InfluxDB, and services like Amazon CloudWatch or Azure Monitor.
+In Grafana, queries fetch and transform data from data sources, which include databases like MySQL or PostgreSQL, time series databases like Prometheus or InfluxDB, and services like Amazon CloudWatch or Azure Monitor.

-A query specifies the data to extract from a data source, with the syntax varying based on the type of data source used.
+An alert rule defines the following components:

-In Alerting, an alert rule defines of one or more queries and expressions that select the data you want to measure and a [condition](#alert-condition) that needs to be met before an alert rule fires.
+- A [query](#data-source-queries) that specifies the data to retrieve from a data source, with the syntax depending on the type of data source used.
+- A [condition](#alert-condition) that must be met before the alert rule fires.
+- Optional [expressions](#advanced-options-expressions) to perform transformations on the retrieved data.
+
+Alerting periodically runs the queries and expressions, evaluating the condition. If the condition is breached, an alert instance is triggered for each time series.

 ## Data source queries

-Alerting queries are the same type of queries available in Grafana panels. Queries in Grafana can be applied in various ways, depending on the data source and query language being used. However, not all [data sources support Alerting](ref:data-source-alerting).
+Alerting queries are the same as the queries used in Grafana panels, but Grafana-managed alerts are limited to querying [data sources that have Alerting enabled](ref:data-source-alerting).

-Each data source’s query editor provides a customized user interface to help you write queries that take advantage of its unique capabilities. For additional information about queries in Grafana, refer to [Query and transform data](ref:query-transform-data).
+Queries in Grafana can be applied in various ways, depending on the data source and query language being used. Each data source’s query editor provides a customized user interface to help you write queries that take advantage of its unique capabilities.

-Some common types of query components include:
+For more details about queries in Grafana, refer to [Query and transform data](ref:query-transform-data).

-**Metrics or data fields**: Specify the specific metrics or data fields you want to retrieve, such as CPU usage, network traffic, or sensor readings.
+{{< figure src="/media/docs/alerting/alerting-query-conditions-default-options.png" max-width="750px" caption="Define alert query and alert condition" >}}

-**Time range**: Define the time range for which you want to fetch data, such as the last hour, a specific day, or a custom time range.
+## Alert condition

-**Filters**: Apply filters to narrow down the data based on specific criteria, such as filtering data by a specific tag, host, or application.
+The alert condition is the query or expression that determines whether the alert fires or not depending whether the value satisfies the specified comparison. There can be only one condition which determines the triggering of the alert.

-**Aggregations**: Perform aggregations on the data to calculate metrics like averages, sums, or counts over a given time period.
+If the queried data meets the defined condition, Grafana fires the alert.

-**Grouping**: Group the data by specific dimensions or tags to create aggregated views or breakdowns.
+When using **Default options**, the `When` input [reduces the query data](#reduce), and the last input defines the threshold condition.

-{{% admonition type="note" %}}
-Grafana doesn't support alert queries with template variables. More details [here](https://community.grafana.com/t/template-variables-are-not-supported-in-alert-queries-while-setting-up-alert/2514).
-{{% /admonition %}}
+When using **Advanced options**, you have to choose one of your queries or expressions as the alert condition.

-## Expression queries
+## Advanced options: Expressions

-In Grafana, an expression is used to perform calculations, transformations, or aggregations on the data source queried data. It allows you to create custom metrics or modify existing metrics based on mathematical operations, functions, or logical expressions.
+Expressions are only available for Grafana-managed alerts and when the **Advanced options** are enabled.

-By leveraging expression queries, users can perform tasks such as calculating the percentage change between two values, applying functions like logarithmic or trigonometric functions, aggregating data over specific time ranges or dimensions, and implementing conditional logic to handle different scenarios.
+In Grafana, expressions allow you to perform calculations, transformations, or aggregations on queried data. They modify existing metrics through mathematical operations, functions, or logical expressions.

-In Alerting, you can only use expressions for Grafana-managed alert rules. For each expression, you can choose from the math, reduce, and resample expressions. These are called multi-dimensional rules, because they generate an alert instance for each series.
+With expression queries, you can perform tasks such as calculating the percentage change between two values, applying functions like logarithmic or trigonometric functions, aggregating data over specific time ranges or dimensions, and implementing conditional logic to handle different scenarios.

-**Reduce**
+{{< figure src="/media/docs/alerting/alert-rule-expressions.png" max-width="750px" caption="Alert rule expressions" >}}

-Aggregates time series values in the selected time range into a single value. It's not necessary for [rules using numeric data](#alert-on-numeric-data).
+The following expressions are available:

-**Math**
+### Reduce

-Performs free-form math functions/operations on time series and number data. Can be used to preprocess time series data or to define an alert condition for number data. For example:
+Aggregates time series values within the selected time range into a single number.
+
+Reduce takes one or more time series and transform each series into a single number, which can then be compared in the alert condition.
+
+The following aggregations functions are included: `Min`, `Max`, `Mean`, `Mediam`, `Sum`, `Count`, and `Last`.
+
+### Math
+
+Performs free-form math functions/operations on time series data and numbers. For instance, `$A + 1` or `$A * 100`.
+
+You can also use a Math expression to define the alert condition for numbers. For example:

 - `$B > 70` should fire if the value of B (query or expression) is more than 70.
 - `$B < $C * 100` should fire if the value of B is less than the value of C multiplied by 100.

 If queries being compared have multiple series in their results, series from different queries are matched if they have the same labels or one is a subset of the other.

-**Resample**
+### Resample

 Realigns a time range to a new set of timestamps, this is useful when comparing time series data from different data sources where the timestamps would otherwise not align.

-**Threshold**
+### Threshold

-Checks if any time series data matches the threshold condition.
+Compares single numbers from previous queries or expressions (e.g., `$A`, `$B`) to a specified condition. It's often used to define the alert condition.

-The threshold expression allows you to compare two single values. It returns `0` when the condition is false and `1` if the condition is true. The following threshold functions are available:
+The threshold expression allows the comparison between two single values. Available threshold functions are:

- Is above (x > y)
- Is below (x < y)
- Is within range (x > y1 AND x < y2)
- Is outside range (x < y1 OR x > y2)
+- **Is above**: `$A > 5`
+- **Is below**: `$B < 3`
+- **Is within range**: `$A > 0 AND $A < 10`
+- **Is outside range**: `$B < 0 OR $B > 100`

-**Classic condition (legacy)**
+A threshold returns `0` when the condition is false and `1` when true.
+
+If the threshold is set as the alert condition, the alert fires when the threshold returns `1`.
+
+#### Recovery threshold
+
+To reduce the noise from flapping alerts, you can set a recovery threshold different to the alert threshold.
+
+Flapping alerts occur when a metric hovers around the alert threshold condition and may lead to frequent state changes, resulting in too many notifications.
+
+The value of a flapping metric can continually go above and below a threshold, resulting in a series of firing-resolved-firing notifications and a noisy alert state history.
+
+For example, if you have an alert for latency with a threshold of 1000ms and the number fluctuates around 1000 (say 980 -> 1010 -> 990 -> 1020, and so on), then each of those might trigger a notification:
+
+- 980 -> 1010 triggers a firing alert.
+- 1010 -> 990 triggers a resolving alert.
+- 990 -> 1020 triggers a firing alert again.
+
+To prevent this, you can set a recovery threshold to define two thresholds instead of one:
+
+1. An alert is triggered when the first threshold is crossed.
+1. An alert is resolved only when the second (recovery) threshold is crossed.
+
+In the previous example, setting the recovery threshold to 900ms means the alert only resolves when the latency falls below 900ms:
+
+- 980 -> 1010 triggers a firing alert.
+- 1010 -> 990 does not resolve the alert, keeping it in the firing state.
+- 990 -> 1020 keeps the alert in the firing state.
+
+The recovery threshold mitigates unnecessary alert state changes and reduces alert noise.
+
+{{< collapse title="Classic condition (legacy)" >}}
+
+#### Classic condition (legacy)

 Classic conditions exist mainly for compatibility reasons and should be avoided if possible.

@@ -113,66 +153,35 @@ Classic condition checks if any time series data matches the alert condition. It

 | Condition operators | How it works                                                                                                                                                                                                                                                |
 | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| and                 | Two conditions before and after must be true for the overall condition to be true.                                                                                                                                                                          |
-| or                  | If one of conditions before and after are true, the overall condition is true.                                                                                                                                                                              |
-| logic-or            | If the condition before `logic-or` is true, the overall condition is immediately true, without evaluating subsequent conditions. For instance, `TRUE and TRUE logic-or FALSE and FALSE` evaluate to `TRUE`, because the preceding condition returns `TRUE`. |
+| `and`               | Two conditions before and after must be true for the overall condition to be true.                                                                                                                                                                          |
+| `or`                | If one of conditions before and after are true, the overall condition is true.                                                                                                                                                                              |
+| `logic-or`          | If the condition before `logic-or` is true, the overall condition is immediately true, without evaluating subsequent conditions. For instance, `TRUE and TRUE logic-or FALSE and FALSE` evaluate to `TRUE`, because the preceding condition returns `TRUE`. |

-## Aggregations
+The following aggregation functions are also available to further refine your query.

-Grafana Alerting provides the following aggregation functions to enable you to further refine your query.
+| Function           | What it does                                                                    |
+| ------------------ | ------------------------------------------------------------------------------- |
+| `avg`              | Displays the average of the values                                              |
+| `min`              | Displays the lowest value                                                       |
+| `max`              | Displays the highest value                                                      |
+| `sum`              | Displays the sum of all values                                                  |
+| `count`            | Counts the number of values in the result                                       |
+| `last`             | Displays the last value                                                         |
+| `median`           | Displays the median value                                                       |
+| `diff`             | Displays the difference between the newest and oldest value                     |
+| `diff_abs`         | Displays the absolute value of diff                                             |
+| `percent_diff`     | Displays the percentage value of the difference between newest and oldest value |
+| `percent_diff_abs` | Displays the absolute value of `percent_diff`                                   |
+| `count_non_null`   | Displays a count of values in the result set that aren't `null`                 |

-These functions are available for **Reduce** and **Classic condition** expressions only.
-
-| Function         | Expression       | What it does                                                                    |
-| ---------------- | ---------------- | ------------------------------------------------------------------------------- |
-| avg              | Reduce / Classic | Displays the average of the values                                              |
-| min              | Reduce / Classic | Displays the lowest value                                                       |
-| max              | Reduce / Classic | Displays the highest value                                                      |
-| sum              | Reduce / Classic | Displays the sum of all values                                                  |
-| count            | Reduce / Classic | Counts the number of values in the result                                       |
-| last             | Reduce / Classic | Displays the last value                                                         |
-| median           | Reduce / Classic | Displays the median value                                                       |
-| diff             | Classic          | Displays the difference between the newest and oldest value                     |
-| diff_abs         | Classic          | Displays the absolute value of diff                                             |
-| percent_diff     | Classic          | Displays the percentage value of the difference between newest and oldest value |
-| percent_diff_abs | Classic          | Displays the absolute value of percent_diff                                     |
-| count_non_null   | Classic          | Displays a count of values in the result set that aren't `null`                 |
-
-## Alert condition
-
-An alert condition is the query or expression that determines whether the alert fires or not depending on the value it yields. There can be only one condition which determines the triggering of the alert.
-
-After you have defined your queries and expressions, choose one of them as the alert rule condition. By default, the last expression added is used as the alert condition.
-
-When the queried data satisfies the defined condition, Grafana triggers the associated alert, which can be configured to send notifications through various channels like email, Slack, or PagerDuty.
-
-For details about how the alert evaluation triggers notifications, refer to [Alert rule evaluation](ref:alert-rule-evaluation).
-
-## Recovery threshold
-
-To reduce the noise of flapping alerts, you can set a recovery threshold different to the alert threshold.
-
-Flapping alerts occur when a metric hovers around the alert threshold condition and may lead to frequent state changes, resulting in too many notifications being generated.
-
-It can be tricky to create an alert rule for a noisy metric. That is, when the value of a metric continually goes above and below a threshold. This is called flapping and results in a series of firing - resolved - firing notifications and a noisy alert state history.
-
-For example, if you have an alert for latency with a threshold of 1000ms and the number fluctuates around 1000 (say 980 ->1010 -> 990 -> 1020, and so on) then each of those triggers a notification.
-
-To solve this problem, you can set a (custom) recovery threshold, which basically means having two thresholds instead of one:
-
-1. An alert is triggered when the first threshold is crossed.
-2. An alert is resolved only when the second threshold is crossed.
-
-For example, you could set a threshold of 1000ms and a recovery threshold of 900ms. This way, an alert rule only stops firing when it goes under 900ms and flapping is reduced.
-
-For details about how the alert evaluation triggers notifications, refer to [Alert rule evaluation](ref:alert-rule-evaluation).
+{{< /collapse >}}

 ## Alert on numeric data

 Among certain data sources numeric data that is not time series can be directly alerted on, or passed into Server Side Expressions (SSE). This allows for more processing and resulting efficiency within the data source, and it can also simplify alert rules.
-When alerting on numeric data instead of time series data, there is no need to reduce each labeled time series into a single number. Instead labeled numbers are returned to Grafana instead.
+When alerting on numeric data instead of time series data, there is no need to [reduce](#reduce) each labeled time series into a single number. Instead labeled numbers are returned to Grafana instead.

-### Tabular Data
+#### Tabular Data

 This feature is supported with backend data sources that query tabular data:

@@ -126,7 +126,7 @@ Grafana Alerting provides advanced notification capabilities that you’ll find

 For instance, you can customize notifications with shared [templates](ref:templates) that provide actionable alert information and can be reused for multiple notifications.

-Additionally, you can use [silences](ref:silences) and [mute timings](ref:mute-timings) to pause notifications for a given time window or at regular intervals, respectively.
+Additionally, you can use [silences](ref:silences) and [mute timings](ref:mute-timings) to pause or suppress notifications without interrupting alert evaluation.

 ## Architecture

@@ -188,7 +188,7 @@ Create a notification policy if you want to handle metrics returned by alert rul

   This new child policy routes alerts that match the label `device=desktop` to the Webhook contact point.

-1. **Repeat the steps above to create a second child policy** to match another alert instance. For labels use: `device=mobile`. Use the Webhook integration for the contact point. Alternatively, experiment by using a different Webhook endpoint or a [different integration](https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/#list-of-supported-integrations).
+1. **Repeat the steps above to create a second child policy** to match another alert instance. For labels use: `device=mobile`. Use the Webhook integration for the contact point. Alternatively, experiment by using a different Webhook endpoint or a [different integration](https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/#supported-contact-point-integrations).

 <!-- INTERACTIVE ignore END -->

@@ -206,7 +206,7 @@ Create a notification policy if you want to handle metrics returned by alert rul

   This new child policy routes alerts that match the label `device=desktop` to the Webhook contact point.

-1. **Repeat the steps above to create a second child policy** to match another alert instance. For labels use: `device=mobile`. Use the Webhook integration for the contact point. Alternatively, experiment by using a different Webhook endpoint or a [different integration](https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/#list-of-supported-integrations).
+1. **Repeat the steps above to create a second child policy** to match another alert instance. For labels use: `device=mobile`. Use the Webhook integration for the contact point. Alternatively, experiment by using a different Webhook endpoint or a [different integration](https://grafana.com/docs/grafana/latest/alerting/configure-notifications/manage-contact-points/#supported-contact-point-integrations).

 {{< /docs/ignore >}}

@@ -1,3 +1,4 @@
+import { ScopedVars } from './ScopedVars';
 import { ExploreCorrelationHelperData, ExplorePanelsState } from './explore';
 import { InterpolateFunction } from './panel';
 import { DataQuery } from './query';
@@ -80,7 +81,7 @@ export interface DataLinkTransformationConfig {

 /** @internal */
 export interface InternalDataLink<T extends DataQuery = any> {
-  query: T;
+  query: T | ((options: { replaceVariables: InterpolateFunction; scopedVars: ScopedVars }) => T);
  datasourceUid: string;
  datasourceName: string; // used as a title if `DataLink.title` is empty
  panelsState?: ExplorePanelsState;
@@ -38,7 +38,11 @@ export type LinkToExploreOptions = {
 export function mapInternalLinkToExplore(options: LinkToExploreOptions): LinkModel<Field> {
  const { onClickFn, replaceVariables, link, scopedVars, range, field, internalLink } = options;

-  const interpolatedQuery = interpolateObject(link.internal?.query, scopedVars, replaceVariables);
+  const query =
+    typeof link.internal?.query === 'function'
+      ? link.internal.query({ replaceVariables, scopedVars })
+      : internalLink.query;
+  const interpolatedQuery = interpolateObject(query, scopedVars, replaceVariables);
  const interpolatedPanelsState = interpolateObject(link.internal?.panelsState, scopedVars, replaceVariables);
  const interpolatedCorrelationData = interpolateObject(link.meta?.correlationData, scopedVars, replaceVariables);
  const title = link.title ? link.title : internalLink.datasourceName;
@@ -44,4 +44,7 @@ export enum NodeGraphDataFrameFieldNames {

  // Supplies a fixed Y position for the node to have in the finished graph.
  fixedY = 'fixedy',
+
+  // Whether the node is instrumented or not
+  isInstrumented = 'isinstrumented',
 }
@@ -6,7 +6,7 @@ import { ComponentProps } from 'react';
 import { Field } from '../Forms/Field';

 import { MultiCombobox } from './MultiCombobox';
-import { generateOptions, fakeSearchAPI } from './storyUtils';
+import { generateOptions, fakeSearchAPI, generateGroupingOptions } from './storyUtils';
 import { ComboboxOption } from './types';

 const meta: Meta<typeof MultiCombobox> = {
@@ -107,6 +107,40 @@ export const ManyOptions: StoryObj<ManyOptionsArgs> = {
  render: ManyOptionsStory,
 };

+const ManyOptionsGroupedStory: StoryFn<ManyOptionsArgs> = ({ numberOfOptions = 1e5, ...args }) => {
+  const [dynamicArgs, setArgs] = useArgs();
+
+  const [options, setOptions] = useState<ComboboxOption[]>([]);
+
+  useEffect(() => {
+    setTimeout(async () => {
+      const options = await generateGroupingOptions(numberOfOptions);
+      setOptions(options);
+    }, 1000);
+  }, [numberOfOptions]);
+  const { onChange, ...rest } = args;
+  return (
+    <MultiCombobox
+      {...rest}
+      {...dynamicArgs}
+      options={options}
+      onChange={(opts) => {
+        setArgs({ value: opts });
+        onChangeAction(opts);
+      }}
+    />
+  );
+};
+
+export const ManyOptionsGrouped: StoryObj<ManyOptionsArgs> = {
+  args: {
+    numberOfOptions: 1e4,
+    options: undefined,
+    value: undefined,
+  },
+  render: ManyOptionsGroupedStory,
+};
+
 function loadOptionsWithLabels(inputValue: string) {
  loadOptionsAction(inputValue);
  return fakeSearchAPI(`http://example.com/search?errorOnQuery=break&query=${inputValue}`);
@@ -246,8 +246,18 @@ export const MultiCombobox = <T extends string | number>(props: MultiComboboxPro
  const virtualizerOptions = {
    count: options.length,
    getScrollElement: () => scrollRef.current,
-    estimateSize: (index: number) =>
-      'description' in options[index] ? MENU_OPTION_HEIGHT_DESCRIPTION : MENU_OPTION_HEIGHT,
+    estimateSize: (index: number) => {
+      const firstGroupItem = isNewGroup(options[index], index > 0 ? options[index - 1] : undefined);
+      const hasDescription = 'description' in options[index];
+      let itemHeight = MENU_OPTION_HEIGHT;
+      if (hasDescription) {
+        itemHeight = MENU_OPTION_HEIGHT_DESCRIPTION;
+      }
+      if (firstGroupItem) {
+        itemHeight += MENU_OPTION_HEIGHT;
+      }
+      return itemHeight;
+    },
    overscan: VIRTUAL_OVERSCAN_ITEMS,
  };

@@ -337,6 +347,7 @@ export const MultiCombobox = <T extends string | number>(props: MultiComboboxPro
            <ScrollContainer showScrollIndicators maxHeight="inherit" ref={scrollRef}>
              <ul style={{ height: rowVirtualizer.getTotalSize() }} className={styles.menuUlContainer}>
                {rowVirtualizer.getVirtualItems().map((virtualRow) => {
+                  const startingNewGroup = isNewGroup(options[virtualRow.index], options[virtualRow.index - 1]);
                  const index = virtualRow.index;
                  const item = options[index];
                  const itemProps = getItemProps({ item, index });
@@ -354,29 +365,46 @@ export const MultiCombobox = <T extends string | number>(props: MultiComboboxPro
                      key={`${item.value}-${index}`}
                      data-index={index}
                      {...itemProps}
-                      className={cx(styles.option, { [styles.optionFocused]: highlightedIndex === index })}
+                      className={styles.optionBasic}
                      style={{ height: virtualRow.size, transform: `translateY(${virtualRow.start}px)` }}
                    >
-                      <Stack direction="row" alignItems="center">
-                        <Checkbox
-                          key={id}
-                          value={allItemsSelected || isSelected}
-                          indeterminate={isAll && selectedItems.length > 0 && !allItemsSelected}
-                          aria-labelledby={id}
-                          onClick={(e) => {
-                            e.stopPropagation();
-                          }}
-                        />
-                        <OptionListItem
-                          label={
-                            isAll
-                              ? (item.label ?? item.value.toString()) +
-                                (isAll && inputValue !== '' ? ` (${options.length - 1})` : '')
-                              : (item.label ?? item.value.toString())
-                          }
-                          description={item?.description}
-                          id={id}
-                        />
+                      <Stack direction="column" justifyContent="space-between" width={'100%'} height={'100%'} gap={0}>
+                        {startingNewGroup && (
+                          <div className={styles.optionGroup}>
+                            <OptionListItem
+                              label={item.group ?? t('combobox.group.undefined', 'No group')}
+                              id={id}
+                              isGroup={true}
+                            />
+                          </div>
+                        )}
+                        <div
+                          className={cx(styles.option, {
+                            [styles.optionFocused]: highlightedIndex === index,
+                          })}
+                        >
+                          <Stack direction="row" alignItems="center">
+                            <Checkbox
+                              key={id}
+                              value={allItemsSelected || isSelected}
+                              indeterminate={isAll && selectedItems.length > 0 && !allItemsSelected}
+                              aria-labelledby={id}
+                              onClick={(e) => {
+                                e.stopPropagation();
+                              }}
+                            />
+                            <OptionListItem
+                              label={
+                                isAll
+                                  ? (item.label ?? item.value.toString()) +
+                                    (isAll && inputValue !== '' ? ` (${options.length - 1})` : '')
+                                  : (item.label ?? item.value.toString())
+                              }
+                              description={item?.description}
+                              id={id}
+                            />
+                          </Stack>
+                        </div>
                      </Stack>
                    </li>
                  );
@@ -425,3 +453,17 @@ function isComboboxOptions<T extends string | number>(
 ): value is Array<ComboboxOption<T>> {
  return typeof value[0] === 'object';
 }
+
+const isNewGroup = <T extends string | number>(option: ComboboxOption<T>, prevOption?: ComboboxOption<T>) => {
+  const currentGroup = option.group;
+
+  if (!currentGroup) {
+    return prevOption?.group ? true : false;
+  }
+
+  if (!prevOption) {
+    return true;
+  }
+
+  return prevOption.group !== currentGroup;
+};
@@ -1,3 +1,5 @@
+import { cx } from '@emotion/css';
+
 import { useStyles2 } from '../../themes';

 import { getComboboxStyles } from './getComboboxStyles';
@@ -6,13 +8,14 @@ interface Props {
  label: string;
  description?: string;
  id: string;
+  isGroup?: boolean;
 }

-export const OptionListItem = ({ label, description, id }: Props) => {
+export const OptionListItem = ({ label, description, id, isGroup = false }: Props) => {
  const styles = useStyles2(getComboboxStyles);
  return (
-    <div className={styles.optionBody}>
-      <span className={styles.optionLabel} id={id}>
+    <div className={styles.optionBody} aria-disabled={isGroup}>
+      <span className={cx(styles.optionLabel, { [styles.optionLabelGroup]: isGroup })} id={id}>
        {label}
      </span>
      {description && <span className={styles.optionDescription}>{description}</span>}
@@ -32,9 +32,8 @@ export const getComboboxStyles = (theme: GrafanaTheme2) => {
      label: 'combobox-menu-ul-container',
      listStyle: 'none',
    }),
-    option: css({
+    optionBasic: css({
      label: 'combobox-option',
-      padding: MENU_ITEM_PADDING,
      position: 'absolute',
      display: 'flex',
      alignItems: 'center',
@@ -43,7 +42,11 @@ export const getComboboxStyles = (theme: GrafanaTheme2) => {
      whiteSpace: 'nowrap',
      width: '100%',
      overflow: 'hidden',
+    }),
+    option: css({
+      padding: MENU_ITEM_PADDING,
      cursor: 'pointer',
+      width: '100%',
      '&:hover': {
        background: theme.colors.action.hover,
        '@media (forced-colors: active), (prefers-contrast: more)': {
@@ -51,6 +54,11 @@ export const getComboboxStyles = (theme: GrafanaTheme2) => {
        },
      },
    }),
+    optionGroup: css({
+      cursor: 'default',
+      padding: MENU_ITEM_PADDING,
+      borderTop: `1px solid ${theme.colors.border.weak}`,
+    }),
    optionBody: css({
      label: 'combobox-option-body',
      display: 'flex',
@@ -67,6 +75,12 @@ export const getComboboxStyles = (theme: GrafanaTheme2) => {
      fontWeight: MENU_ITEM_FONT_WEIGHT,
      letterSpacing: 0, // pr todo: text in grafana has a slightly different letter spacing, which causes measureText() to be ~5% off
    }),
+    optionLabelGroup: css({
+      label: 'combobox-option-label-group',
+      color: theme.colors.text.secondary,
+      fontSize: theme.typography.bodySmall.fontSize,
+      fontWeight: theme.typography.fontWeightLight,
+    }),
    optionDescription: css({
      label: 'combobox-option-description',
      fontWeight: theme.typography.fontWeightRegular,
@@ -37,3 +37,11 @@ export async function generateOptions(amount: number): Promise<ComboboxOption[]>
    value: index.toString(),
  }));
 }
+
+export async function generateGroupingOptions(amount: number): Promise<ComboboxOption[]> {
+  return Array.from({ length: amount }, (_, index) => ({
+    label: 'Option ' + index,
+    value: index.toString(),
+    group: index % 9 !== 0 ? 'Group ' + Math.floor(index / 10) : undefined,
+  }));
+}
@@ -4,4 +4,5 @@ export type ComboboxOption<T extends string | number = string> = {
  label?: string;
  value: T;
  description?: string;
+  group?: string;
 };
@@ -95,16 +95,39 @@ export function useOptions<T extends string | number>(rawOptions: AsyncOptions<T
    [debouncedLoadOptions, isAsync]
  );

-  const finalOptions = useMemo(() => {
-    let currentOptions = [];
-    if (isAsync) {
-      currentOptions = addCustomValue(asyncOptions);
-    } else {
-      currentOptions = addCustomValue(rawOptions.filter(itemFilter(userTypedSearch)));
+  const organizeOptionsByGroup = useCallback((options: Array<ComboboxOption<T>>) => {
+    const groupedOptions = new Map<string | undefined, Array<ComboboxOption<T>>>();
+    for (const option of options) {
+      const groupExists = groupedOptions.has(option.group);
+      if (groupExists) {
+        groupedOptions.get(option.group)?.push(option);
+      } else {
+        groupedOptions.set(option.group, [option]);
+      }
    }

-    return currentOptions;
-  }, [isAsync, addCustomValue, asyncOptions, rawOptions, userTypedSearch]);
+    // Reorganize options to have groups first, then undefined group
+    const reorganizeOptions = [];
+    for (const [group, groupOptions] of groupedOptions) {
+      if (!group) {
+        continue;
+      }
+      reorganizeOptions.push(...groupOptions);
+    }
+
+    const undefinedGroupOptions = groupedOptions.get(undefined);
+    if (undefinedGroupOptions) {
+      reorganizeOptions.push(...undefinedGroupOptions);
+    }
+    return reorganizeOptions;
+  }, []);
+
+  const finalOptions = useMemo(() => {
+    const currentOptions = isAsync ? asyncOptions : rawOptions.filter(itemFilter(userTypedSearch));
+    const currentOptionsOrganised = organizeOptionsByGroup(currentOptions);
+
+    return addCustomValue(currentOptionsOrganised);
+  }, [isAsync, organizeOptionsByGroup, addCustomValue, asyncOptions, rawOptions, userTypedSearch]);

  return { options: finalOptions, updateOptions, asyncLoading, asyncError };
 }
@@ -15,6 +15,8 @@ import {
  DataQueryRequest,
  getTimeZone,
  PluginMetaInfo,
+  DataLink,
+  NodeGraphDataFrameFieldNames,
 } from '@grafana/data';
 import {
  BackendDataSourceResponse,
@@ -544,6 +546,32 @@ describe('Tempo service graph view', () => {
    expect(response.data[1].fields[0]?.config?.links?.length).toBeGreaterThan(0);
    expect(response.data[1].fields[0]?.config?.links).toEqual(serviceGraphLinks);

+    const viewServicesLink = response.data[1].fields[0]?.config?.links.find(
+      (link: DataLink) => link.title === 'View traces'
+    );
+    expect(viewServicesLink).toBeDefined();
+    expect(viewServicesLink.internal.query({ replaceVariables: replaceVariablesInstrumented })).toEqual({
+      refId: 'A',
+      queryType: 'traceqlSearch',
+      filters: [
+        {
+          id: 'service-name',
+          operator: '=',
+          scope: 'resource',
+          tag: 'service.name',
+          value: 'my-service',
+          valueType: 'string',
+        },
+      ],
+    });
+    expect(viewServicesLink.internal.query({ replaceVariables: replaceVariablesUninstrumented })).toEqual({
+      refId: 'A',
+      queryType: 'traceql',
+      filters: [],
+      query:
+        '{span.db.name="my-service" || span.db.system="my-service" || span.peer.service="my-service" || span.messaging.system="my-service" || span.net.peer.name="my-service"}',
+    });
+
    expect(response.data[2].name).toBe('Edges');
    expect(response.data[2].fields[0].values.length).toBe(2);
  });
@@ -584,14 +612,26 @@ describe('Tempo service graph view', () => {
      'sum by (client, server) (rate(traces_service_graph_request_server_seconds_sum{ foo="bar" }[$__range]))'
    );
    expect(nthQuery(0).targets[1].expr).toBe(
-      'sum by (client, server) (rate(traces_service_graph_request_total{ foo="bar" }[$__range]))'
+      'group by (client, connection_type, server) (traces_service_graph_request_server_seconds_sum{ foo="bar" })'
    );
    expect(nthQuery(0).targets[2].expr).toBe(
-      'sum by (client, server) (rate(traces_service_graph_request_failed_total{ foo="bar" }[$__range]))'
+      'sum by (client, server) (rate(traces_service_graph_request_total{ foo="bar" }[$__range]))'
    );
    expect(nthQuery(0).targets[3].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_total{ foo="bar" })'
+    );
+    expect(nthQuery(0).targets[4].expr).toBe(
+      'sum by (client, server) (rate(traces_service_graph_request_failed_total{ foo="bar" }[$__range]))'
+    );
+    expect(nthQuery(0).targets[5].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_failed_total{ foo="bar" })'
+    );
+    expect(nthQuery(0).targets[6].expr).toBe(
      'sum by (client, server) (rate(traces_service_graph_request_server_seconds_bucket{ foo="bar" }[$__range]))'
    );
+    expect(nthQuery(0).targets[7].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_server_seconds_bucket{ foo="bar" })'
+    );
  });

  it('runs correct queries with multiple serviceMapQuery defined', async () => {
@@ -632,14 +672,26 @@ describe('Tempo service graph view', () => {
      'sum by (client, server) (rate(traces_service_graph_request_server_seconds_sum{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_server_seconds_sum{baz="bad"}[$__range]))'
    );
    expect(nthQuery(0).targets[1].expr).toBe(
-      'sum by (client, server) (rate(traces_service_graph_request_total{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_total{baz="bad"}[$__range]))'
+      'group by (client, connection_type, server) (traces_service_graph_request_server_seconds_sum{ foo="bar" }) OR group by (client, connection_type, server) (traces_service_graph_request_server_seconds_sum{baz="bad"})'
    );
    expect(nthQuery(0).targets[2].expr).toBe(
-      'sum by (client, server) (rate(traces_service_graph_request_failed_total{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_failed_total{baz="bad"}[$__range]))'
+      'sum by (client, server) (rate(traces_service_graph_request_total{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_total{baz="bad"}[$__range]))'
    );
    expect(nthQuery(0).targets[3].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_total{ foo="bar" }) OR group by (client, connection_type, server) (traces_service_graph_request_total{baz="bad"})'
+    );
+    expect(nthQuery(0).targets[4].expr).toBe(
+      'sum by (client, server) (rate(traces_service_graph_request_failed_total{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_failed_total{baz="bad"}[$__range]))'
+    );
+    expect(nthQuery(0).targets[5].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_failed_total{ foo="bar" }) OR group by (client, connection_type, server) (traces_service_graph_request_failed_total{baz="bad"})'
+    );
+    expect(nthQuery(0).targets[6].expr).toBe(
      'sum by (client, server) (rate(traces_service_graph_request_server_seconds_bucket{ foo="bar" }[$__range])) OR sum by (client, server) (rate(traces_service_graph_request_server_seconds_bucket{baz="bad"}[$__range]))'
    );
+    expect(nthQuery(0).targets[7].expr).toBe(
+      'group by (client, connection_type, server) (traces_service_graph_request_server_seconds_bucket{ foo="bar" }) OR group by (client, connection_type, server) (traces_service_graph_request_server_seconds_bucket{baz="bad"})'
+    );
  });

  it('should build expr correctly', () => {
@@ -803,27 +855,33 @@ describe('Tempo service graph view', () => {
          url: '',
          title: 'View traces',
          internal: {
-            query: {
-              refId: 'A',
-              queryType: 'traceqlSearch',
-              filters: [
-                {
-                  id: 'service-name',
-                  operator: '=',
-                  scope: 'resource',
-                  tag: 'service.name',
-                  value: '${__data.fields.target}',
-                  valueType: 'string',
-                },
-              ],
-            },
-            datasourceUid: 'EbPO1fYnz',
            datasourceName: '',
+            datasourceUid: 'EbPO1fYnz',
+            query: expect.any(Function),
          },
        },
      ],
    };
    expect(fieldConfig).toStrictEqual(resultObj);
+
+    const viewServicesLink: DataLink | undefined = fieldConfig.links.find(
+      (link: DataLink) => link.title === 'View traces'
+    );
+    expect(viewServicesLink).toBeDefined();
+    expect(viewServicesLink!.internal!.query({ replaceVariables: replaceVariablesInstrumented })).toEqual({
+      refId: 'A',
+      queryType: 'traceqlSearch',
+      filters: [
+        {
+          id: 'service-name',
+          operator: '=',
+          scope: 'resource',
+          tag: 'service.name',
+          value: 'my-service',
+          valueType: 'string',
+        },
+      ],
+    });
  });

  it('should get field config correctly when namespaces are present', () => {
@@ -894,35 +952,41 @@ describe('Tempo service graph view', () => {
          url: '',
          title: 'View traces',
          internal: {
-            query: {
-              queryType: 'traceqlSearch',
-              refId: 'A',
-              filters: [
-                {
-                  id: 'service-namespace',
-                  operator: '=',
-                  scope: 'resource',
-                  tag: 'service.namespace',
-                  value: '${__data.fields.targetNamespace}',
-                  valueType: 'string',
-                },
-                {
-                  id: 'service-name',
-                  operator: '=',
-                  scope: 'resource',
-                  tag: 'service.name',
-                  value: '${__data.fields.targetName}',
-                  valueType: 'string',
-                },
-              ],
-            },
-            datasourceUid: 'EbPO1fYnz',
            datasourceName: '',
+            datasourceUid: 'EbPO1fYnz',
+            query: expect.any(Function),
          },
        },
      ],
    };
    expect(fieldConfig).toStrictEqual(resultObj);
+
+    const viewServicesLink: DataLink | undefined = fieldConfig.links.find(
+      (link: DataLink) => link.title === 'View traces'
+    );
+    expect(viewServicesLink).toBeDefined();
+    expect(viewServicesLink!.internal!.query({ replaceVariables: replaceVariablesInstrumented })).toEqual({
+      refId: 'A',
+      queryType: 'traceqlSearch',
+      filters: [
+        {
+          id: 'service-namespace',
+          operator: '=',
+          scope: 'resource',
+          tag: 'service.namespace',
+          value: 'my-namespace',
+          valueType: 'string',
+        },
+        {
+          id: 'service-name',
+          operator: '=',
+          scope: 'resource',
+          tag: 'service.name',
+          value: 'my-service',
+          valueType: 'string',
+        },
+      ],
+    });
  });

  it('should get rate aligned values correctly', () => {
@@ -1435,26 +1499,31 @@ const serviceGraphLinks = [
    url: '',
    title: 'View traces',
    internal: {
-      query: {
-        refId: 'A',
-        queryType: 'traceqlSearch',
-        filters: [
-          {
-            id: 'service-name',
-            operator: '=',
-            scope: 'resource',
-            tag: 'service.name',
-            value: '${__data.fields.id}',
-            valueType: 'string',
-          },
-        ],
-      } as TempoQuery,
+      query: expect.any(Function),
      datasourceUid: 'gdev-tempo',
      datasourceName: 'Tempo',
    },
  },
 ];

+const replaceVariablesInstrumented = (variable: string): string => {
+  const variables: Record<string, string> = {
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.title}}`]: 'my-service',
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.subTitle}}`]: 'my-namespace',
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.isInstrumented}}`]: 'true',
+  };
+  return variables[variable];
+};
+
+const replaceVariablesUninstrumented = (variable: string): string => {
+  const variables: Record<string, string> = {
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.title}}`]: 'my-service',
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.subTitle}}`]: 'my-namespace',
+    [`\${__data.fields.${NodeGraphDataFrameFieldNames.isInstrumented}}`]: 'false',
+  };
+  return variables[variable];
+};
+
 interface PromQuery extends DataQuery {
  expr: string;
 }
@@ -7,6 +7,7 @@ import {
  CoreApp,
  DataFrame,
  DataFrameDTO,
+  DataLink,
  DataQueryRequest,
  DataQueryResponse,
  DataQueryResponseData,
@@ -15,6 +16,7 @@ import {
  dateTime,
  FieldType,
  LoadingState,
+  NodeGraphDataFrameFieldNames,
  rangeUtil,
  ScopedVars,
  SelectableValue,
@@ -1122,13 +1124,7 @@ export function getFieldConfig(
        datasourceUid,
        false
      ),
-      makeTempoLink(
-        'View traces',
-        namespaceFields !== undefined ? `\${${namespaceFields.targetNamespace}}` : '',
-        `\${${targetField}}`,
-        '',
-        tempoDatasourceUid
-      ),
+      makeTempoLinkServiceMap('View traces', tempoDatasourceUid, !!namespaceFields?.targetNamespace),
    ],
  };
 }
@@ -1183,25 +1179,99 @@ export function makeTempoLink(
  };
 }

+function makeTempoLinkServiceMap(
+  title: string,
+  datasourceUid: string,
+  includeNamespace: boolean
+): DataLink<TempoQuery> {
+  return {
+    url: '',
+    title,
+    internal: {
+      datasourceUid,
+      datasourceName: getDataSourceSrv().getInstanceSettings(datasourceUid)?.name ?? '',
+      query: ({ replaceVariables, scopedVars }) => {
+        const serviceName = replaceVariables?.(`\${__data.fields.${NodeGraphDataFrameFieldNames.title}}`, scopedVars);
+        const serviceNamespace = replaceVariables?.(
+          `\${__data.fields.${NodeGraphDataFrameFieldNames.subTitle}}`,
+          scopedVars
+        );
+        const isInstrumented =
+          replaceVariables?.(`\${__data.fields.${NodeGraphDataFrameFieldNames.isInstrumented}}`, scopedVars) !==
+          'false';
+        const query: TempoQuery = { refId: 'A', queryType: 'traceqlSearch', filters: [] };
+
+        // Only do the peer query if service is actively set as not instrumented
+        if (isInstrumented === false) {
+          const filters = ['db.name', 'db.system', 'peer.service', 'messaging.system', 'net.peer.name']
+            .map((peerAttribute) => `span.${peerAttribute}="${serviceName}"`)
+            .join(' || ');
+          query.queryType = 'traceql';
+          query.query = `{${filters}}`;
+        } else {
+          if (includeNamespace && serviceNamespace) {
+            query.filters.push({
+              id: 'service-namespace',
+              scope: TraceqlSearchScope.Resource,
+              tag: 'service.namespace',
+              value: serviceNamespace,
+              operator: '=',
+              valueType: 'string',
+            });
+          }
+          if (serviceName) {
+            query.filters.push({
+              id: 'service-name',
+              scope: TraceqlSearchScope.Resource,
+              tag: 'service.name',
+              value: serviceName,
+              operator: '=',
+              valueType: 'string',
+            });
+          }
+        }
+
+        return query;
+      },
+    },
+  };
+}
+
 function makePromServiceMapRequest(options: DataQueryRequest<TempoQuery>): DataQueryRequest<PromQuery> {
  return {
    ...options,
-    targets: serviceMapMetrics.map((metric) => {
-      const { serviceMapQuery, serviceMapIncludeNamespace: serviceMapIncludeNamespace } = options.targets[0];
-      const extraSumByFields = serviceMapIncludeNamespace ? ', client_service_namespace, server_service_namespace' : '';
-      const queries = Array.isArray(serviceMapQuery) ? serviceMapQuery : [serviceMapQuery];
-      const subExprs = queries.map(
-        (query) => `sum by (client, server${extraSumByFields}) (rate(${metric}${query || ''}[$__range]))`
-      );
-      return {
-        format: 'table',
-        refId: metric,
-        // options.targets[0] is not correct here, but not sure what should happen if you have multiple queries for
-        // service map at the same time anyway
-        expr: subExprs.join(' OR '),
-        instant: true,
-      };
-    }),
+    targets: serviceMapMetrics
+      .map<PromQuery[]>((metric) => {
+        const { serviceMapQuery, serviceMapIncludeNamespace: serviceMapIncludeNamespace } = options.targets[0];
+        const extraSumByFields = serviceMapIncludeNamespace
+          ? ', client_service_namespace, server_service_namespace'
+          : '';
+        const queries = Array.isArray(serviceMapQuery) ? serviceMapQuery : [serviceMapQuery];
+        const sumSubExprs = queries.map(
+          (query) => `sum by (client, server${extraSumByFields}) (rate(${metric}${query || ''}[$__range]))`
+        );
+        const groupSubExprs = queries.map(
+          (query) => `group by (client, connection_type, server${extraSumByFields}) (${metric}${query || ''})`
+        );
+
+        return [
+          {
+            format: 'table',
+            refId: metric,
+            // options.targets[0] is not correct here, but not sure what should happen if you have multiple queries for
+            // service map at the same time anyway
+            expr: sumSubExprs.join(' OR '),
+            instant: true,
+          },
+          {
+            format: 'table',
+            refId: `${metric}_labels`,
+            expr: groupSubExprs.join(' OR '),
+            instant: true,
+          },
+        ];
+      })
+      .flat(),
  };
 }

@@ -20,6 +20,7 @@ it('assigns correct field type even if values are numbers', async () => {
    { name: 'secondarystat', values: [10, 20], type: FieldType.number },
    { name: 'arc__success', values: [1, 1], type: FieldType.number },
    { name: 'arc__failed', values: [0, 0], type: FieldType.number },
+    { name: 'isinstrumented', values: [true, true], type: FieldType.boolean },
  ]);
 });

@@ -41,6 +42,7 @@ it('do not fail on response with empty list', async () => {
    { name: 'secondarystat', values: [], type: FieldType.number },
    { name: 'arc__success', values: [], type: FieldType.number },
    { name: 'arc__failed', values: [], type: FieldType.number },
+    { name: 'isinstrumented', values: [], type: FieldType.boolean },
  ]);
 });

@@ -66,6 +68,7 @@ describe('mapPromMetricsToServiceMap', () => {
      { name: 'secondarystat', values: [10, 20, NaN] },
      { name: 'arc__success', values: [0.8, 0.25, 1] },
      { name: 'arc__failed', values: [0.2, 0.75, 0] },
+      { name: 'isinstrumented', values: [true, true, true] },
    ]);
    expect(edges.fields).toMatchObject([
      { name: 'id', values: ['app_db', 'lb_app'] },
@@ -101,6 +104,7 @@ describe('mapPromMetricsToServiceMap', () => {
      { name: 'secondarystat', values: [10, 20, NaN] },
      { name: 'arc__success', values: [0.8, 0.25, 1] },
      { name: 'arc__failed', values: [0.2, 0.75, 0] },
+      { name: 'isinstrumented', values: [true, true, true] },
    ]);
    expect(edges.fields).toMatchObject([
      { name: 'id', values: ['ns1/app_ns3/db', 'ns2/lb_ns1/app'] },
@@ -138,6 +142,41 @@ describe('mapPromMetricsToServiceMap', () => {
      { name: 'secondarystat', values: [10, 20, NaN] },
      { name: 'arc__success', values: [0, 0, 1] },
      { name: 'arc__failed', values: [1, 1, 0] },
+      { name: 'isinstrumented', values: [true, true, true] },
+    ]);
+  });
+
+  it('handles setting isInstrumented based on the connection_type', () => {
+    const range = {
+      from: dateTime('2000-01-01T00:00:00'),
+      to: dateTime('2000-01-01T00:01:00'),
+    };
+    const { nodes } = mapPromMetricsToServiceMap(
+      [
+        {
+          data: [
+            totalsPromMetric(true),
+            secondsPromMetric(true),
+            secondsLabelsPromMetric(true),
+            failedPromMetric(true),
+          ],
+        },
+      ],
+      {
+        ...range,
+        raw: range,
+      }
+    );
+
+    expect(nodes.fields).toMatchObject([
+      { name: 'id', values: ['ns3/db', 'ns1/app', 'ns2/lb'] },
+      { name: 'title', values: ['db', 'app', 'lb'] },
+      { name: 'subtitle', values: ['ns3', 'ns1', 'ns2'] },
+      { name: 'mainstat', values: [1000, 2000, NaN] },
+      { name: 'secondarystat', values: [10, 20, NaN] },
+      { name: 'arc__success', values: [0.8, 0.25, 1] },
+      { name: 'arc__failed', values: [0.2, 0.75, 0] },
+      { name: 'isinstrumented', values: [true, false, true] },
    ]);
  });
 });
@@ -182,6 +221,27 @@ const secondsPromMetric = (namespace?: boolean) =>
    ],
  });

+const secondsLabelsPromMetric = (namespace?: boolean) =>
+  createDataFrame({
+    refId: 'traces_service_graph_request_server_seconds_sum_labels',
+    fields: [
+      { name: 'Time', values: [1628169788000, 1628169788000] },
+      { name: 'client', values: ['app', 'lb'] },
+      { name: 'instance', values: ['127.0.0.1:12345', '127.0.0.1:12345'] },
+      { name: 'job', values: ['local_scrape', 'local_scrape'] },
+      { name: 'server', values: ['db', 'app'] },
+      { name: 'tempo_config', values: ['default', 'default'] },
+      { name: 'Value #traces_service_graph_request_server_seconds_sum_label', values: [1, 1] },
+      { name: 'connection_type', values: ['messaging_system', 'virtual_node'] },
+      ...(namespace
+        ? [
+            { name: 'client_service_namespace', values: ['ns1', 'ns2'] },
+            { name: 'server_service_namespace', values: ['ns3', 'ns1'] },
+          ]
+        : []),
+    ],
+  });
+
 const failedPromMetric = (namespace?: boolean) =>
  createDataFrame({
    refId: 'traces_service_graph_request_failed_total',
@@ -61,6 +61,10 @@ export function mapPromMetricsToServiceMap(
  collectMetricData(frames[secondsMetric], 'seconds', secondsMetric, nodesMap, edgesMap);
  collectMetricData(frames[failedMetric], 'failed', failedMetric, nodesMap, edgesMap);

+  collectIsInstrumented(frames[`${totalsMetric}_labels`], nodesMap);
+  collectIsInstrumented(frames[`${secondsMetric}_labels`], nodesMap);
+  collectIsInstrumented(frames[`${failedMetric}_labels`], nodesMap);
+
  return convertToDataFrames(nodesMap, edgesMap, range);
 }

@@ -97,6 +101,11 @@ function createServiceMapDataFrames() {
      config: { displayName: 'Failed', color: { fixedColor: 'red', mode: FieldColorModeId.Fixed } },
      values: [],
    },
+    {
+      name: Fields.isInstrumented,
+      type: FieldType.boolean,
+      values: [],
+    },
  ]);
  const edges = createDF('Edges', [
    { name: Fields.id, type: FieldType.string, values: [] },
@@ -145,6 +154,7 @@ type ServiceMapStatistics = {
 type NodeObject = ServiceMapStatistics & {
  name: string;
  namespace?: string;
+  isInstrumented?: boolean;
 };

 type EdgeObject = ServiceMapStatistics & {
@@ -240,6 +250,21 @@ function collectMetricData(
  }
 }

+function collectIsInstrumented(frame: DataFrameView | undefined, nodesMap: Record<string, NodeObject>) {
+  if (!frame) {
+    return;
+  }
+
+  for (let i = 0; i < frame.length; i++) {
+    const row = frame.get(i);
+    const serverId = row.server_service_namespace ? `${row.server_service_namespace}/${row.server}` : row.server;
+
+    if (nodesMap[serverId] && nodesMap[serverId].isInstrumented !== true) {
+      nodesMap[serverId].isInstrumented = row.connection_type === '' || row.connection_type === 'messaging_system';
+    }
+  }
+}
+
 function convertToDataFrames(
  nodesMap: Record<string, NodeObject>,
  edgesMap: Record<string, EdgeObject>,
@@ -258,6 +283,7 @@ function convertToDataFrames(
      [Fields.secondaryStat]: node.total ? Math.round(node.total * 100) / 100 : Number.NaN, // Request per second (to 2 decimals)
      [Fields.arc + 'success']: node.total ? (node.total - Math.min(node.failed || 0, node.total)) / node.total : 1,
      [Fields.arc + 'failed']: node.total ? Math.min(node.failed || 0, node.total) / node.total : 0,
+      [Fields.isInstrumented]: node.isInstrumented ?? true,
    });
  }
  for (const edgeId of Object.keys(edgesMap)) {
@@ -17,6 +17,7 @@ export type NodeDatum = SimulationNodeDatum & {
  icon?: IconName;
  nodeRadius?: Field;
  highlighted: boolean;
+  isInstrumented?: boolean;
 };

 export type NodeDatumFromEdge = NodeDatum & { mainStatNumeric?: number; secondaryStatNumeric?: number };
@@ -19,7 +19,7 @@ describe('processNodes', () => {

  it('returns proper nodes and edges', async () => {
    const { nodes, edges, legend } = processNodes(
-      makeNodesDataFrame(3),
+      makeNodesDataFrame(3, [{ isinstrumented: false }]),
      makeEdgesDataFrame([
        { source: '0', target: '1' },
        { source: '0', target: '2' },
@@ -28,7 +28,7 @@ describe('processNodes', () => {
    );

    expect(nodes).toEqual([
-      makeNodeDatum(),
+      makeNodeDatum({ isInstrumented: false }),
      makeNodeDatum({ dataFrameRowIndex: 1, id: '1', incoming: 1, title: 'service:1' }),
      makeNodeDatum({ dataFrameRowIndex: 2, id: '2', incoming: 2, title: 'service:2' }),
    ]);
@@ -366,6 +366,7 @@ function makeNodeDatum(options: Partial<NodeDatum> = {}) {
      type: 'number',
      values: [40, 40, 40],
    },
+    isInstrumented: true,
    ...options,
  };
 }
@@ -58,6 +58,7 @@ export type NodeFields = {
  icon?: Field;
  nodeRadius?: Field;
  highlighted?: Field;
+  isInstrumented?: Field;
 };

 export function getNodeFields(nodes: DataFrame): NodeFields {
@@ -80,6 +81,7 @@ export function getNodeFields(nodes: DataFrame): NodeFields {
    highlighted: fieldsCache.getFieldByName(NodeGraphDataFrameFieldNames.highlighted.toLowerCase()),
    fixedX: fieldsCache.getFieldByName(NodeGraphDataFrameFieldNames.fixedX.toLowerCase()),
    fixedY: fieldsCache.getFieldByName(NodeGraphDataFrameFieldNames.fixedY.toLowerCase()),
+    isInstrumented: fieldsCache.getFieldByName(NodeGraphDataFrameFieldNames.isInstrumented.toLowerCase()),
  };
 }

@@ -364,6 +366,7 @@ function makeNodeDatum(id: string, nodeFields: NodeFields, index: number): NodeD
    highlighted: nodeFields.highlighted?.values[index] || false,
    x: nodeFields.fixedX?.values[index] ?? undefined,
    y: nodeFields.fixedY?.values[index] ?? undefined,
+    isInstrumented: nodeFields.isInstrumented?.values[index] ?? true,
  };
 }

@@ -384,16 +387,19 @@ export function statToString(config: FieldConfig, value: number | string): strin
 * Utilities mainly for testing
 */

-export function makeNodesDataFrame(count: number) {
+export function makeNodesDataFrame(
+  count: number,
+  partialNodes: Array<Partial<Record<NodeGraphDataFrameFieldNames, unknown>>> = []
+) {
  const frame = nodesFrame();
  for (let i = 0; i < count; i++) {
-    frame.add(makeNode(i));
+    frame.add(makeNode(i, partialNodes[i]));
  }

  return frame;
 }

-function makeNode(index: number) {
+function makeNode(index: number, partialNode: Partial<Record<NodeGraphDataFrameFieldNames, unknown>> = {}) {
  return {
    id: index.toString(),
    title: `service:${index}`,
@@ -405,6 +411,8 @@ function makeNode(index: number) {
    color: 0.5,
    icon: 'database',
    noderadius: 40,
+    isinstrumented: true,
+    ...partialNode,
  };
 }

@@ -453,6 +461,10 @@ function nodesFrame() {
      values: [],
      type: FieldType.number,
    },
+    [NodeGraphDataFrameFieldNames.isInstrumented]: {
+      values: [],
+      type: FieldType.boolean,
+    },
  };

  return new MutableDataFrame({
@@ -724,6 +724,9 @@
    "custom-value": {
      "description": "Use custom value"
    },
+    "group": {
+      "undefined": "No group"
+    },
    "options": {
      "no-found": "No options found."
    }
@@ -724,6 +724,9 @@
    "custom-value": {
      "description": "Ůşę čūşŧőm väľūę"
    },
+    "group": {
+      "undefined": "Ńő ģřőūp"
+    },
    "options": {
      "no-found": "Ńő őpŧįőŉş ƒőūŉđ."
    }