Compare commits

..
Author SHA1 Message Date
Georges Chaudy 3c83c64d72 Implement batch operation support in KV storage
- Introduced a new Batch method to execute multiple operations atomically within a single transaction.
- Defined BatchOp types for various operations: put, create, update, and delete, with specific semantics for each.
- Updated the KV interface and implementation to support batch operations, ensuring rollback on failure.
- Enhanced testing suite to cover various batch scenarios, including success, failure, and edge cases, ensuring robust functionality.
2026-01-09 15:24:34 +01:00
Georges Chaudy 6e39b24b6f Enhance comparison functions in KV storage
- Updated the CompareKeyExists function to simplify its usage by removing the 'exists' parameter, making it always succeed if the key exists.
- Introduced a new CompareKeyNotExists function to check for non-existent keys.
- Modified transaction tests to utilize the new comparison functions, ensuring accurate behavior for both existing and non-existing keys.
2025-12-04 16:17:43 +01:00
Georges Chaudy c31c1d8e8d Add transaction support to KV storage
- Introduced transaction operations with compare-and-swap semantics.
- Added types and functions for comparisons (e.g., CompareExists, CompareValue) and transaction operations (e.g., TxnOpPut, TxnOpDelete).
- Implemented Txn method to execute transactions based on comparisons and success/failure operations.
- Added validation for transaction requests to enforce limits on comparisons and operations.
- Enhanced testing suite to cover various transaction scenarios, including success and failure cases, and edge conditions.

This update enhances the KV interface, allowing for more complex data manipulation while ensuring data integrity through transactional guarantees.
2025-12-03 11:33:22 +01:00
Paul Marbach e36ea78771 Suggestions: Deprecate the old API and put external suggestions behind a flag (#114127)
* Suggestions: Deprecate previous API, enable external plugin suggestions behind flag

* fix types for deprecated builder

* restore some support for cloud-onboarding

* add support for cloud-onboarding usage, add test to ensure it keeps working

* refactor to not hardcode on 'core:'

* remove unused import
2025-12-01 23:22:22 +00:00
Steve Simpson b332a108f3 Alerting: Notification history query API. (#114677)
* Alerting: Notification history query API.

First cut at defining a namespace scoped route on the historian.alerting app
to query notification history.

* Address review comments
2025-12-02 00:14:54 +01:00
Todd Treece 1060dd538a CI: Run lint on self-hosted ubuntu-x64-small (#114674) 2025-12-01 22:27:14 +00:00
Todd Treece be8076dee8 CI: Run lint on ubuntu-latest-8-cores (#114673) 2025-12-01 21:40:46 +00:00
Ashley Harrison 7f1ac6188a PanelChrome: Wrapping div needs height: 100% as well (#114655)
wrapping div needs height: 100% as well
2025-12-01 17:39:15 +00:00
Rafael Bortolon Paulovic 31eaf1e898 chore: add log and metric before unified migration enforcement (#114598) 2025-12-01 17:56:59 +01:00
75 changed files with 1794 additions and 1599 deletions
+1 -1
View File
@@ -57,7 +57,7 @@ jobs:
lint-go:
needs: detect-changes
if: needs.detect-changes.outputs.changed == 'true'
runs-on: ubuntu-latest
runs-on: ubuntu-x64-large-io
steps:
- uses: actions/checkout@v5
with:
+9 -23
View File
@@ -1,34 +1,20 @@
package kinds
import (
"github.com/grafana/grafana/apps/alerting/historian/kinds/v0alpha1"
)
manifest: {
appName: "alerting-historian"
groupOverride: "historian.alerting.grafana.app"
versions: {
"v0alpha1": v0alpha1
"v0alpha1": {
kinds: [dummyv0alpha1]
routes: v0alpha1.routes
}
}
}
v0alpha1: {
kinds: [dummyv0alpha1]
routes: {
namespaced: {
// This endpoint is an exact copy of the existing /history endpoint,
// with the exception that error responses will be Kubernetes-style,
// not Grafana-style. It will be replaced in the future with a better
// more schema-friendly API.
"/alertstate/history": {
"GET": {
response: {
body: [string]: _
}
responseMetadata: typeMeta: false
}
}
}
}
}
dummyv0alpha1: {
kind: "Dummy"
schema: {
@@ -37,4 +23,4 @@ dummyv0alpha1: {
dummyField: int
}
}
}
}
@@ -0,0 +1,9 @@
package v0alpha1
#Matcher: {
type: "=" | "!=" | "=~" | "!~" @cuetsy(kind="enum",memberNames="Equal|NotEqual|EqualRegex|NotEqualRegex")
label: string
value: string
}
#Matchers: [...#Matcher]
@@ -0,0 +1,65 @@
package v0alpha1
import (
"time"
)
#NotificationStatus: "firing" | "resolved" @cog(kind="enum",memberNames="Firing|Resolved")
#NotificationOutcome: "success" | "error" @cog(kind="enum",memberNames="Success|Error")
#NotificationQuery: {
// From is the starting timestamp for the query.
from?: time.Time
// To is the starting timestamp for the query.
to?: time.Time
// Limit is the maximum number of entries to return.
limit?: int64
// Receiver optionally filters the entries by receiver title (contact point).
receiver?: string
// Status optionally filters the entries to only either firing or resolved.
status?: #NotificationStatus
// Outcome optionally filters the entries to only either successful or failed attempts.
outcome?: #NotificationOutcome
// RuleUID optionally filters the entries to a specific alert rule.
ruleUID?: string
// GroupLabels optionally filters the entries by matching group labels.
groupLabels?: #Matchers
}
#NotificationQueryResult: {
entries: [...#NotificationEntry]
}
#NotificationEntry: {
// Timestamp is the time at which the notification attempt completed.
timestamp: time.Time
// Receiver is the receiver (contact point) title.
receiver: string
// Status indicates if the notification contains one or more firing alerts.
status: #NotificationStatus
// Outcome indicaes if the notificaion attempt was successful or if it failed.
outcome: #NotificationOutcome
// GroupLabels are the labels uniquely identifying the alert group within a route.
groupLabels: [string]: string
// Alerts are the alerts grouped into the notification.
alerts: [...#NotificationEntryAlert]
// Retry indicates if the attempt was a retried attempt.
retry: bool
// Error is the message returned by the contact point if delivery failed.
error?: string
// Duration is the length of time the notification attempt took in nanoseconds.
duration: int
// PipelineTime is the time at which the flush began.
pipelineTime: time.Time
// GroupKey uniquely idenifies the dispatcher alert group.
groupKey: string
}
#NotificationEntryAlert: {
status: string
labels: [string]: string
annotations: [string]: string
startsAt: time.Time
endsAt: time.Time
}
@@ -0,0 +1,29 @@
package v0alpha1
routes: {
namespaced: {
// This endpoint is an exact copy of the existing /history endpoint,
// with the exception that error responses will be Kubernetes-style,
// not Grafana-style. It will be replaced in the future with a better
// more schema-friendly API.
"/alertstate/history": {
"GET": {
response: {
body: [string]: _
}
responseMetadata: typeMeta: false
}
}
// Query notification history.
"/notification/query": {
"POST": {
request: {
body: #NotificationQuery
}
response: #NotificationQueryResult
responseMetadata: typeMeta: false
}
}
}
}
@@ -0,0 +1,67 @@
// Code generated - EDITING IS FUTILE. DO NOT EDIT.
package v0alpha1
import (
time "time"
)
type CreateNotificationqueryRequestNotificationStatus string
const (
CreateNotificationqueryRequestNotificationStatusFiring CreateNotificationqueryRequestNotificationStatus = "firing"
CreateNotificationqueryRequestNotificationStatusResolved CreateNotificationqueryRequestNotificationStatus = "resolved"
)
type CreateNotificationqueryRequestNotificationOutcome string
const (
CreateNotificationqueryRequestNotificationOutcomeSuccess CreateNotificationqueryRequestNotificationOutcome = "success"
CreateNotificationqueryRequestNotificationOutcomeError CreateNotificationqueryRequestNotificationOutcome = "error"
)
type CreateNotificationqueryRequestMatchers []CreateNotificationqueryRequestMatcher
type CreateNotificationqueryRequestMatcher struct {
Type CreateNotificationqueryRequestMatcherType `json:"type"`
Label string `json:"label"`
Value string `json:"value"`
}
// NewCreateNotificationqueryRequestMatcher creates a new CreateNotificationqueryRequestMatcher object.
func NewCreateNotificationqueryRequestMatcher() *CreateNotificationqueryRequestMatcher {
return &CreateNotificationqueryRequestMatcher{}
}
type CreateNotificationqueryRequestBody struct {
// From is the starting timestamp for the query.
From *time.Time `json:"from,omitempty"`
// To is the starting timestamp for the query.
To *time.Time `json:"to,omitempty"`
// Limit is the maximum number of entries to return.
Limit *int64 `json:"limit,omitempty"`
// Receiver optionally filters the entries by receiver title (contact point).
Receiver *string `json:"receiver,omitempty"`
// Status optionally filters the entries to only either firing or resolved.
Status *CreateNotificationqueryRequestNotificationStatus `json:"status,omitempty"`
// Outcome optionally filters the entries to only either successful or failed attempts.
Outcome *CreateNotificationqueryRequestNotificationOutcome `json:"outcome,omitempty"`
// RuleUID optionally filters the entries to a specific alert rule.
RuleUID *string `json:"ruleUID,omitempty"`
// GroupLabels optionally filters the entries by matching group labels.
GroupLabels *CreateNotificationqueryRequestMatchers `json:"groupLabels,omitempty"`
}
// NewCreateNotificationqueryRequestBody creates a new CreateNotificationqueryRequestBody object.
func NewCreateNotificationqueryRequestBody() *CreateNotificationqueryRequestBody {
return &CreateNotificationqueryRequestBody{}
}
type CreateNotificationqueryRequestMatcherType string
const (
CreateNotificationqueryRequestMatcherTypeEqual CreateNotificationqueryRequestMatcherType = "="
CreateNotificationqueryRequestMatcherTypeNotEqual CreateNotificationqueryRequestMatcherType = "!="
CreateNotificationqueryRequestMatcherTypeEqualRegex CreateNotificationqueryRequestMatcherType = "=~"
CreateNotificationqueryRequestMatcherTypeNotEqualRegex CreateNotificationqueryRequestMatcherType = "!~"
)
@@ -0,0 +1,86 @@
// Code generated - EDITING IS FUTILE. DO NOT EDIT.
package v0alpha1
import (
time "time"
)
// +k8s:openapi-gen=true
type NotificationEntry struct {
// Timestamp is the time at which the notification attempt completed.
Timestamp time.Time `json:"timestamp"`
// Receiver is the receiver (contact point) title.
Receiver string `json:"receiver"`
// Status indicates if the notification contains one or more firing alerts.
Status NotificationStatus `json:"status"`
// Outcome indicaes if the notificaion attempt was successful or if it failed.
Outcome NotificationOutcome `json:"outcome"`
// GroupLabels are the labels uniquely identifying the alert group within a route.
GroupLabels map[string]string `json:"groupLabels"`
// Alerts are the alerts grouped into the notification.
Alerts []NotificationEntryAlert `json:"alerts"`
// Retry indicates if the attempt was a retried attempt.
Retry bool `json:"retry"`
// Error is the message returned by the contact point if delivery failed.
Error *string `json:"error,omitempty"`
// Duration is the length of time the notification attempt took in nanoseconds.
Duration int64 `json:"duration"`
// PipelineTime is the time at which the flush began.
PipelineTime time.Time `json:"pipelineTime"`
// GroupKey uniquely idenifies the dispatcher alert group.
GroupKey string `json:"groupKey"`
}
// NewNotificationEntry creates a new NotificationEntry object.
func NewNotificationEntry() *NotificationEntry {
return &NotificationEntry{
GroupLabels: map[string]string{},
Alerts: []NotificationEntryAlert{},
}
}
// +k8s:openapi-gen=true
type NotificationStatus string
const (
NotificationStatusFiring NotificationStatus = "firing"
NotificationStatusResolved NotificationStatus = "resolved"
)
// +k8s:openapi-gen=true
type NotificationOutcome string
const (
NotificationOutcomeSuccess NotificationOutcome = "success"
NotificationOutcomeError NotificationOutcome = "error"
)
// +k8s:openapi-gen=true
type NotificationEntryAlert struct {
Status string `json:"status"`
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
StartsAt time.Time `json:"startsAt"`
EndsAt time.Time `json:"endsAt"`
}
// NewNotificationEntryAlert creates a new NotificationEntryAlert object.
func NewNotificationEntryAlert() *NotificationEntryAlert {
return &NotificationEntryAlert{
Labels: map[string]string{},
Annotations: map[string]string{},
}
}
// +k8s:openapi-gen=true
type CreateNotificationquery struct {
Entries []NotificationEntry `json:"entries"`
}
// NewCreateNotificationquery creates a new CreateNotificationquery object.
func NewCreateNotificationquery() *CreateNotificationquery {
return &CreateNotificationquery{
Entries: []NotificationEntry{},
}
}
+318 -3
View File
@@ -92,9 +92,321 @@ var appManifestData = app.ManifestData{
},
},
},
"/notification/query": {
Post: &spec3.Operation{
OperationProps: spec3.OperationProps{
OperationId: "createNotificationquery",
RequestBody: &spec3.RequestBody{
RequestBodyProps: spec3.RequestBodyProps{
Content: map[string]*spec3.MediaType{
"application/json": {
MediaTypeProps: spec3.MediaTypeProps{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"from": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Format: "date-time",
Description: "From is the starting timestamp for the query.",
},
},
"groupLabels": {
SchemaProps: spec.SchemaProps{
Description: "GroupLabels optionally filters the entries by matching group labels.",
Ref: spec.MustCreateRef("#/components/schemas/createNotificationqueryMatchers"),
},
},
"limit": {
SchemaProps: spec.SchemaProps{
Type: []string{"integer"},
Description: "Limit is the maximum number of entries to return.",
},
},
"outcome": {
SchemaProps: spec.SchemaProps{
Description: "Outcome optionally filters the entries to only either successful or failed attempts.",
Ref: spec.MustCreateRef("#/components/schemas/createNotificationqueryNotificationOutcome"),
},
},
"receiver": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Description: "Receiver optionally filters the entries by receiver title (contact point).",
},
},
"ruleUID": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Description: "RuleUID optionally filters the entries to a specific alert rule.",
},
},
"status": {
SchemaProps: spec.SchemaProps{
Description: "Status optionally filters the entries to only either firing or resolved.",
Ref: spec.MustCreateRef("#/components/schemas/createNotificationqueryNotificationStatus"),
},
},
"to": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Format: "date-time",
Description: "To is the starting timestamp for the query.",
},
},
},
}},
}},
},
}},
Responses: &spec3.Responses{
ResponsesProps: spec3.ResponsesProps{
Default: &spec3.Response{
ResponseProps: spec3.ResponseProps{
Description: "Default OK response",
Content: map[string]*spec3.MediaType{
"application/json": {
MediaTypeProps: spec3.MediaTypeProps{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"entries": {
SchemaProps: spec.SchemaProps{
Type: []string{"array"},
},
},
},
Required: []string{
"entries",
},
}},
}},
},
},
},
}},
},
},
},
},
Cluster: map[string]spec3.PathProps{},
Schemas: map[string]spec.Schema{},
Schemas: map[string]spec.Schema{
"createNotificationqueryMatcher": {
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"label": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
},
},
"type": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Enum: []interface{}{
"=",
"!=",
"=~",
"!~",
},
},
},
"value": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
},
},
},
Required: []string{
"type",
"label",
"value",
},
},
},
"createNotificationqueryMatchers": {
SchemaProps: spec.SchemaProps{
Type: []string{"array"},
},
},
"createNotificationqueryNotificationEntry": {
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"alerts": {
SchemaProps: spec.SchemaProps{
Type: []string{"array"},
Description: "Alerts are the alerts grouped into the notification.",
},
},
"duration": {
SchemaProps: spec.SchemaProps{
Type: []string{"integer"},
Description: "Duration is the length of time the notification attempt took in nanoseconds.",
},
},
"error": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Description: "Error is the message returned by the contact point if delivery failed.",
},
},
"groupKey": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Description: "GroupKey uniquely idenifies the dispatcher alert group.",
},
},
"groupLabels": {
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Description: "GroupLabels are the labels uniquely identifying the alert group within a route.",
AdditionalProperties: &spec.SchemaOrBool{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
},
},
},
},
},
"outcome": {
SchemaProps: spec.SchemaProps{
Description: "Outcome indicaes if the notificaion attempt was successful or if it failed.",
Ref: spec.MustCreateRef("#/components/schemas/createNotificationqueryNotificationOutcome"),
},
},
"pipelineTime": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Format: "date-time",
Description: "PipelineTime is the time at which the flush began.",
},
},
"receiver": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Description: "Receiver is the receiver (contact point) title.",
},
},
"retry": {
SchemaProps: spec.SchemaProps{
Type: []string{"boolean"},
Description: "Retry indicates if the attempt was a retried attempt.",
},
},
"status": {
SchemaProps: spec.SchemaProps{
Description: "Status indicates if the notification contains one or more firing alerts.",
Ref: spec.MustCreateRef("#/components/schemas/createNotificationqueryNotificationStatus"),
},
},
"timestamp": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Format: "date-time",
Description: "Timestamp is the time at which the notification attempt completed.",
},
},
},
Required: []string{
"timestamp",
"receiver",
"status",
"outcome",
"groupLabels",
"alerts",
"retry",
"duration",
"pipelineTime",
"groupKey",
},
},
},
"createNotificationqueryNotificationEntryAlert": {
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"annotations": {
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
AdditionalProperties: &spec.SchemaOrBool{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
},
},
},
},
},
"endsAt": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Format: "date-time",
},
},
"labels": {
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
AdditionalProperties: &spec.SchemaOrBool{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
},
},
},
},
},
"startsAt": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Format: "date-time",
},
},
"status": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
},
},
},
Required: []string{
"status",
"labels",
"annotations",
"startsAt",
"endsAt",
},
},
},
"createNotificationqueryNotificationOutcome": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Enum: []interface{}{
"success",
"error",
},
},
},
"createNotificationqueryNotificationStatus": {
SchemaProps: spec.SchemaProps{
Type: []string{"string"},
Enum: []interface{}{
"firing",
"resolved",
},
},
},
},
},
},
},
@@ -120,7 +432,8 @@ func ManifestGoTypeAssociator(kind, version string) (goType resource.Kind, exist
}
var customRouteToGoResponseType = map[string]any{
"v0alpha1||<namespace>/alertstate/history|GET": v0alpha1.GetAlertstatehistory{},
"v0alpha1||<namespace>/alertstate/history|GET": v0alpha1.GetAlertstatehistory{},
"v0alpha1||<namespace>/notification/query|POST": v0alpha1.CreateNotificationquery{},
}
// ManifestCustomRouteResponsesAssociator returns the associated response go type for a given kind, version, custom route path, and method, if one exists.
@@ -145,7 +458,9 @@ func ManifestCustomRouteQueryAssociator(kind, version, path, verb string) (goTyp
return goType, exists
}
var customRouteToGoRequestBodyType = map[string]any{}
var customRouteToGoRequestBodyType = map[string]any{
"v0alpha1||<namespace>/notification/query|POST": v0alpha1.CreateNotificationqueryRequestBody{},
}
func ManifestCustomRouteRequestBodyAssociator(kind, version, path, verb string) (goType any, exists bool) {
if len(path) > 0 && path[0] == '/' {
+20
View File
@@ -1,8 +1,13 @@
package app
import (
"context"
"net/http"
"github.com/grafana/grafana-app-sdk/app"
"github.com/grafana/grafana-app-sdk/simple"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/grafana/grafana/apps/alerting/historian/pkg/apis/alertinghistorian/v0alpha1"
"github.com/grafana/grafana/apps/alerting/historian/pkg/app/config"
@@ -21,6 +26,11 @@ func New(cfg app.Config) (app.App, error) {
Path: "/alertstate/history",
Method: "GET",
}: runtimeConfig.GetAlertStateHistoryHandler,
{
Namespaced: true,
Path: "/notification/query",
Method: "POST",
}: UnimplementedHandler,
},
},
// TODO: Remove when SDK is fixed.
@@ -43,3 +53,13 @@ func New(cfg app.Config) (app.App, error) {
return a, nil
}
func UnimplementedHandler(ctx context.Context, writer app.CustomRouteResponseWriter, request *app.CustomRouteRequest) error {
return &apierrors.StatusError{
ErrStatus: metav1.Status{
Status: metav1.StatusFailure,
Code: http.StatusUnprocessableEntity,
Message: "unimplemented",
},
}
}
+1
View File
@@ -53,6 +53,7 @@ pluginMetaV0Alpha1: {
skipDataQuery?: bool
state?: "alpha" | "beta"
streaming?: bool
suggestions?: bool
tracing?: bool
iam?: #IAM
// +listType=atomic
@@ -40,6 +40,7 @@ type PluginMetaJSONData struct {
SkipDataQuery *bool `json:"skipDataQuery,omitempty"`
State *PluginMetaJSONDataState `json:"state,omitempty"`
Streaming *bool `json:"streaming,omitempty"`
Suggestions *bool `json:"suggestions,omitempty"`
Tracing *bool `json:"tracing,omitempty"`
Iam *PluginMetaIAM `json:"iam,omitempty"`
// +listType=atomic
File diff suppressed because one or more lines are too long
@@ -341,6 +341,10 @@
"type": "boolean",
"description": "Initialize plugin on startup. By default, the plugin initializes on first use, but when preload is set to true the plugin loads when the Grafana web app loads the first time. Only applicable to app plugins. When setting to `true`, implement [frontend code splitting](https://grafana.com/developers/plugin-tools/get-started/best-practices#app-plugins) to minimise performance implications."
},
"suggestions": {
"type": "boolean",
"description": "For panel plugins. If set to true, the plugin's suggestions supplier will be invoked and any suggestions returned will be included in the Suggestions pane in the Panel Editor."
},
"queryOptions": {
"type": "object",
"description": "For data source plugins. There is a query options section in the plugin's query editor and these options can be turned on if needed.",
@@ -1,68 +0,0 @@
---
aliases:
- ../../../panels-visualizations/query-transform-data/ # /docs/grafana/next/panels-visualizations/query-transform-data/
- ../../../panels-visualizations/query-transform-data/expression-queries/ # /docs/grafana/next/panels-visualizations/query-transform-data/expression-queries/
- ../../../panels/query-a-data-source/use-expressions-to-manipulate-data/ # /docs/grafana/next/panels/query-a-data-source/use-expressions-to-manipulate-data/
- ../../../panels/query-a-data-source/use-expressions-to-manipulate-data/about-expressions/ # /docs/grafana/next/panels/query-a-data-source/use-expressions-to-manipulate-data/about-expressions/
- ../../../panels/query-a-data-source/use-expressions-to-manipulate-data/write-an-expression/ # /docs/grafana/next/panels/query-a-data-source/use-expressions-to-manipulate-data/write-an-expression/
labels:
products:
- cloud
- enterprise
- oss
menuTitle: Expressions
title: Grafana expressions
description: Write server-side expressions to manipulate data using math and other operations
weight: 40
refs:
no-data-and-error-handling:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-grafana-managed-rule/#configure-no-data-and-error-handling
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-grafana-managed-rule/#configure-no-data-and-error-handling
multiple-dimensional-data:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/fundamentals/timeseries-dimensions/
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/fundamentals/timeseries-dimensions/
grafana-alerting:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/
labels:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/fundamentals/timeseries-dimensions/#labels
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/fundamentals/timeseries-dimensions/#labels
---
# Grafana expressions
An expression is a server-side operation that takes query results from one or more data sources and transforms them into new data. Expressions perform calculations like math operations, aggregations, or timestamp alignments without modifying the original data source results. This lets you derive metrics, combine data from different sources, and perform transformations your data sources can't do on their own.
By running on the server, expressions also enable features like alerting to continue working even when no user is viewing a dashboard.
## What problems do expressions solve?
Expressions fill the gap between what your data sources can produce and what your visualizations or alerts need.
They address several common challenges:
- **Cross-data-source calculations:** Combine results from different data sources that can't query each other directly. For example, calculate error rates by dividing HTTP errors from Prometheus by total requests from an SQL database.
- **Derived metrics:** Compute values your data source doesn't provide, such as percentage changes, moving averages, ratios, or conditional logic based on thresholds.
- **Alerting on complex conditions:** Apply math, reductions, and comparisons to drive alert rules when your data source lacks the necessary functions or when you need to alert across multiple data sources.
- **Post-query transformations:** Align timestamps between series, resample data to consistent intervals, filter out non-numeric values, or reduce time series to single summary values.
- **Multi-dimensional data operations:** Perform calculations across multiple series while preserving their label identities. For example, apply the same formula to dozens of host metrics without writing individual queries for each host.
- **Label-based series matching:** Automatically join and combine series based on their labels. For example, match CPU metrics and memory metrics for the same hosts by joining on common labels like `host` or `region`.
- **Data quality handling:** Clean your data by filtering out, replacing, or detecting problematic values such as null, NaN, or infinity values before performing calculations or creating alerts.
Without expressions, you'd need to either modify your data source queries (when possible), use client-side transformations (which don't work for alerting), or export and process data externally.
## Get started
Explore these resources to start using expressions:
- [Create and use expressions](create-use-expressions/) - Learn how to create expressions and use Math, Reduce, and Resample operations.
- [Expression examples](expression-examples/) - Practical examples from basic to advanced for common monitoring scenarios.
- [Troubleshoot expressions](troubleshoot-expressions/) - Debug and resolve common expression issues.
@@ -1,253 +0,0 @@
---
aliases:
labels:
products:
- cloud
- enterprise
- oss
menuTitle: Create and use expressions
title: Create and use expressions
description: Learn how to create expressions and use Math, Reduce, and Resample operations
weight: 41
refs:
multiple-dimensional-data:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/fundamentals/timeseries-dimensions/
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/fundamentals/timeseries-dimensions/
grafana-alerting:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/
---
# Create and use expressions
Expressions are most commonly used for [Grafana Alerting](ref:grafana-alerting), where server-side processing ensures alerts continue working even when no user is viewing a dashboard.
You can also use expressions with backend data sources in visualizations.
## Understand expression data
Before creating expressions, understand the data types and special values you'll work with.
### Data types
Expressions work with two types of data from backend data sources:
- **Time series:** Collections of timestamped values, typically returned by time series databases like Prometheus or InfluxDB.
- **Numbers:** Individual numeric values, such as aggregated results from SQL queries or reduced time series.
Expressions also operate on [multiple-dimensional data](ref:multiple-dimensional-data), where each series or number is identified by labels or tags.
For example, a single query can return CPU metrics for multiple hosts, with each series labeled by its hostname.
### Special values
When working with expressions, you'll encounter special values that represent problematic or undefined data:
- **null:** Represents missing or absent data. Common when a data point doesn't exist or wasn't recorded.
- **NaN (Not a Number):** Represents an undefined or invalid mathematical result, such as dividing zero by zero or taking the logarithm of a negative number. NaN is unique because it doesn't equal itself, which is why expressions include the `is_nan()` function.
- **Infinity (Inf):** Represents values too large to represent as numbers. Can be positive (`Inf`) or negative (`-Inf`). Often results from dividing by zero.
Expressions provide functions like `is_null()`, `is_nan()`, `is_inf()`, and `is_number()` to detect and handle these special values in your data.
### Reference queries and expressions
Each query or expression in Grafana has a unique identifier called a RefID (Reference ID).
RefIDs appear as letters (`A`, `B`, `C`) or custom names in the query editor, and they let you reference the output of one query in another expression.
To use a query or expression in a math operation, prefix its RefID with a dollar sign: `$A`, `$B`, `$C`.
**Example:**
If query `A` returns CPU usage and query `B` returns CPU capacity, you can create an expression `$A / $B * 100` to calculate CPU percentage.
The expression automatically uses the data from queries A and B based on their RefIDs.
## Create an expression
To add an expression to a panel:
1. Open the panel in edit mode.
1. Below your existing queries, click **Expression**.
1. In the **Operation** field, select **Math**, **Reduce**, or **Resample**.
1. Configure the expression based on the operation type.
1. Click **Apply** to save your changes.
The expression appears in your query list with its own RefID and can be referenced by other expressions.
## Expression operations
Expressions provide three core operations that you can combine to transform your data: Math, Reduce, and Resample.
Each operation solves specific data transformation challenges.
### Math
Math operations let you perform calculations on your query results using standard arithmetic, comparison, and logical operators.
Use math expressions to derive new metrics, calculate percentages, or implement conditional logic.
**Common use cases:**
- Calculate error rates: `$errors / $total_requests * 100`
- Convert units: `$bytes / 1024 / 1024` (bytes to megabytes)
- Implement thresholds: `$cpu_usage > 80` (returns 1 for true, 0 for false)
- Calculate capacity remaining: `$max_capacity - $current_usage`
#### Syntax and operators
Reference queries and expressions using their RefID prefixed with a dollar sign: `$A`, `$B`, `$C`.
If a RefID contains spaces, use brace syntax: `${my query}`.
**Supported operators:**
- **Arithmetic:** `+`, `-`, `*`, `/`, `%` (modulo), `**` (exponent)
- **Comparison:** `<`, `>`, `==`, `!=`, `>=`, `<=` (return 1 for true, 0 for false)
- **Logical:** `&&` (and), `||` (or), `!` (not)
**Numeric constants:**
- Decimal: `2.24`, `-0.8e-2`
- Octal: `072` (leading zero)
- Hexadecimal: `0x2A` (leading 0x)
#### How operations work with different data types
Math operations behave differently depending on whether you're working with numbers or time series:
- **Number + Number:** Performs the operation on the two values. Example: `5 + 3 = 8`
- **Number + Time series:** Applies the operation to every point in the series. Example: `$cpu_series * 100` multiplies each CPU value by 100
- **Time series + Time series:** Performs the operation on matching timestamps. Example: `$series_A + $series_B` adds values at each timestamp that exists in both series
If time series have different timestamps, use the Resample operation to align them first.
#### Label-based series matches
When working with multiple series, expressions automatically match series based on their labels.
If query `$A` returns CPU usage for multiple hosts (each with a `{host=...}` label) and query `$B` returns memory usage for the same hosts, the expression `$A + $B` automatically matches each host's CPU and memory values.
**Matching rules:**
- Series with identical labels match automatically
- A series with no labels matches any other series
- Series with subset labels match (for example, `{host=web01}` matches `{host=web01, region=us-east}`)
- If both variables contain only one series, they always match
#### Available functions
Math expressions include functions for common operations and data quality checks.
All functions work with both individual numbers and time series.
**Mathematical functions:**
- `abs(x)` - Returns absolute value. Example: `abs($temperature_diff)`
- `log(x)` - Returns natural logarithm. Returns NaN for negative values. Example: `log($growth_rate)`
- `round(x)` - Rounds to nearest integer. Example: `round($average)`
- `ceil(x)` - Rounds up to nearest integer. Example: `ceil(3.2)` returns `4`
- `floor(x)` - Rounds down to nearest integer. Example: `floor(3.8)` returns `3`
**Data quality functions:**
These functions help you detect and handle problematic values in your data:
- `is_number(x)` - Returns 1 for valid numbers, 0 for null, NaN, or infinity. Example: `is_number($A)`
- `is_null(x)` - Returns 1 for null values, 0 otherwise. Example: `is_null($A)`
- `is_nan(x)` - Returns 1 for NaN values, 0 otherwise. Useful because NaN doesn't equal itself. Example: `is_nan($A)`
- `is_inf(x)` - Returns 1 for positive or negative infinity, 0 otherwise. Example: `is_inf($A)`
**Test functions:**
- `null()`, `nan()`, `inf()`, `infn()` - Return the named special value. Primarily for testing.
### Reduce
Reduce operations convert time series into single numeric values while preserving their labels.
Use reduce to create summary statistics, single-value panels, or alert conditions based on time series data.
**Common use cases:**
- Create alert thresholds: Reduce CPU time series to average and alert if it exceeds 80%
- Display current values: Show the last recorded temperature from a sensor
- Calculate totals: Sum all errors across a time range
- Find extremes: Identify maximum memory usage in the last hour
**Available reduction functions:**
- **Last:** Returns the most recent value. Useful for "current state" displays.
- **Mean:** Returns the average of all values. Use for typical behavior over time.
- **Min / Max:** Returns the smallest or largest value. Useful for capacity planning or finding anomalies.
- **Sum:** Returns the total of all values. Useful for counting events or totaling metrics.
- **Count:** Returns the number of data points. Useful for checking data completeness.
**Example:**
If query `$A` returns CPU usage time series for three hosts over the last hour, applying `Reduce(Mean)` produces three numbers: the average CPU for each host, each labeled with its hostname.
#### Handle non-numeric values
Reduce operations let you control how null, NaN, and infinity values are handled:
- **Strict:** Returns NaN if any non-numeric values exist. Use when data quality is critical.
- **Drop non-numeric:** Filters out problematic values before calculating. Use when occasional bad data points are acceptable.
- **Replace non-numeric:** Replaces bad values with a specified number. Use when you want to substitute a default value.
### Resample
Resample operations align time series to a consistent time interval, enabling you to perform math operations between series with mismatched timestamps.
**Why resample:**
When combining time series from different data sources, their timestamps rarely align perfectly.
One series might report every 15 seconds while another reports every minute.
Resampling normalizes both series to the same interval so you can add, subtract, or compare them.
**Example use case:**
You want to calculate `$errors / $requests` but your error logs report every 10 seconds while your request metrics report every 30 seconds.
Resample both series to 30-second intervals, then perform the division.
**Configuration:**
- **Resample to:** The target interval. Use `s` (seconds), `m` (minutes), `h` (hours), `d` (days), `w` (weeks), or `y` (years). Example: `10s`, `1m`, `1h`
- **Downsample:** How to handle multiple data points in one interval. Choose a reduction function like Mean, Max, Min, or Sum. Example: If resampling from 10s to 30s intervals and you have 3 values, Mean averages them.
- **Upsample:** How to fill intervals with no data points:
- **Pad:** Uses the last known value (forward fill)
- **Backfill:** Uses the next known value (backward fill)
- **fillna:** Inserts NaN for missing intervals
## Best practices
Follow these guidelines to build efficient and maintainable expressions.
### Process data in the data source when possible
Perform aggregations, filtering, and complex calculations inside your data source rather than in expressions when you can.
Data sources are optimized for processing their own data, and moving large volumes of data to Grafana for simple operations is inefficient.
**Use expressions for:**
- Operations your data source doesn't support
- Cross-data-source calculations
- Lightweight post-processing
- Alerting logic that needs server-side evaluation
**Avoid expressions for:**
- Simple aggregations your data source can perform
- Processing millions of data points
- Operations that could be handled by recording rules or continuous queries
### Understand backend data source requirements
Expressions only work with backend (server-side) data sources. Browser-based data sources can't be used in expressions.
**Supported:** Prometheus, Loki, InfluxDB, MySQL, PostgreSQL, CloudWatch, and other backend data sources.
**Not supported:** TestData, browser-based plugins, or client-side data sources.
### Use alerting-compatible configurations
Expressions work differently in alerting contexts than in panels:
- Alerting requires expressions to evaluate server-side.
- Most alert conditions need single values (use Reduce operations).
- Test your expressions with the same time ranges your alerts will use.
- Legacy dashboard alerts don't support expressions - use [Grafana Alerting](ref:grafana-alerting) instead.
@@ -1,523 +0,0 @@
---
aliases:
labels:
products:
- cloud
- enterprise
- oss
menuTitle: Expressions examples
title: Expressions examples
description: Practical expression examples from basic to advanced for common monitoring scenarios
weight: 55
refs:
grafana-expressions:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/visualizations/panels-visualizations/query-transform-data/expression-queries/
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/visualizations/panels-visualizations/query-transform-data/expression-queries/
grafana-alerting:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/
---
# Expressions examples
This document provides practical expression examples for common monitoring and visualization scenarios.
Examples progress from basic to advanced, showing you how to solve real-world problems with Grafana Expressions.
For foundational concepts, refer to [Grafana expressions](ref:grafana-expressions).
## Basic examples
Start here if you're new to expressions. These examples demonstrate fundamental patterns you'll use frequently.
### Convert units
**Scenario:** Your metrics are in bytes, but you want to display them in gigabytes.
**Setup:**
- Query A (Prometheus): `node_memory_MemTotal_bytes`
- Expression B (Math): `$A / 1024 / 1024 / 1024`
**Result:** Memory values converted from bytes to gigabytes.
**Variations:**
- Bytes to megabytes: `$A / 1024 / 1024`
- Bytes to terabytes: `$A / 1024 / 1024 / 1024 / 1024`
- Milliseconds to seconds: `$A / 1000`
- Celsius to Fahrenheit: `$A * 9 / 5 + 32`
---
### Calculate a simple percentage
**Scenario:** Show what percentage of total memory is being used.
**Setup:**
- Query A (Prometheus): `node_memory_MemTotal_bytes`
- Query B (Prometheus): `node_memory_MemAvailable_bytes`
- Expression C (Math): `($A - $B) / $A * 100`
**Result:** Memory usage as a percentage (0-100).
**Tip:** This pattern works for any "used / total \* 100" calculation.
---
### Get the current (latest) value
**Scenario:** Display the most recent temperature reading in a stat panel.
**Setup:**
- Query A (InfluxDB): Temperature sensor time series data
- Expression B (Reduce): Input `$A`, Function: **Last**
**Result:** Single number showing the most recent value from the time series.
**When to use:** Stat panels, gauges, or any visualization that needs a single current value.
---
### Calculate an average over time
**Scenario:** Show the average CPU usage over the dashboard time range.
**Setup:**
- Query A (Prometheus): `node_cpu_seconds_total{mode="idle"}`
- Expression B (Reduce): Input `$A`, Function: **Mean**
**Result:** Average CPU value across the selected time range.
**Note:** Each series (each CPU core, each host) produces its own average, preserving labels.
---
### Find maximum or minimum values
**Scenario:** Identify the peak memory usage in the last 24 hours.
**Setup:**
- Query A (Prometheus): `node_memory_MemUsed_bytes` (last 24 hours)
- Expression B (Reduce): Input `$A`, Function: **Max**
**Result:** Peak memory usage value for each host.
**Variations:**
- Use **Min** to find the lowest value
- Use **Count** to see how many data points exist
---
### Simple threshold check
**Scenario:** Create a binary indicator showing whether CPU is above 80%.
**Setup:**
- Query A (Prometheus): `100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)`
- Expression B (Math): `$A > 80`
**Result:** Returns `1` when CPU exceeds 80%, `0` otherwise. Useful for alerting or status indicators.
---
## Intermediate examples
These examples combine multiple operations and handle more complex scenarios.
### Calculate error rate percentage
**Scenario:** Display HTTP error rate as a percentage of total requests.
**Setup:**
- Query A (Prometheus): `sum(rate(http_requests_total{status=~"5.."}[5m]))`
- Query B (Prometheus): `sum(rate(http_requests_total[5m]))`
- Expression C (Math): `$A / $B * 100`
**Result:** Error rate percentage across all endpoints.
**Handling division by zero:** If there are zero requests, this produces infinity. To handle this:
- Expression C (Math): `$B > 0 ? ($A / $B * 100) : 0`
This returns 0 when there are no requests instead of infinity.
---
### Calculate available disk space
**Scenario:** Show available disk space as a percentage for capacity planning.
**Setup:**
- Query A (Prometheus): `node_filesystem_size_bytes{mountpoint="/"}`
- Query B (Prometheus): `node_filesystem_avail_bytes{mountpoint="/"}`
- Expression C (Math): `$B / $A * 100`
**Result:** Percentage of disk space available (not used) for each host's root filesystem.
**For alerting:** Add an alert when available space drops below 10%:
- Expression D (Math): `$C < 10`
---
### Aggregate across multiple servers
**Scenario:** Calculate total requests per second across all web servers.
**Setup:**
- Query A (Prometheus): `rate(http_requests_total{job="webservers"}[5m])`
- Expression B (Reduce): Input `$A`, Function: **Sum**
**Result:** Total requests per second across all servers combined into a single value.
**Alternative:** To get the average per server instead:
- Expression B (Reduce): Input `$A`, Function: **Mean**
---
### Combine metrics from different data sources
**Scenario:** Calculate efficiency by dividing application throughput (Prometheus) by infrastructure cost metric (CloudWatch).
**Setup:**
- Query A (Prometheus): `sum(rate(processed_jobs_total[5m]))`
- Query B (CloudWatch): EC2 instance cost metric
- Expression C (Resample): Input `$A`, Resample to: `1m`, Downsample: Mean
- Expression D (Resample): Input `$B`, Resample to: `1m`, Downsample: Mean
- Expression E (Math): `$C / $D`
**Result:** Jobs processed per dollar (or cost unit), showing application efficiency.
**Why resample:** Different data sources often have different collection intervals. Resampling ensures timestamps align for math operations.
---
### Compare hosts to fleet average
**Scenario:** Identify hosts performing worse than the fleet average.
**Setup:**
- Query A (Prometheus): `node_cpu_usage_percent` (returns one series per host)
- Expression B (Reduce): Input `$A`, Function: **Mean** (fleet average)
- Expression C (Math): `$A - $B`
**Result:** Each host shows how much above or below the fleet average they are. Positive values indicate above-average CPU usage.
---
### Filter invalid data
**Scenario:** Calculate average response time, ignoring any null or NaN values in the data.
**Setup:**
- Query A (Time series): Response time data with occasional gaps
- Expression B (Reduce): Input `$A`, Function: **Mean**, Mode: **Drop non-numeric**
**Result:** Clean average that ignores invalid data points.
**Alternative modes:**
- **Strict:** Returns NaN if any value is invalid (use when data quality matters)
- **Replace non-numeric:** Substitutes a specific value for invalid data points
---
### Calculate rate of change
**Scenario:** Show how quickly memory usage is increasing or decreasing.
**Setup:**
- Query A (Prometheus): `node_memory_MemUsed_bytes`
- Query B (Prometheus): `node_memory_MemUsed_bytes offset 5m`
- Expression C (Math): `$A - $B`
**Result:** Bytes of memory change over the last 5 minutes. Positive = increasing, negative = decreasing.
**As a percentage change:**
- Expression C (Math): `($A - $B) / $B * 100`
---
## Advanced examples
These examples demonstrate complex multi-step calculations and sophisticated alerting patterns.
### Compare current value to 24-hour average
**Scenario:** Highlight when current traffic is significantly above or below the daily norm.
**Setup:**
- Query A (Prometheus): `sum(rate(http_requests_total[24h]))` (historical average)
- Query B (Prometheus): `sum(rate(http_requests_total[5m]))` (current rate)
- Expression C (Reduce): Input `$A`, Function: **Mean**
- Expression D (Math): `($B - $C) / $C * 100`
**Result:** Percentage difference from the 24-hour average. +50 means 50% above normal, -30 means 30% below normal.
**Use cases:**
- Detect traffic anomalies
- Identify unusual load patterns
- Trigger alerts for significant deviations
---
### Calculate service level indicator (SLI)
**Scenario:** Calculate the percentage of requests meeting your latency target (under 200ms).
**Setup:**
- Query A (Prometheus): `sum(rate(http_request_duration_seconds_bucket{le="0.2"}[5m]))`
- Query B (Prometheus): `sum(rate(http_request_duration_seconds_count[5m]))`
- Expression C (Math): `$A / $B * 100`
**Result:** Percentage of requests completing in under 200ms (your SLI).
**For SLO alerting:** Alert when SLI drops below 99%:
- Expression D (Reduce): Input `$C`, Function: **Mean**
- Expression E (Math): `$D < 99`
---
### Multi-host alerts with reduction
**Scenario:** Alert when average CPU across all production servers exceeds 80%.
**Setup:**
- Query A (Prometheus): `100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle",env="production"}[5m])) * 100)`
- Expression B (Reduce): Input `$A`, Function: **Mean** (average across all hosts)
- Expression C (Math): `$B > 80`
**Result:** Single alert that fires when the fleet average crosses the threshold, not individual host alerts.
**Alternative - alert on any host:**
- Expression B (Reduce): Input `$A`, Function: **Max**
This alerts when any single host exceeds 80%.
---
### Calculate compound metrics
**Scenario:** Calculate Apdex score (Application Performance Index) from response time buckets.
**Setup:**
- Query A (Prometheus): `sum(rate(http_request_duration_seconds_bucket{le="0.5"}[5m]))` (satisfied: <500ms)
- Query B (Prometheus): `sum(rate(http_request_duration_seconds_bucket{le="2.0"}[5m]))` (tolerating: <2s)
- Query C (Prometheus): `sum(rate(http_request_duration_seconds_count[5m]))` (total)
- Expression D (Math): `($A + ($B - $A) / 2) / $C`
**Result:** Apdex score from 0 to 1, where 1 is perfect user satisfaction.
**Formula explained:** Apdex = (Satisfied + Tolerating/2) / Total
---
### Detect sustained conditions
**Scenario:** Alert only when CPU has been high for at least 5 minutes, not just a brief spike.
**Setup:**
- Query A (Prometheus): `avg_over_time(node_cpu_usage_percent[5m])`
- Expression B (Reduce): Input `$A`, Function: **Mean**
- Expression C (Math): `$B > 80`
**Result:** Alerts only fire when the 5-minute average exceeds the threshold, filtering out brief spikes.
**Alternative approach using count:**
- Query A: `node_cpu_usage_percent`
- Expression B (Math): `$A > 80`
- Expression C (Reduce): Input `$B`, Function: **Sum** (counts "1" values where condition is true)
- Expression D (Math): `$C > 5`
This alerts when more than 5 data points in the range exceed the threshold.
---
### Correlate metrics across systems
**Scenario:** Calculate orders processed per database query to measure backend efficiency.
**Setup:**
- Query A (Prometheus - App metrics): `sum(rate(orders_processed_total[5m]))`
- Query B (MySQL data source): Database queries per second from performance schema
- Expression C (Resample): Input `$A`, Resample to: `30s`, Downsample: Mean
- Expression D (Resample): Input `$B`, Resample to: `30s`, Downsample: Mean
- Expression E (Math): `$C / $D`
**Result:** Orders per database query, showing how efficiently your backend processes orders.
**Lower is better:** Fewer queries per order means more efficient database usage.
---
### Ratio-based alerts with baseline
**Scenario:** Alert when error ratio increases by more than 2x compared to yesterday's baseline.
**Setup:**
- Query A (Prometheus): `sum(rate(http_errors_total[5m]))` (current errors)
- Query B (Prometheus): `sum(rate(http_requests_total[5m]))` (current requests)
- Query C (Prometheus): `sum(rate(http_errors_total[5m] offset 24h))` (yesterday's errors)
- Query D (Prometheus): `sum(rate(http_requests_total[5m] offset 24h))` (yesterday's requests)
- Expression E (Math): `$A / $B` (current error rate)
- Expression F (Math): `$C / $D` (baseline error rate)
- Expression G (Reduce): Input `$E`, Function: **Mean**
- Expression H (Reduce): Input `$F`, Function: **Mean**
- Expression I (Math): `$G / $H > 2`
**Result:** Alerts when today's error rate is more than double yesterday's rate.
**Why this matters:** Absolute thresholds don't account for normal variation. Ratio-based alerting adapts to your system's baseline behavior.
---
### Calculate percentile-based thresholds
**Scenario:** Alert when response time exceeds the 95th percentile baseline.
**Setup:**
- Query A (Prometheus): `histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))`
- Query B (Prometheus): `histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[1h])) by (le))`
- Expression C (Reduce): Input `$A`, Function: **Last** (current p95)
- Expression D (Reduce): Input `$B`, Function: **Mean** (baseline p95)
- Expression E (Math): `$C > $D * 1.5`
**Result:** Alerts when current p95 latency exceeds 1.5x the hourly baseline.
---
### Weighted scores across metrics
**Scenario:** Create a composite health score from multiple metrics (CPU, memory, disk, network).
**Setup:**
- Query A: CPU usage percentage (0-100)
- Query B: Memory usage percentage (0-100)
- Query C: Disk usage percentage (0-100)
- Query D: Network saturation percentage (0-100)
- Expression E (Reduce): Input `$A`, Function: **Mean**
- Expression F (Reduce): Input `$B`, Function: **Mean**
- Expression G (Reduce): Input `$C`, Function: **Mean**
- Expression H (Reduce): Input `$D`, Function: **Mean**
- Expression I (Math): `($E * 0.3) + ($F * 0.25) + ($G * 0.25) + ($H * 0.2)`
**Result:** Weighted health score from 0-100 where lower is healthier. Weights reflect relative importance (CPU 30%, Memory 25%, Disk 25%, Network 20%).
**For alerting:**
- Expression J (Math): `$I > 70`
Alert when composite score indicates degraded health.
---
### Conditional logic with fallbacks
**Scenario:** Show error rate, but display 0 instead of infinity when there are no requests.
**Setup:**
- Query A (Prometheus): `sum(rate(http_errors_total[5m]))`
- Query B (Prometheus): `sum(rate(http_requests_total[5m]))`
- Expression C (Math): `$B > 0 ? ($A / $B * 100) : 0`
**Result:** Error rate percentage that safely handles zero-request periods.
**Conditional syntax:** `condition ? value_if_true : value_if_false`
**More examples:**
- Cap values at 100: `$A > 100 ? 100 : $A`
- Convert negative to zero: `$A < 0 ? 0 : $A`
- Binary classification: `$A > threshold ? 1 : 0`
---
### Time-window comparison for trend detection
**Scenario:** Detect if metrics are trending up or down by comparing recent data to slightly older data.
**Setup:**
- Query A (Prometheus): `avg_over_time(http_requests_total[5m])`
- Query B (Prometheus): `avg_over_time(http_requests_total[5m] offset 10m)`
- Expression C (Reduce): Input `$A`, Function: **Mean**
- Expression D (Reduce): Input `$B`, Function: **Mean**
- Expression E (Math): `($C - $D) / $D * 100`
**Result:** Percentage change in requests between the last 5 minutes and the previous 5-minute window.
**Interpretation:**
- Positive values: Traffic increasing
- Negative values: Traffic decreasing
- Values near 0: Traffic stable
**Use case:** Detect rapid traffic changes that might indicate problems or attacks.
---
## Tips for expression development
Follow these best practices to build reliable, maintainable expressions in your visualizations and alerts.
### Start simple and iterate
Begin with basic operations and verify each step works before adding complexity. Use the Query Inspector to see intermediate results.
### Name your queries clearly
While RefIDs default to letters, you can use descriptive names. Referencing `${errors}` and `${total_requests}` is clearer than `$A` and `$B`.
### Test with realistic time ranges
Expressions may behave differently with various time ranges. Test with the same ranges you'll use in production dashboards or alerts.
### Handle edge cases
Consider what happens when:
- Data is missing (NoData)
- Values are zero (division by zero)
- Metrics haven't been collected yet
- Time series have different numbers of points
### Document complex expressions
Add panel descriptions or annotation text explaining what complex expressions calculate and why.
### Monitor expression performance
If dashboards become slow, check if expressions are processing too much data. Consider moving heavy aggregations to recording rules or data source queries.
@@ -0,0 +1,263 @@
---
aliases:
- ../../../panels-visualizations/query-transform-data/ # /docs/grafana/next/panels-visualizations/query-transform-data/
- ../../../panels-visualizations/query-transform-data/expression-queries/ # /docs/grafana/next/panels-visualizations/query-transform-data/expression-queries/
- ../../../panels/query-a-data-source/use-expressions-to-manipulate-data/ # /docs/grafana/next/panels/query-a-data-source/use-expressions-to-manipulate-data/
- ../../../panels/query-a-data-source/use-expressions-to-manipulate-data/about-expressions/ # /docs/grafana/next/panels/query-a-data-source/use-expressions-to-manipulate-data/about-expressions/
- ../../../panels/query-a-data-source/use-expressions-to-manipulate-data/write-an-expression/ # /docs/grafana/next/panels/query-a-data-source/use-expressions-to-manipulate-data/write-an-expression/
labels:
products:
- cloud
- enterprise
- oss
menuTitle: Write expression queries
title: Write expression queries
description: Write server-side expressions to manipulate data using math and other operations
weight: 40
refs:
no-data-and-error-handling:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-grafana-managed-rule/#configure-no-data-and-error-handling
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/alerting-rules/create-grafana-managed-rule/#configure-no-data-and-error-handling
multiple-dimensional-data:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/fundamentals/timeseries-dimensions/
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/fundamentals/timeseries-dimensions/
grafana-alerting:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/alerting/
labels:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/fundamentals/timeseries-dimensions/#labels
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/fundamentals/timeseries-dimensions/#labels
---
# Write expression queries
Server-side expressions enable you to manipulate data returned from queries with math and other operations. Expressions create new data and do not manipulate the data returned by data sources.
## About expressions
Server-side expressions allow you to manipulate data returned from queries with math and other operations. Expressions create new data and do not manipulate the data returned by data sources, aside from some minor data restructuring to make the data acceptable input for expressions.
### Using expressions
Expressions are most commonly used for [Grafana Alerting](ref:grafana-alerting). The processing is done server-side, so expressions can operate without a browser session. However, expressions can also be used with backend data sources and visualization.
{{< admonition type="note" >}}
Expressions do not work with legacy dashboard alerts.
{{< /admonition >}}
Expressions are meant to augment data sources by enabling queries from different data sources to be combined or by providing operations unavailable in a data source.
{{< admonition type="note" >}}
When possible, you should do data processing inside the data source. Copying data from storage to the Grafana server for processing is inefficient, so expressions are targeted at lightweight data processing.
{{< /admonition >}}
Expressions work with data source queries that return time series or number data. They also operate on [multiple-dimensional data](ref:multiple-dimensional-data). For example, a query that returns multiple series, where each series is identified by labels or tags.
An individual expression takes one or more queries or other expressions as input and adds data to the result. Each individual expression or query is represented by a variable that is a named identifier known as its RefID (e.g., the default letter `A` or `B`).
To reference the output of an individual expression or a data source query in another expression, this identifier is used as a variable.
### Types of expressions
Expressions work with two types of data.
- A collection of time series.
- A collection of numbers, where each number is an item.
Each collection is returned from a single data source query or expression and represented by the RefID. Each collection is a set, where each item in the set is uniquely identified by its dimensions which are stored as [labels](ref:labels) or key-value pairs.
### Data source queries
Server-side expressions only support data source queries for backend data sources. The data is generally assumed to be labeled time series data. In the future we intend to add an assertion of the query return type (number or time series) data so expressions can handle errors better.
Data source queries, when used with expressions, are executed by the expression engine. When it does this, it restructures data to be either one time series or one number per data frame. So for example if using a data source that returns multiple series on one frame in the table view, you might notice it looks different when executed with expressions.
Currently, the only non-time series format (number) is supported when you're using data frames and you have a table response that returns a data frame with no time, string columns, and one number column:
| Loc | Host | Avg_CPU |
| --- | ---- | ------- |
| MIA | A | 1 |
| NYC | B | 2 |
The example above will produce a number that works with expressions. The string columns become labels and the number column the corresponding value. For example `{"Loc": "MIA", "Host": "A"}` with a value of 1.
### Operations
You can use the following operations in expressions: math, reduce, and resample.
#### Math
Math is for free-form math formulas on time series or number data. Math operations take numbers and time series as input and change them to different numbers and time series.
Data from other queries or expressions are referenced with the RefID prefixed with a dollar sign, for example `$A`. If the variable has spaces in the name, then you can use a brace syntax like `${my variable}`.
Numeric constants may be in decimal (`2.24`), octal (with a leading zero like `072`), or hex (with a leading 0x like `0x2A`). Exponentials and signs are also supported (e.g., `-0.8e-2`).
##### Operators
The arithmetic (`+`, binary and unary `-`, `*`, `/`, `%`, exponent `**`), relational (`<`, `>`, `==`, `!=`, `>=`, `<=`), and logical (`&&`, `||`, and unary `!`) operators are supported.
How the operation behaves with data depends on if it is a number or time series data.
With binary operations, such as `$A + $B` or `$A || $B`, the operator is applied in the following ways depending on the type of data:
- If both `$A` and `$B` are a number, then the operation is performed between the two numbers.
- If one variable is a number, and the other variable is a time series, then the operation between the value of each point in the time series and the number is performed.
- If both `$A` and `$B` are time series data, then the operation between each value in the two series is performed for each time stamp that exists in both `$A` and `$B`. The Resample operation can be used to line up time stamps. (**Note:** in the future, we plan to add options to the Math operation for different behaviors).
Summary:
- Number OP number = number
- Number OP series = series
- Series OP series = series
Because expressions work with multiple series or numbers represented by a single variable, binary operations also perform a union (join) between the two variables. This is done based on the identifying labels associated with each individual series or number.
So if you have numbers with labels like `{host=web01}` in `$A` and another number in `$B` with the same labels then the operation is performed between those two items within each variable, and the result will share the same labels. The rules for the behavior of this union are as follows:
- An item with no labels will join to anything.
- If both `$A` and `$B` each contain only one item (one series, or one number), they will join.
- If labels are exact match they will join.
- If labels are a subset of the other, for example and item in `$A` is labeled `{host=A,dc=MIA}` and item in `$B` is labeled `{host=A}` they will join.
- Currently, if within a variable such as `$A` there are different tag _keys_ for each item, the join behavior is undefined.
The relational and logical operators return 0 for false 1 for true.
##### Math Functions
While most functions exist in the own expression operations, the math operation does have some functions similar to math operators or symbols. When functions can take either numbers or series, than the same type as the argument will be returned. When it is a series, the operation of performed for the value of each point in the series.
###### abs
abs returns the absolute value of its argument which can be a number or a series. For example `abs(-1)` or `abs($A)`.
###### is_inf
is_inf takes a number or a series and returns `1` for `Inf` values (negative or positive) and `0` for other values. For example `is_inf($A)`.
{{< admonition type="note" >}}
If you need to specifically check for negative infinity for example, you can do a comparison like `$A == infn()`.
{{< /admonition >}}
###### is_nan
is_nan takes a number or a series and returns `1` for `NaN` values and `0` for other values. For example `is_nan($A)`. This function exists because `NaN` is not equal to `NaN`.
###### is_null
is_null takes a number or a series and returns `1` for `null` values and `0` for other values. For example `is_null($A)`.
###### is_number
is_number takes a number or a series and returns `1` for all real number values and `0` for other values (which are `null`, `Inf+`, `Inf-`, and `NaN`). For example `is_number($A)`.
###### log
Log returns the natural logarithm of of its argument which can be a number or a series. If the value is less than 0, NaN is returned. For example `log(-1)` or `log($A)`.
###### inf, infn, nan, and null
The inf, infn, nan, and null functions all return a single value of the name. They primarily exist for testing. Example: `null()`.
###### round
Round returns a rounded integer value. For example, `round(3.123)` or `round($A)`. (This function should probably take an argument so it can add precision to the rounded value).
###### ceil
Ceil rounds the number up to the nearest integer value. For example, `ceil(3.123)` returns 4.
###### floor
Floor rounds the number down to the nearest integer value. For example, `floor(3.123)` returns 3.
#### Reduce
Reduce takes one or more time series returned from a query or an expression and turns each series into a single number. The labels of the time series are kept as labels on each outputted reduced number.
**Fields:**
- **Function -** The reduction function to use
- **Input -** The variable (refID (such as `A`)) to resample
- **Mode -** Allows control behavior of reduction function when a series contains non-numerical values (null, NaN, +\-Inf)
##### Reduction Functions
###### Count
Count returns the number of points in each series.
###### Mean
Mean returns the total of all values in each series divided by the number of points in that series. In `strict` mode if any values in the series are null or nan, or if the series is empty, NaN is returned.
###### Min and Max
Min and Max return the smallest or largest value in the series respectively. In `strict` mode if any values in the series are null or nan, or if the series is empty, NaN is returned.
###### Sum
Sum returns the total of all values in the series. If series is of zero length, the sum will be 0. In `strict` mode if there are any NaN or Null values in the series, NaN is returned.
##### Last
Last returns the last number in the series. If the series has no values then returns NaN.
##### Reduction Modes
###### Strict
In Strict mode the input series is processed as is. If any values in the series are non-numeric (null, NaN or +\-Inf), NaN is returned.
###### Drop Non-Numeric
In this mode all non-numeric values (null, NaN or +\-Inf) in the input series are filtered out before executing the reduction function.
###### Replace Non-Numeric
In this mode all non-numeric values are replaced by a pre-defined value.
#### Resample
Resample changes the time stamps in each time series to have a consistent time interval. The main use case is so you can resample time series that do not share the same timestamps so math can be performed between them. This can be done by resample each of the two series, and then in a Math operation referencing the resampled variables.
**Fields:**
- **Input -** The variable of time series data (refID (such as `A`)) to resample
- **Resample to -** The duration of time to resample to, for example `10s`. Units may be `s` seconds, `m` for minutes, `h` for hours, `d` for days, `w` for weeks, and `y` of years.
- **Downsample -** The reduction function to use when there are more than one data point per window sample. See the reduction operation for behavior details.
- **Upsample -** The method to use to fill a window sample that has no data points.
- **pad** fills with the last know value
- **backfill** with next known value
- **fillna** to fill empty sample windows with NaNs
## Write an expression
If your data source supports them, then Grafana displays the **Expression** button and shows any existing expressions in the query editor list.
For more information about expressions, refer to [About expressions](#about-expressions).
1. Open the panel.
1. Below the query, click **Expression**.
1. In the **Operation** field, select the type of expression you want to write.
For more information about expression operations, refer to [About expressions](#about-expressions).
1. Write the expression.
1. Click **Apply**.
## Special cases
When any queried data source returns no series or numbers, the expression engine returns `NoData`. For example, if a request contains two data source queries that are merged by an expression, if `NoData` is returned by at least one of the data source queries, then the returned result for the entire query is `NoData`.
For more information about how [Grafana Alerting](ref:grafana-alerting) processes `NoData` results, refer to [No data and error handling](ref:no-data-and-error-handling).
In the case of using an expression on multiple queries, the expression engine requires that all of the queries return an identical timestamp. For example, if using math to combine the results of multiple SQL queries which each use `SELECT NOW() AS "time"`, the expression will only work if all queries evaluate `NOW()` to an identical timestamp; which does not always happen. To resolve this, you can replace `NOW()` with an arbitrary time, such as `SELECT 1 AS "time"`, or any other valid UNIX timestamp.
@@ -1,514 +0,0 @@
---
aliases:
labels:
products:
- cloud
- enterprise
- oss
menuTitle: Troubleshoot expressions
title: Troubleshoot Grafana expressions
description: Debug and resolve common issues when working with Grafana Expressions
weight: 50
refs:
grafana-expressions:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/visualizations/panels-visualizations/query-transform-data/expression-queries/
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/visualizations/panels-visualizations/query-transform-data/expression-queries/
transformations:
- pattern: /docs/grafana/
destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/query-transform-data/transform-data/
- pattern: /docs/grafana-cloud/
destination: /docs/grafana/<GRAFANA_VERSION>/panels-visualizations/query-transform-data/transform-data/
---
# Troubleshoot Grafana expressions
This guide helps you diagnose and resolve common issues when working with expressions.
## Debug expressions
When an expression doesn't produce the expected results, use these strategies to identify the problem.
### Test expressions step by step
Break complex expressions into smaller pieces and verify each step:
1. **Test individual queries first:** Ensure each data source query returns the expected data before adding expressions.
1. **Add expressions incrementally:** Start with a simple expression and gradually add complexity.
1. **Use separate panels for testing:** Create a temporary panel to test expressions in isolation.
1. **Check intermediate results:** Add expressions at each step of your calculation to see intermediate values.
**Example:**
Instead of creating `($A - $B) / $C * 100` immediately, build it incrementally:
- Expression D: `$A - $B` (verify the subtraction works)
- Expression E: `$D / $C` (verify the division works)
- Expression F: `$E * 100` (final percentage)
Once working, you can collapse them into a single expression if desired.
### Verify RefID references
Ensure you're referencing the correct queries and expressions:
- RefIDs are case-sensitive: `$A` is different from `$a`
- Check that RefIDs haven't changed after reordering queries
- Use `${RefID}` syntax for RefIDs with spaces or special characters
### Check data types
Expressions expect specific data types. Verify your queries return time series or numbers, not tables or other formats.
**Common issues:**
- SQL queries returning multiple columns (expressions need one value column)
- Queries returning string data instead of numbers
- Empty result sets that appear as NoData
### Inspect labels
Use the Table view in panels to see the labels on your series and verify they match as expected.
**What to check:**
- Do series from different queries have compatible labels for joining?
- Are label names spelled consistently across queries?
- Are there unexpected extra labels preventing matches?
## Common errors and solutions
Following are common errors and how to troubleshoot them.
### "NoData" result
**Problem:** Your expression returns NoData even though some queries have data.
**Causes and solutions:**
- **One query returns no data:** If any query in an expression returns NoData, the entire expression returns NoData. Check that all queries have data for the selected time range.
- **Mismatched time ranges:** Ensure all queries use compatible time ranges. A query with "Last 5 minutes" can't combine with a query using "Last 24 hours" without adjustments.
- **Backend data source required:** Expressions only work with backend data sources. Check that you're not using browser-based data sources.
**Solution:**
Test each query independently to identify which one returns NoData, then investigate why that query has no data.
### No series match in math operations
**Problem:** Math expression like `$A + $B` returns no data, but both queries return data.
**Causes and solutions:**
- **Label mismatch:** Series from `$A` and `$B` have different labels that prevent automatic matching.
**Example:** `$A` has `{host="web01", region="us-east"}` but `$B` has `{server="web01", region="us-east"}`. The different label names (`host` vs `server`) prevent matching.
**Solution:** Modify your queries to use consistent label names, or ensure one set of series has no labels (which matches anything).
- **No overlapping timestamps:** Time series need matching timestamps for math operations.
**Solution:** Use the Resample operation to align timestamps to a common interval.
### Timestamp mismatch errors
**Problem:** Combining results from multiple SQL queries fails because timestamps don't align.
**Example:**
```sql
-- Query A
SELECT NOW() AS "time", COUNT(*) as "errors" FROM error_log;
-- Query B
SELECT NOW() AS "time", COUNT(*) as "requests" FROM request_log;
```
These queries may execute at slightly different times, producing different timestamps.
**Solution 1 - Use fixed timestamps:**
```sql
-- Query A
SELECT 1 AS "time", COUNT(*) as "errors" FROM error_log;
-- Query B
SELECT 1 AS "time", COUNT(*) as "requests" FROM request_log;
```
**Solution 2 - Use consistent time references:**
Ensure all queries evaluate time identically by using the same timestamp variable or function.
**Solution 3 - Use Resample:**
Add Resample operations to align both series to a common interval before performing math.
### Math operations produce unexpected nulls or NaN
**Problem:** Expression results contain null or NaN values unexpectedly.
**Causes and solutions:**
- **Division by zero:** Dividing by zero produces infinity. Use conditional logic: `$A > 0 ? $B / $A : 0`
- **Logarithm of negative numbers:** `log()` of negative values returns NaN.
- **Operations on null values:** Math operations involving null typically produce null.
**Solution:**
Use data quality functions to filter or handle problematic values:
```
is_number($A) ? $A : 0
```
Or use Reduce with "Drop non-numeric" mode to clean data before calculations.
### Reduce returns NaN in strict mode
**Problem:** Reduce operation returns NaN even though most data points are valid.
**Cause:** Strict mode returns NaN if _any_ value in the series is null, NaN, or infinity.
**Solution:**
Change the reduction mode:
- **Drop non-numeric:** Ignores invalid values and calculates from valid ones
- **Replace non-numeric:** Substitutes a specific value for invalid data points
Use Strict mode only when data quality is critical and you want to know if any values are invalid.
### Expression works in panel but fails in alerting
**Problem:** Expression displays correctly in a panel but produces errors or unexpected results in alert rules.
**Causes and solutions:**
- **Time range differences:** Alerts use specific time ranges that may differ from your panel's time range. Verify the alert's time range settings.
- **Data availability:** Data may be available when viewing the panel but missing when the alert evaluates.
- **Reduce required for alerting:** Most alert conditions need single values. Ensure you're using Reduce to convert time series to numbers for threshold comparisons.
**Solution:**
Test your expression in a panel using the same time range as your alert rule.
## Work with timestamps
Timestamps can be a common source of issues when working with expressions. Here's how to handle them effectively.
### Understand timestamp alignment
Math operations between time series require matching timestamps. If series `$A` has points at `10:00:00`, `10:00:30`, `10:01:00` and series `$B` has points at `10:00:15`, `10:00:45`, `10:01:15`, the operation `$A + $B` produces no results because no timestamps match exactly.
### When to resample
Use Resample when:
- Combining data from sources with different collection intervals
- One data source reports irregularly while another reports at fixed intervals
- You need to ensure timestamps align for math operations
- You want to normalize data to a consistent interval for visualization
### Resample strategies
**Downsample (reducing frequency):**
When going from higher to lower frequency (for example, 10s intervals to 1m intervals), choose an appropriate reduction function:
- **Mean:** For averaging values (CPU percentage, temperature)
- **Max:** For peak values (maximum memory usage)
- **Min:** For minimum values (lowest throughput)
- **Sum:** For accumulating values (request counts, error totals)
**Upsample (increasing frequency):**
When going from lower to higher frequency (for example, 1m intervals to 10s intervals), choose a fill strategy:
- **Pad (forward fill):** Assumes value stays constant until next measurement (good for state data)
- **Backfill:** Uses next known value (less common, use when future values inform past state)
- **fillna:** Inserts NaN for unknown intervals (explicit about missing data)
### SQL timestamp best practices
When writing SQL queries for use with expressions:
**Do:**
- Use consistent timestamp columns across queries
- Round or truncate timestamps to a common interval if needed
- Use fixed timestamps for non-time-based aggregations
```sql
-- Good: Consistent time bucket
SELECT
DATE_TRUNC('minute', timestamp) AS "time",
COUNT(*) as "value"
FROM events
GROUP BY 1
ORDER BY 1;
```
**Don't:**
- Use `NOW()` or `CURRENT_TIMESTAMP` which vary between query executions
- Mix different timestamp columns in related queries
- Return data without a time column for time series expressions
## Handle missing data
Understanding how expressions handle missing data helps you build robust dashboards and alerts.
### NoData propagation
When any query in an expression returns NoData, the entire expression result is NoData. This is by design to prevent calculations on incomplete data.
**Example:**
```
Expression: $A / $B
- Query A returns: 100
- Query B returns: NoData
- Expression result: NoData (not 100, not error)
```
### Strategies for missing data
**1. Use default values:**
Modify your data source queries to return zero or a default value instead of no data.
**2. Build conditional logic:**
Use multiple expressions to check for data availability before performing calculations.
**3. Adjust time ranges:**
Ensure queries use time ranges likely to have data. If a service only reports every 5 minutes, don't query the last 1 minute.
**4. Configure alert NoData handling:**
In alerting, you can configure how NoData is treated (for example, trigger alert, don't trigger, or mark as special state).
### Missing data points vs NoData
**Missing data points:** Some points in a time series are null or absent, but the series exists.
- Handle with Reduce modes (Drop non-numeric, Replace non-numeric)
- Use data quality functions: `is_null($A)`, `is_number($A)`
**NoData:** No series returned at all from a query.
- Check query syntax and time range
- Verify data exists in the data source
- Ensure data source is reachable
## Performance considerations
Expressions run on the Grafana server, so understanding performance implications helps you build efficient dashboards and alerts.
### When expressions are inefficient
**Large data volumes:**
- Pulling millions of data points to Grafana for simple aggregations
- Better: Perform aggregation in the data source query
**Repeated operations:**
- Running the same calculation across many panels
- Better: Consider recording rules (Prometheus) or continuous queries (InfluxDB)
**Complex nested expressions:**
- Long chains of expressions that could be simplified
- Better: Simplify the expression or move logic to data source
### Optimization strategies
**1. Push processing to data sources:**
Instead of:
```
Query A: SELECT value FROM metrics
Expression B: Reduce(Mean, $A)
Expression C: $B > 100
```
Do in data source:
```
Query A: SELECT AVG(value) FROM metrics
Expression B: $A > 100
```
**2. Use appropriate time ranges:**
- Don't query years of data when hours suffice
- Match time ranges to your actual analysis needs
- Use relative time ranges for consistent performance
**3. Reduce data points before math:**
If you only need a single value for alerting, reduce first then perform math rather than calculating across every point:
**Less efficient:**
```
Expression A: $QueryA * 100 (multiplies every point)
Expression B: Reduce(Mean, $A)
```
**More efficient:**
```
Expression A: Reduce(Mean, $QueryA)
Expression B: $A * 100 (multiplies one value)
```
**4. Limit label cardinality:**
High-cardinality labels (many unique values) multiply the number of series. If querying metrics with thousands of unique host labels, consider aggregating in the data source.
### Monitor expression performance
Watch for these warning signs:
- Panels take more than 2-3 seconds to load
- Query inspector shows expressions processing thousands of series
- Grafana server CPU spikes when loading dashboards
- Alert evaluation takes significant time
If you see these issues, review your expressions for optimization opportunities.
## Expressions vs transformations
Both expressions and transformations manipulate query data, but they serve different purposes and have different capabilities.
### When to use expressions
Use expressions when:
- **Server-side processing required:** Alerting requires server-side evaluation
- **Cross-data-source operations:** Combining data from different data sources
- **Label-based matching:** Automatic series matching based on labels
- **Simple math and aggregations:** Basic calculations and reductions
- **Backend data sources:** Working with backend/server-side data sources
**Advantages:**
- Work in alerting rules
- Operate on data before visualization
- Support cross-data-source calculations
- Preserve label-based series relationships
**Limitations:**
- Only work with backend data sources
- Limited operation types (Math, Reduce, Resample)
- Less flexible than transformations for complex data reshaping
- Can't modify table structures significantly
### When to use transformations
Use transformations when:
- **Complex data reshaping:** Pivoting, merging, or restructuring data
- **Table operations:** Working with tabular data formats
- **Field manipulation:** Renaming, organizing, or filtering fields
- **Client-side only needed:** Visualization changes that don't affect alerting
- **Advanced processing:** Operations not available in expressions
**Advantages:**
- More operation types available
- Better for complex table manipulations
- Work with any data source (including browser-based)
- More flexible field and column operations
- Can dramatically reshape data structures
**Limitations:**
- Don't work in alerting (client-side only)
- Can't combine different data sources
- Process data after query execution
- Don't preserve complex label relationships
### Comparison table
| Feature | Expressions | Transformations |
| --------------------- | -------------------------------- | ---------------------- |
| Works in alerts | Yes | No |
| Combines data sources | Yes | No |
| Available operations | 3 types (Math, Reduce, Resample) | 20+ types |
| Execution | Server-side | Client-side (browser) |
| Data source support | Backend only | All data sources |
| Label matching | Automatic | Manual |
| Table operations | Limited | Extensive |
| Performance | Uses server resources | Uses browser resources |
### Use both together
You can use expressions and transformations in the same panel:
1. Expressions run first (server-side)
1. Transformations run after (client-side)
**Example workflow:**
- Query A: Prometheus metric
- Query B: SQL query
- Expression C: Combine `$A` and `$B` (server-side)
- Transformation: Rename fields, organize columns (client-side)
This approach lets you leverage the strengths of both systems.
### Migration considerations
**From transformations to expressions:**
Consider this when:
- You need the same logic in alerting
- You're combining data sources
- Server-side processing would improve performance
**Limitations:**
- May need to redesign complex transformations
- Some transformation operations have no expression equivalent
- Need backend data sources
**From expressions to transformations:**
Consider this when:
- You need more complex data manipulation
- You're working with browser-based data sources
- You need advanced table operations
**Limitations:**
- Can't use in alerting
- Can't combine different data sources
- May need to change query structure
## Get help
If you're still experiencing issues after trying these troubleshooting steps:
1. **Check the Query Inspector:** Click the Query Inspector button to see raw query results and expression outputs
1. **Review Grafana logs:** Server-side expression errors appear in Grafana server logs
1. **Simplify and isolate:** Create a minimal example that reproduces the issue
1. **Community resources:** Search or post in the Grafana community forums
1. **Documentation:** Refer to [Grafana Expressions](ref:grafana-expressions) for detailed operation documentation
When asking for help, include:
- Grafana version
- Data source type and version
- Simplified example of your queries and expressions
- Expected vs actual results
- Any error messages from Query Inspector or logs
+1 -3
View File
@@ -715,11 +715,9 @@ export {
export {
type VisualizationSuggestion,
type VisualizationSuggestionsSupplier,
type VisualizationSuggestionsSupplierFn,
type PanelPluginVisualizationSuggestion,
type VisualizationSuggestionsBuilder,
VisualizationSuggestionScore,
VisualizationSuggestionsBuilder,
VisualizationSuggestionsListAppender,
} from './types/suggestions';
export {
type MatcherConfig,
@@ -1,14 +1,18 @@
import { createDataFrame } from '../dataframe/processDataFrame';
import { identityOverrideProcessor } from '../field/overrides/processors';
import {
StandardEditorsRegistryItem,
standardEditorsRegistry,
standardFieldConfigEditorRegistry,
} from '../field/standardFieldConfigEditorRegistry';
import { FieldType } from '../types/dataFrame';
import { FieldConfigProperty, FieldConfigPropertyItem } from '../types/fieldOverrides';
import { PanelMigrationModel } from '../types/panel';
import { VisualizationSuggestionsBuilder, VisualizationSuggestionScore } from '../types/suggestions';
import { PanelOptionsEditorBuilder } from '../utils/OptionsUIBuilders';
import { PanelPlugin } from './PanelPlugin';
import { getPanelDataSummary } from './suggestions/getPanelDataSummary';
describe('PanelPlugin', () => {
describe('declarative options', () => {
@@ -483,4 +487,107 @@ describe('PanelPlugin', () => {
});
});
});
describe('suggestions', () => {
it('should register a suggestions supplier', () => {
const panel = new PanelPlugin(() => <div>Panel</div>);
panel.meta = panel.meta || {};
panel.meta.id = 'test-panel';
panel.meta.name = 'Test Panel';
panel.setSuggestionsSupplier((ds) => {
if (!ds.hasFieldType(FieldType.number)) {
return;
}
return [
{
name: 'Number Panel',
score: VisualizationSuggestionScore.Good,
},
];
});
const suggestions = panel.getSuggestions(
getPanelDataSummary([createDataFrame({ fields: [{ type: FieldType.number, name: 'Value' }] })])
);
expect(suggestions).toHaveLength(1);
expect(suggestions![0].pluginId).toBe(panel.meta.id);
expect(suggestions![0].name).toBe('Number Panel');
expect(
panel.getSuggestions(
getPanelDataSummary([createDataFrame({ fields: [{ type: FieldType.string, name: 'Value' }] })])
)
).toBeUndefined();
});
it('should not throw for the old syntax, but also should not register suggestions', () => {
jest.spyOn(console, 'warn').mockImplementation();
class DeprecatedSuggestionsSupplier {
getSuggestionsForData(builder: VisualizationSuggestionsBuilder): void {
const appender = builder.getListAppender({
name: 'Deprecated Suggestion',
pluginId: 'deprecated-plugin',
options: {},
});
if (builder.dataSummary.hasNumberField) {
appender.append({});
}
}
}
const panel = new PanelPlugin(() => <div>Panel</div>);
expect(() => {
panel.setSuggestionsSupplier(new DeprecatedSuggestionsSupplier());
}).not.toThrow();
expect(console.warn).toHaveBeenCalled();
expect(
panel.getSuggestions(
getPanelDataSummary([
createDataFrame({
fields: [{ type: FieldType.number, name: 'Value', values: [1, 2, 3, 4, 5] }],
}),
])
)
).toBeUndefined();
});
it('should support the deprecated pattern of getSuggestionsSupplier with builder', () => {
jest.spyOn(console, 'warn').mockImplementation();
const panel = new PanelPlugin(() => <div>Panel</div>).setSuggestionsSupplier((ds) => {
if (!ds.hasFieldType(FieldType.number)) {
return;
}
return [
{
name: 'Number Panel',
score: VisualizationSuggestionScore.Good,
},
];
});
const oldSupplier = panel.getSuggestionsSupplier();
const builder1 = new VisualizationSuggestionsBuilder([
createDataFrame({ fields: [{ type: FieldType.number, name: 'Value' }] }),
]);
oldSupplier.getSuggestionsForData(builder1);
const suggestions1 = builder1.getList();
expect(suggestions1).toHaveLength(1);
expect(suggestions1![0].pluginId).toBe(panel.meta.id);
expect(suggestions1![0].name).toBe('Number Panel');
const builder2 = new VisualizationSuggestionsBuilder([
createDataFrame({ fields: [{ type: FieldType.string, name: 'Value' }] }),
]);
oldSupplier.getSuggestionsForData(builder2);
const suggestions2 = builder2.getList();
expect(suggestions2).toHaveLength(0);
});
});
});
+72 -36
View File
@@ -1,4 +1,4 @@
import { set } from 'lodash';
import { defaultsDeep, set } from 'lodash';
import { ComponentClass, ComponentType } from 'react';
import { FieldConfigOptionsRegistry } from '../field/FieldConfigOptionsRegistry';
@@ -14,11 +14,19 @@ import {
PanelPluginDataSupport,
} from '../types/panel';
import { GrafanaPlugin } from '../types/plugin';
import { VisualizationSuggestionsSupplierFn, VisualizationSuggestionsSupplier } from '../types/suggestions';
import {
getSuggestionHash,
PanelPluginVisualizationSuggestion,
VisualizationSuggestion,
VisualizationSuggestionsSupplierDeprecated,
VisualizationSuggestionsSupplier,
VisualizationSuggestionsBuilder,
} from '../types/suggestions';
import { FieldConfigEditorBuilder, PanelOptionsEditorBuilder } from '../utils/OptionsUIBuilders';
import { deprecationWarning } from '../utils/deprecationWarning';
import { createFieldConfigRegistry } from './registryFactories';
import { PanelDataSummary } from './suggestions/getPanelDataSummary';
/** @beta */
export type StandardOptionConfig = {
@@ -109,7 +117,7 @@ export class PanelPlugin<
};
private optionsSupplier?: PanelOptionsSupplier<TOptions>;
private suggestionsSupplier?: VisualizationSuggestionsSupplier;
private suggestionsSupplier?: VisualizationSuggestionsSupplier<TOptions, TFieldConfigOptions>;
panel: ComponentType<PanelProps<TOptions>> | null;
editor?: ComponentClass<PanelEditorProps<TOptions>>;
@@ -363,56 +371,84 @@ export class PanelPlugin<
}
/**
* @deprecated use VisualizationSuggestionsSupplierFn
* @deprecated use VisualizationSuggestionsSupplier
*/
setSuggestionsSupplier(supplier: VisualizationSuggestionsSupplier): this;
setSuggestionsSupplier(supplier: VisualizationSuggestionsSupplierDeprecated): this;
/**
* @alpha
* sets function that can return visualization examples and suggestions.
*/
setSuggestionsSupplier(supplier: VisualizationSuggestionsSupplierFn<TOptions, TFieldConfigOptions>): this;
setSuggestionsSupplier(supplier: VisualizationSuggestionsSupplier<TOptions, TFieldConfigOptions>): this;
setSuggestionsSupplier(
supplier: VisualizationSuggestionsSupplier | VisualizationSuggestionsSupplierFn<TOptions, TFieldConfigOptions>
supplier:
| VisualizationSuggestionsSupplier<TOptions, TFieldConfigOptions>
| VisualizationSuggestionsSupplierDeprecated
): this {
this.suggestionsSupplier =
typeof supplier === 'function'
? {
getSuggestionsForData: (builder) => {
const appender = builder.getListAppender<TOptions, TFieldConfigOptions>({
pluginId: this.meta.id,
name: this.meta.name,
options: {},
fieldConfig: {
defaults: {},
overrides: [],
},
});
const result = supplier(builder.dataSummary);
if (Array.isArray(result)) {
appender.appendAll(result);
}
},
}
: supplier;
if (typeof supplier !== 'function') {
deprecationWarning(
'PanelPlugin',
'plugin.setSuggestionsSupplier(new Supplier())',
'plugin.setSuggestionsSupplier(dataSummary => [...])'
);
return this;
}
this.suggestionsSupplier = supplier;
return this;
}
/**
* Returns the suggestions supplier
* @alpha
* get suggestions based on the PanelDataSummary
*/
getSuggestionsSupplier(): VisualizationSuggestionsSupplier | undefined {
return this.suggestionsSupplier;
getSuggestions(
panelDataSummary: PanelDataSummary
): Array<PanelPluginVisualizationSuggestion<TOptions, TFieldConfigOptions>> | void {
const withDefaults = (
suggestion: VisualizationSuggestion<TOptions, TFieldConfigOptions>
): Omit<PanelPluginVisualizationSuggestion<TOptions, TFieldConfigOptions>, 'hash'> =>
defaultsDeep(suggestion, {
pluginId: this.meta.id,
name: this.meta.name,
options: {},
fieldConfig: {
defaults: {},
overrides: [],
},
} satisfies Omit<PanelPluginVisualizationSuggestion<TOptions, TFieldConfigOptions>, 'hash'>);
return this.suggestionsSupplier?.(panelDataSummary)?.map(
(s): PanelPluginVisualizationSuggestion<TOptions, TFieldConfigOptions> => {
const suggestionWithDefaults = withDefaults(s);
return Object.assign(suggestionWithDefaults, { hash: getSuggestionHash(suggestionWithDefaults) });
}
);
}
/**
* @alpha
* returns whether the plugin has configured suggestions
* @deprecated use getSuggestions
* we have to keep this method intact to support cloud-onboarding plugin.
*/
hasSuggestions(): boolean {
return this.suggestionsSupplier !== undefined;
getSuggestionsSupplier() {
const withDefaults = (
suggestion: VisualizationSuggestion<TOptions, TFieldConfigOptions>
): Omit<PanelPluginVisualizationSuggestion<TOptions, TFieldConfigOptions>, 'hash'> =>
defaultsDeep(suggestion, {
pluginId: this.meta.id,
name: this.meta.name,
options: {},
fieldConfig: {
defaults: {},
overrides: [],
},
} satisfies Omit<PanelPluginVisualizationSuggestion<TOptions, TFieldConfigOptions>, 'hash'>);
return {
getSuggestionsForData: (builder: VisualizationSuggestionsBuilder) => {
deprecationWarning('PanelPlugin', 'getSuggestionsSupplier()', 'getSuggestions(panelDataSummary)');
this.suggestionsSupplier?.(builder.dataSummary)?.forEach((s) => {
builder.getListAppender(withDefaults(s)).append(s);
});
},
};
}
hasPluginId(pluginId: string) {
+5
View File
@@ -1143,6 +1143,11 @@ export interface FeatureToggles {
*/
newVizSuggestions?: boolean;
/**
* Enable all plugins to supply visualization suggestions (including 3rd party plugins)
* @default false
*/
externalVizSuggestions?: boolean;
/**
* Restrict PanelChrome contents with overflow: hidden;
* @default true
*/
+2
View File
@@ -20,6 +20,8 @@ export type InterpolateFunction = (value: string, scopedVars?: ScopedVars, forma
export interface PanelPluginMeta extends PluginMeta {
/** Indicates that panel does not issue queries */
skipDataQuery?: boolean;
/** Indicates that the panel implements suggestions */
suggestions?: boolean;
/** Indicates that panel should not be available in visualisation picker */
hideFromList?: boolean;
/** Sort order */
+33 -55
View File
@@ -2,11 +2,10 @@ import { defaultsDeep } from 'lodash';
import { DataTransformerConfig } from '@grafana/schema';
import { PanelDataSummary, getPanelDataSummary } from '../panel/suggestions/getPanelDataSummary';
import { getPanelDataSummary, PanelDataSummary } from '../panel/suggestions/getPanelDataSummary';
import { PanelModel } from './dashboard';
import { DataFrame } from './dataFrame';
import { FieldConfigSource } from './fieldOverrides';
import { PanelData } from './panel';
/**
* @internal
@@ -108,35 +107,6 @@ export enum VisualizationSuggestionScore {
OK = 50,
}
/**
* @internal
* TODO this will move into the grafana app code once suppliers are migrated.
*/
export class VisualizationSuggestionsBuilder {
/** Summary stats for current data */
dataSummary: PanelDataSummary;
private list: PanelPluginVisualizationSuggestion[] = [];
constructor(
/** Current data */
public data?: PanelData,
/** Current panel & options */
public panel?: PanelModel
) {
this.dataSummary = getPanelDataSummary(data?.series);
}
getListAppender<TOptions extends unknown, TFieldConfig extends {} = {}>(
defaults: Omit<PanelPluginVisualizationSuggestion<TOptions, TFieldConfig>, 'hash'>
) {
return new VisualizationSuggestionsListAppender<TOptions, TFieldConfig>(this.list, defaults);
}
getList() {
return this.list;
}
}
/**
* @alpha
* TODO: this name is temporary; it will become just "VisualizationSuggestionsSupplier" when the other interface is deleted.
@@ -147,40 +117,48 @@ export class VisualizationSuggestionsBuilder {
* - returns an array of VisualizationSuggestions
* - boolean return equates to "show a single suggestion card for this panel plugin with the default options" (true = show, false or void = hide)
*/
export type VisualizationSuggestionsSupplierFn<TOptions extends unknown, TFieldConfig extends {} = {}> = (
export type VisualizationSuggestionsSupplier<TOptions extends unknown, TFieldConfig extends {} = {}> = (
panelDataSummary: PanelDataSummary
) => Array<VisualizationSuggestion<TOptions, TFieldConfig>> | void;
/**
* @deprecated use VisualizationSuggestionsSupplierFn instead.
* DEPRECATED - the below exports need to remain in the code base to help make the transition for the Polystat plugin, which implements
* suggestions using the old API. These should be removed for Grafana 13.
*/
export type VisualizationSuggestionsSupplier = {
/**
* Adds suitable suggestions for the current data
*/
/**
* @deprecated use VisualizationSuggestionsSupplier
*/
export interface VisualizationSuggestionsSupplierDeprecated {
getSuggestionsForData: (builder: VisualizationSuggestionsBuilder) => void;
};
}
/**
* @internal
* TODO this will move into the grafana app code once suppliers are migrated.
* @deprecated use VisualizationSuggestionsSupplier
*/
export class VisualizationSuggestionsListAppender<TOptions extends unknown, TFieldConfig extends {} = {}> {
constructor(
private list: VisualizationSuggestion[],
private defaults: Partial<PanelPluginVisualizationSuggestion<TOptions, TFieldConfig>> = {}
) {}
export class VisualizationSuggestionsBuilder {
public dataSummary: PanelDataSummary;
public list: PanelPluginVisualizationSuggestion[] = [];
append(suggestion: VisualizationSuggestion<TOptions, TFieldConfig>) {
this.appendAll([suggestion]);
constructor(dataFrames: DataFrame[]) {
this.dataSummary = getPanelDataSummary(dataFrames);
}
appendAll(suggestions: Array<VisualizationSuggestion<TOptions, TFieldConfig>>) {
this.list.push(
...suggestions.map((s): PanelPluginVisualizationSuggestion<TOptions, TFieldConfig> => {
const suggestionWithDefaults = defaultsDeep(s, this.defaults);
return Object.assign(suggestionWithDefaults, { hash: getSuggestionHash(suggestionWithDefaults) });
})
);
getList(): PanelPluginVisualizationSuggestion[] {
return this.list;
}
getListAppender(suggestionDefaults: Omit<PanelPluginVisualizationSuggestion, 'hash'>) {
const withDefaults = (suggestion: VisualizationSuggestion): PanelPluginVisualizationSuggestion => {
const s = defaultsDeep({}, suggestion, suggestionDefaults);
return {
...s,
hash: getSuggestionHash(s),
};
};
return {
append: (suggestion: VisualizationSuggestion) => {
this.list.push(withDefaults(suggestion));
},
};
}
}
@@ -518,6 +518,7 @@ const getStyles = (theme: GrafanaTheme2) => {
return {
container: css({
height: '100%',
position: 'relative',
}),
panel: css({
+1
View File
@@ -164,6 +164,7 @@ func (hs *HTTPServer) getFrontendSettings(c *contextmodel.ReqContext) (*dtos.Fro
ModuleHash: hs.pluginAssets.ModuleHash(c.Req.Context(), panel),
BaseURL: panel.BaseURL,
SkipDataQuery: panel.SkipDataQuery,
Suggestions: panel.Suggestions,
HideFromList: panel.HideFromList,
ReleaseState: string(panel.State),
Signature: string(panel.Signature),
+11
View File
@@ -222,6 +222,10 @@ var (
// MStatTotalRepositories is a metric total amount of repositories
MStatTotalRepositories prometheus.Gauge
// MUnifiedStorageMigrationStatus indicates the migration status for unified storage in this instance.
// Possible values: 0 (default/undefined), 1 (migration disabled), 2 (migration would run).
MUnifiedStorageMigrationStatus prometheus.Gauge
)
const (
@@ -691,6 +695,12 @@ func init() {
Help: "total amount of repositories",
Namespace: ExporterName,
})
MUnifiedStorageMigrationStatus = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "unified_storage_migration_status",
Help: "indicates whether this instance would run unified storage migrations (0=undefined, 1=migration disabled, 2=would run)",
Namespace: ExporterName,
})
}
// SetBuildInformation sets the build information for this binary
@@ -829,5 +839,6 @@ func initMetricVars(reg prometheus.Registerer) {
MStatTotalRepositories,
MFolderIDsAPICount,
MFolderIDsServiceCount,
MUnifiedStorageMigrationStatus,
)
}
+1
View File
@@ -319,6 +319,7 @@ type PanelDTO struct {
HideFromList bool `json:"hideFromList"`
Sort int `json:"sort"`
SkipDataQuery bool `json:"skipDataQuery"`
Suggestions bool `json:"suggestions,omitempty"`
ReleaseState string `json:"state"`
BaseURL string `json:"baseUrl"`
Signature string `json:"signature"`
+1
View File
@@ -105,6 +105,7 @@ type JSONData struct {
// Panel settings
SkipDataQuery bool `json:"skipDataQuery"`
Suggestions bool `json:"suggestions,omitempty"`
// App settings
AutoEnabled bool `json:"autoEnabled"`
+8
View File
@@ -1884,6 +1884,14 @@ var (
Owner: grafanaDatavizSquad,
Expression: "false",
},
{
Name: "externalVizSuggestions",
Description: "Enable all plugins to supply visualization suggestions (including 3rd party plugins)",
Stage: FeatureStageExperimental,
FrontendOnly: true,
Owner: grafanaDatavizSquad,
Expression: "false",
},
{
Name: "preventPanelChromeOverflow",
Description: "Restrict PanelChrome contents with overflow: hidden;",
+1
View File
@@ -256,6 +256,7 @@ cdnPluginsUrls,experimental,@grafana/plugins-platform-backend,false,false,false
pluginInstallAPISync,experimental,@grafana/plugins-platform-backend,false,false,false
newGauge,experimental,@grafana/dataviz-squad,false,false,true
newVizSuggestions,preview,@grafana/dataviz-squad,false,false,true
externalVizSuggestions,experimental,@grafana/dataviz-squad,false,false,true
preventPanelChromeOverflow,preview,@grafana/grafana-frontend-platform,false,false,true
jaegerEnableGrpcEndpoint,experimental,@grafana/oss-big-tent,false,false,false
pluginStoreServiceLoading,experimental,@grafana/plugins-platform-backend,false,false,false
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
256 pluginInstallAPISync experimental @grafana/plugins-platform-backend false false false
257 newGauge experimental @grafana/dataviz-squad false false true
258 newVizSuggestions preview @grafana/dataviz-squad false false true
259 externalVizSuggestions experimental @grafana/dataviz-squad false false true
260 preventPanelChromeOverflow preview @grafana/grafana-frontend-platform false false true
261 jaegerEnableGrpcEndpoint experimental @grafana/oss-big-tent false false false
262 pluginStoreServiceLoading experimental @grafana/plugins-platform-backend false false false
+14
View File
@@ -1383,6 +1383,20 @@
"codeowner": "@grafana/identity-access-team"
}
},
{
"metadata": {
"name": "externalVizSuggestions",
"resourceVersion": "1763498528748",
"creationTimestamp": "2025-11-18T20:42:08Z"
},
"spec": {
"description": "Enable all plugins to supply visualization suggestions (including 3rd party plugins)",
"stage": "experimental",
"codeowner": "@grafana/dataviz-squad",
"frontend": true,
"expression": "false"
}
},
{
"metadata": {
"name": "extraThemes",
+9 -5
View File
@@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana/pkg/util/osutil"
)
// nolint:unused
var migratedUnifiedResources = []string{
//"playlists.playlist.grafana.app",
"folders.folder.grafana.app",
@@ -58,14 +59,16 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
// Set indexer config for unified storage
section := cfg.Raw.Section("unified_storage")
// TODO: Re-enable once migrations are ready and disabled on cloud
//cfg.DisableDataMigrations = section.Key("disable_data_migrations").MustBool(false)
cfg.DisableDataMigrations = true
cfg.DisableDataMigrations = section.Key("disable_data_migrations").MustBool(false)
if !cfg.DisableDataMigrations && cfg.getUnifiedStorageType() == "unified" {
cfg.enforceMigrationToUnifiedConfigs()
// Helper log to find instances running migrations in the future
cfg.Logger.Info("Unified migration configs not yet enforced")
//cfg.enforceMigrationToUnifiedConfigs() // TODO: uncomment when ready for release
} else {
cfg.EnableSearch = section.Key("enable_search").MustBool(false)
// Helper log to find instances disabling migration
cfg.Logger.Info("Unified migration configs enforcement disabled", "storage_type", cfg.getUnifiedStorageType(), "disable_data_migrations", cfg.DisableDataMigrations)
}
cfg.EnableSearch = section.Key("enable_search").MustBool(false)
cfg.MaxPageSizeBytes = section.Key("max_page_size_bytes").MustInt(0)
cfg.IndexPath = section.Key("index_path").String()
cfg.IndexWorkers = section.Key("index_workers").MustInt(10)
@@ -102,6 +105,7 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
cfg.MinFileIndexBuildVersion = section.Key("min_file_index_build_version").MustString("")
}
// nolint:unused
// enforceMigrationToUnifiedConfigs enforces configurations required to run migrated resources in mode 5
// All migrated resources in MigratedUnifiedResources are set to mode 5 and unified search is enabled
func (cfg *Cfg) enforceMigrationToUnifiedConfigs() {
@@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/metrics"
sqlstoremigrator "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/unified/migrations/contract"
@@ -55,8 +56,12 @@ func (p *UnifiedStorageMigrationServiceImpl) Run(ctx context.Context) error {
// skip migrations if disabled in config
if p.cfg.DisableDataMigrations {
metrics.MUnifiedStorageMigrationStatus.Set(1)
logger.Info("Data migrations are disabled, skipping")
return nil
} else {
metrics.MUnifiedStorageMigrationStatus.Set(2)
logger.Info("Data migrations not yet enforced, skipping")
}
// TODO: Re-enable once migrations are ready
+104
View File
@@ -35,6 +35,33 @@ type ListOptions struct {
Limit int64 // maximum number of results to return. 0 means no limit.
}
// BatchOpMode controls the semantics of each operation in a batch
type BatchOpMode int
const (
// BatchOpPut performs an upsert: create or update (never fails on key state)
BatchOpPut BatchOpMode = iota
// BatchOpCreate creates a new key, fails if the key already exists
BatchOpCreate
// BatchOpUpdate updates an existing key, fails if the key doesn't exist
BatchOpUpdate
// BatchOpDelete removes a key, idempotent (never fails on key state)
BatchOpDelete
)
// BatchOp represents a single operation in an atomic batch
type BatchOp struct {
Mode BatchOpMode
Key string
Value []byte // For Put/Create/Update operations, nil for Delete
}
// Maximum limit for batch operations
const MaxBatchOps = 20
// ErrKeyAlreadyExists is returned when BatchOpCreate is used on an existing key
var ErrKeyAlreadyExists = errors.New("key already exists")
type KV interface {
// Keys returns all the keys in the store
Keys(ctx context.Context, section string, opt ListOptions) iter.Seq2[string, error]
@@ -60,6 +87,17 @@ type KV interface {
// UnixTimestamp returns the current time in seconds since Epoch.
// This is used to ensure the server and client are not too far apart in time.
UnixTimestamp(ctx context.Context) (int64, error)
// Batch executes all operations atomically within a single transaction.
// If any operation fails, all operations are rolled back.
// Operations are executed in order; the batch stops on first failure.
//
// Operation semantics:
// - BatchOpPut: Upsert (create or update), never fails on key state
// - BatchOpCreate: Fail with ErrKeyAlreadyExists if key exists
// - BatchOpUpdate: Fail with ErrNotFound if key doesn't exist
// - BatchOpDelete: Idempotent, never fails on key state
Batch(ctx context.Context, section string, ops []BatchOp) error
}
var _ KV = &badgerKV{}
@@ -360,3 +398,69 @@ func (k *badgerKV) BatchDelete(ctx context.Context, section string, keys []strin
return txn.Commit()
}
func (k *badgerKV) Batch(ctx context.Context, section string, ops []BatchOp) error {
if k.db.IsClosed() {
return fmt.Errorf("database is closed")
}
if section == "" {
return fmt.Errorf("section is required")
}
if len(ops) > MaxBatchOps {
return fmt.Errorf("too many operations: %d > %d", len(ops), MaxBatchOps)
}
txn := k.db.NewTransaction(true)
defer txn.Discard()
for _, op := range ops {
keyWithSection := section + "/" + op.Key
switch op.Mode {
case BatchOpCreate:
// Check that key doesn't exist, then set
_, err := txn.Get([]byte(keyWithSection))
if err == nil {
return ErrKeyAlreadyExists
}
if !errors.Is(err, badger.ErrKeyNotFound) {
return err
}
if err := txn.Set([]byte(keyWithSection), op.Value); err != nil {
return err
}
case BatchOpUpdate:
// Check that key exists, then set
_, err := txn.Get([]byte(keyWithSection))
if errors.Is(err, badger.ErrKeyNotFound) {
return ErrNotFound
}
if err != nil {
return err
}
if err := txn.Set([]byte(keyWithSection), op.Value); err != nil {
return err
}
case BatchOpPut:
// Upsert: create or update
if err := txn.Set([]byte(keyWithSection), op.Value); err != nil {
return err
}
case BatchOpDelete:
// Idempotent delete - don't error if not found
if err := txn.Delete([]byte(keyWithSection)); err != nil {
return err
}
default:
return fmt.Errorf("unknown operation mode: %d", op.Mode)
}
}
return txn.Commit()
}
+258
View File
@@ -28,6 +28,7 @@ const (
TestKVUnixTimestamp = "unix timestamp"
TestKVBatchGet = "batch get operations"
TestKVBatchDelete = "batch delete operations"
TestKVBatch = "batch operations"
)
// NewKVFunc is a function that creates a new KV instance for testing
@@ -69,6 +70,7 @@ func RunKVTest(t *testing.T, newKV NewKVFunc, opts *KVTestOptions) {
{TestKVUnixTimestamp, runTestKVUnixTimestamp},
{TestKVBatchGet, runTestKVBatchGet},
{TestKVBatchDelete, runTestKVBatchDelete},
{TestKVBatch, runTestKVBatch},
}
for _, tc := range cases {
@@ -801,3 +803,259 @@ func saveKVHelper(t *testing.T, kv resource.KV, ctx context.Context, section, ke
err = writer.Close()
require.NoError(t, err)
}
func runTestKVBatch(t *testing.T, kv resource.KV, nsPrefix string) {
ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))
section := nsPrefix + "-batch"
t.Run("batch with empty section", func(t *testing.T) {
err := kv.Batch(ctx, "", nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "section is required")
})
t.Run("batch with empty ops succeeds", func(t *testing.T) {
err := kv.Batch(ctx, section, nil)
require.NoError(t, err)
})
t.Run("batch put creates new key", func(t *testing.T) {
ops := []resource.BatchOp{
{Mode: resource.BatchOpPut, Key: "put-key", Value: []byte("put-value")},
}
err := kv.Batch(ctx, section, ops)
require.NoError(t, err)
// Verify the key was created
reader, err := kv.Get(ctx, section, "put-key")
require.NoError(t, err)
value, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "put-value", string(value))
err = reader.Close()
require.NoError(t, err)
})
t.Run("batch put updates existing key", func(t *testing.T) {
// First create a key
saveKVHelper(t, kv, ctx, section, "put-update-key", strings.NewReader("original-value"))
ops := []resource.BatchOp{
{Mode: resource.BatchOpPut, Key: "put-update-key", Value: []byte("updated-value")},
}
err := kv.Batch(ctx, section, ops)
require.NoError(t, err)
// Verify the key was updated
reader, err := kv.Get(ctx, section, "put-update-key")
require.NoError(t, err)
value, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "updated-value", string(value))
err = reader.Close()
require.NoError(t, err)
})
t.Run("batch create succeeds for new key", func(t *testing.T) {
ops := []resource.BatchOp{
{Mode: resource.BatchOpCreate, Key: "create-new-key", Value: []byte("new-value")},
}
err := kv.Batch(ctx, section, ops)
require.NoError(t, err)
// Verify the key was created
reader, err := kv.Get(ctx, section, "create-new-key")
require.NoError(t, err)
value, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "new-value", string(value))
err = reader.Close()
require.NoError(t, err)
})
t.Run("batch create fails for existing key", func(t *testing.T) {
// First create a key
saveKVHelper(t, kv, ctx, section, "create-exists-key", strings.NewReader("existing-value"))
ops := []resource.BatchOp{
{Mode: resource.BatchOpCreate, Key: "create-exists-key", Value: []byte("new-value")},
}
err := kv.Batch(ctx, section, ops)
assert.ErrorIs(t, err, resource.ErrKeyAlreadyExists)
// Verify the original value is unchanged
reader, err := kv.Get(ctx, section, "create-exists-key")
require.NoError(t, err)
value, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "existing-value", string(value))
err = reader.Close()
require.NoError(t, err)
})
t.Run("batch update succeeds for existing key", func(t *testing.T) {
// First create a key
saveKVHelper(t, kv, ctx, section, "update-exists-key", strings.NewReader("original-value"))
ops := []resource.BatchOp{
{Mode: resource.BatchOpUpdate, Key: "update-exists-key", Value: []byte("updated-value")},
}
err := kv.Batch(ctx, section, ops)
require.NoError(t, err)
// Verify the key was updated
reader, err := kv.Get(ctx, section, "update-exists-key")
require.NoError(t, err)
value, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "updated-value", string(value))
err = reader.Close()
require.NoError(t, err)
})
t.Run("batch update fails for non-existent key", func(t *testing.T) {
ops := []resource.BatchOp{
{Mode: resource.BatchOpUpdate, Key: "update-nonexistent-key", Value: []byte("new-value")},
}
err := kv.Batch(ctx, section, ops)
assert.ErrorIs(t, err, resource.ErrNotFound)
// Verify the key was not created
_, err = kv.Get(ctx, section, "update-nonexistent-key")
assert.ErrorIs(t, err, resource.ErrNotFound)
})
t.Run("batch delete removes existing key", func(t *testing.T) {
// First create a key
saveKVHelper(t, kv, ctx, section, "delete-exists-key", strings.NewReader("to-be-deleted"))
ops := []resource.BatchOp{
{Mode: resource.BatchOpDelete, Key: "delete-exists-key"},
}
err := kv.Batch(ctx, section, ops)
require.NoError(t, err)
// Verify the key was deleted
_, err = kv.Get(ctx, section, "delete-exists-key")
assert.ErrorIs(t, err, resource.ErrNotFound)
})
t.Run("batch delete is idempotent for non-existent key", func(t *testing.T) {
ops := []resource.BatchOp{
{Mode: resource.BatchOpDelete, Key: "delete-nonexistent-key"},
}
err := kv.Batch(ctx, section, ops)
require.NoError(t, err) // Should succeed even though key doesn't exist
})
t.Run("batch multiple operations atomic success", func(t *testing.T) {
ops := []resource.BatchOp{
{Mode: resource.BatchOpPut, Key: "multi-key1", Value: []byte("value1")},
{Mode: resource.BatchOpPut, Key: "multi-key2", Value: []byte("value2")},
{Mode: resource.BatchOpPut, Key: "multi-key3", Value: []byte("value3")},
}
err := kv.Batch(ctx, section, ops)
require.NoError(t, err)
// Verify all keys were created
for i := 1; i <= 3; i++ {
key := fmt.Sprintf("multi-key%d", i)
reader, err := kv.Get(ctx, section, key)
require.NoError(t, err)
value, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, fmt.Sprintf("value%d", i), string(value))
err = reader.Close()
require.NoError(t, err)
}
})
t.Run("batch multiple operations atomic rollback on failure", func(t *testing.T) {
// First create a key that will cause the batch to fail
saveKVHelper(t, kv, ctx, section, "rollback-exists", strings.NewReader("existing"))
ops := []resource.BatchOp{
{Mode: resource.BatchOpPut, Key: "rollback-new1", Value: []byte("value1")},
{Mode: resource.BatchOpCreate, Key: "rollback-exists", Value: []byte("should-fail")}, // This will fail
{Mode: resource.BatchOpPut, Key: "rollback-new2", Value: []byte("value2")},
}
err := kv.Batch(ctx, section, ops)
assert.ErrorIs(t, err, resource.ErrKeyAlreadyExists)
// Verify rollback: the first operation should NOT have persisted
_, err = kv.Get(ctx, section, "rollback-new1")
assert.ErrorIs(t, err, resource.ErrNotFound)
// Verify the third operation was not executed
_, err = kv.Get(ctx, section, "rollback-new2")
assert.ErrorIs(t, err, resource.ErrNotFound)
})
t.Run("batch mixed operations", func(t *testing.T) {
// Setup: create a key to update and one to delete
saveKVHelper(t, kv, ctx, section, "mixed-update", strings.NewReader("original"))
saveKVHelper(t, kv, ctx, section, "mixed-delete", strings.NewReader("to-delete"))
ops := []resource.BatchOp{
{Mode: resource.BatchOpCreate, Key: "mixed-create", Value: []byte("created")},
{Mode: resource.BatchOpUpdate, Key: "mixed-update", Value: []byte("updated")},
{Mode: resource.BatchOpDelete, Key: "mixed-delete"},
{Mode: resource.BatchOpPut, Key: "mixed-put", Value: []byte("put")},
}
err := kv.Batch(ctx, section, ops)
require.NoError(t, err)
// Verify create
reader, err := kv.Get(ctx, section, "mixed-create")
require.NoError(t, err)
value, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "created", string(value))
err = reader.Close()
require.NoError(t, err)
// Verify update
reader, err = kv.Get(ctx, section, "mixed-update")
require.NoError(t, err)
value, err = io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "updated", string(value))
err = reader.Close()
require.NoError(t, err)
// Verify delete
_, err = kv.Get(ctx, section, "mixed-delete")
assert.ErrorIs(t, err, resource.ErrNotFound)
// Verify put
reader, err = kv.Get(ctx, section, "mixed-put")
require.NoError(t, err)
value, err = io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "put", string(value))
err = reader.Close()
require.NoError(t, err)
})
t.Run("batch too many operations", func(t *testing.T) {
ops := make([]resource.BatchOp, resource.MaxBatchOps+1)
for i := range ops {
ops[i] = resource.BatchOp{Mode: resource.BatchOpPut, Key: fmt.Sprintf("key-%d", i), Value: []byte("value")}
}
err := kv.Batch(ctx, section, ops)
assert.Error(t, err)
assert.Contains(t, err.Error(), "too many operations")
})
}
@@ -31,7 +31,6 @@ import { UNCONFIGURED_PANEL_PLUGIN_ID } from '../scene/UnconfiguredPanel';
import { DashboardGridItem } from '../scene/layout-default/DashboardGridItem';
import { DashboardLayoutItem, isDashboardLayoutItem } from '../scene/types/DashboardLayoutItem';
import { vizPanelToPanel } from '../serialization/transformSceneToSaveModel';
import { PanelModelCompatibilityWrapper } from '../utils/PanelModelCompatibilityWrapper';
import {
activateSceneObjectAndParentTree,
getDashboardSceneFor,
@@ -121,8 +120,7 @@ export class PanelEditor extends SceneObjectBase<PanelEditorState> {
dataObject.subscribeToState(async () => {
const { data } = dataObject.state;
if (hasData(data) && panel.state.pluginId === UNCONFIGURED_PANEL_PLUGIN_ID) {
const panelModel = new PanelModelCompatibilityWrapper(panel);
const suggestions = await getAllSuggestions(data, panelModel);
const suggestions = await getAllSuggestions(data);
if (suggestions.length > 0) {
const defaultFirstSuggestion = suggestions[0];
@@ -25,7 +25,7 @@ const MIN_COLUMN_SIZE = 260;
export function VisualizationSuggestions({ onChange, data, panel }: Props) {
const styles = useStyles2(getStyles);
const { value: suggestions } = useAsync(() => getAllSuggestions(data, panel), [data, panel]);
const { value: suggestions } = useAsync(async () => await getAllSuggestions(data), [data]);
const [suggestionHash, setSuggestionHash] = useState<string | null>(null);
const [firstCardRef, { width }] = useMeasure<HTMLDivElement>();
const [firstCardHash, setFirstCardHash] = useState<string | null>(null);
@@ -0,0 +1,19 @@
export const panelsToCheckFirst = [
'timeseries',
'barchart',
'gauge',
'stat',
'piechart',
'bargauge',
'table',
'state-timeline',
'status-history',
'logs',
'candlestick',
'flamegraph',
'traces',
'nodeGraph',
'heatmap',
'histogram',
'geomap',
];
@@ -2,11 +2,13 @@ import {
DataFrame,
FieldType,
getDefaultTimeRange,
getPanelDataSummary,
LoadingState,
PanelData,
PanelPluginMeta,
PanelPluginVisualizationSuggestion,
PluginType,
toDataFrame,
VisualizationSuggestionScore,
} from '@grafana/data';
import {
BarGaugeDisplayMode,
@@ -18,26 +20,69 @@ import {
} from '@grafana/schema';
import { config } from 'app/core/config';
import { getAllSuggestions, panelsToCheckFirst } from './getAllSuggestions';
import { panelsToCheckFirst } from './consts';
import { getAllSuggestions, sortSuggestions } from './getAllSuggestions';
config.featureToggles.externalVizSuggestions = true;
let idx = 0;
for (const pluginId of panelsToCheckFirst) {
if (pluginId === 'geomap') {
continue;
}
config.panels[pluginId] = {
module: `core:plugin/${pluginId}`,
id: pluginId,
} as PanelPluginMeta;
module: `core:plugin/${pluginId}`,
sort: idx++,
name: pluginId,
type: PluginType.panel,
baseUrl: 'public/app/plugins/panel',
suggestions: true,
info: {
version: '1.0.0',
updated: '2025-01-01',
links: [],
screenshots: [],
author: {
name: 'Grafana Labs',
},
description: pluginId,
logos: { small: 'small/logo', large: 'large/logo' },
},
};
}
const SCALAR_PLUGINS = ['gauge', 'stat', 'bargauge', 'piechart', 'radialbar'];
config.panels['text'] = {
config.panels.text = {
id: 'text',
module: 'core:plugin/text',
sort: idx++,
name: 'Text',
type: PluginType.panel,
baseUrl: 'public/app/plugins/panel',
skipDataQuery: true,
suggestions: false,
info: {
description: 'pretty decent plugin',
version: '1.0.0',
updated: '2025-01-01',
links: [],
screenshots: [],
author: {
name: 'Grafana Labs',
},
description: 'Text panel',
logos: { small: 'small/logo', large: 'large/logo' },
},
} as PanelPluginMeta;
};
jest.mock('../state/util', () => {
const originalModule = jest.requireActual('../state/util');
return {
...originalModule,
getAllPanelPluginMeta: jest.fn().mockImplementation(() => [...Object.values(config.panels)]),
};
});
const SCALAR_PLUGINS = ['gauge', 'stat', 'bargauge', 'piechart', 'radialbar'];
class ScenarioContext {
data: DataFrame[] = [];
@@ -289,10 +334,8 @@ scenario('Single frame with string and number field', (ctx) => {
pluginId: 'stat',
options: expect.objectContaining({ colorMode: BigValueColorMode.Background }),
}),
expect.objectContaining({
pluginId: 'bargauge',
options: expect.objectContaining({ displayMode: BarGaugeDisplayMode.Basic }),
}),
expect.objectContaining({
pluginId: 'bargauge',
@@ -447,6 +490,70 @@ scenario('Given a preferredVisualisationType', (ctx) => {
});
});
describe('sortSuggestions', () => {
it('should sort suggestions correctly by score', () => {
const suggestions = [
{ pluginId: 'timeseries', name: 'Time series', hash: 'b', score: VisualizationSuggestionScore.OK },
{ pluginId: 'table', name: 'Table', hash: 'a', score: VisualizationSuggestionScore.OK },
{ pluginId: 'stat', name: 'Stat', hash: 'c', score: VisualizationSuggestionScore.Good },
] satisfies PanelPluginVisualizationSuggestion[];
const dataSummary = getPanelDataSummary([
toDataFrame({
fields: [
{ name: 'Time', type: FieldType.time, values: [1, 2, 3, 4, 5] },
{ name: 'ServerA', type: FieldType.number, values: [1, 10, 50, 2, 5] },
{ name: 'ServerB', type: FieldType.number, values: [1, 10, 50, 2, 5] },
],
}),
]);
sortSuggestions(suggestions, dataSummary);
expect(suggestions[0].pluginId).toBe('stat');
expect(suggestions[1].pluginId).toBe('timeseries');
expect(suggestions[2].pluginId).toBe('table');
});
it('should sort suggestions based on core module', () => {
const suggestions = [
{
pluginId: 'fake-external-panel',
name: 'Time series',
hash: 'b',
score: VisualizationSuggestionScore.Good,
},
{
pluginId: 'fake-external-panel',
name: 'Time series',
hash: 'd',
score: VisualizationSuggestionScore.Best,
},
{ pluginId: 'timeseries', name: 'Table', hash: 'a', score: VisualizationSuggestionScore.OK },
{ pluginId: 'stat', name: 'Stat', hash: 'c', score: VisualizationSuggestionScore.Good },
] satisfies PanelPluginVisualizationSuggestion[];
const dataSummary = getPanelDataSummary([
toDataFrame({
fields: [
{ name: 'Time', type: FieldType.time, values: [1, 2, 3, 4, 5] },
{ name: 'ServerA', type: FieldType.number, values: [1, 10, 50, 2, 5] },
{ name: 'ServerB', type: FieldType.number, values: [1, 10, 50, 2, 5] },
],
}),
]);
sortSuggestions(suggestions, dataSummary);
expect(suggestions[0].pluginId).toBe('stat');
expect(suggestions[1].pluginId).toBe('timeseries');
expect(suggestions[2].pluginId).toBe('fake-external-panel');
expect(suggestions[2].hash).toBe('d');
expect(suggestions[3].pluginId).toBe('fake-external-panel');
expect(suggestions[3].hash).toBe('b');
});
});
function repeatFrame(count: number, frame: DataFrame): DataFrame[] {
const frames: DataFrame[] = [];
for (let i = 0; i < count; i++) {
@@ -1,33 +1,52 @@
import {
getPanelDataSummary,
PanelData,
PanelDataSummary,
PanelPlugin,
PanelPluginVisualizationSuggestion,
VisualizationSuggestionsBuilder,
PanelModel,
VisualizationSuggestionScore,
PreferredVisualisationType,
VisualizationSuggestionScore,
} from '@grafana/data';
import { config } from '@grafana/runtime';
import { importPanelPlugin } from 'app/features/plugins/importPanelPlugin';
import { importPanelPlugin, isBuiltInPlugin } from 'app/features/plugins/importPanelPlugin';
export const panelsToCheckFirst = [
'timeseries',
'barchart',
'gauge',
'stat',
'piechart',
'bargauge',
'table',
'state-timeline',
'status-history',
'logs',
'candlestick',
'flamegraph',
'traces',
'nodeGraph',
'heatmap',
'histogram',
'geomap',
];
import { getAllPanelPluginMeta } from '../state/util';
import { panelsToCheckFirst } from './consts';
/**
* gather and cache the plugins which provide visualization suggestions so they can be invoked to build suggestions
*/
let _pluginCache: PanelPlugin[] | null = null;
async function getPanelsWithSuggestions(): Promise<PanelPlugin[]> {
if (!_pluginCache) {
_pluginCache = [];
// list of plugins to load is determined by the feature flag
const pluginIds: string[] = config.featureToggles.externalVizSuggestions
? getAllPanelPluginMeta()
.filter((panel) => panel.suggestions)
.map((m) => m.id)
: panelsToCheckFirst;
// import the plugins in parallel using Promise.allSettled
const settledPromises = await Promise.allSettled(pluginIds.map((id) => importPanelPlugin(id)));
for (let i = 0; i < settledPromises.length; i++) {
const settled = settledPromises[i];
if (settled.status === 'fulfilled') {
_pluginCache.push(settled.value);
}
// TODO: do we want to somehow log if there were errors loading some of the plugins?
}
}
if (_pluginCache.length === 0) {
throw new Error('No panel plugins with visualization suggestions found');
}
return _pluginCache;
}
/**
* some of the PreferredVisualisationTypes do not match the panel plugin ids, so we have to map them. d'oh.
@@ -44,24 +63,54 @@ const mapPreferredVisualisationTypeToPlugin = (type: string): PreferredVisualisa
return PLUGIN_ID_TO_PREFERRED_VIZ_TYPE[type];
};
export async function getAllSuggestions(
data?: PanelData,
panel?: PanelModel
): Promise<PanelPluginVisualizationSuggestion[]> {
const builder = new VisualizationSuggestionsBuilder(data, panel);
/**
* given a list of suggestions, sort them in place based on score and preferred visualisation type
*/
export function sortSuggestions(suggestions: PanelPluginVisualizationSuggestion[], dataSummary: PanelDataSummary) {
suggestions.sort((a, b) => {
// if one of these suggestions is from a built-in panel and the other isn't, prioritize the core panel.
const isPluginABuiltIn = isBuiltInPlugin(a.pluginId);
const isPluginBBuiltIn = isBuiltInPlugin(b.pluginId);
if (isPluginABuiltIn && !isPluginBBuiltIn) {
return -1;
}
if (isPluginBBuiltIn && !isPluginABuiltIn) {
return 1;
}
for (const pluginId of panelsToCheckFirst) {
const plugin = await importPanelPlugin(pluginId);
const supplier = plugin.getSuggestionsSupplier();
// if a preferred visualisation type matches the data, prioritize it
const mappedA = mapPreferredVisualisationTypeToPlugin(a.pluginId);
if (mappedA && dataSummary.hasPreferredVisualisationType(mappedA)) {
return -1;
}
const mappedB = mapPreferredVisualisationTypeToPlugin(a.pluginId);
if (mappedB && dataSummary.hasPreferredVisualisationType(mappedB)) {
return 1;
}
if (supplier) {
supplier.getSuggestionsForData(builder);
// compare scores directly if there are no other factors
return (b.score ?? VisualizationSuggestionScore.OK) - (a.score ?? VisualizationSuggestionScore.OK);
});
}
/**
* given PanelData, return a sorted list of Suggestions from all plugins which support it.
* @param {PanelData} data queried and transformed data for the panel
* @returns {PanelPluginVisualizationSuggestion[]} sorted list of suggestions
*/
export async function getAllSuggestions(data?: PanelData): Promise<PanelPluginVisualizationSuggestion[]> {
const dataSummary = getPanelDataSummary(data?.series);
const list: PanelPluginVisualizationSuggestion[] = [];
const plugins = await getPanelsWithSuggestions();
for (const plugin of plugins) {
const suggestions = plugin.getSuggestions(dataSummary);
if (suggestions) {
list.push(...suggestions);
}
}
const list = builder.getList();
if (builder.dataSummary.fieldCount === 0) {
if (dataSummary.fieldCount === 0) {
for (const plugin of Object.values(config.panels)) {
if (!plugin.skipDataQuery || plugin.hideFromList) {
continue;
@@ -79,15 +128,7 @@ export async function getAllSuggestions(
}
}
return list.sort((a, b) => {
const mappedA = mapPreferredVisualisationTypeToPlugin(a.pluginId);
if (mappedA && builder.dataSummary.hasPreferredVisualisationType(mappedA)) {
return -1;
}
const mappedB = mapPreferredVisualisationTypeToPlugin(a.pluginId);
if (mappedB && builder.dataSummary.hasPreferredVisualisationType(mappedB)) {
return 1;
}
return (b.score ?? VisualizationSuggestionScore.OK) - (a.score ?? VisualizationSuggestionScore.OK);
});
sortSuggestions(list, dataSummary);
return list;
}
@@ -115,4 +115,8 @@ const builtInPlugins: Record<string, System.Module | (() => Promise<System.Modul
'core:plugin/radialbar': radialBar,
};
export function isBuiltinPluginPath(path: string): path is keyof typeof builtInPlugins {
return Boolean(builtInPlugins[path]);
}
export default builtInPlugins;
@@ -1,6 +1,7 @@
import { PanelPlugin, PanelPluginMeta } from '@grafana/data';
import config from 'app/core/config';
import builtInPlugins, { isBuiltinPluginPath } from './built_in_plugins';
import { pluginImporter } from './importer/pluginImporter';
const promiseCache: Record<string, Promise<PanelPlugin>> = {};
@@ -25,6 +26,14 @@ export function importPanelPlugin(id: string): Promise<PanelPlugin> {
return promiseCache[id];
}
export function isBuiltInPlugin(id?: string): id is keyof typeof builtInPlugins {
if (!id) {
return false;
}
const meta = getPanelPluginMeta(id);
return Boolean(meta != null && isBuiltinPluginPath(meta.module));
}
export function hasPanelPlugin(id: string): boolean {
return !!getPanelPluginMeta(id);
}
@@ -2,7 +2,7 @@ import { DEFAULT_LANGUAGE } from '@grafana/i18n';
import { getResolvedLanguage } from '@grafana/i18n/internal';
import { config } from '@grafana/runtime';
import builtInPlugins from '../built_in_plugins';
import builtInPlugins, { isBuiltinPluginPath } from '../built_in_plugins';
import { registerPluginInfoInCache } from '../loader/pluginInfoCache';
import { SystemJS } from '../loader/systemjs';
import { resolveModulePath } from '../loader/utils';
@@ -35,8 +35,8 @@ export async function importPluginModule({
});
}
const builtIn = builtInPlugins[path];
if (builtIn) {
if (isBuiltinPluginPath(path)) {
const builtIn = builtInPlugins[path];
// for handling dynamic imports
if (typeof builtIn === 'function') {
return await builtIn();
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Bar chart",
"id": "barchart",
"suggestions": true,
"info": {
"description": "Categorical charts with group support",
"author": {
@@ -1,6 +1,6 @@
import { defaultsDeep } from 'lodash';
import { FieldType, VisualizationSuggestion, VisualizationSuggestionsSupplierFn, VizOrientation } from '@grafana/data';
import { FieldType, VisualizationSuggestion, VisualizationSuggestionsSupplier, VizOrientation } from '@grafana/data';
import { t } from '@grafana/i18n';
import { LegendDisplayMode, StackingMode, VisibilityMode } from '@grafana/schema';
@@ -32,7 +32,7 @@ const withDefaults = (suggestion: VisualizationSuggestion<Options, FieldConfig>)
},
} satisfies VisualizationSuggestion<Options, FieldConfig>);
export const barchartSuggestionsSupplier: VisualizationSuggestionsSupplierFn<Options, FieldConfig> = (dataSummary) => {
export const barchartSuggestionsSupplier: VisualizationSuggestionsSupplier<Options, FieldConfig> = (dataSummary) => {
if (dataSummary.frameCount !== 1) {
return;
}
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Bar gauge",
"id": "bargauge",
"suggestions": true,
"info": {
"description": "Horizontal and vertical gauges",
"author": {
@@ -4,7 +4,7 @@ import {
FieldColorModeId,
FieldType,
VisualizationSuggestion,
VisualizationSuggestionsSupplierFn,
VisualizationSuggestionsSupplier,
VizOrientation,
} from '@grafana/data';
import { t } from '@grafana/i18n';
@@ -31,7 +31,7 @@ const withDefaults = (suggestion: VisualizationSuggestion<Options>): Visualizati
const BAR_LIMIT = 30;
export const barGaugeSugggestionsSupplier: VisualizationSuggestionsSupplierFn<Options> = (dataSummary) => {
export const barGaugeSugggestionsSupplier: VisualizationSuggestionsSupplier<Options> = (dataSummary) => {
if (!dataSummary.hasData || !dataSummary.hasFieldType(FieldType.number)) {
return;
}
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Candlestick",
"id": "candlestick",
"suggestions": true,
"info": {
"description": "Graphical representation of price movements of a security, derivative, or currency.",
"keywords": ["financial", "price", "currency", "k-line"],
@@ -1,10 +1,10 @@
import { FieldType, VisualizationSuggestionScore, VisualizationSuggestionsSupplierFn } from '@grafana/data';
import { FieldType, VisualizationSuggestionScore, VisualizationSuggestionsSupplier } from '@grafana/data';
import { config } from '@grafana/runtime';
import { prepareCandlestickFields } from './fields';
import { defaultOptions, Options } from './types';
export const candlestickSuggestionSupplier: VisualizationSuggestionsSupplierFn<Options> = (dataSummary) => {
export const candlestickSuggestionSupplier: VisualizationSuggestionsSupplier<Options> = (dataSummary) => {
if (
!dataSummary.rawFrames ||
!dataSummary.hasData ||
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Flame Graph",
"id": "flamegraph",
"suggestions": true,
"info": {
"author": {
"name": "Grafana Labs",
+1 -1
View File
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Gauge",
"id": "gauge",
"suggestions": true,
"info": {
"description": "Standard gauge visualization",
"author": {
@@ -1,6 +1,6 @@
import { defaultsDeep } from 'lodash';
import { ThresholdsMode, FieldType, VisualizationSuggestion, VisualizationSuggestionsSupplierFn } from '@grafana/data';
import { ThresholdsMode, FieldType, VisualizationSuggestion, VisualizationSuggestionsSupplier } from '@grafana/data';
import { t } from '@grafana/i18n';
import { defaultNumericVizOptions } from 'app/features/panel/suggestions/utils';
@@ -33,7 +33,7 @@ const withDefaults = (suggestion: VisualizationSuggestion<Options>): Visualizati
const GAUGE_LIMIT = 10;
export const gaugeSuggestionsSupplier: VisualizationSuggestionsSupplierFn<Options> = (dataSummary) => {
export const gaugeSuggestionsSupplier: VisualizationSuggestionsSupplier<Options> = (dataSummary) => {
if (!dataSummary.hasData || !dataSummary.hasFieldType(FieldType.number)) {
return;
}
+1 -1
View File
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Geomap",
"id": "geomap",
"suggestions": true,
"info": {
"description": "Geomap panel",
"author": {
@@ -1,12 +1,10 @@
import { VisualizationSuggestionScore, VisualizationSuggestionsSupplierFn } from '@grafana/data';
import { VisualizationSuggestionScore, VisualizationSuggestionsSupplier } from '@grafana/data';
import { GraphFieldConfig } from '@grafana/ui';
import { getGeometryField, getDefaultLocationMatchers } from 'app/features/geo/utils/location';
import { Options } from './panelcfg.gen';
export const geomapSuggestionsSupplier: VisualizationSuggestionsSupplierFn<Options, GraphFieldConfig> = (
dataSummary
) => {
export const geomapSuggestionsSupplier: VisualizationSuggestionsSupplier<Options, GraphFieldConfig> = (dataSummary) => {
if (!dataSummary.hasData || !dataSummary.rawFrames) {
return;
}
+1 -1
View File
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Heatmap",
"id": "heatmap",
"suggestions": true,
"info": {
"description": "Like a histogram over time",
"author": {
@@ -3,7 +3,7 @@ import {
FieldType,
PanelDataSummary,
VisualizationSuggestionScore,
VisualizationSuggestionsSupplierFn,
VisualizationSuggestionsSupplier,
} from '@grafana/data';
import { config } from '@grafana/runtime';
import { GraphFieldConfig } from '@grafana/schema';
@@ -43,7 +43,7 @@ function determineScore(dataSummary: PanelDataSummary): VisualizationSuggestionS
return VisualizationSuggestionScore.OK;
}
export const heatmapSuggestionsSupplier: VisualizationSuggestionsSupplierFn<Options, GraphFieldConfig> = (
export const heatmapSuggestionsSupplier: VisualizationSuggestionsSupplier<Options, GraphFieldConfig> = (
dataSummary: PanelDataSummary
) => {
if (
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Histogram",
"id": "histogram",
"suggestions": true,
"info": {
"description": "Distribution of values presented as a bar chart.",
"keywords": ["distribution", "bar chart", "frequency", "proportional"],
+1 -1
View File
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Logs",
"id": "logs",
"suggestions": true,
"info": {
"author": {
"name": "Grafana Labs",
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Node Graph",
"id": "nodeGraph",
"suggestions": true,
"info": {
"author": {
"name": "Grafana Labs",
@@ -1,4 +1,4 @@
import { DataFrame, FieldType, VisualizationSuggestionScore, VisualizationSuggestionsSupplierFn } from '@grafana/data';
import { DataFrame, FieldType, VisualizationSuggestionScore, VisualizationSuggestionsSupplier } from '@grafana/data';
import { Options } from './panelcfg.gen';
@@ -44,7 +44,7 @@ function frameHasCorrectFields(frames: DataFrame[]): boolean {
return hasNodesFrame && hasEdgesFrame;
}
export const nodeGraphSuggestionsSupplier: VisualizationSuggestionsSupplierFn<Options> = (dataSummary) => {
export const nodeGraphSuggestionsSupplier: VisualizationSuggestionsSupplier<Options> = (dataSummary) => {
if (!dataSummary.rawFrames) {
return;
}
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Pie chart",
"id": "piechart",
"suggestions": true,
"info": {
"description": "The new core pie chart visualization",
"author": {
@@ -4,7 +4,7 @@ import {
FieldType,
VisualizationSuggestion,
VisualizationSuggestionScore,
VisualizationSuggestionsSupplierFn,
VisualizationSuggestionsSupplier,
} from '@grafana/data';
import { t } from '@grafana/i18n';
import { LegendDisplayMode } from '@grafana/schema';
@@ -29,7 +29,7 @@ const withDefaults = (suggestion: VisualizationSuggestion<Options>): Visualizati
const SLICE_MAX = 30;
const SLICE_MIN = 2;
export const piechartSuggestionsSupplier: VisualizationSuggestionsSupplierFn<Options> = (dataSummary) => {
export const piechartSuggestionsSupplier: VisualizationSuggestionsSupplier<Options> = (dataSummary) => {
if (!dataSummary.hasFieldType(FieldType.number)) {
return;
}
@@ -3,6 +3,7 @@
"name": "New Gauge",
"id": "radialbar",
"state": "alpha",
"suggestions": false,
"info": {
"description": "Standard gauge visualization",
"author": {
@@ -1,11 +1,6 @@
import { defaultsDeep } from 'lodash';
import {
FieldColorModeId,
FieldType,
VisualizationSuggestion,
VisualizationSuggestionsSupplierFn,
} from '@grafana/data';
import { FieldColorModeId, FieldType, VisualizationSuggestion, VisualizationSuggestionsSupplier } from '@grafana/data';
import { t } from '@grafana/i18n';
import { GraphFieldConfig } from '@grafana/ui';
import { defaultNumericVizOptions } from 'app/features/panel/suggestions/utils';
@@ -40,7 +35,7 @@ const withDefaults = (
const MAX_GAUGES = 10;
export const radialBarSuggestionsSupplier: VisualizationSuggestionsSupplierFn<Options, GraphFieldConfig> = (
export const radialBarSuggestionsSupplier: VisualizationSuggestionsSupplier<Options, GraphFieldConfig> = (
dataSummary
) => {
if (!dataSummary.hasData || !dataSummary.hasFieldType(FieldType.number)) {
+1 -1
View File
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Stat",
"id": "stat",
"suggestions": true,
"info": {
"description": "Big stat values & sparklines",
"author": {
+2 -2
View File
@@ -1,6 +1,6 @@
import { defaultsDeep } from 'lodash';
import { FieldType, VisualizationSuggestion, VisualizationSuggestionsSupplierFn } from '@grafana/data';
import { FieldType, VisualizationSuggestion, VisualizationSuggestionsSupplier } from '@grafana/data';
import { t } from '@grafana/i18n';
import { BigValueColorMode, BigValueGraphMode } from '@grafana/schema';
@@ -24,7 +24,7 @@ const withDefaults = (s: VisualizationSuggestion<Options>): VisualizationSuggest
},
} satisfies VisualizationSuggestion<Options>);
export const statSuggestionsSupplier: VisualizationSuggestionsSupplierFn<Options> = (ds) => {
export const statSuggestionsSupplier: VisualizationSuggestionsSupplier<Options> = (ds) => {
if (!ds.hasData) {
return;
}
@@ -2,7 +2,7 @@
"type": "panel",
"name": "State timeline",
"id": "state-timeline",
"suggestions": true,
"info": {
"description": "State changes and durations",
"author": {
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Status history",
"id": "status-history",
"suggestions": true,
"info": {
"description": "Periodic status history",
"author": {
+1 -1
View File
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Table",
"id": "table",
"suggestions": true,
"info": {
"description": "Supports many column styles",
"author": {
@@ -1,4 +1,4 @@
import { PanelDataSummary, VisualizationSuggestionScore, VisualizationSuggestionsSupplierFn } from '@grafana/data';
import { PanelDataSummary, VisualizationSuggestionScore, VisualizationSuggestionsSupplier } from '@grafana/data';
import icnTablePanelSvg from 'app/plugins/panel/table/img/icn-table-panel.svg';
import { Options, FieldConfig } from './panelcfg.gen';
@@ -16,7 +16,7 @@ function getTableSuggestionScore(dataSummary: PanelDataSummary): VisualizationSu
return VisualizationSuggestionScore.OK;
}
export const tableSuggestionsSupplier: VisualizationSuggestionsSupplierFn<Options, FieldConfig> = (dataSummary) => [
export const tableSuggestionsSupplier: VisualizationSuggestionsSupplier<Options, FieldConfig> = (dataSummary) => [
{
score: getTableSuggestionScore(dataSummary),
cardOptions: {
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Time series",
"id": "timeseries",
"suggestions": true,
"info": {
"description": "Time based line, area and bar charts",
"author": {
@@ -7,7 +7,7 @@ import {
PanelPluginVisualizationSuggestion,
VisualizationSuggestion,
VisualizationSuggestionScore,
VisualizationSuggestionsSupplierFn,
VisualizationSuggestionsSupplier,
} from '@grafana/data';
import { t } from '@grafana/i18n';
import {
@@ -83,7 +83,7 @@ const barChart = (name: string, stacking?: StackingMode) => ({
// TODO: all "gradient color scheme" suggestions have been removed. they will be re-added as part of the "styles" feature.
export const timeseriesSuggestionsSupplier: VisualizationSuggestionsSupplierFn<Options, GraphFieldConfig> = (
export const timeseriesSuggestionsSupplier: VisualizationSuggestionsSupplier<Options, GraphFieldConfig> = (
dataSummary
) => {
if (
+1 -1
View File
@@ -2,7 +2,7 @@
"type": "panel",
"name": "Traces",
"id": "traces",
"suggestions": true,
"info": {
"author": {
"name": "Grafana Labs",
+1 -1
View File
@@ -4,7 +4,7 @@
"id": "trend",
"state": "beta",
"suggestions": true,
"info": {
"description": "Like timeseries, but when x != time",
"author": {