Alerting: New alert list filter improvements (#103107)

* Move filtering code to generators for performance reasons

Discarding rules and groups early in the iterable chain limits the number of promises we need to wait for which improves performance significantly

* Add error handling for generators

* Add support for data source filter for GMA rules

* search WIP fix

* Fix datasource filter

* Move filtering back to filtered rules hook, use paged groups for improved performance

* Add queriedDatasources field to grafana managed rules and update filtering logic to rely on it

- Introduced a new field `queriedDatasources` in the AlertingRule struct to track data sources used in rules.
- Updated the Prometheus API to populate `queriedDatasources` when creating alerting rules.
- Modified filtering logic in the ruleFilter function to utilize the new `queriedDatasources` field for improved data source matching.
- Adjusted related tests to reflect changes in rule structure and filtering behavior.

* Add FilterView performance logging

* Improve GMA Prometheus types, rename queried datasources property

* Use custom generator helpers for flattening and filtering rule groups

* Fix lint errors, add missing translations

* Revert test condition

* Refactor api prom changes

* Fix lint errors

* Update backend tests

* Refactor rule list components to improve error handling and data source management

- Enhanced error handling in FilterViewResults by logging errors before returning an empty iterable.
- Simplified conditional rendering in GrafanaRuleLoader for better readability.
- Updated data source handling in PaginatedDataSourceLoader and PaginatedGrafanaLoader to use new individual rule group generator.
- Renamed toPageless function to toIndividualRuleGroups for clarity in prometheusGroupsGenerator.
- Improved filtering logic in useFilteredRulesIterator to utilize a dedicated function for data source type validation.
- Added isRulesDataSourceType utility function for better data source type checks.
- Removed commented-out code in PromRuleDTOBase for cleaner interface definition.

* Fix abort controller on FilterView

* Improve generators filtering

* fix abort controller

* refactor cancelSearch

* make states exclusive

* Load full page in one loadResultPage call

* Update tests, update translations

* Refactor filter status into separate component

* hoist hook

* Use the new function for supported rules source type

---------

Co-authored-by: Gilles De Mey <gilles.de.mey@gmail.com>

pkg/services/ngalert/api/api_prometheus_test.go

@@ -368,6 +368,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
 				"folderUid": "namespaceUID",
 				"uid": "RuleUID",
 				"query": "vector(1)",
+				"queriedDatasourceUIDs": ["AUID"],
 				"alerts": [{
 					"labels": {
 						"job": "prometheus"
@@ -433,6 +434,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
 				"state": "inactive",
 				"name": "AlwaysFiring",
 				"query": "vector(1)",
+				"queriedDatasourceUIDs": ["AUID"],
 				"folderUid": "namespaceUID",
 				"uid": "RuleUID",
 				"alerts": [{
@@ -499,6 +501,7 @@ func TestRouteGetRuleStatuses(t *testing.T) {
 				"state": "inactive",
 				"name": "AlwaysFiring",
 				"query": "vector(1) | vector(1)",
+				"queriedDatasourceUIDs": ["AUID", "BUID"],
 				"folderUid": "namespaceUID",
 				"uid": "RuleUID",
 				"alerts": [{

pkg/services/ngalert/api/prometheus/api_prometheus.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/apimachinery/identity"
+	"github.com/grafana/grafana/pkg/expr"
 	"github.com/grafana/grafana/pkg/infra/log"
 	contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
 	"github.com/grafana/grafana/pkg/services/folder"
@@ -544,13 +545,16 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, sr StatusRe
 			}
 		}
 
+		queriedDatasourceUIDs := extractDatasourceUIDs(rule)
+
 		alertingRule := apimodels.AlertingRule{
-			State:         "inactive",
-			Name:          rule.Title,
-			Query:         ruleToQuery(log, rule),
-			Duration:      rule.For.Seconds(),
-			KeepFiringFor: rule.KeepFiringFor.Seconds(),
-			Annotations:   apimodels.LabelsFromMap(rule.Annotations),
+			State:                 "inactive",
+			Name:                  rule.Title,
+			Query:                 ruleToQuery(log, rule),
+			QueriedDatasourceUIDs: queriedDatasourceUIDs,
+			Duration:              rule.For.Seconds(),
+			KeepFiringFor:         rule.KeepFiringFor.Seconds(),
+			Annotations:           apimodels.LabelsFromMap(rule.Annotations),
 		}
 
 		newRule := apimodels.Rule{
@@ -663,6 +667,19 @@ func toRuleGroup(log log.Logger, manager state.AlertInstanceManager, sr StatusRe
 	return newGroup, rulesTotals
 }
 
+// extractDatasourceUIDs extracts datasource UIDs from a rule
+func extractDatasourceUIDs(rule *ngmodels.AlertRule) []string {
+	queriedDatasourceUIDs := make([]string, 0, len(rule.Data))
+	for _, query := range rule.Data {
+		// Skip expression datasources (UID -100 or __expr__)
+		if expr.IsDataSource(query.DatasourceUID) {
+			continue
+		}
+		queriedDatasourceUIDs = append(queriedDatasourceUIDs, query.DatasourceUID)
+	}
+	return queriedDatasourceUIDs
+}
+
 // ruleToQuery attempts to extract the datasource queries from the alert query model.
 // Returns the whole JSON model as a string if it fails to extract a minimum of 1 query.
 func ruleToQuery(logger log.Logger, rule *ngmodels.AlertRule) string {

pkg/services/ngalert/api/tooling/definitions/prom.go

@@ -152,9 +152,10 @@ type AlertingRule struct {
 	// required: true
 	Name string `json:"name,omitempty"`
 	// required: true
-	Query         string  `json:"query,omitempty"`
-	Duration      float64 `json:"duration,omitempty"`
-	KeepFiringFor float64 `json:"keepFiringFor,omitempty"`
+	Query                 string   `json:"query,omitempty"`
+	QueriedDatasourceUIDs []string `json:"queriedDatasourceUIDs,omitempty"`
+	Duration              float64  `json:"duration,omitempty"`
+	KeepFiringFor         float64  `json:"keepFiringFor,omitempty"`
 	// required: true
 	Annotations promlabels.Labels `json:"annotations,omitempty"`
 	// required: true
@@ -168,12 +169,10 @@ type AlertingRule struct {
 // adapted from cortex
 // swagger:model
 type Rule struct {
+	UID string `json:"uid,omitempty"`
 	// required: true
-	UID string `json:"uid"`
-	// required: true
-	Name string `json:"name"`
-	// required: true
-	FolderUID string `json:"folderUid"`
+	Name      string `json:"name"`
+	FolderUID string `json:"folderUid,omitempty"`
 	// required: true
 	Query  string            `json:"query"`
 	Labels promlabels.Labels `json:"labels,omitempty"`