[v11.0.x] Annotations: Optimize search by tags (#93609)

Annotations: Optimize search by tags (#93547)

* Annotations: Optimize search on large number of dashboards

* refactor

* fix batch size

* Return early if no annotations found

* revert go.mod

* return nil in case of error

* Move default limit to the API package

* fix empty access control filter

* Set default limit to 100

* optimize query when number of annotations is less than limit

* Update pkg/services/annotations/annotationsimpl/annotations.go

Co-authored-by: Ieva <ieva.vasiljeva@grafana.com>

* remove limit from store since it's set in API

* set default limit in Find method (do not break tests)

* Only add limit to the query if it's set

* use limit trick for all searches without dashboard filter

* set default page if not provided

---------

Co-authored-by: Ieva <ieva.vasiljeva@grafana.com>
(cherry picked from commit 5e713673e1)

Co-authored-by: Alexander Zobnin <alexanderzobnin@gmail.com>

pkg/services/annotations/accesscontrol/accesscontrol.go

@@ -39,7 +39,7 @@ func NewAuthService(db db.DB, features featuremgmt.FeatureToggles) *AuthService
 }
 
 // Authorize checks if the user has permission to read annotations, then returns a struct containing dashboards and scope types that the user has access to.
-func (authz *AuthService) Authorize(ctx context.Context, orgID int64, query *annotations.ItemQuery) (*AccessResources, error) {
+func (authz *AuthService) Authorize(ctx context.Context, query *annotations.ItemQuery) (*AccessResources, error) {
 	user := query.SignedInUser
 	if user == nil || user.IsNil() {
 		return nil, ErrReadForbidden.Errorf("missing user")
@@ -60,14 +60,14 @@ func (authz *AuthService) Authorize(ctx context.Context, orgID int64, query *ann
 	var err error
 	if canAccessDashAnnotations {
 		if query.AnnotationID != 0 {
-			annotationDashboardID, err := authz.getAnnotationDashboard(ctx, query, orgID)
+			annotationDashboardID, err := authz.getAnnotationDashboard(ctx, query)
 			if err != nil {
 				return nil, ErrAccessControlInternal.Errorf("failed to fetch annotations: %w", err)
 			}
 			query.DashboardID = annotationDashboardID
 		}
 
-		visibleDashboards, err = authz.dashboardsWithVisibleAnnotations(ctx, query, orgID)
+		visibleDashboards, err = authz.dashboardsWithVisibleAnnotations(ctx, query)
 		if err != nil {
 			return nil, ErrAccessControlInternal.Errorf("failed to fetch dashboards: %w", err)
 		}
@@ -80,7 +80,7 @@ func (authz *AuthService) Authorize(ctx context.Context, orgID int64, query *ann
 	}, nil
 }
 
-func (authz *AuthService) getAnnotationDashboard(ctx context.Context, query *annotations.ItemQuery, orgID int64) (int64, error) {
+func (authz *AuthService) getAnnotationDashboard(ctx context.Context, query *annotations.ItemQuery) (int64, error) {
 	var items []annotations.Item
 	params := make([]any, 0)
 	err := authz.db.WithDbSession(ctx, func(sess *db.Session) error {
@@ -92,7 +92,7 @@ func (authz *AuthService) getAnnotationDashboard(ctx context.Context, query *ann
 			FROM annotation as a
 			WHERE a.org_id = ? AND a.id = ?
 			`
-		params = append(params, orgID, query.AnnotationID)
+		params = append(params, query.OrgID, query.AnnotationID)
 
 		return sess.SQL(sql, params...).Find(&items)
 	})
@@ -106,7 +106,7 @@ func (authz *AuthService) getAnnotationDashboard(ctx context.Context, query *ann
 	return items[0].DashboardID, nil
 }
 
-func (authz *AuthService) dashboardsWithVisibleAnnotations(ctx context.Context, query *annotations.ItemQuery, orgID int64) (map[string]int64, error) {
+func (authz *AuthService) dashboardsWithVisibleAnnotations(ctx context.Context, query *annotations.ItemQuery) (map[string]int64, error) {
 	recursiveQueriesSupported, err := authz.db.RecursiveQueriesAreSupported()
 	if err != nil {
 		return nil, err
@@ -119,7 +119,7 @@ func (authz *AuthService) dashboardsWithVisibleAnnotations(ctx context.Context,
 
 	filters := []any{
 		permissions.NewAccessControlDashboardPermissionFilter(query.SignedInUser, dashboardaccess.PERMISSION_VIEW, filterType, authz.features, recursiveQueriesSupported),
-		searchstore.OrgFilter{OrgId: orgID},
+		searchstore.OrgFilter{OrgId: query.OrgID},
 	}
 
 	if query.DashboardUID != "" {
@@ -134,32 +134,25 @@ func (authz *AuthService) dashboardsWithVisibleAnnotations(ctx context.Context,
 	}
 
 	sb := &searchstore.Builder{Dialect: authz.db.GetDialect(), Filters: filters, Features: authz.features}
+	// This is a limit for a batch size, not for the end query result.
+	var limit int64 = 1000
+	if query.Page == 0 {
+		query.Page = 1
+	}
+	sql, params := sb.ToSQL(limit, query.Page)
 
 	visibleDashboards := make(map[string]int64)
+	var res []dashboardProjection
 
-	var page int64 = 1
-	var limit int64 = 1000
-	for {
-		var res []dashboardProjection
-		sql, params := sb.ToSQL(limit, page)
+	err = authz.db.WithDbSession(ctx, func(sess *db.Session) error {
+		return sess.SQL(sql, params...).Find(&res)
+	})
+	if err != nil {
+		return nil, err
+	}
 
-		err = authz.db.WithDbSession(ctx, func(sess *db.Session) error {
-			return sess.SQL(sql, params...).Find(&res)
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		for _, p := range res {
-			visibleDashboards[p.UID] = p.ID
-		}
-
-		// if the result is less than the limit, we have reached the end
-		if len(res) < int(limit) {
-			break
-		}
-
-		page++
+	for _, p := range res {
+		visibleDashboards[p.UID] = p.ID
 	}
 
 	return visibleDashboards, nil

pkg/services/annotations/accesscontrol/accesscontrol_test.go

@@ -175,8 +175,8 @@ func TestIntegrationAuthorize(t *testing.T) {
 
 			authz := NewAuthService(sql, featuremgmt.WithFeatures(tc.featureToggle))
 
-			query := &annotations.ItemQuery{SignedInUser: u}
-			resources, err := authz.Authorize(context.Background(), 1, query)
+			query := &annotations.ItemQuery{SignedInUser: u, OrgID: 1}
+			resources, err := authz.Authorize(context.Background(), query)
 			require.NoError(t, err)
 
 			if tc.expectedResources.Dashboards != nil {

pkg/services/annotations/accesscontrol/models.go

@@ -8,6 +8,8 @@ type AccessResources struct {
 	CanAccessDashAnnotations bool
 	// CanAccessOrgAnnotations true if the user is allowed to access organization annotations
 	CanAccessOrgAnnotations bool
+	// Skip filtering
+	SkipAccessControlFilter bool
 }
 
 type dashboardProjection struct {