Annotations: Optimize search by tags (#93547)

* Annotations: Optimize search on large number of dashboards

* refactor

* fix batch size

* Return early if no annotations found

* revert go.mod

* return nil in case of error

* Move default limit to the API package

* fix empty access control filter

* Set default limit to 100

* optimize query when number of annotations is less than limit

* Update pkg/services/annotations/annotationsimpl/annotations.go

Co-authored-by: Ieva <ieva.vasiljeva@grafana.com>

* remove limit from store since it's set in API

* set default limit in Find method (do not break tests)

* Only add limit to the query if it's set

* use limit trick for all searches without dashboard filter

* set default page if not provided

---------

Co-authored-by: Ieva <ieva.vasiljeva@grafana.com>

pkg/services/annotations/annotationsimpl/annotations.go

@@ -72,12 +72,50 @@ func (r *RepositoryImpl) Update(ctx context.Context, item *annotations.Item) err
 }
 
 func (r *RepositoryImpl) Find(ctx context.Context, query *annotations.ItemQuery) ([]*annotations.ItemDTO, error) {
-	resources, err := r.authZ.Authorize(ctx, query.OrgID, query)
-	if err != nil {
-		return make([]*annotations.ItemDTO, 0), err
+	if query.Limit == 0 {
+		query.Limit = 100
 	}
 
-	return r.reader.Get(ctx, query, resources)
+	// Search without dashboard UID filter is expensive, so check without access control first
+	if query.DashboardID == 0 && query.DashboardUID == "" {
+		// Return early if no annotations found, it's not necessary to perform expensive access control filtering
+		res, err := r.reader.Get(ctx, query, &accesscontrol.AccessResources{
+			SkipAccessControlFilter: true,
+		})
+		if err != nil || len(res) == 0 {
+			return []*annotations.ItemDTO{}, err
+		}
+		// If number of resources is less than limit, it makes sense to set query limit to this
+		// value, otherwise query will be iterating over all user's dashboards since original
+		// query limit is never reached.
+		query.Limit = int64(len(res))
+	}
+
+	results := make([]*annotations.ItemDTO, 0, query.Limit)
+	query.Page = 1
+
+	// Iterate over available annotations until query limit is reached
+	// or all available dashboards are checked
+	for len(results) < int(query.Limit) {
+		resources, err := r.authZ.Authorize(ctx, query)
+		if err != nil {
+			return nil, err
+		}
+
+		res, err := r.reader.Get(ctx, query, resources)
+		if err != nil {
+			return nil, err
+		}
+
+		results = append(results, res...)
+		query.Page++
+		// All user's dashboards are fetched
+		if len(resources.Dashboards) < int(query.Limit) {
+			break
+		}
+	}
+
+	return results, nil
 }
 
 func (r *RepositoryImpl) Delete(ctx context.Context, params *annotations.DeleteParams) error {