public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Collect and log search stats. (#114107)

Browse Source 
* Collect and log search stats.

* Fix compilation problems.
This commit is contained in:
Peter Štibraný
2025-11-19 16:52:47 +01:00
committed by GitHub
parent 1fcf15d05f
commit 42db6324c9
9 changed files with 225 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/search/bleve.go
+28 -3
View File
@@ -866,7 +866,7 @@ func (b *bleveIndex) BuildInfo() (resource.IndexBuildInfo, error) {
}, nil
}
func (b *bleveIndex) ListManagedObjects(ctx context.Context, req *resourcepb.ListManagedObjectsRequest) (*resourcepb.ListManagedObjectsResponse, error) {
func (b *bleveIndex) ListManagedObjects(ctx context.Context, req *resourcepb.ListManagedObjectsRequest, stats *resource.SearchStats) (*resourcepb.ListManagedObjectsResponse, error) {
if req.NextPageToken != "" {
return nil, fmt.Errorf("next page not implemented yet")
}
@@ -881,6 +881,7 @@ func (b *bleveIndex) ListManagedObjects(ctx context.Context, req *resourcepb.Lis
}, nil
}
start := time.Now()
q := bleve.NewBooleanQuery()
q.AddMust(&query.TermQuery{
Term: req.Kind,
@@ -890,6 +891,7 @@ func (b *bleveIndex) ListManagedObjects(ctx context.Context, req *resourcepb.Lis
Term: req.Id,
FieldVal: resource.SEARCH_FIELD_MANAGER_ID,
})
stats.AddResultsConversionTime(time.Since(start))
found, err := b.index.SearchInContext(ctx, &bleve.SearchRequest{
Query: q,
@@ -916,6 +918,10 @@ func (b *bleveIndex) ListManagedObjects(ctx context.Context, req *resourcepb.Lis
return nil, err
}
stats.AddTotalHits(int(found.Total))
stats.AddSearchTime(found.Took)
stats.AddReturnedDocuments(len(found.Hits))
asString := func(v any) string {
if v == nil {
return ""
@@ -947,6 +953,7 @@ func (b *bleveIndex) ListManagedObjects(ctx context.Context, req *resourcepb.Lis
return 0
}
start = time.Now()
rsp := &resourcepb.ListManagedObjectsResponse{}
for _, hit := range found.Hits {
item := &resourcepb.ListManagedObjectsResponse_Item{
@@ -963,10 +970,11 @@ func (b *bleveIndex) ListManagedObjects(ctx context.Context, req *resourcepb.Lis
}
rsp.Items = append(rsp.Items, item)
}
stats.AddResultsConversionTime(time.Since(start))
return rsp, nil
}
func (b *bleveIndex) CountManagedObjects(ctx context.Context) ([]*resourcepb.CountManagedObjectsResponse_ResourceCount, error) {
func (b *bleveIndex) CountManagedObjects(ctx context.Context, stats *resource.SearchStats) ([]*resourcepb.CountManagedObjectsResponse_ResourceCount, error) {
found, err := b.index.SearchInContext(ctx, &bleve.SearchRequest{
Query: bleve.NewMatchAllQuery(),
Size: 0,
@@ -977,6 +985,11 @@ func (b *bleveIndex) CountManagedObjects(ctx context.Context) ([]*resourcepb.Cou
if err != nil {
return nil, err
}
stats.AddSearchTime(found.Took)
stats.AddTotalHits(int(found.Total))
stats.AddReturnedDocuments(len(found.Hits))
vals := make([]*resourcepb.CountManagedObjectsResponse_ResourceCount, 0)
f, ok := found.Facets["count"]
if ok && f.Terms != nil {
@@ -1003,6 +1016,7 @@ func (b *bleveIndex) Search(
access authlib.AccessClient,
req *resourcepb.ResourceSearchRequest,
federate []resource.ResourceIndex, // For federated queries, these will match the values in req.federate
stats *resource.SearchStats,
) (*resourcepb.ResourceSearchResponse, error) {
ctx, span := b.tracing.Start(ctx, tracingPrexfixBleve+"Search")
defer span.End()
@@ -1026,6 +1040,7 @@ func (b *bleveIndex) Search(
return nil, err
}
conversionStarts := time.Now()
// convert protobuf request to bleve request
searchrequest, e := b.toBleveSearchRequest(ctx, req, access)
if e != nil {
@@ -1050,6 +1065,7 @@ func (b *bleveIndex) Search(
}
}
}
stats.AddRequestConversionTime(time.Since(conversionStarts))
res, err := index.SearchInContext(ctx, searchrequest)
if err != nil {
@@ -1059,7 +1075,11 @@ func (b *bleveIndex) Search(
response.TotalHits = int64(res.Total)
response.QueryCost = float64(res.Cost)
response.MaxScore = res.MaxScore
stats.AddSearchTime(res.Took)
stats.AddTotalHits(int(res.Total))
stats.AddReturnedDocuments(len(res.Hits))
resultsConversionStart := time.Now()
response.Results, err = b.hitsToTable(ctx, searchrequest.Fields, res.Hits, req.Explain)
if err != nil {
return nil, err
@@ -1073,10 +1093,11 @@ func (b *bleveIndex) Search(
}
response.Facet[k] = f
}
stats.AddResultsConversionTime(time.Since(resultsConversionStart))
return response, nil
}
func (b *bleveIndex) DocCount(ctx context.Context, folder string) (int64, error) {
func (b *bleveIndex) DocCount(ctx context.Context, folder string, stats *resource.SearchStats) (int64, error) {
ctx, span := b.tracing.Start(ctx, tracingPrexfixBleve+"DocCount")
defer span.End()
@@ -1097,6 +1118,10 @@ func (b *bleveIndex) DocCount(ctx context.Context, folder string) (int64, error)
if rsp == nil {
return 0, err
}
if stats != nil {
stats.AddTotalHits(int(rsp.Total))
stats.AddSearchTime(rsp.Took)
}
return int64(rsp.Total), err
}
pkg/storage/unified/search/bleve_performance_test.go
+1 -1
View File
@@ -57,7 +57,7 @@ func runBenchmark(b *testing.B, testIndex resource.ResourceIndex) {
var memStatsBefore, memStatsAfter runtime.MemStats
runtime.ReadMemStats(&memStatsBefore)
_, err := testIndex.Search(context.Background(), nil, searchRequest, nil)
_, err := testIndex.Search(context.Background(), nil, searchRequest, nil, nil)
elapsed := time.Since(start) // Calculate elapsed time
runtime.ReadMemStats(&memStatsAfter)
pkg/storage/unified/search/bleve_search_test.go
+1 -1
View File
@@ -43,7 +43,7 @@ func indexDocumentsWithTitles(t *testing.T, index resource.ResourceIndex, key re
}
func checkSearchQuery(t *testing.T, index resource.ResourceIndex, query *resourcepb.ResourceSearchRequest, orderedExpectedNames []string) {
res, err := index.Search(context.Background(), nil, query, nil)
res, err := index.Search(context.Background(), nil, query, nil, nil)
require.NoError(t, err)
require.Equal(t, int64(len(orderedExpectedNames)), res.TotalHits)
for ix, name := range orderedExpectedNames {
pkg/storage/unified/search/bleve_test.go
+21 -21
View File
@@ -213,7 +213,7 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
Limit: 100,
},
},
}, nil)
}, nil, nil)
require.NoError(t, err)
require.Nil(t, rsp.Error)
require.NotNil(t, rsp.Results)
@@ -242,10 +242,10 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
]
}`, string(disp))
count, _ := index.DocCount(ctx, "")
count, _ := index.DocCount(ctx, "", nil)
assert.Equal(t, int64(3), count)
count, _ = index.DocCount(ctx, "zzz")
count, _ = index.DocCount(ctx, "zzz", nil)
assert.Equal(t, int64(1), count)
rsp, err = index.Search(ctx, NewStubAccessClient(map[string]bool{"dashboards": true}), &resourcepb.ResourceSearchRequest{
@@ -258,7 +258,7 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
}},
},
Limit: 100000,
}, nil)
}, nil, nil)
require.NoError(t, err)
require.Equal(t, int64(2), rsp.TotalHits)
require.Equal(t, []string{"aaa", "bbb"}, []string{
@@ -276,7 +276,7 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
SortBy: []*resourcepb.ResourceSearchRequest_Sort{
{Field: "fields." + DASHBOARD_VIEWS_LAST_1_DAYS, Desc: true},
},
}, nil)
}, nil, nil)
require.NoError(t, err)
require.Equal(t, 2, len(rsp.Results.Columns))
require.Equal(t, DASHBOARD_ERRORS_TODAY, rsp.Results.Columns[0].Name)
@@ -296,7 +296,7 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
SortBy: []*resourcepb.ResourceSearchRequest_Sort{
{Field: "fields." + DASHBOARD_VIEWS_LAST_1_DAYS, Desc: true},
},
}, nil)
}, nil, nil)
require.NoError(t, err)
require.Equal(t, 0, len(rsp.Results.Rows))
@@ -304,7 +304,7 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
found, err := index.ListManagedObjects(ctx, &resourcepb.ListManagedObjectsRequest{
Kind: "repo",
Id: "repo-1",
})
}, nil)
require.NoError(t, err)
jj, err := json.MarshalIndent(found, "", " ")
require.NoError(t, err)
@@ -341,7 +341,7 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
]
}`, string(jj))
counts, err := index.CountManagedObjects(ctx)
counts, err := index.CountManagedObjects(ctx, nil)
require.NoError(t, err)
jj, err = json.MarshalIndent(counts, "", " ")
require.NoError(t, err)
@@ -433,7 +433,7 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
Key: key,
},
Limit: 100000,
}, nil)
}, nil, nil)
require.NoError(t, err)
require.Nil(t, rsp.Error)
require.NotNil(t, rsp.Results)
@@ -468,7 +468,7 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
Limit: 100,
},
},
}, []resource.ResourceIndex{foldersIndex}) // << note the folder index matches the federation request
}, []resource.ResourceIndex{foldersIndex}, nil) // << note the folder index matches the federation request
require.NoError(t, err)
require.Nil(t, rsp.Error)
require.NotNil(t, rsp.Results)
@@ -532,7 +532,7 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
Limit: 100,
},
},
}, []resource.ResourceIndex{foldersIndex}) // << note the folder index matches the federation request
}, []resource.ResourceIndex{foldersIndex}, nil) // << note the folder index matches the federation request
require.NoError(t, err)
require.Equal(t, 3, len(rsp.Results.Rows))
@@ -561,7 +561,7 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
Limit: 100,
},
},
}, []resource.ResourceIndex{foldersIndex}) // << note the folder index matches the federation request
}, []resource.ResourceIndex{foldersIndex}, nil) // << note the folder index matches the federation request
require.NoError(t, err)
require.Equal(t, 2, len(rsp.Results.Rows))
@@ -589,7 +589,7 @@ func testBleveBackend(t *testing.T, backend *bleveBackend) {
Limit: 100,
},
},
}, []resource.ResourceIndex{foldersIndex}) // << note the folder index matches the federation request
}, []resource.ResourceIndex{foldersIndex}, nil) // << note the folder index matches the federation request
require.NoError(t, err)
require.Equal(t, 0, len(rsp.Results.Rows))
@@ -888,7 +888,7 @@ func TestBuildIndexExpiration(t *testing.T) {
idx := backend.GetIndex(ns)
require.Nil(t, idx)
_, err = builtIndex.DocCount(context.Background(), "")
_, err = builtIndex.DocCount(context.Background(), "", nil)
require.ErrorIs(t, err, bleve.ErrorIndexClosed)
// Verify that there are no open indexes.
@@ -897,7 +897,7 @@ func TestBuildIndexExpiration(t *testing.T) {
idx := backend.GetIndex(ns)
require.NotNil(t, idx)
cnt, err := builtIndex.DocCount(context.Background(), "")
cnt, err := builtIndex.DocCount(context.Background(), "", nil)
require.NoError(t, err)
require.Equal(t, int64(1), cnt)
@@ -971,7 +971,7 @@ func TestBuildIndex(t *testing.T) {
idx, err := newBackend.BuildIndex(context.Background(), ns, secondIndexDocsCount, nil, "test", indexTestDocs(ns, secondIndexDocsCount, 100), nil, rebuild)
require.NoError(t, err)
cnt, err := idx.DocCount(context.Background(), "")
cnt, err := idx.DocCount(context.Background(), "", nil)
require.NoError(t, err)
if rebuild {
require.Equal(t, int64(secondIndexDocsCount), cnt, "Index has been not rebuilt")
@@ -1033,10 +1033,10 @@ func TestRebuildingIndexClosesPreviousCachedIndex(t *testing.T) {
// Verify that first and second index are different, and first one is now closed.
require.NotEqual(t, firstIndex, secondIndex)
_, err = firstIndex.DocCount(context.Background(), "")
_, err = firstIndex.DocCount(context.Background(), "", nil)
require.ErrorIs(t, err, bleve.ErrorIndexClosed)
cnt, err := secondIndex.DocCount(context.Background(), "")
cnt, err := secondIndex.DocCount(context.Background(), "", nil)
require.NoError(t, err)
require.Equal(t, int64(secondSize), cnt)
@@ -1341,7 +1341,7 @@ func TestConcurrentIndexUpdateSearchAndRebuild(t *testing.T) {
Fields: []string{"title"},
Query: "Document",
Limit: 10,
}, nil)
}, nil, nil)
if err != nil {
if errors.Is(err, bleve.ErrorIndexClosed) || errors.Is(err, context.Canceled) {
continue
@@ -1574,13 +1574,13 @@ func searchTitle(t *testing.T, idx resource.ResourceIndex, query string, limit i
Fields: []string{"title"},
Query: query,
Limit: int64(limit),
}, nil)
}, nil, nil)
require.NoError(t, err)
return resp
}
func docCount(t *testing.T, idx resource.ResourceIndex) int {
cnt, err := idx.DocCount(context.Background(), "")
cnt, err := idx.DocCount(context.Background(), "", nil)
require.NoError(t, err)
return int(cnt)
}
pkg/storage/unified/search/user_test.go
+4 -3
View File
@@ -7,6 +7,9 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/selection"
iamv0 "github.com/grafana/grafana/apps/iam/pkg/apis/iam/v0alpha1"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/infra/tracing"
@@ -14,8 +17,6 @@ import (
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
"github.com/grafana/grafana/pkg/storage/unified/search"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/selection"
)
func TestUserDocumentBuilder(t *testing.T) {
@@ -175,7 +176,7 @@ func indexUserDocuments(t *testing.T, index resource.ResourceIndex, key resource
func checkUserSearchQuery(t *testing.T, index resource.ResourceIndex, query *resourcepb.ResourceSearchRequest, orderedExpectedNames []string) {
t.Helper()
res, err := index.Search(context.Background(), nil, query, nil)
res, err := index.Search(context.Background(), nil, query, nil, nil)
require.NoError(t, err)
require.Equal(t, int64(len(orderedExpectedNames)), res.TotalHits)
names := make([]string, len(res.Results.Rows))