public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Unified Storage Search: Add sprinkles (#97820)

Browse Source 
* Wire up sprinkles to oss and enterprise. Fetching sprinkles not implemented yet.

* Adds wireset for initializing document builders. Had to init it when creating the service to avoid cyclical imports.

* updates to int64 for stats

* adds config for sprinklesApiServer and gets sprinkles from there when its present

* add comment for later

* adds feature toggle for sprinkles. returns empty results when flag not enabled.

* adds unified storage config setting for sprinkles apiserver page limit

* fixes bug where dashboard uid was not getting set

* when creating dashboard summary, use metadata.name as the dashboard uid

* cleans up wire. use existing oss and enterprise sets to generate doc builders

* remove old wireset

* fix linter - adds missing arg for doc builders

* update dashboard stats in tests

* updates test-data dashboards

* log a warning instead of returning an error if we can't get sprinkles for a namespace

* dont read uid from dashboard json
This commit is contained in:
owensmallwood
2024-12-18 11:00:54 -06:00
committed by GitHub
parent 54e603e552
commit 4837585cab
18 changed files with 108 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/search/bleve_test.go
+1 -1
View File
@@ -49,7 +49,7 @@ func TestBleveBackend(t *testing.T) {
return &DashboardDocumentBuilder{
Namespace: namespace,
Blob: blob,
Stats: NewDashboardStatsLookup(nil), // empty stats
Stats: make(map[string]map[string]int64), // empty stats
DatasourceLookup: dashboard.CreateDatasourceLookup([]*dashboard.DatasourceQueryResult{{}}),
}, nil
})
pkg/storage/unified/search/dashboard.go
+11 -14
View File
@@ -75,7 +75,7 @@ func DashboardBuilder(namespaced resource.NamespacedDocumentSupplier) (resource.
return &DashboardDocumentBuilder{
Namespace: namespace,
Blob: blob,
Stats: NewDashboardStatsLookup(nil),
Stats: nil,
DatasourceLookup: dashboard.CreateDatasourceLookup([]*dashboard.DatasourceQueryResult{
// empty values (does not resolve anything)
}),
@@ -94,8 +94,8 @@ type DashboardDocumentBuilder struct {
Namespace string
// Cached stats for this namespace
// TODO, load this from apiserver request
Stats DashboardStatsLookup
// maps dashboard UID to stats
Stats map[string]map[string]int64
// data source lookup
DatasourceLookup dashboard.DatasourceLookup
@@ -104,17 +104,12 @@ type DashboardDocumentBuilder struct {
Blob resource.BlobSupport
}
type DashboardStatsLookup = func(ctx context.Context, uid string) map[string]int64
func NewDashboardStatsLookup(stats map[string]map[string]int64) DashboardStatsLookup {
return func(ctx context.Context, uid string) map[string]int64 {
if stats == nil {
return nil
}
return stats[uid]
}
type DashboardStats interface {
GetStats(ctx context.Context, namespace string) (map[string]map[string]int64, error)
}
type DashboardStatsLookup = func(ctx context.Context, uid string) map[string]int64
var _ resource.DocumentBuilder = &DashboardDocumentBuilder{}
func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resource.ResourceKey, rv int64, value []byte) (*resource.IndexableDocument, error) {
@@ -150,6 +145,9 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou
return nil, err
}
// metadata name is the dashboard uid
summary.UID = obj.GetName()
doc := resource.NewIndexableDocument(key, rv, obj)
doc.Title = summary.Title
doc.Description = summary.Description
@@ -211,8 +209,7 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou
}
// Add the stats fields
stats := s.Stats(ctx, key.Name) // summary.UID
for k, v := range stats {
for k, v := range s.Stats[summary.UID] {
doc.Fields[k] = v
}
pkg/storage/unified/search/dashboard_stats.go
+15
View File
@@ -0,0 +1,15 @@
package search
import (
"context"
)
type OssDashboardStats struct{}
func ProvideDashboardStats() *OssDashboardStats {
return &OssDashboardStats{}
}
func (s *OssDashboardStats) GetStats(ctx context.Context, namespace string) (map[string]map[string]int64, error) {
return nil, nil
}
pkg/storage/unified/search/document.go
+18 -4
View File
@@ -5,23 +5,25 @@ import (
"github.com/grafana/authlib/claims"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/store/kind/dashboard"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
// The default list of open source document builders
type StandardDocumentBuilders struct {
sql db.DB
sql db.DB
sprinkles DashboardStats
}
// Hooked up so wire can fill in different sprinkles
func ProvideDocumentBuilders(sql db.DB) resource.DocumentBuilderSupplier {
return &StandardDocumentBuilders{sql}
func ProvideDocumentBuilders(sql db.DB, sprinkles DashboardStats) resource.DocumentBuilderSupplier {
return &StandardDocumentBuilders{sql, sprinkles}
}
func (s *StandardDocumentBuilders) GetDocumentBuilders() ([]resource.DocumentBuilderInfo, error) {
dashboards, err := DashboardBuilder(func(ctx context.Context, namespace string, blob resource.BlobSupport) (resource.DocumentBuilder, error) {
stats := NewDashboardStatsLookup(nil) // empty stats
logger := log.New("dashboard_builder", "namespace", namespace)
dsinfo := []*dashboard.DatasourceQueryResult{{}}
ns, err := claims.ParseNamespace(namespace)
if err != nil && s.sql != nil {
@@ -43,6 +45,18 @@ func (s *StandardDocumentBuilders) GetDocumentBuilders() ([]resource.DocumentBui
dsinfo = append(dsinfo, info)
}
}
// Fetch dashboard sprinkles for the namespace
// This could take a while if namespace has a lot of dashboards
var stats map[string]map[string]int64
if s.sprinkles != nil {
stats, err = s.sprinkles.GetStats(ctx, namespace)
if err != nil {
// only log a warning. Don't need to fail the indexer if we can't get sprinkles
logger.Warn("Failed to get sprinkles", "error", err)
}
}
return &DashboardDocumentBuilder{
Namespace: namespace,
Blob: blob,
pkg/storage/unified/search/document_test.go
+2 -2
View File
@@ -56,12 +56,12 @@ func TestDashboardDocumentBuilder(t *testing.T) {
return &DashboardDocumentBuilder{
Namespace: namespace,
Blob: blob,
Stats: NewDashboardStatsLookup(map[string]map[string]int64{
Stats: map[string]map[string]int64{
"aaa": {
DASHBOARD_ERRORS_LAST_1_DAYS: 1,
DASHBOARD_ERRORS_LAST_7_DAYS: 1,
},
}),
},
DatasourceLookup: dashboard.CreateDatasourceLookup([]*dashboard.DatasourceQueryResult{{
Name: "TheDisplayName", // used to be the unique ID!
Type: "my-custom-plugin",
pkg/storage/unified/sql/service.go
+6 -7
View File
@@ -20,7 +20,6 @@ import (
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/storage/unified/resource/grpc"
"github.com/grafana/grafana/pkg/storage/unified/search"
)
var (
@@ -51,6 +50,8 @@ type service struct {
log log.Logger
reg prometheus.Registerer
docBuilders resource.DocumentBuilderSupplier
}
func ProvideUnifiedStorageGrpcService(
@@ -59,6 +60,7 @@ func ProvideUnifiedStorageGrpcService(
db infraDB.DB,
log log.Logger,
reg prometheus.Registerer,
docBuilders resource.DocumentBuilderSupplier,
) (UnifiedStorageGrpcService, error) {
tracingCfg, err := tracing.ProvideTracingConfig(cfg)
if err != nil {
@@ -78,7 +80,7 @@ func ProvideUnifiedStorageGrpcService(
// FIXME: This is a temporary solution while we are migrating to the new authn interceptor
// grpcutils.NewGrpcAuthenticator should be used instead.
authn, err := grpcutils.NewGrpcAuthenticatorWithFallback(cfg, prometheus.DefaultRegisterer, tracing, &grpc.Authenticator{})
authn, err := grpcutils.NewGrpcAuthenticatorWithFallback(cfg, reg, tracing, &grpc.Authenticator{})
if err != nil {
return nil, err
}
@@ -92,6 +94,7 @@ func ProvideUnifiedStorageGrpcService(
db: db,
log: log,
reg: reg,
docBuilders: docBuilders,
}
// This will be used when running as a dskit service
@@ -106,11 +109,7 @@ func (s *service) start(ctx context.Context) error {
return err
}
// TODO, for standalone this will need to be started from enterprise
// Connecting to the correct remote services (cloudconfig for DS info and usage stats)
docs := search.ProvideDocumentBuilders(nil)
server, err := NewResourceServer(ctx, s.db, s.cfg, s.features, docs, s.tracing, s.reg, authzClient)
server, err := NewResourceServer(ctx, s.db, s.cfg, s.features, s.docBuilders, s.tracing, s.reg, authzClient)
if err != nil {
return err
}
pkg/storage/unified/sql/test/integration_test.go
+1 -1
View File
@@ -366,7 +366,7 @@ func TestClientServer(t *testing.T) {
features := featuremgmt.WithFeatures()
svc, err := sql.ProvideUnifiedStorageGrpcService(cfg, features, dbstore, nil, prometheus.NewPedanticRegistry())
svc, err := sql.ProvideUnifiedStorageGrpcService(cfg, features, dbstore, nil, prometheus.NewPedanticRegistry(), nil)
require.NoError(t, err)
var client resource.ResourceStoreClient