UnifiedSearch: Use ResourceIndex from dashboards apiserver (v0alpha1 only) (#96939)
This commit is contained in:
Ryan McKinley
@@ -3,18 +3,18 @@ package sql
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/grafana/authlib/claims"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/identity"
|
||||
infraDB "github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/infra/tracing"
|
||||
"github.com/grafana/grafana/pkg/services/authz"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/search"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
|
||||
)
|
||||
|
||||
@@ -54,12 +54,18 @@ func NewResourceServer(ctx context.Context, db infraDB.DB, cfg *setting.Cfg,
|
||||
opts.Backend = store
|
||||
opts.Diagnostics = store
|
||||
opts.Lifecycle = store
|
||||
opts.Search = resource.SearchOptions{
|
||||
Resources: docs,
|
||||
}
|
||||
|
||||
// Setup the search server
|
||||
if features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorageSearch) {
|
||||
opts.Index = resource.NewResourceIndexServer(cfg, tracer)
|
||||
opts.Search = resource.SearchOptions{
|
||||
Backend: search.NewBleveBackend(search.BleveOptions{
|
||||
Root: filepath.Join(cfg.DataPath, "unified-search", "bleve"),
|
||||
FileThreshold: 10, // fewer than X items will use a memory index
|
||||
BatchSize: 500, // This is the batch size for how many objects to add to the index at once
|
||||
}, tracer, reg),
|
||||
Resources: docs,
|
||||
WorkerThreads: 5, // from cfg?
|
||||
}
|
||||
}
|
||||
|
||||
rs, err := resource.NewResourceServer(opts)
|
||||
@@ -67,29 +73,5 @@ func NewResourceServer(ctx context.Context, db infraDB.DB, cfg *setting.Cfg,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Initialize the indexer if one is configured
|
||||
if opts.Index != nil {
|
||||
// TODO: Create a proper identity for the indexer
|
||||
orgId := int64(1)
|
||||
ctx = identity.WithRequester(ctx, &identity.StaticRequester{
|
||||
Type: claims.TypeServiceAccount, // system:apiserver
|
||||
UserID: 1,
|
||||
OrgID: int64(1),
|
||||
Name: "admin",
|
||||
Login: "admin",
|
||||
OrgRole: identity.RoleAdmin,
|
||||
IsGrafanaAdmin: true,
|
||||
Permissions: map[int64]map[string][]string{
|
||||
orgId: {
|
||||
"*": {"*"}, // all resources, all scopes
|
||||
},
|
||||
},
|
||||
})
|
||||
_, err = rs.(resource.ResourceIndexer).Index(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return rs, nil
|
||||
}
|
||||
|
||||
@@ -102,8 +102,8 @@ func (s *service) start(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// TODO, for standalone this will need to be started from enterprise
|
||||
// Connecting to the correct remote services
|
||||
docs := search.ProvideDocumentBuilders()
|
||||
// Connecting to the correct remote services (cloudconfig for DS info and usage stats)
|
||||
docs := search.ProvideDocumentBuilders(nil)
|
||||
|
||||
server, err := NewResourceServer(ctx, s.db, s.cfg, s.features, docs, s.tracing, s.reg, authzClient)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql"
|
||||
"github.com/grafana/grafana/pkg/util/testutil"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// addResource is a helper to create a resource in unified storage
|
||||
func addResource(t *testing.T, ctx context.Context, backend sql.Backend, resourceName string, data string) {
|
||||
ir, err := resource.NewIndexedResource([]byte(data))
|
||||
require.NoError(t, err)
|
||||
_, err = backend.WriteEvent(ctx, resource.WriteEvent{
|
||||
Type: resource.WatchEvent_ADDED,
|
||||
Value: []byte(data),
|
||||
Key: &resource.ResourceKey{
|
||||
Namespace: ir.Namespace,
|
||||
Group: ir.Group,
|
||||
Resource: resourceName,
|
||||
Name: ir.Name,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestIntegrationIndexerSearch(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test")
|
||||
}
|
||||
|
||||
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second))
|
||||
cfg := setting.NewCfg()
|
||||
cfg.IndexWorkers = 1
|
||||
cfg.IndexMaxBatchSize = 100
|
||||
cfg.IndexListLimit = 100
|
||||
backend, server := newServer(t, cfg)
|
||||
|
||||
playlist1 := `{
|
||||
"kind": "Playlist",
|
||||
"apiVersion": "playlist.grafana.app/v0alpha1",
|
||||
"metadata": {
|
||||
"name": "playlist dogs",
|
||||
"namespace": "tenant1",
|
||||
"uid": "1fe028dc-81bb-4268-a3ff-20899ff0a16f",
|
||||
"resourceVersion": "1",
|
||||
"creationTimestamp": "2024-01-01T12:00:00Z"
|
||||
},
|
||||
"spec": {
|
||||
"interval": "5m",
|
||||
"title": "dogs"
|
||||
}
|
||||
}`
|
||||
playlist2 := `{
|
||||
"kind": "Playlist",
|
||||
"apiVersion": "playlist.grafana.app/v0alpha1",
|
||||
"metadata": {
|
||||
"name": "playlist cats",
|
||||
"namespace": "tenant1",
|
||||
"uid": "1fe028dc-81bb-4268-a3ff-20899ff0a16f123",
|
||||
"resourceVersion": "2",
|
||||
"creationTimestamp": "2024-01-02T12:00:00Z"
|
||||
},
|
||||
"spec": {
|
||||
"interval": "5m",
|
||||
"title": "cats"
|
||||
}
|
||||
}`
|
||||
|
||||
// add playlist1 and playlist2 to unified storage
|
||||
addResource(t, ctx, backend, "playlists", playlist1)
|
||||
addResource(t, ctx, backend, "playlists", playlist2)
|
||||
|
||||
// initialize and build the search index
|
||||
indexer, ok := server.(resource.ResourceIndexer)
|
||||
if !ok {
|
||||
t.Fatal("server does not implement ResourceIndexer")
|
||||
}
|
||||
_, err := indexer.Index(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
// run search tests against the index
|
||||
t.Run("can search for all resources", func(t *testing.T) {
|
||||
res, err := server.Search(ctx, &resource.SearchRequest{
|
||||
Tenant: "tenant1",
|
||||
Query: "*",
|
||||
Limit: 10,
|
||||
Offset: 0,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res.Items, 2)
|
||||
})
|
||||
|
||||
t.Run("can search for resources by title", func(t *testing.T) {
|
||||
res, err := server.Search(ctx, &resource.SearchRequest{
|
||||
Tenant: "tenant1",
|
||||
Query: "Spec.title:dogs",
|
||||
Limit: 10,
|
||||
Offset: 0,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res.Items, 1)
|
||||
})
|
||||
|
||||
t.Run("can filter resources by created time", func(t *testing.T) {
|
||||
res, err := server.Search(ctx, &resource.SearchRequest{
|
||||
Tenant: "tenant1",
|
||||
Query: "CreatedAt:>=\"2024-01-02\"",
|
||||
Limit: 10,
|
||||
Offset: 0,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, res.Items, 1)
|
||||
ir := resource.IndexedResource{}
|
||||
err = json.Unmarshal(res.Items[0].Value, &ir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "playlist cats", ir.Name)
|
||||
})
|
||||
}
|
||||
@@ -56,7 +56,6 @@ func newServer(t *testing.T, cfg *setting.Cfg) (sql.Backend, resource.ResourceSe
|
||||
Backend: ret,
|
||||
Diagnostics: ret,
|
||||
Lifecycle: ret,
|
||||
Index: resource.NewResourceIndexServer(cfg, tracing.NewNoopTracerService()),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, server)
|
||||
|
||||
Reference in New Issue
Block a user