Search: use SQL search as a fallback during bluge's initial indexing (#54095) (#54276)

* Search: use SQL search as a fallback when bluge indexing is ongoing

* Search: lint

* Search: feedback fixes - return an empty frame with a special name

* Search: revert readiness check query type

* Search: remove println

* remove sleep, get coffee

(cherry picked from commit 74158ed66b)

pkg/services/searchV2/index.go

@@ -82,30 +82,78 @@ func (i *orgIndex) readerForIndex(idxType indexType) (*bluge.Reader, func(), err
 }
 
 type searchIndex struct {
-	mu             sync.RWMutex
-	loader         dashboardLoader
-	perOrgIndex    map[int64]*orgIndex
-	eventStore     eventStore
-	logger         log.Logger
-	buildSignals   chan buildSignal
-	extender       DocumentExtender
-	folderIdLookup folderUIDLookup
-	syncCh         chan chan struct{}
+	mu                      sync.RWMutex
+	loader                  dashboardLoader
+	perOrgIndex             map[int64]*orgIndex
+	initializedOrgs         map[int64]bool
+	initialIndexingComplete bool
+	initializationMutex     sync.RWMutex
+	eventStore              eventStore
+	logger                  log.Logger
+	buildSignals            chan buildSignal
+	extender                DocumentExtender
+	folderIdLookup          folderUIDLookup
+	syncCh                  chan chan struct{}
 }
 
 func newSearchIndex(dashLoader dashboardLoader, evStore eventStore, extender DocumentExtender, folderIDs folderUIDLookup) *searchIndex {
 	return &searchIndex{
-		loader:         dashLoader,
-		eventStore:     evStore,
-		perOrgIndex:    map[int64]*orgIndex{},
-		logger:         log.New("searchIndex"),
-		buildSignals:   make(chan buildSignal),
-		extender:       extender,
-		folderIdLookup: folderIDs,
-		syncCh:         make(chan chan struct{}),
+		loader:          dashLoader,
+		eventStore:      evStore,
+		perOrgIndex:     map[int64]*orgIndex{},
+		initializedOrgs: map[int64]bool{},
+		logger:          log.New("searchIndex"),
+		buildSignals:    make(chan buildSignal),
+		extender:        extender,
+		folderIdLookup:  folderIDs,
+		syncCh:          make(chan chan struct{}),
 	}
 }
 
+func (i *searchIndex) isInitialized(_ context.Context, orgId int64) IsSearchReadyResponse {
+	i.initializationMutex.RLock()
+	orgInitialized := i.initializedOrgs[orgId]
+	initialInitComplete := i.initialIndexingComplete
+	i.initializationMutex.RUnlock()
+
+	if orgInitialized && initialInitComplete {
+		return IsSearchReadyResponse{IsReady: true}
+	}
+
+	if !initialInitComplete {
+		return IsSearchReadyResponse{IsReady: false, Reason: "initial-indexing-ongoing"}
+	}
+
+	i.triggerBuildingOrgIndex(orgId)
+	return IsSearchReadyResponse{IsReady: false, Reason: "org-indexing-ongoing"}
+}
+
+func (i *searchIndex) triggerBuildingOrgIndex(orgId int64) {
+	go func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+		defer cancel()
+
+		doneIndexing := make(chan error, 1)
+		signal := buildSignal{orgID: orgId, done: doneIndexing}
+		select {
+		case i.buildSignals <- signal:
+		case <-ctx.Done():
+			i.logger.Warn("Failed to send a build signal to initialize org index", "orgId", orgId)
+			return
+		}
+		select {
+		case err := <-doneIndexing:
+			if err != nil {
+				i.logger.Error("Failed to build org index", "orgId", orgId, "error", err)
+			} else {
+				i.logger.Debug("Successfully built org index", "orgId", orgId)
+			}
+		case <-ctx.Done():
+			i.logger.Warn("Building org index timeout", "orgId", orgId)
+		}
+	}()
+}
+
 func (i *searchIndex) sync(ctx context.Context) error {
 	doneCh := make(chan struct{}, 1)
 	select {
@@ -150,6 +198,10 @@ func (i *searchIndex) run(ctx context.Context, orgIDs []int64, reIndexSignalCh c
 	// Channel to handle signals about asynchronous full re-indexing completion.
 	reIndexDoneCh := make(chan int64, 1)
 
+	i.initializationMutex.Lock()
+	i.initialIndexingComplete = true
+	i.initializationMutex.Unlock()
+
 	for {
 		select {
 		case doneCh := <-i.syncCh:
@@ -422,6 +474,10 @@ func (i *searchIndex) buildOrgIndex(ctx context.Context, orgID int64) (int, erro
 	i.perOrgIndex[orgID] = index
 	i.mu.Unlock()
 
+	i.initializationMutex.Lock()
+	i.initializedOrgs[orgID] = true
+	i.initializationMutex.Unlock()
+
 	if orgID == 1 {
 		go func() {
 			if reader, cancel, err := index.readerForIndex(indexTypeDashboard); err == nil {

pkg/services/searchV2/search_service_mock.go

@@ -45,6 +45,20 @@ func (_m *MockSearchService) IsDisabled() bool {
 	return r0
 }
 
+// IsReady provides a mock function with given fields: ctx, orgId
+func (_m *MockSearchService) IsReady(ctx context.Context, orgId int64) IsSearchReadyResponse {
+	ret := _m.Called(ctx, orgId)
+
+	var r0 IsSearchReadyResponse
+	if rf, ok := ret.Get(0).(func(context.Context, int64) IsSearchReadyResponse); ok {
+		r0 = rf(ctx, orgId)
+	} else {
+		r0 = ret.Get(0).(IsSearchReadyResponse)
+	}
+
+	return r0
+}
+
 // RegisterDashboardIndexExtender provides a mock function with given fields: ext
 func (_m *MockSearchService) RegisterDashboardIndexExtender(ext DashboardIndexExtender) {
 	_m.Called(ext)

pkg/services/searchV2/service.go

@@ -62,6 +62,10 @@ type StandardSearchService struct {
 	reIndexCh      chan struct{}
 }
 
+func (s *StandardSearchService) IsReady(ctx context.Context, orgId int64) IsSearchReadyResponse {
+	return s.dashboardIndex.isInitialized(ctx, orgId)
+}
+
 func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore store.EntityEventsService, ac accesscontrol.AccessControl) SearchService {
 	extender := &NoopExtender{}
 	s := &StandardSearchService{

pkg/services/searchV2/stub.go

@@ -10,6 +10,10 @@ import (
 type stubSearchService struct {
 }
 
+func (s *stubSearchService) IsReady(ctx context.Context, orgId int64) IsSearchReadyResponse {
+	return IsSearchReadyResponse{}
+}
+
 func (s *stubSearchService) IsDisabled() bool {
 	return true
 }

pkg/services/searchV2/types.go

@@ -31,11 +31,17 @@ type DashboardQuery struct {
 	From               int          `json:"from,omitempty"`       // for paging
 }
 
+type IsSearchReadyResponse struct {
+	IsReady bool
+	Reason  string // initial-indexing-ongoing, org-indexing-ongoing
+}
+
 //go:generate mockery --name SearchService --structname MockSearchService --inpackage --filename search_service_mock.go
 type SearchService interface {
 	registry.CanBeDisabled
 	registry.BackgroundService
 	DoDashboardQuery(ctx context.Context, user *backend.User, orgId int64, query DashboardQuery) *backend.DataResponse
+	IsReady(ctx context.Context, orgId int64) IsSearchReadyResponse
 	RegisterDashboardIndexExtender(ext DashboardIndexExtender)
 	TriggerReIndex()
 }