K8s: Provisioned dashboard support (#98843)

pkg/registry/apis/dashboard/legacy/sql_dashboards.go

@@ -22,6 +22,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
 	gapiutil "github.com/grafana/grafana/pkg/services/apiserver/utils"
 	"github.com/grafana/grafana/pkg/services/dashboards"
+	"github.com/grafana/grafana/pkg/services/dashboards/service"
 	"github.com/grafana/grafana/pkg/services/provisioning"
 	"github.com/grafana/grafana/pkg/storage/legacysql"
 	"github.com/grafana/grafana/pkg/storage/unified/resource"
@@ -278,21 +279,24 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows) (*dashboardRow, error) {
 		if origin_name.String != "" {
 			ts := time.Unix(origin_ts.Int64, 0)
 
-			resolvedPath := a.provisioning.GetDashboardProvisionerResolvedPath(origin_name.String)
-			originPath, err := filepath.Rel(
-				resolvedPath,
-				origin_path.String,
-			)
-			if err != nil {
-				return nil, err
-			}
-
-			meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{
+			repo := &utils.ResourceRepositoryInfo{
 				Name:      origin_name.String,
-				Path:      originPath,
 				Hash:      origin_hash.String,
 				Timestamp: &ts,
-			})
+			}
+			// if the reader cannot be found, it may be an orphaned provisioned dashboard
+			resolvedPath := a.provisioning.GetDashboardProvisionerResolvedPath(origin_name.String)
+			if resolvedPath != "" {
+				originPath, err := filepath.Rel(
+					resolvedPath,
+					origin_path.String,
+				)
+				if err != nil {
+					return nil, err
+				}
+				repo.Path = originPath
+			}
+			meta.SetRepositoryInfo(repo)
 		} else if plugin_id != "" {
 			meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{
 				Name: "plugin",
@@ -390,6 +394,7 @@ func (a *dashboardSqlAccess) SaveDashboard(ctx context.Context, orgId int64, das
 	}
 	out, err := a.dashStore.SaveDashboard(ctx, dashboards.SaveDashboardCommand{
 		OrgID:     orgId,
+		PluginID:  service.GetPluginIDFromMeta(meta),
 		Dashboard: simplejson.NewFromAny(dash.Spec.UnstructuredContent()),
 		FolderUID: meta.GetFolder(),
 		Overwrite: true, // already passed the revisionVersion checks!

pkg/services/dashboards/dashboard.go

@@ -62,6 +62,7 @@ type DashboardProvisioningService interface {
 //go:generate mockery --name Store --structname FakeDashboardStore --inpackage --filename store_mock.go
 type Store interface {
 	DeleteDashboard(ctx context.Context, cmd *DeleteDashboardCommand) error
+	CleanupAfterDelete(ctx context.Context, cmd *DeleteDashboardCommand) error
 	DeleteAllDashboards(ctx context.Context, orgID int64) error
 	DeleteOrphanedProvisionedDashboards(ctx context.Context, cmd *DeleteOrphanedProvisionedDashboardsCommand) error
 	FindDashboards(ctx context.Context, query *FindPersistedDashboardsQuery) ([]DashboardSearchProjection, error)
@@ -75,7 +76,7 @@ type Store interface {
 	GetProvisionedDataByDashboardID(ctx context.Context, dashboardID int64) (*DashboardProvisioning, error)
 	GetProvisionedDataByDashboardUID(ctx context.Context, orgID int64, dashboardUID string) (*DashboardProvisioning, error)
 	SaveDashboard(ctx context.Context, cmd SaveDashboardCommand) (*Dashboard, error)
-	SaveProvisionedDashboard(ctx context.Context, cmd SaveDashboardCommand, provisioning *DashboardProvisioning) (*Dashboard, error)
+	SaveProvisionedDashboard(ctx context.Context, dash *Dashboard, provisioning *DashboardProvisioning) error
 	UnprovisionDashboard(ctx context.Context, id int64) error
 	// ValidateDashboardBeforeSave validates a dashboard before save.
 	ValidateDashboardBeforeSave(ctx context.Context, dashboard *Dashboard, overwrite bool) (bool, error)

pkg/services/dashboards/database/database.go

@@ -145,25 +145,19 @@ func (d *dashboardStore) GetProvisionedDashboardData(ctx context.Context, name s
 	return result, err
 }
 
-func (d *dashboardStore) SaveProvisionedDashboard(ctx context.Context, cmd dashboards.SaveDashboardCommand, provisioning *dashboards.DashboardProvisioning) (*dashboards.Dashboard, error) {
+func (d *dashboardStore) SaveProvisionedDashboard(ctx context.Context, dash *dashboards.Dashboard, provisioning *dashboards.DashboardProvisioning) error {
 	ctx, span := tracer.Start(ctx, "dashboards.database.SaveProvisionedDashboard")
 	defer span.End()
 
-	var result *dashboards.Dashboard
-	var err error
-	err = d.store.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
-		result, err = saveDashboard(sess, &cmd, d.emitEntityEvent())
-		if err != nil {
-			return err
-		}
-
+	err := d.store.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
 		if provisioning.Updated == 0 {
-			provisioning.Updated = result.Updated.Unix()
+			provisioning.Updated = dash.Updated.Unix()
 		}
 
-		return saveProvisionedData(sess, provisioning, result)
+		return saveProvisionedData(sess, provisioning, dash)
 	})
-	return result, err
+
+	return err
 }
 
 func (d *dashboardStore) SaveDashboard(ctx context.Context, cmd dashboards.SaveDashboardCommand) (*dashboards.Dashboard, error) {
@@ -656,6 +650,45 @@ func (d *dashboardStore) deleteDashboard(cmd *dashboards.DeleteDashboardCommand,
 	return nil
 }
 
+func (d *dashboardStore) CleanupAfterDelete(ctx context.Context, cmd *dashboards.DeleteDashboardCommand) error {
+	type statement struct {
+		SQL  string
+		args []any
+	}
+	sqlStatements := []statement{
+		{SQL: "DELETE FROM dashboard_tag WHERE dashboard_uid = ? AND org_id = ?", args: []any{cmd.UID, cmd.OrgID}},
+		{SQL: "DELETE FROM star WHERE dashboard_uid = ? AND org_id = ?", args: []any{cmd.UID, cmd.OrgID}},
+		{SQL: "DELETE FROM playlist_item WHERE type = 'dashboard_by_id' AND value = ?", args: []any{cmd.ID}},
+		{SQL: "DELETE FROM dashboard_version WHERE dashboard_id = ?", args: []any{cmd.ID}},
+		{SQL: "DELETE FROM dashboard_provisioning WHERE dashboard_id = ?", args: []any{cmd.ID}},
+		{SQL: "DELETE FROM dashboard_acl WHERE dashboard_id = ?", args: []any{cmd.ID}},
+		{SQL: "DELETE FROM annotation WHERE dashboard_id = ? AND org_id = ?", args: []any{cmd.ID, cmd.OrgID}},
+	}
+
+	err := d.store.WithTransactionalDbSession(ctx, func(sess *db.Session) error {
+		err := d.store.WithDbSession(ctx, func(sess *db.Session) error {
+			for _, stmnt := range sqlStatements {
+				_, err := sess.Exec(append([]any{stmnt.SQL}, stmnt.args...)...)
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		if err := d.deleteResourcePermissions(sess, cmd.OrgID, ac.GetResourceScopeUID("dashboards", cmd.UID)); err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	return err
+}
+
 func (d *dashboardStore) DeleteAllDashboards(ctx context.Context, orgID int64) error {
 	ctx, span := tracer.Start(ctx, "dashboards.database.DeleteAllDashboards")
 	defer span.End()

pkg/services/dashboards/database/database_provisioning_test.go

@@ -52,13 +52,15 @@ func TestIntegrationDashboardProvisioningTest(t *testing.T) {
 			ExternalID: "/var/grafana.json",
 			Updated:    now.Unix(),
 		}
-
-		dash, err := dashboardStore.SaveProvisionedDashboard(context.Background(), saveDashboardCmd, provisioning)
-		require.Nil(t, err)
+		dash, err := dashboardStore.SaveDashboard(context.Background(), saveDashboardCmd)
+		require.NoError(t, err)
 		require.NotNil(t, dash)
 		require.NotEqual(t, 0, dash.ID)
 		dashId := dash.ID
 
+		err = dashboardStore.SaveProvisionedDashboard(context.Background(), dash, provisioning)
+		require.Nil(t, err)
+
 		t.Run("Deleting orphaned provisioned dashboards", func(t *testing.T) {
 			saveCmd := dashboards.SaveDashboardCommand{
 				OrgID:     1,
@@ -69,13 +71,16 @@ func TestIntegrationDashboardProvisioningTest(t *testing.T) {
 					"title": "another_dashboard",
 				}),
 			}
+			anotherDash, err := dashboardStore.SaveDashboard(context.Background(), saveCmd)
+			require.NoError(t, err)
+
 			provisioning := &dashboards.DashboardProvisioning{
 				Name:       "another_reader",
 				ExternalID: "/var/grafana.json",
 				Updated:    now.Unix(),
 			}
 
-			anotherDash, err := dashboardStore.SaveProvisionedDashboard(context.Background(), saveCmd, provisioning)
+			err = dashboardStore.SaveProvisionedDashboard(context.Background(), anotherDash, provisioning)
 			require.Nil(t, err)
 
 			query := &dashboards.GetDashboardsQuery{DashboardIDs: []int64{anotherDash.ID}}

pkg/services/dashboards/database/database_test.go

@@ -238,6 +238,38 @@ func TestIntegrationDashboardDataAccess(t *testing.T) {
 		require.NotContains(t, terms, "delete this")
 	})
 
+	t.Run("Should delete associated provisioning info, even without the dashboard existing in the db", func(t *testing.T) {
+		setup()
+		provisioningData := &dashboards.DashboardProvisioning{
+			ID:          1,
+			DashboardID: 200,
+			Name:        "test",
+			CheckSum:    "123",
+			Updated:     54321,
+			ExternalID:  "/path/to/dashboard",
+		}
+		err := dashboardStore.SaveProvisionedDashboard(context.Background(), &dashboards.Dashboard{
+			ID: 200,
+		}, provisioningData)
+		require.NoError(t, err)
+
+		res, err := dashboardStore.GetProvisionedDashboardData(context.Background(), "test")
+		require.NoError(t, err)
+		require.Len(t, res, 1)
+		require.Equal(t, res[0], provisioningData)
+
+		err = dashboardStore.CleanupAfterDelete(context.Background(), &dashboards.DeleteDashboardCommand{
+			ID:    200,
+			OrgID: 1,
+			UID:   "test",
+		})
+		require.NoError(t, err)
+
+		res, err = dashboardStore.GetProvisionedDashboardData(context.Background(), "test")
+		require.NoError(t, err)
+		require.Len(t, res, 0)
+	})
+
 	t.Run("Should be able to delete all dashboards for an org", func(t *testing.T) {
 		setup()
 		dash1 := insertTestDashboard(t, dashboardStore, "delete me", 1, 0, "", false, "delete this")

pkg/services/dashboards/models.go

@@ -429,6 +429,10 @@ type FindPersistedDashboardsQuery struct {
 	Sort       model.SortOption
 	IsDeleted  bool
 
+	ProvisionedRepo       string
+	ProvisionedPath       string
+	ProvisionedReposNotIn []string
+
 	Filters []any
 
 	// Skip access control checks. This field is used by OpenFGA search implementation.

pkg/services/dashboards/service/dashboard_service.go

@@ -8,15 +8,19 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel"
 	"golang.org/x/exp/slices"
+	"golang.org/x/sync/errgroup"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/selection"
 	k8sUser "k8s.io/apiserver/pkg/authentication/user"
 	k8sRequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/client-go/dynamic"
@@ -165,7 +169,7 @@ func (dr *DashboardServiceImpl) Count(ctx context.Context, scopeParams *quota.Sc
 
 		total := int64(0)
 		for _, org := range orgs {
-			ctx = identity.WithRequester(ctx, getQuotaRequester(org.ID))
+			ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(org.ID))
 			dashs, err := dr.listDashboardsThroughK8s(ctx, org.ID)
 			if err != nil {
 				return u, err
@@ -213,13 +217,13 @@ func readQuotaConfig(cfg *setting.Cfg) (*quota.Map, error) {
 	return limits, nil
 }
 
-func getQuotaRequester(orgId int64) *identity.StaticRequester {
+func getDashboardBackgroundRequester(orgId int64) *identity.StaticRequester {
 	return &identity.StaticRequester{
 		Type:   claims.TypeServiceAccount,
 		UserID: 1,
 		OrgID:  orgId,
-		Name:   "quota-requester",
-		Login:  "quota-requester",
+		Name:   "dashboard-background",
+		Login:  "dashboard-background",
 		Permissions: map[int64]map[string][]string{
 			orgId: {
 				"*": {"*"},
@@ -229,15 +233,103 @@ func getQuotaRequester(orgId int64) *identity.StaticRequester {
 }
 
 func (dr *DashboardServiceImpl) GetProvisionedDashboardData(ctx context.Context, name string) ([]*dashboards.DashboardProvisioning, error) {
+	if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) {
+		orgs, err := dr.orgService.Search(ctx, &org.SearchOrgsQuery{})
+		if err != nil {
+			return nil, err
+		}
+
+		results := []*dashboards.DashboardProvisioning{}
+		var mu sync.Mutex
+		g, ctx := errgroup.WithContext(ctx)
+		for _, org := range orgs {
+			func(orgID int64) {
+				g.Go(func() error {
+					res, err := dr.searchProvisionedDashboardsThroughK8s(ctx, dashboards.FindPersistedDashboardsQuery{
+						ProvisionedRepo: name,
+						OrgId:           orgID,
+					})
+					if err != nil {
+						return err
+					}
+
+					mu.Lock()
+					for _, r := range res {
+						results = append(results, &r.DashboardProvisioning)
+					}
+					mu.Unlock()
+					return nil
+				})
+			}(org.ID)
+		}
+
+		if err := g.Wait(); err != nil {
+			return nil, err
+		}
+
+		return results, nil
+	}
+
 	return dr.dashboardStore.GetProvisionedDashboardData(ctx, name)
 }
 
 func (dr *DashboardServiceImpl) GetProvisionedDashboardDataByDashboardID(ctx context.Context, dashboardID int64) (*dashboards.DashboardProvisioning, error) {
+	if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) {
+		// if dashboard id is 0, it is a new dashboard
+		if dashboardID == 0 {
+			return nil, nil
+		}
+
+		orgs, err := dr.orgService.Search(ctx, &org.SearchOrgsQuery{})
+		if err != nil {
+			return nil, err
+		}
+
+		for _, org := range orgs {
+			res, err := dr.searchProvisionedDashboardsThroughK8s(ctx, dashboards.FindPersistedDashboardsQuery{
+				OrgId:        org.ID,
+				DashboardIds: []int64{dashboardID},
+			})
+			if err != nil {
+				return nil, err
+			}
+
+			if len(res) == 1 {
+				return &res[0].DashboardProvisioning, nil
+			} else if len(res) > 1 {
+				return nil, fmt.Errorf("found more than one provisioned dashboard with ID %d", dashboardID)
+			}
+		}
+
+		return nil, nil
+	}
+
 	return dr.dashboardStore.GetProvisionedDataByDashboardID(ctx, dashboardID)
 }
 
 func (dr *DashboardServiceImpl) GetProvisionedDashboardDataByDashboardUID(ctx context.Context, orgID int64, dashboardUID string) (*dashboards.DashboardProvisioning, error) {
-	// TODO: make this go through the k8s cli too under the feature toggle. First get dashboard through unistore & then get provisioning data
+	if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) {
+		if dashboardUID == "" {
+			return nil, nil
+		}
+
+		res, err := dr.searchProvisionedDashboardsThroughK8s(ctx, dashboards.FindPersistedDashboardsQuery{
+			OrgId:         orgID,
+			DashboardUIDs: []string{dashboardUID},
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		if len(res) == 1 {
+			return &res[0].DashboardProvisioning, nil
+		} else if len(res) > 1 {
+			return nil, fmt.Errorf("found more than one provisioned dashboard with UID %s", dashboardUID)
+		}
+
+		return nil, nil
+	}
+
 	return dr.dashboardStore.GetProvisionedDataByDashboardUID(ctx, orgID, dashboardUID)
 }
 
@@ -452,7 +544,34 @@ func (dr *DashboardServiceImpl) ValidateDashboardBeforeSave(ctx context.Context,
 }
 
 func (dr *DashboardServiceImpl) DeleteOrphanedProvisionedDashboards(ctx context.Context, cmd *dashboards.DeleteOrphanedProvisionedDashboardsCommand) error {
-	// TODO: once we can search in unistore by id, go through k8s cli too
+	if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) {
+		// check each org for orphaned provisioned dashboards
+		orgs, err := dr.orgService.Search(ctx, &org.SearchOrgsQuery{})
+		if err != nil {
+			return err
+		}
+
+		for _, org := range orgs {
+			// find all dashboards in the org that have a file repo set that is not in the given readers list
+			foundDashs, err := dr.searchProvisionedDashboardsThroughK8s(ctx, dashboards.FindPersistedDashboardsQuery{
+				ProvisionedReposNotIn: cmd.ReaderNames,
+				OrgId:                 org.ID,
+			})
+			if err != nil {
+				return err
+			}
+
+			// delete them
+			for _, foundDash := range foundDashs {
+				ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(org.ID))
+				if err = dr.deleteDashboard(ctx, foundDash.DashboardID, foundDash.DashboardUID, org.ID, false); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+
 	return dr.dashboardStore.DeleteOrphanedProvisionedDashboards(ctx, cmd)
 }
 
@@ -520,15 +639,30 @@ func (dr *DashboardServiceImpl) SaveProvisionedDashboard(ctx context.Context, dt
 	}
 
 	dto.User = accesscontrol.BackgroundUser("dashboard_provisioning", dto.OrgID, org.RoleAdmin, provisionerPermissions)
+	ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(dto.OrgID))
 
 	cmd, err := dr.BuildSaveDashboardCommand(ctx, dto, false)
 	if err != nil {
 		return nil, err
 	}
 
-	// dashboard
-	// TODO: make this go through the k8s cli too under the feature toggle. First save dashboard & then save provisioning data
-	dash, err := dr.dashboardStore.SaveProvisionedDashboard(ctx, *cmd, provisioning)
+	var dash *dashboards.Dashboard
+	if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) {
+		// save the dashboard but then do NOT return
+		// we want to save the provisioning data to the dashboard_provisioning table still
+		// to ensure we can safely rollback to mode2 if needed
+		dash, err = dr.saveProvisionedDashboardThroughK8s(ctx, cmd, provisioning, false)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		dash, err = dr.saveDashboard(ctx, cmd)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	err = dr.dashboardStore.SaveProvisionedDashboard(ctx, dash, provisioning)
 	if err != nil {
 		return nil, err
 	}
@@ -545,6 +679,8 @@ func (dr *DashboardServiceImpl) SaveFolderForProvisionedDashboards(ctx context.C
 	defer span.End()
 
 	dto.SignedInUser = accesscontrol.BackgroundUser("dashboard_provisioning", dto.OrgID, org.RoleAdmin, provisionerPermissions)
+	ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(dto.OrgID))
+
 	f, err := dr.folderService.Create(ctx, dto)
 	if err != nil {
 		dr.log.Error("failed to create folder for provisioned dashboards", "folder", dto.Title, "org", dto.OrgID, "err", err)
@@ -687,6 +823,7 @@ func (dr *DashboardServiceImpl) GetDashboardByPublicUid(ctx context.Context, das
 
 // DeleteProvisionedDashboard removes dashboard from the DB even if it is provisioned.
 func (dr *DashboardServiceImpl) DeleteProvisionedDashboard(ctx context.Context, dashboardId int64, orgId int64) error {
+	ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(orgId))
 	return dr.deleteDashboard(ctx, dashboardId, "", orgId, false)
 }
 
@@ -697,7 +834,12 @@ func (dr *DashboardServiceImpl) deleteDashboard(ctx context.Context, dashboardId
 	cmd := &dashboards.DeleteDashboardCommand{OrgID: orgId, ID: dashboardId, UID: dashboardUID}
 
 	if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) {
-		return dr.deleteDashboardThroughK8s(ctx, cmd, validateProvisionedDashboard)
+		err := dr.deleteDashboardThroughK8s(ctx, cmd, validateProvisionedDashboard)
+		if err != nil {
+			return err
+		}
+		// cleanup things related to dashboards that are not stored in unistore yet
+		return dr.dashboardStore.CleanupAfterDelete(ctx, cmd)
 	}
 
 	if validateProvisionedDashboard {
@@ -744,12 +886,46 @@ func (dr *DashboardServiceImpl) ImportDashboard(ctx context.Context, dto *dashbo
 // UnprovisionDashboard removes info about dashboard being provisioned. Used after provisioning configs are changed
 // and provisioned dashboards are left behind but not deleted.
 func (dr *DashboardServiceImpl) UnprovisionDashboard(ctx context.Context, dashboardId int64) error {
-	// TODO: once we can search by dashboard ID in unistore, go through k8s cli too
+	if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) {
+		orgs, err := dr.orgService.Search(ctx, &org.SearchOrgsQuery{})
+		if err != nil {
+			return err
+		}
+
+		for _, org := range orgs {
+			ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(org.ID))
+			dash, err := dr.getDashboardThroughK8s(ctx, &dashboards.GetDashboardQuery{OrgID: org.ID, ID: dashboardId})
+			if err != nil {
+				// if we can't find it in this org, try the next one
+				continue
+			}
+
+			_, err = dr.saveProvisionedDashboardThroughK8s(ctx, &dashboards.SaveDashboardCommand{
+				OrgID:     org.ID,
+				PluginID:  dash.PluginID,
+				FolderUID: dash.FolderUID,
+				FolderID:  dash.FolderID, // nolint:staticcheck
+				UpdatedAt: time.Now(),
+				Dashboard: dash.Data,
+			}, nil, true)
+
+			return err
+		}
+
+		return dashboards.ErrDashboardNotFound
+	}
+
 	return dr.dashboardStore.UnprovisionDashboard(ctx, dashboardId)
 }
 
 func (dr *DashboardServiceImpl) GetDashboardsByPluginID(ctx context.Context, query *dashboards.GetDashboardsByPluginIDQuery) ([]*dashboards.Dashboard, error) {
-	// TODO: once we can do this search in unistore, go through k8s cli too
+	if dr.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) {
+		return dr.searchDashboardsThroughK8s(ctx, &dashboards.FindPersistedDashboardsQuery{
+			OrgId:           query.OrgID,
+			ProvisionedRepo: pluginIDRepoName,
+			ProvisionedPath: query.PluginID,
+		})
+	}
 	return dr.dashboardStore.GetDashboardsByPluginID(ctx, query)
 }
 
@@ -1220,10 +1396,16 @@ func (dr *DashboardServiceImpl) CleanUpDeletedDashboards(ctx context.Context) (i
 // -----------------------------------------------------------------------------------------
 
 func (dk8s *dashk8sHandler) getClient(ctx context.Context, orgID int64) (dynamic.ResourceInterface, bool) {
-	dyn, err := dynamic.NewForConfig(dk8s.restConfigProvider.GetRestConfig(ctx))
+	cfg := dk8s.restConfigProvider.GetRestConfig(ctx)
+	if cfg == nil {
+		return nil, false
+	}
+
+	dyn, err := dynamic.NewForConfig(cfg)
 	if err != nil {
 		return nil, false
 	}
+
 	return dyn.Resource(dk8s.gvr).Namespace(dk8s.getNamespace(orgID)), true
 }
 
@@ -1305,15 +1487,81 @@ func (dr *DashboardServiceImpl) getDashboardThroughK8s(ctx context.Context, quer
 	}
 
 	out, err := client.Get(newCtx, query.UID, v1.GetOptions{}, subresource)
-	if err != nil {
+	if err != nil && !apierrors.IsNotFound(err) {
 		return nil, err
-	} else if out == nil {
+	} else if err != nil || out == nil {
 		return nil, dashboards.ErrDashboardNotFound
 	}
 
 	return dr.UnstructuredToLegacyDashboard(ctx, out, query.OrgID)
 }
 
+func (dr *DashboardServiceImpl) saveProvisionedDashboardThroughK8s(ctx context.Context, cmd *dashboards.SaveDashboardCommand, provisioning *dashboards.DashboardProvisioning, unprovision bool) (*dashboards.Dashboard, error) {
+	// default to 1 if not set
+	if cmd.OrgID == 0 {
+		cmd.OrgID = 1
+	}
+
+	// create a new context - prevents issues when the request stems from the k8s api itself
+	// otherwise the context goes through the handlers twice and causes issues
+	newCtx, cancel, err := dr.getK8sContext(ctx)
+	if err != nil {
+		return nil, err
+	} else if cancel != nil {
+		defer cancel()
+	}
+
+	client, ok := dr.k8sclient.getClient(newCtx, cmd.OrgID)
+	if !ok {
+		return nil, nil
+	}
+
+	obj, err := LegacySaveCommandToUnstructured(cmd, dr.k8sclient.getNamespace(cmd.OrgID))
+	if err != nil {
+		return nil, err
+	}
+
+	annotations := obj.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	if unprovision {
+		delete(annotations, utils.AnnoKeyRepoName)
+		delete(annotations, utils.AnnoKeyRepoPath)
+		delete(annotations, utils.AnnoKeyRepoHash)
+		delete(annotations, utils.AnnoKeyRepoTimestamp)
+	} else {
+		annotations[utils.AnnoKeyRepoName] = provisionedFileNameWithPrefix(provisioning.Name)
+		annotations[utils.AnnoKeyRepoPath] = provisioning.ExternalID
+		annotations[utils.AnnoKeyRepoHash] = provisioning.CheckSum
+		annotations[utils.AnnoKeyRepoTimestamp] = time.Unix(provisioning.Updated, 0).UTC().Format(time.RFC3339)
+	}
+	obj.SetAnnotations(annotations)
+
+	var out *unstructured.Unstructured
+	current, err := client.Get(newCtx, obj.GetName(), v1.GetOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return nil, err
+	} else if current == nil || (err != nil && apierrors.IsNotFound(err)) {
+		out, err = client.Create(newCtx, &obj, v1.CreateOptions{})
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		out, err = client.Update(newCtx, &obj, v1.UpdateOptions{})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	finalDash, err := dr.UnstructuredToLegacyDashboard(ctx, out, cmd.OrgID)
+	if err != nil {
+		return nil, err
+	}
+
+	return finalDash, nil
+}
+
 func (dr *DashboardServiceImpl) saveDashboardThroughK8s(ctx context.Context, cmd *dashboards.SaveDashboardCommand, orgID int64) (*dashboards.Dashboard, error) {
 	// create a new context - prevents issues when the request stems from the k8s api itself
 	// otherwise the context goes through the handlers twice and causes issues
@@ -1334,6 +1582,8 @@ func (dr *DashboardServiceImpl) saveDashboardThroughK8s(ctx context.Context, cmd
 		return nil, err
 	}
 
+	setPluginID(obj, cmd.PluginID)
+
 	var out *unstructured.Unstructured
 	current, err := client.Get(newCtx, obj.GetName(), v1.GetOptions{})
 	if current == nil || err != nil {
@@ -1466,40 +1716,64 @@ func (dr *DashboardServiceImpl) searchDashboardsThroughK8sRaw(ctx context.Contex
 
 	request := &resource.ResourceSearchRequest{
 		Options: &resource.ListOptions{
-			Key: dashboardskey,
+			Key:    dashboardskey,
+			Fields: []*resource.Requirement{},
+			Labels: []*resource.Requirement{},
 		},
 		Limit: 100000}
 
 	if len(query.DashboardUIDs) > 0 {
 		request.Options.Fields = []*resource.Requirement{{
 			Key:      "key.name",
-			Operator: "in",
+			Operator: string(selection.In),
 			Values:   query.DashboardUIDs,
 		}}
 	} else if len(query.DashboardIds) > 0 {
 		values := make([]string, len(query.DashboardIds))
-		for _, id := range query.DashboardIds {
-			values = append(values, strconv.FormatInt(id, 10))
+		for i, id := range query.DashboardIds {
+			values[i] = strconv.FormatInt(id, 10)
 		}
 
-		request.Options.Labels = []*resource.Requirement{{
+		request.Options.Labels = append(request.Options.Labels, &resource.Requirement{
 			Key:      utils.LabelKeyDeprecatedInternalID, // nolint:staticcheck
-			Operator: "in",
+			Operator: string(selection.In),
 			Values:   values,
-		}}
+		})
 	}
 
 	if len(query.FolderUIDs) > 0 {
 		req := []*resource.Requirement{{
 			Key:      "folder",
-			Operator: "in",
+			Operator: string(selection.In),
 			Values:   query.FolderUIDs,
 		}}
-		if len(request.Options.Fields) == 0 {
-			request.Options.Fields = req
-		} else {
-			request.Options.Fields = append(request.Options.Fields, req...)
-		}
+		request.Options.Fields = append(request.Options.Fields, req...)
+	}
+
+	if query.ProvisionedRepo != "" {
+		req := []*resource.Requirement{{
+			Key:      "repo.name",
+			Operator: string(selection.In),
+			Values:   []string{query.ProvisionedRepo},
+		}}
+		request.Options.Fields = append(request.Options.Fields, req...)
+	}
+
+	if len(query.ProvisionedReposNotIn) > 0 {
+		req := []*resource.Requirement{{
+			Key:      "repo.name",
+			Operator: string(selection.NotIn),
+			Values:   query.ProvisionedReposNotIn,
+		}}
+		request.Options.Fields = append(request.Options.Fields, req...)
+	}
+	if query.ProvisionedPath != "" {
+		req := []*resource.Requirement{{
+			Key:      "repo.path",
+			Operator: string(selection.In),
+			Values:   []string{query.ProvisionedPath},
+		}}
+		request.Options.Fields = append(request.Options.Fields, req...)
 	}
 
 	// note: this does not allow for partial matching
@@ -1509,28 +1783,19 @@ func (dr *DashboardServiceImpl) searchDashboardsThroughK8sRaw(ctx context.Contex
 	if query.Title != "" {
 		req := []*resource.Requirement{{
 			Key:      "title",
-			Operator: "in",
+			Operator: string(selection.In),
 			Values:   []string{query.Title},
 		}}
-		if len(request.Options.Fields) == 0 {
-			request.Options.Fields = req
-		} else {
-			request.Options.Fields = append(request.Options.Fields, req...)
-		}
+		request.Options.Fields = append(request.Options.Fields, req...)
 	}
 
 	if len(query.Tags) > 0 {
 		req := []*resource.Requirement{{
 			Key:      "tags",
-			Operator: "in",
+			Operator: string(selection.In),
 			Values:   query.Tags,
 		}}
-
-		if len(request.Options.Fields) == 0 {
-			request.Options.Fields = req
-		} else {
-			request.Options.Fields = append(request.Options.Fields, req...)
-		}
+		request.Options.Fields = append(request.Options.Fields, req...)
 	}
 
 	res, err := dr.k8sclient.getSearcher().Search(ctx, request)
@@ -1541,6 +1806,100 @@ func (dr *DashboardServiceImpl) searchDashboardsThroughK8sRaw(ctx context.Contex
 	return ParseResults(res, 0)
 }
 
+type dashboardProvisioningWithUID struct {
+	dashboards.DashboardProvisioning
+	DashboardUID string
+}
+
+func (dr *DashboardServiceImpl) searchProvisionedDashboardsThroughK8s(ctx context.Context, query dashboards.FindPersistedDashboardsQuery) ([]*dashboardProvisioningWithUID, error) {
+	ctx = identity.WithRequester(ctx, getDashboardBackgroundRequester(query.OrgId))
+
+	if query.ProvisionedRepo != "" {
+		query.ProvisionedRepo = provisionedFileNameWithPrefix(query.ProvisionedRepo)
+	}
+
+	if len(query.ProvisionedReposNotIn) > 0 {
+		repos := make([]string, len(query.ProvisionedReposNotIn))
+		for i, v := range query.ProvisionedReposNotIn {
+			repos[i] = provisionedFileNameWithPrefix(v)
+		}
+		query.ProvisionedReposNotIn = repos
+	}
+
+	searchResults, err := dr.searchDashboardsThroughK8sRaw(ctx, &query)
+	if err != nil {
+		return nil, err
+	}
+
+	newCtx, cancel, err := dr.getK8sContext(ctx)
+	if err != nil {
+		return nil, err
+	} else if cancel != nil {
+		defer cancel()
+	}
+
+	client, ok := dr.k8sclient.getClient(newCtx, query.OrgId)
+	if !ok {
+		return nil, nil
+	}
+
+	// loop through all hits concurrently to get the repo information (if set due to file provisioning)
+	dashs := make([]*dashboardProvisioningWithUID, 0)
+	var mu sync.Mutex
+	g, ctx := errgroup.WithContext(ctx)
+	for _, h := range searchResults.Hits {
+		func(hit v0alpha1.DashboardHit) {
+			g.Go(func() error {
+				out, err := client.Get(ctx, hit.Name, v1.GetOptions{}, "")
+				if err != nil {
+					return err
+				} else if out == nil {
+					return dashboards.ErrDashboardNotFound
+				}
+
+				meta, err := utils.MetaAccessor(out)
+				if err != nil {
+					return err
+				}
+
+				// ensure the repo is set due to file provisioning, otherwise skip it
+				fileRepo, found := getProvisionedFileNameFromMeta(meta)
+				if !found {
+					return nil
+				}
+
+				provisioning := &dashboardProvisioningWithUID{
+					DashboardUID: hit.Name,
+				}
+				provisioning.Name = fileRepo
+				provisioning.ExternalID = meta.GetRepositoryPath()
+				provisioning.CheckSum = meta.GetRepositoryHash()
+				provisioning.DashboardID = meta.GetDeprecatedInternalID() // nolint:staticcheck
+
+				updated, err := meta.GetRepositoryTimestamp()
+				if err != nil {
+					return err
+				}
+				if updated != nil {
+					provisioning.Updated = updated.Unix()
+				}
+
+				mu.Lock()
+				dashs = append(dashs, provisioning)
+				mu.Unlock()
+
+				return nil
+			})
+		}(h)
+	}
+
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+
+	return dashs, nil
+}
+
 func (dr *DashboardServiceImpl) searchDashboardsThroughK8s(ctx context.Context, query *dashboards.FindPersistedDashboardsQuery) ([]*dashboards.Dashboard, error) {
 	response, err := dr.searchDashboardsThroughK8sRaw(ctx, query)
 	if err != nil {
@@ -1681,6 +2040,8 @@ func (dr *DashboardServiceImpl) UnstructuredToLegacyDashboard(ctx context.Contex
 		out.Deleted = obj.GetDeletionTimestamp().Time
 	}
 
+	out.PluginID = GetPluginIDFromMeta(obj)
+
 	creator, err := dr.getUserFromMeta(ctx, obj.GetCreatedBy())
 	if err != nil {
 		return nil, err
@@ -1705,10 +2066,6 @@ func (dr *DashboardServiceImpl) UnstructuredToLegacyDashboard(ctx context.Contex
 		out.GnetID = gnetID
 	}
 
-	if pluginID, ok := spec["plugin_id"].(string); ok {
-		out.PluginID = pluginID
-	}
-
 	if isFolder, ok := spec["is_folder"].(bool); ok {
 		out.IsFolder = isFolder
 	}
@@ -1747,6 +2104,42 @@ func (dr *DashboardServiceImpl) getUser(ctx context.Context, uid string) (*user.
 	return dr.userService.GetByUID(ctx, &user.GetUserByUIDQuery{UID: uid})
 }
 
+var pluginIDRepoName = "plugin"
+var fileProvisionedRepoPrefix = "file:"
+
+func setPluginID(obj unstructured.Unstructured, pluginID string) {
+	if pluginID == "" {
+		return
+	}
+
+	annotations := obj.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	annotations[utils.AnnoKeyRepoName] = pluginIDRepoName
+	annotations[utils.AnnoKeyRepoPath] = pluginID
+	obj.SetAnnotations(annotations)
+}
+
+func provisionedFileNameWithPrefix(name string) string {
+	if name == "" {
+		return ""
+	}
+
+	return fileProvisionedRepoPrefix + name
+}
+
+func getProvisionedFileNameFromMeta(obj utils.GrafanaMetaAccessor) (string, bool) {
+	return strings.CutPrefix(obj.GetRepositoryName(), fileProvisionedRepoPrefix)
+}
+
+func GetPluginIDFromMeta(obj utils.GrafanaMetaAccessor) string {
+	if obj.GetRepositoryName() == pluginIDRepoName {
+		return obj.GetRepositoryPath()
+	}
+	return ""
+}
+
 func LegacySaveCommandToUnstructured(cmd *dashboards.SaveDashboardCommand, namespace string) (unstructured.Unstructured, error) {
 	uid := cmd.GetDashboardModel().UID
 	if uid == "" {

pkg/services/dashboards/service/dashboard_service_test.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+	"fmt"
 	"reflect"
 	"testing"
 	"time"
@@ -135,7 +136,8 @@ func TestDashboardService(t *testing.T) {
 			dto := &dashboards.SaveDashboardDTO{}
 
 			t.Run("Should not return validation error if dashboard is provisioned", func(t *testing.T) {
-				fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.AnythingOfType("dashboards.SaveDashboardCommand"), mock.AnythingOfType("*dashboards.DashboardProvisioning")).Return(&dashboards.Dashboard{Data: simplejson.New()}, nil).Once()
+				fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.AnythingOfType("*dashboards.DashboardProvisioning")).Return(nil).Once()
+				fakeStore.On("SaveDashboard", mock.Anything, mock.AnythingOfType("dashboards.SaveDashboardCommand")).Return(&dashboards.Dashboard{Data: simplejson.New()}, nil).Once()
 
 				dto.Dashboard = dashboards.NewDashboard("Dash")
 				dto.Dashboard.SetID(3)
@@ -145,7 +147,8 @@ func TestDashboardService(t *testing.T) {
 			})
 
 			t.Run("Should override invalid refresh interval if dashboard is provisioned", func(t *testing.T) {
-				fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.AnythingOfType("dashboards.SaveDashboardCommand"), mock.AnythingOfType("*dashboards.DashboardProvisioning")).Return(&dashboards.Dashboard{Data: simplejson.New()}, nil).Once()
+				fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.AnythingOfType("*dashboards.DashboardProvisioning")).Return(nil).Once()
+				fakeStore.On("SaveDashboard", mock.Anything, mock.AnythingOfType("dashboards.SaveDashboardCommand")).Return(&dashboards.Dashboard{Data: simplejson.New()}, nil).Once()
 
 				oldRefreshInterval := service.cfg.MinRefreshInterval
 				service.cfg.MinRefreshInterval = "5m"
@@ -246,7 +249,10 @@ func (m *mockDashK8sCli) getClient(ctx context.Context, orgID int64) (dynamic.Re
 }
 
 func (m *mockDashK8sCli) getNamespace(orgID int64) string {
-	return "default"
+	if orgID == 1 {
+		return "default"
+	}
+	return fmt.Sprintf("orgs-%d", orgID)
 }
 
 func (m *mockDashK8sCli) getSearcher() resource.ResourceIndexClient {
@@ -567,6 +573,617 @@ func TestGetAllDashboardsByOrgId(t *testing.T) {
 	})
 }
 
+func TestGetProvisionedDashboardData(t *testing.T) {
+	fakeStore := dashboards.FakeDashboardStore{}
+	defer fakeStore.AssertExpectations(t)
+	service := &DashboardServiceImpl{
+		cfg:            setting.NewCfg(),
+		dashboardStore: &fakeStore,
+		orgService: &orgtest.FakeOrgService{
+			ExpectedOrgs: []*org.OrgDTO{{ID: 1}, {ID: 2}},
+		},
+	}
+
+	t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) {
+		service.features = featuremgmt.WithFeatures()
+		fakeStore.On("GetProvisionedDashboardData", mock.Anything, "test").Return([]*dashboards.DashboardProvisioning{}, nil).Once()
+		dashboard, err := service.GetProvisionedDashboardData(context.Background(), "test")
+		require.NoError(t, err)
+		require.NotNil(t, dashboard)
+		fakeStore.AssertExpectations(t)
+	})
+
+	t.Run("Should use Kubernetes client if feature flags are enabled and get from relevant org", func(t *testing.T) {
+		ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service)
+		k8sClientMock.On("getClient", mock.Anything, mock.Anything).Return(k8sResourceMock, true)
+		k8sResourceMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{
+			"metadata": map[string]any{
+				"name": "uid",
+				"labels": map[string]any{
+					utils.LabelKeyDeprecatedInternalID: "1", // nolint:staticcheck
+				},
+				"annotations": map[string]any{
+					utils.AnnoKeyRepoName:      fileProvisionedRepoPrefix + "test",
+					utils.AnnoKeyRepoHash:      "hash",
+					utils.AnnoKeyRepoPath:      "path/to/file",
+					utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
+				},
+			},
+			"spec": map[string]any{
+				"test":    "test",
+				"version": int64(1),
+				"title":   "testing slugify",
+			},
+		}}, nil).Once()
+		repo := "test"
+		k8sClientMock.searcher.On("Search",
+			mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
+				// ensure the prefix is added to the query
+				return req.Options.Key.Namespace == "default" && req.Options.Fields[0].Values[0] == provisionedFileNameWithPrefix(repo)
+			})).Return(&resource.ResourceSearchResponse{
+			Results: &resource.ResourceTable{
+				Columns: []*resource.ResourceTableColumnDefinition{},
+				Rows:    []*resource.ResourceTableRow{},
+			},
+			TotalHits: 0,
+		}, nil).Once()
+		k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
+			// ensure the prefix is added to the query
+			return req.Options.Key.Namespace == "orgs-2" && req.Options.Fields[0].Values[0] == provisionedFileNameWithPrefix(repo)
+		})).Return(&resource.ResourceSearchResponse{
+			Results: &resource.ResourceTable{
+				Columns: []*resource.ResourceTableColumnDefinition{
+					{
+						Name: "title",
+					},
+					{
+						Name: "folder",
+					},
+				},
+				Rows: []*resource.ResourceTableRow{
+					{
+						Key: &resource.ResourceKey{
+							Name:     "uid",
+							Resource: "dashboard",
+						},
+						Cells: [][]byte{
+							[]byte("Dashboard 1"),
+							[]byte("folder 1"),
+						},
+					},
+				},
+			},
+			TotalHits: 1,
+		}, nil).Once()
+		dashes, err := service.GetProvisionedDashboardData(ctx, repo)
+		require.NoError(t, err)
+		require.Len(t, dashes, 1)
+		require.Equal(t, dashes[0], &dashboards.DashboardProvisioning{
+			ID:          0,
+			DashboardID: 1,
+			Name:        "test",
+			ExternalID:  "path/to/file",
+			CheckSum:    "hash",
+			Updated:     1735689600,
+		})
+		k8sClientMock.AssertExpectations(t)
+	})
+}
+
+func TestGetProvisionedDashboardDataByDashboardID(t *testing.T) {
+	fakeStore := dashboards.FakeDashboardStore{}
+	defer fakeStore.AssertExpectations(t)
+	service := &DashboardServiceImpl{
+		cfg:            setting.NewCfg(),
+		dashboardStore: &fakeStore,
+		orgService: &orgtest.FakeOrgService{
+			ExpectedOrgs: []*org.OrgDTO{{ID: 1}, {ID: 2}},
+		},
+	}
+
+	t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) {
+		service.features = featuremgmt.WithFeatures()
+		fakeStore.On("GetProvisionedDataByDashboardID", mock.Anything, int64(1)).Return(&dashboards.DashboardProvisioning{}, nil).Once()
+		dashboard, err := service.GetProvisionedDashboardDataByDashboardID(context.Background(), 1)
+		require.NoError(t, err)
+		require.NotNil(t, dashboard)
+		fakeStore.AssertExpectations(t)
+	})
+
+	t.Run("Should use Kubernetes client if feature flags are enabled and get from whatever org it is in", func(t *testing.T) {
+		ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service)
+		k8sClientMock.On("getClient", mock.Anything, mock.Anything).Return(k8sResourceMock, true)
+		k8sResourceMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{
+			"metadata": map[string]any{
+				"name": "uid",
+				"labels": map[string]any{
+					utils.LabelKeyDeprecatedInternalID: "1", // nolint:staticcheck
+				},
+				"annotations": map[string]any{
+					utils.AnnoKeyRepoName:      fileProvisionedRepoPrefix + "test",
+					utils.AnnoKeyRepoHash:      "hash",
+					utils.AnnoKeyRepoPath:      "path/to/file",
+					utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
+				},
+			},
+			"spec": map[string]any{
+				"test":    "test",
+				"version": int64(1),
+				"title":   "testing slugify",
+			},
+		}}, nil)
+		k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
+			return req.Options.Key.Namespace == "default"
+		})).Return(&resource.ResourceSearchResponse{
+			Results: &resource.ResourceTable{
+				Columns: []*resource.ResourceTableColumnDefinition{},
+				Rows:    []*resource.ResourceTableRow{},
+			},
+			TotalHits: 0,
+		}, nil)
+		k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
+			return req.Options.Key.Namespace == "orgs-2"
+		})).Return(&resource.ResourceSearchResponse{
+			Results: &resource.ResourceTable{
+				Columns: []*resource.ResourceTableColumnDefinition{
+					{
+						Name: "title",
+					},
+					{
+						Name: "folder",
+					},
+				},
+				Rows: []*resource.ResourceTableRow{
+					{
+						Key: &resource.ResourceKey{
+							Name:     "uid",
+							Resource: "dashboard",
+						},
+						Cells: [][]byte{
+							[]byte("Dashboard 1"),
+							[]byte("folder 1"),
+						},
+					},
+				},
+			},
+			TotalHits: 1,
+		}, nil)
+		dash, err := service.GetProvisionedDashboardDataByDashboardID(ctx, 1)
+		require.NoError(t, err)
+		require.Equal(t, dash, &dashboards.DashboardProvisioning{
+			ID:          0,
+			DashboardID: 1,
+			Name:        "test",
+			ExternalID:  "path/to/file",
+			CheckSum:    "hash",
+			Updated:     1735689600,
+		})
+		k8sClientMock.AssertExpectations(t)
+	})
+}
+
+func TestGetProvisionedDashboardDataByDashboardUID(t *testing.T) {
+	fakeStore := dashboards.FakeDashboardStore{}
+	defer fakeStore.AssertExpectations(t)
+	service := &DashboardServiceImpl{
+		cfg:            setting.NewCfg(),
+		dashboardStore: &fakeStore,
+		orgService: &orgtest.FakeOrgService{
+			ExpectedOrgs: []*org.OrgDTO{{ID: 1}, {ID: 2}},
+		},
+	}
+
+	t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) {
+		service.features = featuremgmt.WithFeatures()
+		fakeStore.On("GetProvisionedDataByDashboardUID", mock.Anything, int64(1), "test").Return(&dashboards.DashboardProvisioning{}, nil).Once()
+		dashboard, err := service.GetProvisionedDashboardDataByDashboardUID(context.Background(), 1, "test")
+		require.NoError(t, err)
+		require.NotNil(t, dashboard)
+		fakeStore.AssertExpectations(t)
+	})
+
+	t.Run("Should use Kubernetes client if feature flags are enabled", func(t *testing.T) {
+		ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service)
+		k8sClientMock.On("getClient", mock.Anything, mock.Anything).Return(k8sResourceMock, true)
+		k8sResourceMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{
+			"metadata": map[string]any{
+				"name": "uid",
+				"labels": map[string]any{
+					utils.LabelKeyDeprecatedInternalID: "1", // nolint:staticcheck
+				},
+				"annotations": map[string]any{
+					utils.AnnoKeyRepoName:      fileProvisionedRepoPrefix + "test",
+					utils.AnnoKeyRepoHash:      "hash",
+					utils.AnnoKeyRepoPath:      "path/to/file",
+					utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
+				},
+			},
+			"spec": map[string]any{
+				"test":    "test",
+				"version": int64(1),
+				"title":   "testing slugify",
+			},
+		}}, nil).Once()
+		k8sClientMock.searcher.On("Search", mock.Anything).Return(&resource.ResourceSearchResponse{
+			Results: &resource.ResourceTable{
+				Columns: []*resource.ResourceTableColumnDefinition{
+					{
+						Name: "title",
+					},
+					{
+						Name: "folder",
+					},
+				},
+				Rows: []*resource.ResourceTableRow{
+					{
+						Key: &resource.ResourceKey{
+							Name:     "uid",
+							Resource: "dashboard",
+						},
+						Cells: [][]byte{
+							[]byte("Dashboard 1"),
+							[]byte("folder 1"),
+						},
+					},
+				},
+			},
+			TotalHits: 1,
+		}, nil).Once()
+		dash, err := service.GetProvisionedDashboardDataByDashboardUID(ctx, 1, "uid")
+		require.NoError(t, err)
+		require.Equal(t, dash, &dashboards.DashboardProvisioning{
+			ID:          0,
+			DashboardID: 1,
+			Name:        "test",
+			ExternalID:  "path/to/file",
+			CheckSum:    "hash",
+			Updated:     1735689600,
+		})
+		k8sClientMock.AssertExpectations(t)
+	})
+}
+
+func TestDeleteOrphanedProvisionedDashboards(t *testing.T) {
+	fakeStore := dashboards.FakeDashboardStore{}
+	defer fakeStore.AssertExpectations(t)
+	service := &DashboardServiceImpl{
+		cfg:            setting.NewCfg(),
+		dashboardStore: &fakeStore,
+		orgService: &orgtest.FakeOrgService{
+			ExpectedOrgs: []*org.OrgDTO{{ID: 1}, {ID: 2}},
+		},
+	}
+
+	t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) {
+		service.features = featuremgmt.WithFeatures()
+		fakeStore.On("DeleteOrphanedProvisionedDashboards", mock.Anything, &dashboards.DeleteOrphanedProvisionedDashboardsCommand{
+			ReaderNames: []string{"test"},
+		}).Return(nil).Once()
+		err := service.DeleteOrphanedProvisionedDashboards(context.Background(), &dashboards.DeleteOrphanedProvisionedDashboardsCommand{
+			ReaderNames: []string{"test"},
+		})
+		require.NoError(t, err)
+		fakeStore.AssertExpectations(t)
+	})
+
+	t.Run("Should use Kubernetes client if feature flags are enabled, delete across all orgs, but only delete file based provisioned dashboards", func(t *testing.T) {
+		_, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service)
+		k8sClientMock.On("getClient", mock.Anything, mock.Anything).Return(k8sResourceMock, true)
+		k8sResourceMock.On("Delete", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
+		fakeStore.On("CleanupAfterDelete", mock.Anything, &dashboards.DeleteDashboardCommand{UID: "uid", OrgID: 1}).Return(nil).Once()
+		fakeStore.On("CleanupAfterDelete", mock.Anything, &dashboards.DeleteDashboardCommand{UID: "uid3", OrgID: 2}).Return(nil).Once()
+		k8sResourceMock.On("Get", mock.Anything, "uid", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{
+			"metadata": map[string]any{
+				"name": "uid",
+				"annotations": map[string]any{
+					utils.AnnoKeyRepoName:      fileProvisionedRepoPrefix + "orphaned",
+					utils.AnnoKeyRepoHash:      "hash",
+					utils.AnnoKeyRepoPath:      "path/to/file",
+					utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
+				},
+			},
+			"spec": map[string]any{},
+		}}, nil).Once()
+		// should not delete this one, because it does not start with "file:"
+		k8sResourceMock.On("Get", mock.Anything, "uid2", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{
+			"metadata": map[string]any{
+				"name": "uid2",
+				"annotations": map[string]any{
+					utils.AnnoKeyRepoName: "plugin",
+					utils.AnnoKeyRepoHash: "app",
+				},
+			},
+			"spec": map[string]any{},
+		}}, nil).Once()
+
+		k8sResourceMock.On("Get", mock.Anything, "uid3", mock.Anything, mock.Anything).Return(&unstructured.Unstructured{Object: map[string]any{
+			"metadata": map[string]any{
+				"name": "uid3",
+				"annotations": map[string]any{
+					utils.AnnoKeyRepoName:      fileProvisionedRepoPrefix + "orphaned",
+					utils.AnnoKeyRepoHash:      "hash",
+					utils.AnnoKeyRepoPath:      "path/to/file",
+					utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
+				},
+			},
+			"spec": map[string]any{},
+		}}, nil).Once()
+		k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
+			return req.Options.Key.Namespace == "default" && req.Options.Fields[0].Key == "repo.name" && req.Options.Fields[0].Values[0] == provisionedFileNameWithPrefix("test") &&
+				req.Options.Fields[0].Operator == "notin"
+		})).Return(&resource.ResourceSearchResponse{
+			Results: &resource.ResourceTable{
+				Columns: []*resource.ResourceTableColumnDefinition{
+					{
+						Name: "title",
+					},
+					{
+						Name: "folder",
+					},
+				},
+				Rows: []*resource.ResourceTableRow{
+					{
+						Key: &resource.ResourceKey{
+							Name:     "uid",
+							Resource: "dashboard",
+						},
+						Cells: [][]byte{
+							[]byte("Dashboard 1"),
+							[]byte("folder 1"),
+						},
+					},
+				},
+			},
+			TotalHits: 1,
+		}, nil).Once()
+
+		k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
+			return req.Options.Key.Namespace == "orgs-2" && req.Options.Fields[0].Key == "repo.name" && req.Options.Fields[0].Values[0] == provisionedFileNameWithPrefix("test") &&
+				req.Options.Fields[0].Operator == "notin"
+		})).Return(&resource.ResourceSearchResponse{
+			Results: &resource.ResourceTable{
+				Columns: []*resource.ResourceTableColumnDefinition{
+					{
+						Name: "title",
+					},
+					{
+						Name: "folder",
+					},
+				},
+				Rows: []*resource.ResourceTableRow{
+					{
+						Key: &resource.ResourceKey{
+							Name:     "uid2",
+							Resource: "dashboard",
+						},
+						Cells: [][]byte{
+							[]byte("Dashboard 2"),
+							[]byte("folder 2"),
+						},
+					},
+					{
+						Key: &resource.ResourceKey{
+							Name:     "uid3",
+							Resource: "dashboard",
+						},
+						Cells: [][]byte{
+							[]byte("Dashboard 3"),
+							[]byte("folder 3"),
+						},
+					},
+				},
+			},
+			TotalHits: 2,
+		}, nil).Once()
+		err := service.DeleteOrphanedProvisionedDashboards(context.Background(), &dashboards.DeleteOrphanedProvisionedDashboardsCommand{
+			ReaderNames: []string{"test"},
+		})
+		require.NoError(t, err)
+		k8sClientMock.AssertExpectations(t)
+	})
+}
+
+func TestUnprovisionDashboard(t *testing.T) {
+	fakeStore := dashboards.FakeDashboardStore{}
+	defer fakeStore.AssertExpectations(t)
+	service := &DashboardServiceImpl{
+		cfg:            setting.NewCfg(),
+		dashboardStore: &fakeStore,
+		orgService: &orgtest.FakeOrgService{
+			ExpectedOrgs: []*org.OrgDTO{{ID: 1}, {ID: 2}},
+		},
+	}
+
+	t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) {
+		service.features = featuremgmt.WithFeatures()
+		fakeStore.On("UnprovisionDashboard", mock.Anything, int64(1)).Return(nil).Once()
+		err := service.UnprovisionDashboard(context.Background(), 1)
+		require.NoError(t, err)
+		fakeStore.AssertExpectations(t)
+	})
+
+	t.Run("Should use Kubernetes client if feature flags are enabled - should remove annotations", func(t *testing.T) {
+		ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service)
+		k8sClientMock.On("getClient", mock.Anything, mock.Anything).Return(k8sResourceMock, true)
+		dash := &unstructured.Unstructured{Object: map[string]any{
+			"metadata": map[string]any{
+				"name": "uid",
+				"annotations": map[string]any{
+					utils.AnnoKeyRepoName:      fileProvisionedRepoPrefix + "test",
+					utils.AnnoKeyRepoHash:      "hash",
+					utils.AnnoKeyRepoPath:      "path/to/file",
+					utils.AnnoKeyRepoTimestamp: "2025-01-01T00:00:00Z",
+				},
+			},
+			"spec": map[string]any{},
+		}}
+		k8sResourceMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(dash, nil)
+		dashWithoutAnnotations := &unstructured.Unstructured{Object: map[string]any{
+			"apiVersion": "dashboard.grafana.app/v0alpha1",
+			"kind":       "Dashboard",
+			"metadata": map[string]any{
+				"name":        "uid",
+				"namespace":   "default",
+				"annotations": map[string]any{},
+			},
+			"spec": map[string]any{
+				"uid":     "uid",
+				"version": 1,
+			},
+		}}
+		// should update it to be without annotations
+		k8sResourceMock.On("Update", mock.Anything, dashWithoutAnnotations, mock.Anything, mock.Anything).Return(dashWithoutAnnotations, nil)
+		k8sClientMock.searcher.On("Search", mock.Anything).Return(&resource.ResourceSearchResponse{
+			Results: &resource.ResourceTable{
+				Columns: []*resource.ResourceTableColumnDefinition{
+					{
+						Name: "title",
+					},
+					{
+						Name: "folder",
+					},
+				},
+				Rows: []*resource.ResourceTableRow{
+					{
+						Key: &resource.ResourceKey{
+							Name:     "uid",
+							Resource: "dashboard",
+						},
+						Cells: [][]byte{
+							[]byte("Dashboard 1"),
+							[]byte("folder 1"),
+						},
+					},
+				},
+			},
+			TotalHits: 1,
+		}, nil)
+		err := service.UnprovisionDashboard(ctx, 1)
+		require.NoError(t, err)
+		k8sClientMock.AssertExpectations(t)
+	})
+}
+
+func TestGetDashboardsByPluginID(t *testing.T) {
+	fakeStore := dashboards.FakeDashboardStore{}
+	defer fakeStore.AssertExpectations(t)
+	service := &DashboardServiceImpl{
+		cfg:            setting.NewCfg(),
+		dashboardStore: &fakeStore,
+	}
+
+	query := &dashboards.GetDashboardsByPluginIDQuery{
+		PluginID: "testing",
+		OrgID:    1,
+	}
+	t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) {
+		service.features = featuremgmt.WithFeatures()
+		fakeStore.On("GetDashboardsByPluginID", mock.Anything, mock.Anything).Return([]*dashboards.Dashboard{}, nil).Once()
+		_, err := service.GetDashboardsByPluginID(context.Background(), query)
+		require.NoError(t, err)
+		fakeStore.AssertExpectations(t)
+	})
+
+	t.Run("Should use Kubernetes client if feature flags are enabled", func(t *testing.T) {
+		ctx, k8sClientMock, _ := setupK8sDashboardTests(service)
+		k8sClientMock.searcher.On("Search", mock.MatchedBy(func(req *resource.ResourceSearchRequest) bool {
+			return req.Options.Fields[0].Key == "repo.name" && req.Options.Fields[0].Values[0] == "plugin" &&
+				req.Options.Fields[1].Key == "repo.path" && req.Options.Fields[1].Values[0] == "testing"
+		})).Return(&resource.ResourceSearchResponse{
+			Results: &resource.ResourceTable{
+				Columns: []*resource.ResourceTableColumnDefinition{
+					{
+						Name: "title",
+					},
+					{
+						Name: "folder",
+					},
+				},
+				Rows: []*resource.ResourceTableRow{
+					{
+						Key: &resource.ResourceKey{
+							Name:     "uid",
+							Resource: "dashboard",
+						},
+						Cells: [][]byte{
+							[]byte("Dashboard 1"),
+							[]byte("folder 1"),
+						},
+					},
+				},
+			},
+			TotalHits: 1,
+		}, nil)
+		dashes, err := service.GetDashboardsByPluginID(ctx, query)
+		require.NoError(t, err)
+		require.Len(t, dashes, 1)
+		k8sClientMock.AssertExpectations(t)
+	})
+}
+
+func TestSaveProvisionedDashboard(t *testing.T) {
+	fakeStore := dashboards.FakeDashboardStore{}
+	defer fakeStore.AssertExpectations(t)
+	service := &DashboardServiceImpl{
+		cfg:            setting.NewCfg(),
+		dashboardStore: &fakeStore,
+		log:            log.NewNopLogger(),
+	}
+
+	origNewDashboardGuardian := guardian.New
+	defer func() { guardian.New = origNewDashboardGuardian }()
+	guardian.MockDashboardGuardian(&guardian.FakeDashboardGuardian{CanSaveValue: true})
+
+	query := &dashboards.SaveDashboardDTO{
+		OrgID: 1,
+		User:  &user.SignedInUser{UserID: 1},
+		Dashboard: &dashboards.Dashboard{
+			UID:   "uid",
+			Title: "testing slugify",
+			Slug:  "testing-slugify",
+			OrgID: 1,
+			Data:  simplejson.NewFromAny(map[string]any{"test": "test", "title": "testing slugify", "uid": "uid"}),
+		},
+	}
+
+	t.Run("Should fallback to dashboard store if Kubernetes feature flags are not enabled", func(t *testing.T) {
+		service.features = featuremgmt.WithFeatures()
+		fakeStore.On("GetDashboard", mock.Anything, mock.Anything).Return(&dashboards.Dashboard{}, nil)
+		fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+		fakeStore.On("SaveDashboard", mock.Anything, mock.Anything, mock.Anything).Return(&dashboards.Dashboard{}, nil)
+		dashboard, err := service.SaveProvisionedDashboard(context.Background(), query, &dashboards.DashboardProvisioning{})
+		require.NoError(t, err)
+		require.NotNil(t, dashboard)
+		fakeStore.AssertExpectations(t)
+	})
+
+	dashboardUnstructured := unstructured.Unstructured{Object: map[string]any{
+		"metadata": map[string]any{
+			"name": "uid",
+		},
+		"spec": map[string]any{
+			"test":    "test",
+			"version": int64(1),
+			"title":   "testing slugify",
+		},
+	}}
+
+	t.Run("Should use Kubernetes create if feature flags are enabled", func(t *testing.T) {
+		ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service)
+		fakeStore.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+		k8sClientMock.On("getClient", mock.Anything, int64(1)).Return(k8sResourceMock, true)
+		k8sResourceMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil)
+		k8sResourceMock.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&dashboardUnstructured, nil)
+
+		dashboard, err := service.SaveProvisionedDashboard(ctx, query, &dashboards.DashboardProvisioning{})
+		require.NoError(t, err)
+		require.NotNil(t, dashboard)
+		k8sClientMock.AssertExpectations(t)
+		// ensure the provisioning data is still saved to the db
+		fakeStore.AssertExpectations(t)
+	})
+}
+
 func TestSaveDashboard(t *testing.T) {
 	fakeStore := dashboards.FakeDashboardStore{}
 	defer fakeStore.AssertExpectations(t)
@@ -667,22 +1284,19 @@ func TestDeleteDashboard(t *testing.T) {
 	t.Run("Should use Kubernetes client if feature flags are enabled", func(t *testing.T) {
 		ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service)
 		k8sClientMock.On("getClient", mock.Anything, int64(1)).Return(k8sResourceMock, true).Once()
-		k8sResourceMock.On("Delete", mock.Anything, mock.Anything, metav1.DeleteOptions{
-			GracePeriodSeconds: nil,
-		}, mock.Anything).Return(nil).Once()
+		k8sResourceMock.On("Delete", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
+		fakeStore.On("CleanupAfterDelete", mock.Anything, mock.Anything).Return(nil).Once()
 
 		err := service.DeleteDashboard(ctx, 1, "uid", 1)
 		require.NoError(t, err)
 		k8sClientMock.AssertExpectations(t)
 	})
 
-	t.Run("When UID is not passed in for provisioned dashboards, should retrieve that first and set grace period to zero", func(t *testing.T) {
+	t.Run("If UID is not passed in, it should retrieve that first", func(t *testing.T) {
 		ctx, k8sClientMock, k8sResourceMock := setupK8sDashboardTests(service)
-		zeroInt64 := int64(0)
 		k8sClientMock.On("getClient", mock.Anything, int64(1)).Return(k8sResourceMock, true).Once()
-		k8sResourceMock.On("Delete", mock.Anything, mock.Anything, metav1.DeleteOptions{
-			GracePeriodSeconds: &zeroInt64,
-		}, mock.Anything).Return(nil).Once()
+		k8sResourceMock.On("Delete", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
+		fakeStore.On("CleanupAfterDelete", mock.Anything, mock.Anything).Return(nil).Once()
 		k8sClientMock.searcher.On("Search", mock.Anything).Return(&resource.ResourceSearchResponse{
 			Results: &resource.ResourceTable{
 				Columns: []*resource.ResourceTableColumnDefinition{
@@ -708,7 +1322,7 @@ func TestDeleteDashboard(t *testing.T) {
 			},
 			TotalHits: 1,
 		}, nil)
-		err := service.DeleteProvisionedDashboard(ctx, 1, 1)
+		err := service.DeleteDashboard(ctx, 1, "", 1)
 		require.NoError(t, err)
 		k8sClientMock.AssertExpectations(t)
 		k8sClientMock.searcher.AssertExpectations(t)

pkg/services/dashboards/store_mock.go

@@ -94,6 +94,24 @@ func (_m *FakeDashboardStore) DeleteDashboard(ctx context.Context, cmd *DeleteDa
 	return r0
 }
 
+// CleanupAfterDelete provides a mock function with given fields: ctx, cmd
+func (_m *FakeDashboardStore) CleanupAfterDelete(ctx context.Context, cmd *DeleteDashboardCommand) error {
+	ret := _m.Called(ctx, cmd)
+
+	if len(ret) == 0 {
+		panic("no return value specified for CleanupAfterDelete")
+	}
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, *DeleteDashboardCommand) error); ok {
+		r0 = rf(ctx, cmd)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
 // DeleteAllDashboards provides a mock function with given fields: ctx, orgID
 func (_m *FakeDashboardStore) DeleteAllDashboards(ctx context.Context, orgID int64) error {
 	ret := _m.Called(ctx, orgID)
@@ -208,7 +226,6 @@ func (_m *FakeDashboardStore) GetAllDashboards(ctx context.Context) ([]*Dashboar
 	return r0, r1
 }
 
-
 // GetAllDashboardsByOrgId provides a mock function with given fields: ctx
 func (_m *FakeDashboardStore) GetAllDashboardsByOrgId(ctx context.Context, orgID int64) ([]*Dashboard, error) {
 	ret := _m.Called(ctx, orgID)
@@ -588,34 +605,22 @@ func (_m *FakeDashboardStore) SaveDashboard(ctx context.Context, cmd SaveDashboa
 	return r0, r1
 }
 
-// SaveProvisionedDashboard provides a mock function with given fields: ctx, cmd, provisioning
-func (_m *FakeDashboardStore) SaveProvisionedDashboard(ctx context.Context, cmd SaveDashboardCommand, provisioning *DashboardProvisioning) (*Dashboard, error) {
-	ret := _m.Called(ctx, cmd, provisioning)
+// SaveProvisionedDashboard provides a mock function with given fields: ctx, dash, provisioning
+func (_m *FakeDashboardStore) SaveProvisionedDashboard(ctx context.Context, dash *Dashboard, provisioning *DashboardProvisioning) error {
+	ret := _m.Called(ctx, dash, provisioning)
 
 	if len(ret) == 0 {
 		panic("no return value specified for SaveProvisionedDashboard")
 	}
 
-	var r0 *Dashboard
-	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, SaveDashboardCommand, *DashboardProvisioning) (*Dashboard, error)); ok {
-		return rf(ctx, cmd, provisioning)
-	}
-	if rf, ok := ret.Get(0).(func(context.Context, SaveDashboardCommand, *DashboardProvisioning) *Dashboard); ok {
-		r0 = rf(ctx, cmd, provisioning)
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, *Dashboard, *DashboardProvisioning) error); ok {
+		r0 = rf(ctx, dash, provisioning)
 	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(*Dashboard)
-		}
+		r0 = ret.Error(0)
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, SaveDashboardCommand, *DashboardProvisioning) error); ok {
-		r1 = rf(ctx, cmd, provisioning)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
+	return r0
 }
 
 // SoftDeleteDashboard provides a mock function with given fields: ctx, orgID, dashboardUid

pkg/storage/unified/search/bleve.go

@@ -625,6 +625,19 @@ func requirementQuery(req *resource.Requirement, prefix string) (query.Query, *r
 
 		return query.NewDisjunctionQuery(disjuncts), nil
 	case selection.NotIn:
+		boolQuery := bleve.NewBooleanQuery()
+
+		var mustNotQueries []query.Query
+		for _, value := range req.Values {
+			mustNotQueries = append(mustNotQueries, bleve.NewMatchQuery(value))
+		}
+		boolQuery.AddMustNot(mustNotQueries...)
+
+		// must still have a value
+		notEmptyQuery := bleve.NewWildcardQuery("*")
+		boolQuery.AddMust(notEmptyQuery)
+
+		return boolQuery, nil
 	}
 	return nil, resource.NewBadRequestError(
 		fmt.Sprintf("unsupported query operation (%s %s %v)", req.Key, req.Operator, req.Values),