Search: support datasource template variables when parsing dashboard JSON models (#51587)

* init

* support template variables

* support variables without curly braces

* add todo for `__all` case

* fix `$__all` case for non-multivalue

* extract some functions

* fix flakinesss

* support `$__all` and `default` template variables

* add todo

* compilation fix

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

pkg/services/export/export_dashboards.go

@@ -11,11 +11,12 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/grafana/grafana/pkg/infra/filestorage"
+	"github.com/grafana/grafana/pkg/services/searchV2"
 	"github.com/grafana/grafana/pkg/services/searchV2/extract"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 )
 
-func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup) error {
+func exportDashboards(helper *commitHelper, job *gitExportJob) error {
 	alias := make(map[string]string, 100)
 	ids := make(map[int64]string, 100)
 	folders := make(map[int64]string, 100)
@@ -25,13 +26,18 @@ func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup)
 		folders[0] = job.cfg.GeneralFolderPath // "general"
 	}
 
+	lookup, err := searchV2.LoadDatasourceLookup(helper.ctx, helper.orgID, job.sql)
+	if err != nil {
+		return err
+	}
+
 	rootDir := path.Join(helper.orgDir, "root")
 	folderStructure := commitOptions{
 		when:    time.Now(),
 		comment: "Exported folder structure",
 	}
 
-	err := job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
+	err = job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
 		type dashDataQueryResult struct {
 			Id       int64
 			UID      string `xorm:"uid"`

pkg/services/export/export_ds.go

@@ -6,41 +6,26 @@ import (
 	"sort"
 
 	"github.com/grafana/grafana/pkg/services/datasources"
-	"github.com/grafana/grafana/pkg/services/searchV2/extract"
 )
 
-type dsLookup func(ref *extract.DataSourceRef) *extract.DataSourceRef
-
-func exportDataSources(helper *commitHelper, job *gitExportJob, save bool) (dsLookup, error) {
+func exportDataSources(helper *commitHelper, job *gitExportJob) error {
 	cmd := &datasources.GetDataSourcesQuery{
 		OrgId: job.orgID,
 	}
 	err := job.sql.GetDataSources(helper.ctx, cmd)
 	if err != nil {
-		return nil, err
+		return nil
 	}
 
 	sort.SliceStable(cmd.Result, func(i, j int) bool {
 		return cmd.Result[i].Created.After(cmd.Result[j].Created)
 	})
 
-	byUID := make(map[string]*extract.DataSourceRef, len(cmd.Result))
-	byName := make(map[string]*extract.DataSourceRef, len(cmd.Result))
 	for _, ds := range cmd.Result {
-		ref := &extract.DataSourceRef{
-			UID:  ds.Uid,
-			Type: ds.Type,
-		}
-		byUID[ds.Uid] = ref
-		byName[ds.Name] = ref
-		if !save {
-			continue
-		}
-
 		ds.OrgId = 0
 		ds.Version = 0
 		ds.SecureJsonData = map[string][]byte{
-			"TODO": []byte("secret store lookup"),
+			"TODO": []byte("XXX"),
 		}
 
 		err := helper.add(commitOptions{
@@ -54,26 +39,9 @@ func exportDataSources(helper *commitHelper, job *gitExportJob, save bool) (dsLo
 			comment: fmt.Sprintf("Add datasource: %s", ds.Name),
 		})
 		if err != nil {
-			return nil, err
+			return err
 		}
 	}
 
-	// Return the lookup function
-	return func(ref *extract.DataSourceRef) *extract.DataSourceRef {
-		if ref == nil || ref.UID == "" {
-			return &extract.DataSourceRef{
-				UID:  "default.uid",
-				Type: "default.type",
-			}
-		}
-		v, ok := byUID[ref.UID]
-		if ok {
-			return v
-		}
-		v, ok = byName[ref.UID]
-		if ok {
-			return v
-		}
-		return nil
-	}, nil
+	return nil
 }

pkg/services/export/git_export_job.go

@@ -172,19 +172,16 @@ func (e *gitExportJob) doExportWithHistory() error {
 
 func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error {
 	include := e.cfg.Include
-	lookup, err := exportDataSources(helper, e, include.DS)
-	if err != nil {
-		return err
-	}
-
-	if include.Dash {
-		err = exportDashboards(helper, e, lookup)
-		if err != nil {
-			return err
-		}
-	}
 
 	exporters := []simpleExporter{}
+	if include.Dash {
+		exporters = append(exporters, exportDashboards)
+	}
+
+	if include.DS {
+		exporters = append(exporters, exportDataSources)
+	}
+
 	if include.Auth {
 		exporters = append(exporters, dumpAuthTables)
 	}
@@ -207,12 +204,12 @@ func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error {
 	}
 
 	for _, fn := range exporters {
-		err = fn(helper, e)
+		err := fn(helper, e)
 		if err != nil {
 			return err
 		}
 	}
-	return err
+	return nil
 }
 
 /**