Scuemata: Add test to validate devenv resources (#35810)

* Add test for devenv resources

* Refactor validation tests for grokkability

* Devenv dashboards error-tracking script

* Refactor to use cueerrors.Details()

* Further test refinement

* Close major elements of dashboard schema

* Centralize dashboard validation tests

General dashboard validation testing belongs in the load package.

* Better names for error context on glue CUE code

* Fixup validate-resource

Do only one of base or dist, and fix copied docs.

* Skip the devenv test

* Remove test for validateResources

* Fix shellcheck

* Backend linter

Co-authored-by: sam boyer <sdboyer@grafana.com>

pkg/schema/load/dashboard.go

@@ -81,7 +81,7 @@ func DistDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
 	// Value.Fill() can't target definitions. Need new method based on cue.Path;
 	// a CL has been merged that creates FillPath and will be in the next
 	// release of CUE.
-	dummy, _ := rt.Compile("mergeStruct", `
+	dummy, _ := rt.Compile("glue-unifyPanelDashboard", `
 	obj: {}
 	dummy: {
 		#Panel: obj
@@ -125,7 +125,11 @@ type compositeDashboardSchema struct {
 
 // Validate checks that the resource is correct with respect to the schema.
 func (cds *compositeDashboardSchema) Validate(r schema.Resource) error {
-	rv, err := rt.Compile("resource", r.Value)
+	name := r.Name
+	if name == "" {
+		name = "resource"
+	}
+	rv, err := rt.Compile(name, r.Value)
 	if err != nil {
 		return err
 	}

pkg/schema/load/generic.go

@@ -102,7 +102,11 @@ type genericVersionedSchema struct {
 
 // Validate checks that the resource is correct with respect to the schema.
 func (gvs *genericVersionedSchema) Validate(r schema.Resource) error {
-	rv, err := rt.Compile("resource", r.Value)
+	name := r.Name
+	if name == "" {
+		name = "resource"
+	}
+	rv, err := rt.Compile(name, r.Value)
 	if err != nil {
 		return err
 	}

pkg/schema/load/load_test.go

@@ -1,13 +1,16 @@
 package load
 
 import (
+	"encoding/json"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
 	"testing"
 	"testing/fstest"
 
+	"cuelang.org/go/cue/errors"
 	"github.com/grafana/grafana/pkg/schema"
 	"github.com/laher/mergefs"
 	"github.com/stretchr/testify/require"
@@ -45,8 +48,11 @@ func TestScuemataBasics(t *testing.T) {
 	}
 }
 
-func TestDashboardValidity(t *testing.T) {
-	validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards"))
+func TestDevenvDashboardValidity(t *testing.T) {
+	// TODO un-skip when tests pass on all devenv dashboards
+	t.Skip()
+	// validdir := os.DirFS(filepath.Join("..", "..", "..", "devenv", "dev-dashboards"))
+	validdir := filepath.Join("..", "..", "..", "devenv", "dev-dashboards")
 
 	dash, err := BaseDashboardFamily(p)
 	require.NoError(t, err, "error while loading base dashboard scuemata")
@@ -54,29 +60,55 @@ func TestDashboardValidity(t *testing.T) {
 	ddash, err := DistDashboardFamily(p)
 	require.NoError(t, err, "error while loading dist dashboard scuemata")
 
-	require.NoError(t, fs.WalkDir(validdir, ".", func(path string, d fs.DirEntry, err error) error {
-		require.NoError(t, err)
+	doTest := func(sch schema.VersionedCueSchema) func(t *testing.T) {
+		return func(t *testing.T) {
+			t.Parallel()
+			require.NoError(t, filepath.Walk(validdir, func(path string, d fs.FileInfo, err error) error {
+				require.NoError(t, err)
 
-		if d.IsDir() || filepath.Ext(d.Name()) != ".json" {
-			return nil
+				if d.IsDir() || filepath.Ext(d.Name()) != ".json" {
+					return nil
+				}
+
+				// Ignore gosec warning G304 since it's a test
+				// nolint:gosec
+				b, err := os.Open(path)
+				require.NoError(t, err, "failed to open dashboard file")
+
+				// Only try to validate dashboards with schemaVersion >= 30
+				jtree := make(map[string]interface{})
+				byt, err := io.ReadAll(b)
+				if err != nil {
+					t.Fatal(err)
+				}
+				require.NoError(t, json.Unmarshal(byt, &jtree))
+				if oldschemav, has := jtree["schemaVersion"]; !has {
+					t.Logf("no schemaVersion in %s", path)
+					return nil
+				} else {
+					if !(oldschemav.(float64) > 29) {
+						t.Logf("schemaVersion is %v, older than 30, skipping %s", oldschemav, path)
+						return nil
+					}
+				}
+
+				t.Run(filepath.Base(path), func(t *testing.T) {
+					err := sch.Validate(schema.Resource{Value: byt, Name: path})
+					if err != nil {
+						// Testify trims errors to short length. We want the full text
+						t.Fatal(errors.Details(err, nil))
+					}
+				})
+
+				return nil
+			}))
 		}
+	}
 
-		t.Run(path, func(t *testing.T) {
-			b, err := validdir.Open(path)
-			require.NoError(t, err, "failed to open dashboard file")
-
-			t.Run("base", func(t *testing.T) {
-				_, err := schema.SearchAndValidate(dash, b)
-				require.NoError(t, err, "dashboard failed validation")
-			})
-			t.Run("dist", func(t *testing.T) {
-				_, err := schema.SearchAndValidate(ddash, b)
-				require.NoError(t, err, "dashboard failed validation")
-			})
-		})
-
-		return nil
-	}))
+	// TODO will need to expand this appropriately when the scuemata contain
+	// more than one schema
+	t.Run("base", doTest(dash))
+	t.Run("dist", doTest(ddash))
 }
 
 func TestPanelValidity(t *testing.T) {

pkg/schema/load/panel.go

@@ -16,7 +16,7 @@ import (
 // Returns a disjunction of structs representing each panel schema version
 // (post-mapping from on-disk #PanelModel form) from each scuemata in the map.
 func disjunctPanelScuemata(scuemap map[string]schema.VersionedCueSchema) (cue.Value, error) {
-	partsi, err := rt.Compile("panelDisjunction", `
+	partsi, err := rt.Compile("glue-panelDisjunction", `
 	allPanels: [Name=_]: {}
 	parts: or([for v in allPanels { v }])
 	`)
@@ -44,7 +44,7 @@ func disjunctPanelScuemata(scuemap map[string]schema.VersionedCueSchema) (cue.Va
 func mapPanelModel(id string, vcs schema.VersionedCueSchema) cue.Value {
 	maj, min := vcs.Version()
 	// Ignore err return, this can't fail to compile
-	inter, _ := rt.Compile("typedPanel", fmt.Sprintf(`
+	inter, _ := rt.Compile(fmt.Sprintf("%s-glue-panelComposition", id), fmt.Sprintf(`
 	in: {
 		type: %q
 		v: {