public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scuemata: Add test to validate devenv resources (#35810)

Browse Source 
* Add test for devenv resources

* Refactor validation tests for grokkability

* Devenv dashboards error-tracking script

* Refactor to use cueerrors.Details()

* Further test refinement

* Close major elements of dashboard schema

* Centralize dashboard validation tests

General dashboard validation testing belongs in the load package.

* Better names for error context on glue CUE code

* Fixup validate-resource

Do only one of base or dist, and fix copied docs.

* Skip the devenv test

* Remove test for validateResources

* Fix shellcheck

* Backend linter

Co-authored-by: sam boyer <sdboyer@grafana.com>
This commit is contained in:
Dimitris Sotirakis
2021-07-15 20:08:03 -04:00
committed by GitHub
co-authored by sam boyer
parent 8de218d5f1
commit 2e0dc835cf
10 changed files with 115 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/schema/load/dashboard.go
+6 -2
View File
@@ -81,7 +81,7 @@ func DistDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
// Value.Fill() can't target definitions. Need new method based on cue.Path;
// a CL has been merged that creates FillPath and will be in the next
// release of CUE.
dummy, _ := rt.Compile("mergeStruct", `
dummy, _ := rt.Compile("glue-unifyPanelDashboard", `
obj: {}
dummy: {
#Panel: obj
@@ -125,7 +125,11 @@ type compositeDashboardSchema struct {
// Validate checks that the resource is correct with respect to the schema.
func (cds *compositeDashboardSchema) Validate(r schema.Resource) error {
rv, err := rt.Compile("resource", r.Value)
name := r.Name
if name == "" {
name = "resource"
}
rv, err := rt.Compile(name, r.Value)
if err != nil {
return err
}
pkg/schema/load/generic.go
+5 -1
View File
@@ -102,7 +102,11 @@ type genericVersionedSchema struct {
// Validate checks that the resource is correct with respect to the schema.
func (gvs *genericVersionedSchema) Validate(r schema.Resource) error {
rv, err := rt.Compile("resource", r.Value)
name := r.Name
if name == "" {
name = "resource"
}
rv, err := rt.Compile(name, r.Value)
if err != nil {
return err
}
pkg/schema/load/load_test.go
+54 -22
View File
@@ -1,13 +1,16 @@
package load
import (
"encoding/json"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"testing"
"testing/fstest"
"cuelang.org/go/cue/errors"
"github.com/grafana/grafana/pkg/schema"
"github.com/laher/mergefs"
"github.com/stretchr/testify/require"
@@ -45,8 +48,11 @@ func TestScuemataBasics(t *testing.T) {
}
}
func TestDashboardValidity(t *testing.T) {
validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards"))
func TestDevenvDashboardValidity(t *testing.T) {
// TODO un-skip when tests pass on all devenv dashboards
t.Skip()
// validdir := os.DirFS(filepath.Join("..", "..", "..", "devenv", "dev-dashboards"))
validdir := filepath.Join("..", "..", "..", "devenv", "dev-dashboards")
dash, err := BaseDashboardFamily(p)
require.NoError(t, err, "error while loading base dashboard scuemata")
@@ -54,29 +60,55 @@ func TestDashboardValidity(t *testing.T) {
ddash, err := DistDashboardFamily(p)
require.NoError(t, err, "error while loading dist dashboard scuemata")
require.NoError(t, fs.WalkDir(validdir, ".", func(path string, d fs.DirEntry, err error) error {
require.NoError(t, err)
doTest := func(sch schema.VersionedCueSchema) func(t *testing.T) {
return func(t *testing.T) {
t.Parallel()
require.NoError(t, filepath.Walk(validdir, func(path string, d fs.FileInfo, err error) error {
require.NoError(t, err)
if d.IsDir() || filepath.Ext(d.Name()) != ".json" {
return nil
if d.IsDir() || filepath.Ext(d.Name()) != ".json" {
return nil
}
// Ignore gosec warning G304 since it's a test
// nolint:gosec
b, err := os.Open(path)
require.NoError(t, err, "failed to open dashboard file")
// Only try to validate dashboards with schemaVersion >= 30
jtree := make(map[string]interface{})
byt, err := io.ReadAll(b)
if err != nil {
t.Fatal(err)
}
require.NoError(t, json.Unmarshal(byt, &jtree))
if oldschemav, has := jtree["schemaVersion"]; !has {
t.Logf("no schemaVersion in %s", path)
return nil
} else {
if !(oldschemav.(float64) > 29) {
t.Logf("schemaVersion is %v, older than 30, skipping %s", oldschemav, path)
return nil
}
}
t.Run(filepath.Base(path), func(t *testing.T) {
err := sch.Validate(schema.Resource{Value: byt, Name: path})
if err != nil {
// Testify trims errors to short length. We want the full text
t.Fatal(errors.Details(err, nil))
}
})
return nil
}))
}
}
t.Run(path, func(t *testing.T) {
b, err := validdir.Open(path)
require.NoError(t, err, "failed to open dashboard file")
t.Run("base", func(t *testing.T) {
_, err := schema.SearchAndValidate(dash, b)
require.NoError(t, err, "dashboard failed validation")
})
t.Run("dist", func(t *testing.T) {
_, err := schema.SearchAndValidate(ddash, b)
require.NoError(t, err, "dashboard failed validation")
})
})
return nil
}))
// TODO will need to expand this appropriately when the scuemata contain
// more than one schema
t.Run("base", doTest(dash))
t.Run("dist", doTest(ddash))
}
func TestPanelValidity(t *testing.T) {
pkg/schema/load/panel.go
+2 -2
View File
@@ -16,7 +16,7 @@ import (
// Returns a disjunction of structs representing each panel schema version
// (post-mapping from on-disk #PanelModel form) from each scuemata in the map.
func disjunctPanelScuemata(scuemap map[string]schema.VersionedCueSchema) (cue.Value, error) {
partsi, err := rt.Compile("panelDisjunction", `
partsi, err := rt.Compile("glue-panelDisjunction", `
allPanels: [Name=_]: {}
parts: or([for v in allPanels { v }])
`)
@@ -44,7 +44,7 @@ func disjunctPanelScuemata(scuemap map[string]schema.VersionedCueSchema) (cue.Va
func mapPanelModel(id string, vcs schema.VersionedCueSchema) cue.Value {
maj, min := vcs.Version()
// Ignore err return, this can't fail to compile
inter, _ := rt.Compile("typedPanel", fmt.Sprintf(`
inter, _ := rt.Compile(fmt.Sprintf("%s-glue-panelComposition", id), fmt.Sprintf(`
in: {
type: %q
v: {
pkg/schema/schema.go
+13 -4
View File
@@ -277,7 +277,11 @@ func Exact(maj, min int) SearchOption {
// that are 1) missing in the Resource AND 2) specified by the schema,
// filled with default values specified by the schema.
func ApplyDefaults(r Resource, scue cue.Value) (Resource, error) {
rv, err := rt.Compile("resource", r.Value)
name := r.Name
if name == "" {
name = "resource"
}
rv, err := rt.Compile(name, r.Value)
if err != nil {
return r, err
}
@@ -306,7 +310,11 @@ func convertCUEValueToString(inputCUE cue.Value) (string, error) {
// in the where the values at those paths are the same as the default value
// given in the schema.
func TrimDefaults(r Resource, scue cue.Value) (Resource, error) {
rvInstance, err := rt.Compile("resource", r.Value)
name := r.Name
if name == "" {
name = "resource"
}
rvInstance, err := rt.Compile(name, r.Value)
if err != nil {
return r, err
}
@@ -329,7 +337,7 @@ func isCueValueEqual(inputdef cue.Value, input cue.Value) bool {
func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool, error) {
// To include all optional fields, we need to use inputdef for iteration,
// since the lookuppath with optional field doesn't work very well
rvInstance, err := rt.Compile("resource", []byte{})
rvInstance, err := rt.Compile("helper", []byte{})
if err != nil {
return input, false, err
}
@@ -421,6 +429,7 @@ func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool,
// TODO this is a terrible way to do this, refactor
type Resource struct {
Value interface{}
Name string
}
// WrapCUEError is a wrapper for cueErrors that occur and are not self explanatory.
@@ -430,7 +439,7 @@ func WrapCUEError(err error) error {
var cErr errs.Error
m := make(map[int]string)
if ok := errors.As(err, &cErr); ok {
for _, e := range errs.Errors(err) {
for _, e := range errs.Errors(cErr) {
if e.Position().File() != nil {
line := e.Position().Line()
m[line] = fmt.Sprintf("%q: in file %s", err, e.Position().File().Name())