public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor dashboard scuemata to introduce composition slots, allow unspecified panels to validate, and re-enable devenv validation (#38727)

Browse Source 
* Re-enable devenv dashboard validation

* Open up dashboard schema composition points

* Introduce composition space at front of scuemata

* Refactor go code to use new composition structure

* Bunch of small cleanups in dashboard.cue

* Enable both base and dist tests of devenv

* Get rid of obsolete CUE loading hacks

* Skip weird failures on these tests

Really don't seem to be testing for what we intend them to be testing
for.
This commit is contained in:
sam boyer
2021-09-06 18:53:42 -04:00
committed by GitHub
parent 0ecc24d1c0
commit 96473004db
8 changed files with 130 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/schema/load/dashboard.go
+37 -25
View File
@@ -36,10 +36,19 @@ func defaultOverlay(p BaseLoadPaths) (map[string]load.Source, error) {
// family: the 0.0 schema. schema.Find() provides easy traversal to newer schema
// versions.
func BaseDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
overlay, err := defaultOverlay(p)
v, err := baseDashboardFamily(p)
if err != nil {
return nil, err
}
return buildGenericScuemata(v)
}
// Helper that gets the entire scuemata family, for reuse by Dist/Instance callers.
func baseDashboardFamily(p BaseLoadPaths) (cue.Value, error) {
overlay, err := defaultOverlay(p)
if err != nil {
return cue.Value{}, err
}
cfg := &load.Config{
Overlay: overlay,
@@ -51,16 +60,16 @@ func BaseDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
if err != nil {
cueError := schema.WrapCUEError(err)
if err != nil {
return nil, cueError
return cue.Value{}, cueError
}
}
famval := inst.Value().LookupPath(cue.MakePath(cue.Str("Family")))
if !famval.Exists() {
return nil, errors.New("dashboard schema family did not exist at expected path in expected file")
return cue.Value{}, errors.New("dashboard schema family did not exist at expected path in expected file")
}
return buildGenericScuemata(famval)
return famval, nil
}
// DistDashboardFamily loads the family of schema representing the "Dist"
@@ -73,38 +82,41 @@ func BaseDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
// family: the 0.0 schema. schema.Find() provides easy traversal to newer schema
// versions.
func DistDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
head, err := BaseDashboardFamily(p)
famval, err := baseDashboardFamily(p)
if err != nil {
return nil, err
}
scuemap, err := readPanelModels(p)
scuemap, err := loadPanelScuemata(p)
if err != nil {
return nil, err
}
dj, err := disjunctPanelScuemata(scuemap)
if err != nil {
return nil, err
}
// Stick this into a dummy struct so that we can unify it into place, as
// Value.Fill() can't target definitions. Need new method based on cue.Path;
// a CL has been merged that creates FillPath and will be in the next
// release of CUE.
dummy, _ := rt.Compile("glue-unifyPanelDashboard", `
obj: {}
dummy: {
#Panel: obj
}
`)
filled := dummy.Value().FillPath(cue.MakePath(cue.Str("obj")), dj)
ddj := filled.LookupPath(cue.MakePath(cue.Str("dummy")))
// TODO see if unifying into the expected form in a loop, then unifying that
// consolidated form improves performance
for typ, fam := range scuemap {
famval = famval.FillPath(cue.MakePath(cue.Str("compose"), cue.Str("Panel"), cue.Str(typ)), fam)
}
head, err := buildGenericScuemata(famval)
if err != nil {
return nil, err
}
// TODO sloppy duplicate logic of what's in readPanelModels(), for now
all := make(map[string]schema.VersionedCueSchema)
for id, val := range scuemap {
fam, err := buildGenericScuemata(val)
if err != nil {
return nil, err
}
all[id] = fam
}
var first, prev *compositeDashboardSchema
for head != nil {
cds := &compositeDashboardSchema{
base: head,
actual: head.CUE().Unify(ddj),
panelFams: scuemap,
actual: head.CUE(),
panelFams: all,
// TODO migrations
migration: terminalMigrationFunc,
}
@@ -118,7 +130,6 @@ func DistDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
prev = cds
head = head.Successor()
}
return first, nil
}
@@ -182,6 +193,7 @@ func (cds *compositeDashboardSchema) LatestPanelSchemaFor(id string) (schema.Ver
}
latest := schema.Find(psch, schema.Latest())
// FIXME this relies on old sloppiness
sch := &genericVersionedSchema{
actual: cds.base.CUE().LookupPath(panelSubpath).Unify(mapPanelModel(id, latest)),
}
pkg/schema/load/load_test.go
+3 -7
View File
@@ -50,8 +50,6 @@ func TestScuemataBasics(t *testing.T) {
}
func TestDevenvDashboardValidity(t *testing.T) {
t.Skip()
validdir := filepath.Join("..", "..", "..", "devenv", "dev-dashboards")
doTest := func(sch schema.VersionedCueSchema) func(t *testing.T) {
@@ -109,11 +107,9 @@ func TestDevenvDashboardValidity(t *testing.T) {
// TODO will need to expand this appropriately when the scuemata contain
// more than one schema
// TODO disabled because base variant validation currently must fail in order for
// dist/instance validation to do closed validation of plugin-specified fields
// t.Run("base", doTest(dash))
// dash, err := BaseDashboardFamily(p)
// require.NoError(t, err, "error while loading base dashboard scuemata")
dash, err := BaseDashboardFamily(p)
require.NoError(t, err, "error while loading base dashboard scuemata")
t.Run("base", doTest(dash))
ddash, err := DistDashboardFamily(p)
require.NoError(t, err, "error while loading dist dashboard scuemata")
pkg/schema/load/panel.go
+4 -33
View File
@@ -13,34 +13,10 @@ import (
"github.com/grafana/grafana/pkg/schema"
)
// Returns a disjunction of structs representing each panel schema version
// (post-mapping from on-disk #PanelModel form) from each scuemata in the map.
func disjunctPanelScuemata(scuemap map[string]schema.VersionedCueSchema) (cue.Value, error) {
partsi, err := rt.Compile("glue-panelDisjunction", `
allPanels: [Name=_]: {}
parts: or([for v in allPanels { v }])
`)
if err != nil {
return cue.Value{}, err
}
parts := partsi.Value()
for id, sch := range scuemap {
for sch != nil {
cv := mapPanelModel(id, sch)
mjv, miv := sch.Version()
parts = parts.FillPath(cue.MakePath(cue.Str("allPanels"), cue.Str(fmt.Sprintf("%s@%v.%v", id, mjv, miv))), cv)
sch = sch.Successor()
}
}
return parts.LookupPath(cue.MakePath(cue.Str("parts"))), nil
}
// mapPanelModel maps a schema from the #PanelModel form in which it's declared
// in a plugin's model.cue to the structure in which it actually appears in the
// dashboard schema.
// TODO remove, this is old sloppy hacks
func mapPanelModel(id string, vcs schema.VersionedCueSchema) cue.Value {
maj, min := vcs.Version()
// Ignore err return, this can't fail to compile
@@ -69,7 +45,7 @@ func mapPanelModel(id string, vcs schema.VersionedCueSchema) cue.Value {
return inter.Value().FillPath(cue.MakePath(cue.Str("in"), cue.Str("model")), vcs.CUE()).LookupPath(cue.MakePath(cue.Str(("result"))))
}
func readPanelModels(p BaseLoadPaths) (map[string]schema.VersionedCueSchema, error) {
func loadPanelScuemata(p BaseLoadPaths) (map[string]cue.Value, error) {
overlay := make(map[string]load.Source)
if err := toOverlay(prefix, p.BaseCueFS, overlay); err != nil {
@@ -89,7 +65,7 @@ func readPanelModels(p BaseLoadPaths) (map[string]schema.VersionedCueSchema, err
return nil, errors.New("could not locate #PanelFamily definition")
}
all := make(map[string]schema.VersionedCueSchema)
all := make(map[string]cue.Value)
err = fs.WalkDir(p.DistPluginCueFS, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
@@ -149,13 +125,8 @@ func readPanelModels(p BaseLoadPaths) (map[string]schema.VersionedCueSchema, err
return err
}
// Create a generic schema family to represent the whole of the
fam, err := buildGenericScuemata(pmod)
if err != nil {
return err
}
all[id] = pmod
all[id] = fam
return nil
})
if err != nil {