public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactory to move trim/apply default into schema.go (#33754)

Browse Source 
* refactory to move trim/apply default into schema.go

* fix comments

* move schema test outside of load folder
This commit is contained in:
ying-jeanne
2021-05-12 15:38:00 +08:00
committed by GitHub
parent 679001051b
commit a7ea0ca849
12 changed files with 250 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/schema/load/applydefault_test.go
-115
View File
@@ -1,115 +0,0 @@
package load
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"path/filepath"
"testing"
"cuelang.org/go/cue"
"github.com/google/go-cmp/cmp"
"github.com/grafana/grafana/pkg/schema"
"golang.org/x/tools/txtar"
)
var CasesDir = filepath.Join("testdata", "artifacts", "dashboards", "trimdefault")
type Case struct {
Name string
CUE string
Full string
Trimed string
}
func TestGenerate(t *testing.T) {
cases, err := loadCases(CasesDir)
if err != nil {
t.Fatal(err)
}
for _, c := range cases {
t.Run(c.Name+" apply default value", func(t *testing.T) {
var r cue.Runtime
scmInstance, err := r.Compile(c.Name+".cue", c.CUE)
if err != nil {
t.Fatal(err)
}
inputResource := schema.Resource{Value: c.Trimed}
scm := genericVersionedSchema{actual: scmInstance.Value()}
out, err := scm.ApplyDefaults(inputResource)
if err != nil {
t.Fatal(err)
}
b := []byte(out.Value.(string))
if s := cmp.Diff(string(b), c.Full); s != "" {
t.Fatal(s)
}
})
}
for _, c := range cases {
t.Run(c.Name+" trim default value", func(t *testing.T) {
var r cue.Runtime
scmInstance, err := r.Compile(c.Name+".cue", c.CUE)
if err != nil {
t.Fatal(err)
}
inputResource := schema.Resource{Value: c.Full}
scm := genericVersionedSchema{actual: scmInstance.Value()}
out, err := scm.TrimDefaults(inputResource)
if err != nil {
t.Fatal(err)
}
b := []byte(out.Value.(string))
if s := cmp.Diff(string(b), c.Trimed); s != "" {
t.Fatal(s)
}
})
}
}
func loadCases(dir string) ([]Case, error) {
files, err := ioutil.ReadDir(dir)
if err != nil {
return nil, err
}
var cases []Case
for _, fi := range files {
file := filepath.Join(dir, fi.Name())
a, err := txtar.ParseFile(file)
if err != nil {
return nil, err
}
if len(a.Files) != 3 {
return nil, fmt.Errorf("Malformed test case '%s': Must contain exactly three files (CUE, Full and Trimed), but has %d", file, len(a.Files))
}
fullBuffer := new(bytes.Buffer)
fullJson := a.Files[1].Data
if err := json.Compact(fullBuffer, fullJson); err != nil {
return nil, err
}
trimBuffer := new(bytes.Buffer)
trimedJson := a.Files[2].Data
if err := json.Compact(trimBuffer, trimedJson); err != nil {
return nil, err
}
cases = append(cases, Case{
Name: fi.Name(),
CUE: string(a.Files[0].Data),
Full: fullBuffer.String(),
Trimed: trimBuffer.String(),
})
}
return cases, nil
}
pkg/schema/load/dashboard.go
-14
View File
@@ -129,20 +129,6 @@ func (cds *compositeDashboardSchema) Validate(r schema.Resource) error {
return cds.actual.Unify(rv.Value()).Validate(cue.Concrete(true))
}
// ApplyDefaults returns a new, concrete copy of the Resource with all paths
// that are 1) missing in the Resource AND 2) specified by the schema,
// filled with default values specified by the schema.
func (cds *compositeDashboardSchema) ApplyDefaults(_ schema.Resource) (schema.Resource, error) {
panic("not implemented") // TODO: Implement
}
// TrimDefaults returns a new, concrete copy of the Resource where all paths
// in the where the values at those paths are the same as the default value
// given in the schema.
func (cds *compositeDashboardSchema) TrimDefaults(_ schema.Resource) (schema.Resource, error) {
panic("not implemented") // TODO: Implement
}
// CUE returns the cue.Value representing the actual schema.
func (cds *compositeDashboardSchema) CUE() cue.Value {
return cds.actual
pkg/schema/load/generic.go
-141
View File
@@ -1,13 +1,8 @@
package load
import (
"bytes"
"fmt"
"strings"
"cuelang.org/go/cue"
"cuelang.org/go/cue/load"
cuejson "cuelang.org/go/pkg/encoding/json"
"github.com/grafana/grafana/pkg/schema"
)
@@ -108,142 +103,6 @@ func (gvs *genericVersionedSchema) Validate(r schema.Resource) error {
return gvs.actual.Unify(rv.Value()).Validate(cue.Concrete(true))
}
// ApplyDefaults returns a new, concrete copy of the Resource with all paths
// that are 1) missing in the Resource AND 2) specified by the schema,
// filled with default values specified by the schema.
func (gvs *genericVersionedSchema) ApplyDefaults(r schema.Resource) (schema.Resource, error) {
rv, err := rt.Compile("resource", r.Value)
if err != nil {
return r, err
}
rvUnified := rv.Value().Unify(gvs.CUE())
re, err := convertCUEValueToString(rvUnified)
if err != nil {
return r, err
}
return schema.Resource{Value: re}, nil
}
func convertCUEValueToString(inputCUE cue.Value) (string, error) {
re, err := cuejson.Marshal(inputCUE)
if err != nil {
return re, err
}
result := []byte(re)
result = bytes.Replace(result, []byte("\\u003c"), []byte("<"), -1)
result = bytes.Replace(result, []byte("\\u003e"), []byte(">"), -1)
result = bytes.Replace(result, []byte("\\u0026"), []byte("&"), -1)
return string(result), nil
}
// TrimDefaults returns a new, concrete copy of the Resource where all paths
// in the where the values at those paths are the same as the default value
// given in the schema.
func (gvs *genericVersionedSchema) TrimDefaults(r schema.Resource) (schema.Resource, error) {
rvInstance, err := rt.Compile("resource", r.Value)
if err != nil {
return r, err
}
rv, _, err := removeDefaultHelper(gvs.CUE(), rvInstance.Value())
if err != nil {
return r, err
}
re, err := convertCUEValueToString(rv)
fmt.Println("the trimed fields would be: ", re)
if err != nil {
return r, err
}
return schema.Resource{Value: re}, nil
}
func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool, error) {
// Since for now, panel definition is open validation,
// we need to loop on the input CUE for trimming
rvInstance, err := rt.Compile("resource", []byte{})
if err != nil {
return input, false, err
}
rv := rvInstance.Value()
switch inputdef.IncompleteKind() {
case cue.StructKind:
// Get all fields including optional fields
iter, err := inputdef.Fields(cue.Optional(true))
if err != nil {
return rv, false, err
}
keySet := make(map[string]bool)
for iter.Next() {
lable, _ := iter.Value().Label()
keySet[lable] = true
lv := input.LookupPath(cue.MakePath(cue.Str(lable)))
if err != nil {
continue
}
if lv.Exists() {
re, isEqual, err := removeDefaultHelper(iter.Value(), lv)
if err == nil && !isEqual {
rv = rv.FillPath(cue.MakePath(cue.Str(lable)), re)
}
}
}
// Get all the fields that are not defined in schema yet for panel
iter, err = input.Fields()
if err != nil {
return rv, false, err
}
for iter.Next() {
lable, _ := iter.Value().Label()
if exists := keySet[lable]; !exists {
rv = rv.FillPath(cue.MakePath(cue.Str(lable)), iter.Value())
}
}
return rv, false, nil
case cue.ListKind:
val, _ := inputdef.Default()
err1 := input.Subsume(val)
err2 := val.Subsume(input)
if val.IsConcrete() && err1 == nil && err2 == nil {
return rv, true, nil
}
ele := inputdef.LookupPath(cue.MakePath(cue.AnyIndex))
if ele.IncompleteKind() == cue.BottomKind {
return rv, true, nil
}
iter, err := input.List()
if err != nil {
return rv, true, nil
}
var iterlist []string
for iter.Next() {
re, isEqual, err := removeDefaultHelper(ele, iter.Value())
if err == nil && !isEqual {
reString, err := convertCUEValueToString(re)
if err != nil {
return rv, true, nil
}
iterlist = append(iterlist, reString)
}
}
iterlistContent := fmt.Sprintf("[%s]", strings.Join(iterlist, ","))
liInstance, err := rt.Compile("resource", []byte(iterlistContent))
if err != nil {
return rv, false, err
}
return liInstance.Value(), false, nil
default:
val, _ := inputdef.Default()
err1 := input.Subsume(val)
err2 := val.Subsume(input)
if val.IsConcrete() && err1 == nil && err2 == nil {
return input, true, nil
}
return input, false, nil
}
}
// CUE returns the cue.Value representing the actual schema.
func (gvs *genericVersionedSchema) CUE() cue.Value {
return gvs.actual
pkg/schema/load/load_test.go
+1 -1
View File
@@ -51,7 +51,7 @@ func TestDashboardValidity(t *testing.T) {
// TODO FIXME remove this once we actually have dashboard schema filled in
// enough that the tests pass, lol
t.Skip()
validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards", "basic"))
validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards"))
dash, err := BaseDashboardFamily(p)
require.NoError(t, err, "error while loading base dashboard scuemata")
pkg/schema/load/testdata/artifacts/dashboards/basic/basic.json → pkg/schema/load/testdata/artifacts/dashboards/basic.json
Vendored
View File
pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test1
Vendored
-35
View File
@@ -1,35 +0,0 @@
Verifies common usecases for trimdefault/applydefault functions:
* Basic cases
* Real dashboard
-- CUE --
{
id?: number
uid: string
gnetId?: string
style: *"light" | "dark"
timezone?: *"browser" | "utc"
editable: bool | *true
schemaVersion: number | *25
version?: number
graphTooltip: >=0 & <=2 | *0
}
-- Full --
{
"id": 42,
"uid": "emal8gQMz",
"style": "light",
"editable": true,
"schemaVersion": 27,
"version": 2,
"graphTooltip": 0
}
-- Trimed --
{
"id": 42,
"uid": "emal8gQMz",
"schemaVersion": 27,
"version": 2
}
pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test2
Vendored
-41
View File
@@ -1,41 +0,0 @@
Verifies common usecases for trimdefault/applydefault functions:
* Basic cases
* Real dashboard
-- CUE --
{
timepicker?: {
collapse: bool | *false
enable: bool | *true
hidden: bool | *false
refresh_intervals: [...string] | *["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"]
}
}
-- Full --
{
"timepicker": {
"collapse": true,
"enable":true,
"hidden":false,
"refresh_intervals":[
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
}
}
-- Trimed --
{
"timepicker": {
"collapse": true
}
}
pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test3
Vendored
-46
View File
@@ -1,46 +0,0 @@
Verifies common usecases for trimdefault/applydefault functions:
* Nested struct
* Simple array
-- CUE --
{
annotations?: list: [...{
builtIn: number | *0
datasource: string
enable?: bool | *true
hide?: bool | *false
iconColor?: string
name?: string
type: string | *"dashboard"
rawQuery?: string
showIn: number | *0
}]
}
-- Full --
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"name": "Annotations & Alerts",
"type": "dashboard",
"showIn": 0
}
]
}
}
-- Trimed --
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"name": "Annotations & Alerts"
}
]
}
}
pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test4
Vendored
-73
View File
@@ -1,73 +0,0 @@
Verifies common usecases for trimdefault/applydefault functions:
* open structure should be kept when fields not present
-- CUE --
{
templating?: list: [...{...}]
}
-- Full --
{
"templating": {
"list": [
{
"allValue": null,
"current": {
"text": "America",
"value": "America"
},
"datasource": "gdev-postgres",
"definition": "",
"hide": 0,
"includeAll": false,
"label": "Datacenter",
"multi": false,
"name": "datacenter",
"options": [],
"query": "SELECT DISTINCT datacenter FROM grafana_metric",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
}
}
-- Trimed --
{
"templating": {
"list": [
{
"allValue": null,
"current": {
"text": "America",
"value": "America"
},
"datasource": "gdev-postgres",
"definition": "",
"hide": 0,
"includeAll": false,
"label": "Datacenter",
"multi": false,
"name": "datacenter",
"options": [],
"query": "SELECT DISTINCT datacenter FROM grafana_metric",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
}
}