public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Schemas: Replace registry generation and github workflow (#83490)

Browse Source 
* Create small registries for core and composable kinds

* Update workflow with new registries

* Fix imports in plugin schemas and deleted old registry generation files

* Remove verification and maturity

* Modify registries and add missing composable information to make schemas in kind-registry work

* Add missing aliases

* Remove unused templates

* Remove kinds verification

* Format generated code

* Add gen header

* Delete unused code and clean path in composable template

* Delete kind-registry loader

* Delete unused code

* Update License link

* Update codeowners path

* Sort imports

* More cleanup

* Remove verify-kinds.yml from codeowners

* Fix lint

* Update composable_kidns

* Fix cue extension

* Restore verify-kinds to avoid to push outdated kind's registry

* Fix composable format

* Restore code owners for verify-kinds

* Remove verify check
This commit is contained in:
Selene
2024-03-13 18:05:21 +02:00
committed by GitHub
parent fd9031ca37
commit 5c7849417b
34 changed files with 1049 additions and 1710 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/codegen/generators.go
+6
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"path/filepath"
"cuelang.org/go/cue"
"github.com/grafana/codejen"
"github.com/grafana/kindsys"
"github.com/grafana/thema"
@@ -59,3 +60,8 @@ type SchemaForGen struct {
// Whether the schema is grouped. See https://github.com/grafana/thema/issues/62
IsGroup bool
}
type CueSchema struct {
CueFile cue.Value
FilePath string
}
pkg/codegen/jenny_basecorereg.go
-64
View File
@@ -1,64 +0,0 @@
package codegen
import (
"bytes"
"fmt"
"path/filepath"
"github.com/grafana/codejen"
"github.com/grafana/kindsys"
)
// BaseCoreRegistryJenny generates a static registry for core kinds that
// only initializes their [kindsys.Kind]. No slot kinds are composed.
//
// Path should be the relative path to the directory that will contain the
// generated registry. kindrelroot should be the repo-root-relative path to the
// parent directory to all directories that contain generated kind bindings
// (e.g. pkg/kind).
func BaseCoreRegistryJenny(path, kindrelroot string) ManyToOne {
return &genBaseRegistry{
path: path,
kindrelroot: kindrelroot,
}
}
type genBaseRegistry struct {
path string
kindrelroot string
}
func (gen *genBaseRegistry) JennyName() string {
return "BaseCoreRegistryJenny"
}
func (gen *genBaseRegistry) Generate(kinds ...kindsys.Kind) (*codejen.File, error) {
cores := make([]kindsys.Core, 0, len(kinds))
for _, d := range kinds {
if corekind, is := d.(kindsys.Core); is {
cores = append(cores, corekind)
}
}
if len(cores) == 0 {
return nil, nil
}
buf := new(bytes.Buffer)
if err := tmpls.Lookup("kind_registry.tmpl").Execute(buf, tvars_kind_registry{
PackageName: filepath.Base(gen.path),
KindPackagePrefix: filepath.ToSlash(filepath.Join("github.com/grafana/grafana", gen.kindrelroot)),
Kinds: cores,
}); err != nil {
return nil, fmt.Errorf("failed executing kind registry template: %w", err)
}
b, err := postprocessGoFile(genGoFile{
path: gen.path,
in: buf.Bytes(),
})
if err != nil {
return nil, err
}
return codejen.NewFile(filepath.Join(gen.path, "base_gen.go"), b, gen), nil
}
pkg/codegen/jenny_core_registry.go
+62
View File
@@ -0,0 +1,62 @@
package codegen
import (
"bytes"
"fmt"
"go/format"
"path/filepath"
"strings"
"cuelang.org/go/cue"
"github.com/grafana/codejen"
)
var registryPath = filepath.Join("pkg", "registry", "schemas")
// CoreRegistryJenny generates a registry with all core kinds.
type CoreRegistryJenny struct {
}
func (jenny *CoreRegistryJenny) JennyName() string {
return "CoreRegistryJenny"
}
func (jenny *CoreRegistryJenny) Generate(cueFiles []CueSchema) (codejen.Files, error) {
schemas := make([]Schema, len(cueFiles))
for i, v := range cueFiles {
name, err := getSchemaName(v.CueFile)
if err != nil {
return nil, err
}
schemas[i] = Schema{
Name: name,
FilePath: v.FilePath,
}
}
buf := new(bytes.Buffer)
if err := tmpls.Lookup("core_registry.tmpl").Execute(buf, tvars_registry{
Schemas: schemas,
}); err != nil {
return nil, fmt.Errorf("failed executing kind registry template: %w", err)
}
b, err := format.Source(buf.Bytes())
if err != nil {
return nil, err
}
file := codejen.NewFile(filepath.Join(registryPath, "core_kind.go"), b, jenny)
return codejen.Files{*file}, nil
}
func getSchemaName(v cue.Value) (string, error) {
name, err := getPackageName(v)
if err != nil {
return "", err
}
name = strings.Replace(name, "-", "_", -1)
return strings.ToLower(name), nil
}
pkg/codegen/jenny_corekind.go
-72
View File
@@ -1,72 +0,0 @@
package codegen
import (
"bytes"
"fmt"
"path/filepath"
"github.com/grafana/codejen"
"github.com/grafana/kindsys"
)
// CoreKindJenny generates the implementation of [kindsys.Core] for the provided
// kind declaration.
//
// gokindsdir should be the relative path to the parent directory that contains
// all generated kinds.
//
// This generator only has output for core structured kinds.
func CoreKindJenny(gokindsdir string, cfg *CoreKindJennyConfig) OneToOne {
if cfg == nil {
cfg = new(CoreKindJennyConfig)
}
if cfg.GenDirName == nil {
cfg.GenDirName = func(def kindsys.Kind) string {
return def.Props().Common().MachineName
}
}
return &coreKindJenny{
gokindsdir: gokindsdir,
cfg: cfg,
}
}
// CoreKindJennyConfig holds configuration options for [CoreKindJenny].
type CoreKindJennyConfig struct {
// GenDirName returns the name of the directory in which the file should be
// generated. Defaults to DefForGen.Lineage().Name() if nil.
GenDirName func(kindsys.Kind) string
}
type coreKindJenny struct {
gokindsdir string
cfg *CoreKindJennyConfig
}
var _ OneToOne = &coreKindJenny{}
func (gen *coreKindJenny) JennyName() string {
return "CoreKindJenny"
}
func (gen *coreKindJenny) Generate(kind kindsys.Kind) (*codejen.File, error) {
if _, is := kind.(kindsys.Core); !is {
return nil, nil
}
path := filepath.Join(gen.gokindsdir, gen.cfg.GenDirName(kind), kind.Props().Common().MachineName+"_kind_gen.go")
buf := new(bytes.Buffer)
if err := tmpls.Lookup("kind_core.tmpl").Execute(buf, kind); err != nil {
return nil, fmt.Errorf("failed executing kind_core template for %s: %w", path, err)
}
b, err := postprocessGoFile(genGoFile{
path: path,
in: buf.Bytes(),
})
if err != nil {
return nil, err
}
return codejen.NewFile(path, b, gen), nil
}
pkg/codegen/jenny_k8_resources.go
+3 -3
View File
@@ -18,15 +18,15 @@ func (jenny *K8ResourcesJenny) JennyName() string {
return "K8ResourcesJenny"
}
func (jenny *K8ResourcesJenny) Generate(cueFiles []cue.Value) (codejen.Files, error) {
func (jenny *K8ResourcesJenny) Generate(cueFiles []CueSchema) (codejen.Files, error) {
files := make(codejen.Files, 0)
for _, val := range cueFiles {
pkg, err := getPackageName(val)
pkg, err := getPackageName(val.CueFile)
if err != nil {
return nil, err
}
resource, err := jenny.genResource(pkg, val)
resource, err := jenny.genResource(pkg, val.CueFile)
if err != nil {
return nil, err
}
pkg/codegen/tmpl.go
+10 -6
View File
@@ -7,7 +7,6 @@ import (
"time"
"github.com/grafana/codejen"
"github.com/grafana/kindsys"
)
// All the parsed templates in the tmpl subdirectory
@@ -33,11 +32,7 @@ type (
From string
Leader string
}
tvars_kind_registry struct {
PackageName string
KindPackagePrefix string
Kinds []kindsys.Core
}
tvars_resource struct {
PackageName string
KindName string
@@ -51,4 +46,13 @@ type (
tvars_status struct {
PackageName string
}
tvars_registry struct {
Schemas []Schema
}
Schema struct {
Name string
FilePath string
}
)
pkg/codegen/tmpl/core_registry.tmpl
+46
View File
@@ -0,0 +1,46 @@
package schemas
import (
"os"
"path/filepath"
"runtime"
"cuelang.org/go/cue"
"cuelang.org/go/cue/cuecontext"
)
type CoreKind struct {
Name string
CueFile cue.Value
}
func GetCoreKinds() ([]CoreKind, error) {
ctx := cuecontext.New()
kinds := make([]CoreKind, 0)
_, caller, _, _ := runtime.Caller(0)
root := filepath.Join(caller, "../../../..")
{{- range .Schemas }}
{{ .Name }}Cue, err := loadCueFile(ctx, filepath.Join(root, "{{ .FilePath }}"))
if err != nil {
return nil, err
}
kinds = append(kinds, CoreKind{
Name: "{{ .Name }}",
CueFile: {{ .Name }}Cue,
})
{{- end }}
return kinds, nil
}
func loadCueFile(ctx *cue.Context, path string) (cue.Value, error) {
cueFile, err := os.ReadFile(path)
if err != nil {
return cue.Value{}, err
}
return ctx.CompileBytes(cueFile), nil
}
pkg/codegen/tmpl/kind_core.tmpl
-70
View File
@@ -1,70 +0,0 @@
package {{ .Props.MachineName }}
import (
"github.com/grafana/kindsys"
"github.com/grafana/thema"
"github.com/grafana/thema/vmux"
"github.com/grafana/grafana/pkg/cuectx"
)
// rootrel is the relative path from the grafana repository root to the
// directory containing the .cue files in which this kind is defined. Necessary
// for runtime errors related to the definition and/or lineage to provide
// a real path to the correct .cue file.
const rootrel string = "kinds/{{ .Props.MachineName }}"
// TODO standard generated docs
type Kind struct {
kindsys.Core
lin thema.ConvergentLineage[*Resource]
jcodec vmux.Codec
valmux vmux.ValueMux[*Resource]
}
// type guard - ensure generated Kind type satisfies the kindsys.Core interface
var _ kindsys.Core = &Kind{}
// TODO standard generated docs
func NewKind(rt *thema.Runtime, opts ...thema.BindOption) (*Kind, error) {
def, err := cuectx.LoadCoreKindDef(rootrel, rt.Context(), nil)
if err != nil {
return nil, err
}
k := &Kind{}
k.Core, err = kindsys.BindCore(rt, def, opts...)
if err != nil {
return nil, err
}
// Get the thema.Schema that the meta says is in the current version (which
// codegen ensures is always the latest)
cursch := thema.SchemaP(k.Core.Lineage(), def.Properties.CurrentVersion)
tsch, err := thema.BindType(cursch, &Resource{})
if err != nil {
// Should be unreachable, modulo bugs in the Thema->Go code generator
return nil, err
}
k.jcodec = vmux.NewJSONCodec("{{ .Props.MachineName }}.json")
k.lin = tsch.ConvergentLineage()
k.valmux = vmux.NewValueMux(k.lin.TypedSchema(), k.jcodec)
return k, nil
}
// ConvergentLineage returns the same [thema.Lineage] as Lineage, but bound (see [thema.BindType])
// to the the {{ .Props.Name }} [Resource] type generated from the current schema, v{{ .Props.CurrentVersion }}.
func (k *Kind) ConvergentLineage() thema.ConvergentLineage[*Resource] {
return k.lin
}
// JSONValueMux is a version multiplexer that maps a []byte containing JSON data
// at any schematized dashboard version to an instance of {{ .Props.Name }} [Resource].
//
// Validation and translation errors emitted from this func will identify the
// input bytes as "dashboard.json".
//
// This is a thin wrapper around Thema's [vmux.ValueMux].
func (k *Kind) JSONValueMux(b []byte) (*Resource, thema.TranslationLacunas, error) {
return k.valmux(b)
}
pkg/codegen/tmpl/kind_registry.tmpl
-61
View File
@@ -1,61 +0,0 @@
package {{ .PackageName }}
import (
"fmt"
"sync"
{{range .Kinds }}
"{{ $.KindPackagePrefix }}/{{ .Props.MachineName }}"{{end}}
"github.com/grafana/grafana/pkg/cuectx"
"github.com/grafana/kindsys"
"github.com/grafana/thema"
)
// Base is a registry of all Grafana core kinds. It is designed for use both inside
// of Grafana itself, and for import by external Go programs wanting to work with Grafana's
// kind system.
//
// The registry provides two modes for accessing core kinds:
// * Per-kind methods, which return the kind-specific type, e.g. Dashboard() returns [dashboard.Dashboard].
// * All(), which returns a slice of [kindsys.Core].
//
// Prefer the individual named methods for use cases where the particular kind(s) that
// are needed are known to the caller. For example, a dashboard linter can know that it
// specifically wants the dashboard kind.
//
// Prefer All() when performing operations generically across all kinds. For example,
// a generic HTTP middleware for validating request bodies expected to contain some
// kind-schematized type.
type Base struct {
all []kindsys.Core
{{- range .Kinds }}
{{ .Props.MachineName }} *{{ .Props.MachineName }}.Kind{{end}}
}
// type guards
var (
{{- range .Kinds }}
_ kindsys.Core = &{{ .Props.MachineName }}.Kind{}{{end}}
)
{{range .Kinds }}
// {{ .Props.Name }} returns the [kindsys.Interface] implementation for the {{ .Props.MachineName }} kind.
func (b *Base) {{ .Props.Name }}() *{{ .Props.MachineName }}.Kind {
return b.{{ .Props.MachineName }}
}
{{end}}
func doNewBase(rt *thema.Runtime) *Base {
var err error
reg := &Base{}
{{range .Kinds }}
reg.{{ .Props.MachineName }}, err = {{ .Props.MachineName }}.NewKind(rt)
if err != nil {
panic(fmt.Sprintf("error while initializing the {{ .Props.MachineName }} Kind: %s", err))
}
reg.all = append(reg.all, reg.{{ .Props.MachineName }})
{{end}}
return reg
}
pkg/codegen/util_go.go
-60
View File
@@ -1,74 +1,14 @@
package codegen
import (
"bytes"
"fmt"
"go/format"
"go/parser"
"go/token"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/dave/dst"
"github.com/dave/dst/decorator"
"github.com/dave/dst/dstutil"
"golang.org/x/tools/imports"
)
type genGoFile struct {
path string
walker dstutil.ApplyFunc
in []byte
}
func postprocessGoFile(cfg genGoFile) ([]byte, error) {
fname := filepath.Base(cfg.path)
buf := new(bytes.Buffer)
fset := token.NewFileSet()
gf, err := decorator.ParseFile(fset, fname, string(cfg.in), parser.ParseComments)
if err != nil {
return nil, fmt.Errorf("error parsing generated file: %w", err)
}
if cfg.walker != nil {
dstutil.Apply(gf, cfg.walker, nil)
err = format.Node(buf, fset, gf)
if err != nil {
return nil, fmt.Errorf("error formatting Go AST: %w", err)
}
} else {
buf = bytes.NewBuffer(cfg.in)
}
byt, err := imports.Process(fname, buf.Bytes(), nil)
if err != nil {
return nil, fmt.Errorf("goimports processing failed: %w", err)
}
// Compare imports before and after; warn about performance if some were added
gfa, _ := parser.ParseFile(fset, fname, string(byt), parser.ParseComments)
imap := make(map[string]bool)
for _, im := range gf.Imports {
imap[im.Path.Value] = true
}
var added []string
for _, im := range gfa.Imports {
if !imap[im.Path.Value] {
added = append(added, im.Path.Value)
}
}
if len(added) != 0 {
// TODO improve the guidance in this error if/when we better abstract over imports to generate
fmt.Fprintf(os.Stderr, "The following imports were added by goimports while generating %s: \n\t%s\nRelying on goimports to find imports significantly slows down code generation. Consider adding these to the relevant template.\n", cfg.path, strings.Join(added, "\n\t"))
}
return byt, nil
}
type prefixmod struct {
prefix string
replace string