Reconcile coremodels, entities, objects under new kind framework (#56492)

* Update thema to latest

* Deal with s/Library/*Runtime/

* Commit new, working results of codegen

* We like pointers now

* Always take runtime arg for NewBase()

* Sketchy handwavy pass at entity meta framework

* Little nibbles

* Update pkg/framework/coremodel/entityframework.cue

Co-authored-by: Artur Wierzbicki <wierzbicki.artur.94@gmail.com>

* Move file into new framework location

* Introduce loaders, Go code

* Complete rename to kind

* Flesh out framework, add svg/dashboard examples

* Cruft removal

* Remove generated kind go files from gitignore

* Refine maturity concept, add SlotKind

* Update embed and go deps

* Export PrefixWithGrafanaCUE

* Make the loader actually work, holy crap

* Many small tweaks to type.cue

* Add Apache 2 licensing exceptions for kinds

* Add new kinds dir, start of generator

* Roll back to earlier oapi-codegen

* Introduce new grafana-specific CUE loaders

* Introduce new tidy code generators framework

* Catch up kind framework with tinkering

* Add slices for the generators

* Add write/verify step to main generator

* Many renames

* Split up kind framework cue files

* Use kind.Decl within generated kinds

* Create kind.SomeDecl wrapper type to cache lineages

* Better names again

* Get one generated implemented, hopefully

* Copy dashboard schema into new kind.cue

* Small fixes to make the initial gen work

* Put svg kind in its new home

* Add generated Go dashboard type

* More renames and cleanups

* Add base kind registry and generator

* Stop blacklisting *_gen.go files

This is not the Go best practice, anyway. All we actually want to ignore
for enterprise is generated wire files.

* Change codegen output directories

pkg/kind -> pkg/kinds
pkg/registry/kindreg -> pkg/registry/corekind

* Rename pkg/framework/kind to pkg/kindsys

* Add core structured kind generator

* Add plural and machine names to kind spec

* Copy playlist over to kind system

* Consolidate kindsys files

* Add raw kind generator

* Update CODEOWNERS for kind framework

* Touch up comments a bit

* More docs tweaks

* Remove generated types to reduce noise for review

* Split each generator into its own file

* Rename Slot kind to Composable kind

* Add handwavy types for customkind loading

* Guard against init calls to framework loader

* First pass at doc on extending the kind system

* Improve attribute example in docs

* Fix wire imports

* Add basic TS types generator

* Fix composable kind category def

* No need for a separate file with generate directive

* Catch dashboard schema up

* Rename generator types to something saner and generic

* Make version configurable in ts/go generators

* Add CommonMeta to ease property access

* Add kindsys prop indicating whether lineage is group

* Put all kind categories back in a single file

* Finish with kindsys group props

* Refactor maturity progression per discussion

- Replace "committed" with "merged"
- All kindcats can use all maturity levels, at least for now

* Convert ts veneer index generator to modular system

* Move over to new jennywrites framework

* Strip down old coremodel generator

* Use public version of jennywrites

* Pull latest thema

* Commit generated Go types

* Add header injection postprocessor

* Move sdboyer/jennywrites to grafana/codejen

* Tweak header output

* Remove dashboard and playlist coremodels

* Fix up backend dashboards devenv test

* Fix TS import patterns to new gen filename

* Update internal imports, remove coremodel registry

* Fix compilation errors, wire generation

* Export and replace the prefix dropper

* More Go struct and field name changes

* Last name fixes, hopefully

* Fix lint errors

* Last lint error

Co-authored-by: Artur Wierzbicki <wierzbicki.artur.94@gmail.com>
This commit is contained in:
sam boyer
2022-11-10 12:36:40 -08:00
committed by GitHub
co-authored by Artur Wierzbicki
parent f92d978386
commit 07e5f8117f
68 changed files with 2702 additions and 1208 deletions
+38 -7
View File
@@ -29,7 +29,7 @@ type FooThing struct {
}`,
out: `package foo
type Model struct {
type Foo struct {
Id int64
Ref Thing
}
@@ -52,7 +52,7 @@ type FooThing struct {
}`,
out: `package foo
type Model struct {
type Foo struct {
Id int64
Ref *Thing
}
@@ -77,7 +77,7 @@ type FooThing struct {
}`,
out: `package foo
type Model struct {
type Foo struct {
Id int64
Ref []Thing
PRef []*Thing
@@ -104,7 +104,7 @@ type FooThing struct {
}`,
out: `package foo
type Model struct {
type Foo struct {
Id int64
KeyRef map[Thing]string
ValRef map[string]Thing
@@ -132,7 +132,7 @@ type FooThing struct {
}`,
out: `package foo
type Model struct {
type Foo struct {
Id int64
KeyRef map[*Thing]string
ValRef map[string]*Thing
@@ -154,7 +154,7 @@ type Foo struct {
}`,
out: `package foo
type Model struct {
type Foo struct {
Id int64
FooRef []string
}
@@ -235,6 +235,37 @@ type Thing string
// of objects, only types, so we shouldn't encounter this case.
skip: true,
},
"comments": {
in: `package foo
// Foo is a thing. It should be Foo still.
type Foo struct {
Id int64
Ref FooThing
}
// FooThing is also a thing. We want [FooThing] to be known properly.
// Even if FooThing
// were not a FooThing, in our minds, forever shall it be FooThing.
type FooThing struct {
Id int64
}`,
out: `package foo
// Foo is a thing. It should be Foo still.
type Foo struct {
Id int64
Ref Thing
}
// Thing is also a thing. We want [Thing] to be known properly.
// Even if Thing
// were not a Thing, in our minds, forever shall it be Thing.
type Thing struct {
Id int64
}
`,
},
}
for name, it := range tt {
@@ -250,7 +281,7 @@ type Thing string
t.Fatal(err)
}
drop := makePrefixDropper("Foo", "Model")
drop := PrefixDropper("Foo")
astutil.Apply(inf, drop, nil)
buf := new(bytes.Buffer)
err = format.Node(buf, fset, inf)
+1 -115
View File
@@ -4,11 +4,9 @@ import (
"bytes"
"errors"
"fmt"
"go/ast"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"testing/fstest"
@@ -21,7 +19,6 @@ import (
"github.com/grafana/grafana/pkg/cuectx"
"github.com/grafana/thema"
"github.com/grafana/thema/encoding/openapi"
"golang.org/x/tools/go/ast/astutil"
)
// CoremodelDeclaration contains the results of statically analyzing a Grafana
@@ -218,7 +215,7 @@ func (cd *CoremodelDeclaration) GenerateGoCoremodel(path string) (WriteDiffer, e
fullp := filepath.Join(path, fmt.Sprintf("%s_gen.go", lin.Name()))
byt, err := postprocessGoFile(genGoFile{
path: fullp,
walker: makePrefixDropper(strings.Title(lin.Name()), "Model"),
walker: PrefixDropper(strings.Title(lin.Name())),
in: buf.Bytes(),
})
if err != nil {
@@ -273,117 +270,6 @@ func (cd *CoremodelDeclaration) GenerateTypescriptCoremodel() (*tsast.File, erro
return tf, nil
}
type prefixDropper struct {
str string
base string
rxp *regexp.Regexp
rxpsuff *regexp.Regexp
}
func makePrefixDropper(str, base string) astutil.ApplyFunc {
return (&prefixDropper{
str: str,
base: base,
rxpsuff: regexp.MustCompile(fmt.Sprintf(`%s([a-zA-Z_]*)`, str)),
rxp: regexp.MustCompile(fmt.Sprintf(`%s([\s.,;-])`, str)),
}).applyfunc
}
func depoint(e ast.Expr) ast.Expr {
if star, is := e.(*ast.StarExpr); is {
return star.X
}
return e
}
func (d prefixDropper) applyfunc(c *astutil.Cursor) bool {
n := c.Node()
// fmt.Printf("%T %s\n", c.Node(), ast.Print(nil, c.Node()))
switch x := n.(type) {
case *ast.ValueSpec:
// fmt.Printf("%T %s\n", c.Node(), ast.Print(nil, c.Node()))
d.handleExpr(x.Type)
for _, id := range x.Names {
d.do(id)
}
case *ast.TypeSpec:
// Always do typespecs
d.do(x.Name)
case *ast.Field:
// Don't rename struct fields. We just want to rename type declarations, and
// field value specifications that reference those types.
d.handleExpr(x.Type)
// return false
case *ast.CommentGroup:
for _, c := range x.List {
c.Text = d.rxp.ReplaceAllString(c.Text, d.base+"$1")
c.Text = d.rxpsuff.ReplaceAllString(c.Text, "$1")
}
}
return true
}
func (d prefixDropper) handleExpr(e ast.Expr) {
// Deref a StarExpr, if there is one
expr := depoint(e)
switch x := expr.(type) {
case *ast.Ident:
d.do(x)
case *ast.ArrayType:
if id, is := depoint(x.Elt).(*ast.Ident); is {
d.do(id)
}
case *ast.MapType:
if id, is := depoint(x.Key).(*ast.Ident); is {
d.do(id)
}
if id, is := depoint(x.Value).(*ast.Ident); is {
d.do(id)
}
}
}
func (d prefixDropper) do(n *ast.Ident) {
if n.Name != d.str {
n.Name = strings.TrimPrefix(n.Name, d.str)
} else {
n.Name = d.base
}
}
// GenerateCoremodelRegistry produces Go files that define a registry with
// references to all the Go code that is expected to be generated from the
// provided lineages.
func GenerateCoremodelRegistry(path string, ecl []*CoremodelDeclaration) (WriteDiffer, error) {
var cml []tplVars
for _, ec := range ecl {
cml = append(cml, ec.toTemplateObj())
}
buf := new(bytes.Buffer)
if err := tmpls.Lookup("coremodel_registry.tmpl").Execute(buf, tvars_coremodel_registry{
Header: tvars_autogen_header{
GeneratorPath: "pkg/framework/coremodel/gen.go", // FIXME hardcoding is not OK
},
Coremodels: cml,
}); err != nil {
return nil, fmt.Errorf("failed executing coremodel registry template: %w", err)
}
byt, err := postprocessGoFile(genGoFile{
path: path,
in: buf.Bytes(),
})
if err != nil {
return nil, err
}
wd := NewWriteDiffer()
wd[path] = byt
return wd, nil
}
var tmplTypedef = `{{range .Types}}
{{ with .Schema.Description }}{{ . }}{{ else }}// {{.TypeName}} is the Go representation of a {{.JsonName}}.{{ end }}
//
+61
View File
@@ -0,0 +1,61 @@
package codegen
import (
"bytes"
"fmt"
"github.com/grafana/codejen"
"github.com/grafana/grafana/pkg/kindsys"
"github.com/grafana/thema"
)
type OneToOne codejen.OneToOne[*DeclForGen]
type OneToMany codejen.OneToMany[*DeclForGen]
type ManyToOne codejen.ManyToOne[*DeclForGen]
type ManyToMany codejen.ManyToMany[*DeclForGen]
// ForGen is a codejen input transformer that converts a pure kindsys.SomeDecl into
// a DeclForGen by binding its contained lineage.
func ForGen(rt *thema.Runtime, decl *kindsys.SomeDecl) (*DeclForGen, error) {
lin, err := decl.BindKindLineage(rt)
if err != nil {
return nil, err
}
return &DeclForGen{
SomeDecl: decl,
lin: lin,
}, nil
}
// DeclForGen wraps [kindsys.SomeDecl] to provide trivial caching of
// the lineage declared by the kind (nil for raw kinds).
type DeclForGen struct {
*kindsys.SomeDecl
lin thema.Lineage
}
func (decl *DeclForGen) Lineage() thema.Lineage {
return decl.lin
}
func SlashHeaderMapper(maingen string) codejen.FileMapper {
return func(f codejen.File) (codejen.File, error) {
b := new(bytes.Buffer)
fmt.Fprintf(b, headerTmpl, maingen, f.FromString())
fmt.Fprint(b, string(f.Data))
f.Data = b.Bytes()
return f, nil
}
}
var headerTmpl = `// THIS FILE IS GENERATED. EDITING IS FUTILE.
//
// Generated by:
// %s
// Using jennies:
// %s
//
// Run 'make gen-cue' from repository root to regenerate.
`
+62
View File
@@ -0,0 +1,62 @@
package codegen
import (
"bytes"
"fmt"
"path/filepath"
"github.com/grafana/codejen"
)
// BaseCoreRegistryJenny generates a static registry for core kinds that
// only initializes their [kindsys.Interface]. No slot kinds are composed.
//
// Path should be the relative path to the directory that will contain the
// generated registry. kindrelroot should be the repo-root-relative path to the
// parent directory to all directories that contain generated kind bindings
// (e.g. pkg/kind).
func BaseCoreRegistryJenny(path, kindrelroot string) ManyToOne {
return &genBaseRegistry{
path: path,
kindrelroot: kindrelroot,
}
}
type genBaseRegistry struct {
path string
kindrelroot string
}
func (gen *genBaseRegistry) JennyName() string {
return "BaseCoreRegistryJenny"
}
func (gen *genBaseRegistry) Generate(decls []*DeclForGen) (*codejen.File, error) {
var numRaw int
for _, k := range decls {
if k.IsRaw() {
numRaw++
}
}
buf := new(bytes.Buffer)
if err := tmpls.Lookup("kind_registry.tmpl").Execute(buf, tvars_kind_registry{
NumRaw: numRaw,
NumStructured: len(decls) - numRaw,
PackageName: filepath.Base(gen.path),
KindPackagePrefix: filepath.ToSlash(filepath.Join("github.com/grafana/grafana", gen.kindrelroot)),
Kinds: decls,
}); err != nil {
return nil, fmt.Errorf("failed executing kind registry template: %w", err)
}
b, err := postprocessGoFile(genGoFile{
path: gen.path,
in: buf.Bytes(),
})
if err != nil {
return nil, err
}
return codejen.NewFile(filepath.Join(gen.path, "base_gen.go"), b, gen), nil
}
+71
View File
@@ -0,0 +1,71 @@
package codegen
import (
"bytes"
"fmt"
"path/filepath"
"github.com/grafana/codejen"
)
// CoreStructuredKindJenny generates the implementation of
// [kindsys.Structured] for the provided kind declaration.
//
// gokindsdir should be the relative path to the parent directory that contains
// all generated kinds.
//
// This generator only has output for core structured kinds.
func CoreStructuredKindJenny(gokindsdir string, cfg *CoreStructuredKindGeneratorConfig) OneToOne {
if cfg == nil {
cfg = new(CoreStructuredKindGeneratorConfig)
}
if cfg.GenDirName == nil {
cfg.GenDirName = func(decl *DeclForGen) string {
return decl.Meta.Common().MachineName
}
}
return &genCoreStructuredKind{
gokindsdir: gokindsdir,
cfg: cfg,
}
}
// CoreStructuredKindGeneratorConfig holds configuration options for [CoreStructuredKindJenny].
type CoreStructuredKindGeneratorConfig struct {
// GenDirName returns the name of the directory in which the file should be
// generated. Defaults to DeclForGen.Lineage().Name() if nil.
GenDirName func(*DeclForGen) string
}
type genCoreStructuredKind struct {
gokindsdir string
cfg *CoreStructuredKindGeneratorConfig
}
var _ OneToOne = &genCoreStructuredKind{}
func (gen *genCoreStructuredKind) JennyName() string {
return "CoreStructuredKindJenny"
}
func (gen *genCoreStructuredKind) Generate(decl *DeclForGen) (*codejen.File, error) {
if !decl.IsCoreStructured() {
return nil, nil
}
path := filepath.Join(gen.gokindsdir, gen.cfg.GenDirName(decl), decl.Meta.Common().MachineName+"_kind_gen.go")
buf := new(bytes.Buffer)
if err := tmpls.Lookup("kind_corestructured.tmpl").Execute(buf, decl); err != nil {
return nil, fmt.Errorf("failed executing kind_corestructured template for %s: %w", path, err)
}
b, err := postprocessGoFile(genGoFile{
path: path,
in: buf.Bytes(),
})
if err != nil {
return nil, err
}
return codejen.NewFile(path, b, gen), nil
}
+98
View File
@@ -0,0 +1,98 @@
package codegen
import (
"fmt"
"path/filepath"
"github.com/grafana/codejen"
"github.com/grafana/thema"
"github.com/grafana/thema/encoding/gocode"
"golang.org/x/tools/go/ast/astutil"
)
// GoTypesJenny creates a [OneToOne] that produces Go types for the latest
// Thema schema in a structured kind's lineage.
//
// At minimum, a gokindsdir must be provided. This should be the path to the parent
// directory of the directory in which the types should be generated, relative
// to the project root. For example, if the types for a kind named "foo"
// should live at pkg/kind/foo/foo_gen.go, relpath should be "pkg/kind".
//
// This generator is a no-op for raw kinds.
func GoTypesJenny(gokindsdir string, cfg *GoTypesGeneratorConfig) OneToOne {
if cfg == nil {
cfg = new(GoTypesGeneratorConfig)
}
if cfg.GenDirName == nil {
cfg.GenDirName = func(decl *DeclForGen) string {
return decl.Meta.Common().MachineName
}
}
return &genGoTypes{
gokindsdir: gokindsdir,
cfg: cfg,
}
}
// GoTypesGeneratorConfig holds configuration options for [GoTypesJenny].
type GoTypesGeneratorConfig struct {
// Apply is an optional AST manipulation func that, if provided, will be run
// against the generated Go file prior to running it through goimports.
Apply astutil.ApplyFunc
// GenDirName returns the name of the parent directory in which the type file
// should be generated. If nil, the DeclForGen.Lineage().Name() will be used.
GenDirName func(*DeclForGen) string
// Version of the schema to generate. If nil, latest is generated.
Version *thema.SyntacticVersion
}
type genGoTypes struct {
gokindsdir string
cfg *GoTypesGeneratorConfig
}
func (gen *genGoTypes) JennyName() string {
return "GoTypesJenny"
}
func (gen *genGoTypes) Generate(decl *DeclForGen) (*codejen.File, error) {
if decl.IsRaw() {
return nil, nil
}
var sch thema.Schema
var err error
lin := decl.Lineage()
if gen.cfg.Version == nil {
sch = lin.Latest()
} else {
sch, err = lin.Schema(*gen.cfg.Version)
if err != nil {
return nil, fmt.Errorf("error in configured version for %s generator: %w", *gen.cfg.Version, err)
}
}
// always drop prefixes.
var appf []astutil.ApplyFunc
if gen.cfg.Apply != nil {
appf = append(appf, gen.cfg.Apply)
}
appf = append(appf, PrefixDropper(decl.Meta.Common().Name))
pdir := gen.cfg.GenDirName(decl)
fpath := filepath.Join(gen.gokindsdir, pdir, lin.Name()+"_types_gen.go")
// TODO allow using name instead of machine name in thema generator
b, err := gocode.GenerateTypesOpenAPI(sch, &gocode.TypeConfigOpenAPI{
PackageName: filepath.Base(pdir),
ApplyFuncs: appf,
})
if err != nil {
return nil, err
}
return codejen.NewFile(fpath, b, gen), nil
}
+68
View File
@@ -0,0 +1,68 @@
package codegen
import (
"bytes"
"fmt"
"path/filepath"
"github.com/grafana/codejen"
)
// RawKindJenny generates the implementation of [kindsys.Raw] for the
// provided kind declaration.
//
// gokindsdir should be the relative path to the parent directory that contains
// all generated kinds.
//
// This generator only has output for raw kinds.
func RawKindJenny(gokindsdir string, cfg *RawKindGeneratorConfig) OneToOne {
if cfg == nil {
cfg = new(RawKindGeneratorConfig)
}
if cfg.GenDirName == nil {
cfg.GenDirName = func(decl *DeclForGen) string {
return decl.Meta.Common().MachineName
}
}
return &genRawKind{
gokindsdir: gokindsdir,
cfg: cfg,
}
}
type genRawKind struct {
gokindsdir string
cfg *RawKindGeneratorConfig
}
type RawKindGeneratorConfig struct {
// GenDirName returns the name of the directory in which the file should be
// generated. Defaults to DeclForGen.Lineage().Name() if nil.
GenDirName func(*DeclForGen) string
}
func (gen *genRawKind) JennyName() string {
return "RawKindJenny"
}
func (gen *genRawKind) Generate(decl *DeclForGen) (*codejen.File, error) {
if !decl.IsRaw() {
return nil, nil
}
path := filepath.Join(gen.gokindsdir, gen.cfg.GenDirName(decl), decl.Meta.Common().MachineName+"_kind_gen.go")
buf := new(bytes.Buffer)
if err := tmpls.Lookup("kind_raw.tmpl").Execute(buf, decl); err != nil {
return nil, fmt.Errorf("failed executing kind_raw template for %s: %w", path, err)
}
b, err := postprocessGoFile(genGoFile{
path: path,
in: buf.Bytes(),
})
if err != nil {
return nil, err
}
return codejen.NewFile(path, b, gen), nil
}
+85
View File
@@ -0,0 +1,85 @@
package codegen
import (
"fmt"
"path/filepath"
"github.com/grafana/codejen"
"github.com/grafana/thema"
"github.com/grafana/thema/encoding/typescript"
)
// TSTypesJenny creates a [OneToOne] that produces TypeScript types and
// defaults for the latest Thema schema in a structured kind's lineage.
//
// At minimum, a tskindsdir must be provided. This should be the path to the parent
// directory of the directory in which the types should be generated, relative
// to the project root. For example, if the types for a kind named "foo"
// should live at packages/grafana-schema/src/raw/foo, relpath should be "pkg/kind".
//
// This generator is a no-op for raw kinds.
func TSTypesJenny(tskindsdir string, cfg *TSTypesGeneratorConfig) OneToOne {
if cfg == nil {
cfg = new(TSTypesGeneratorConfig)
}
if cfg.GenDirName == nil {
cfg.GenDirName = func(decl *DeclForGen) string {
return decl.Meta.Common().MachineName
}
}
return &genTSTypes{
tskindsdir: tskindsdir,
cfg: cfg,
}
}
// TSTypesGeneratorConfig holds configuration options for [TSTypesJenny].
type TSTypesGeneratorConfig struct {
// GenDirName returns the name of the parent directory in which the type file
// should be generated. If nil, the DeclForGen.Lineage().Name() will be used.
GenDirName func(*DeclForGen) string
// Version of the schema to generate. If nil, latest is generated.
Version *thema.SyntacticVersion
}
type genTSTypes struct {
tskindsdir string
cfg *TSTypesGeneratorConfig
}
func (gen *genTSTypes) JennyName() string {
return "TSTypesJenny"
}
func (gen *genTSTypes) Generate(decl *DeclForGen) (*codejen.File, error) {
if decl.IsRaw() {
return nil, nil
}
var sch thema.Schema
var err error
lin := decl.Lineage()
if gen.cfg.Version == nil {
sch = lin.Latest()
} else {
sch, err = lin.Schema(*gen.cfg.Version)
if err != nil {
return nil, fmt.Errorf("error in configured version for %s generator: %w", *gen.cfg.Version, err)
}
}
// TODO allow using name instead of machine name in thema generator
f, err := typescript.GenerateTypes(sch, &typescript.TypeConfig{
RootName: decl.Meta.Common().Name,
Group: decl.Meta.Common().LineageIsGroup,
})
if err != nil {
return nil, err
}
return codejen.NewFile(
filepath.Join(gen.tskindsdir, gen.cfg.GenDirName(decl), lin.Name()+"_types.gen.ts"),
[]byte(f.String()),
gen), nil
}
+325
View File
@@ -0,0 +1,325 @@
package codegen
import (
"fmt"
"path/filepath"
"sort"
"strings"
"cuelang.org/go/cue"
"cuelang.org/go/cue/errors"
"github.com/grafana/codejen"
"github.com/grafana/cuetsy/ts"
"github.com/grafana/cuetsy/ts/ast"
"github.com/grafana/grafana/pkg/kindsys"
"github.com/grafana/thema"
"github.com/grafana/thema/encoding/typescript"
)
// TSVeneerIndexJenny generates an index.gen.ts file with references to all
// generated TS types. Elements with the attribute @grafana(TSVeneer="type") are
// exported from a handwritten file, rather than the raw generated types.
//
// The provided dir is the path, relative to the grafana root, to the directory
// that should contain the generated index.
//
// Implicitly depends on output patterns in TSTypesJenny.
// TODO this is wasteful; share-nothing generator model entails re-running the cuetsy gen that TSTypesJenny already did
func TSVeneerIndexJenny(dir string) ManyToOne {
return &genTSVeneerIndex{
dir: dir,
}
}
type genTSVeneerIndex struct {
dir string
}
func (gen *genTSVeneerIndex) JennyName() string {
return "TSVeneerIndexJenny"
}
func (gen *genTSVeneerIndex) Generate(decls []*DeclForGen) (*codejen.File, error) {
tsf := new(ast.File)
for _, decl := range decls {
if decl.IsRaw() {
continue
}
sch := decl.Lineage().Latest()
f, err := typescript.GenerateTypes(sch, &typescript.TypeConfig{
RootName: decl.Meta.Common().Name,
Group: decl.Meta.Common().LineageIsGroup,
})
if err != nil {
return nil, fmt.Errorf("%s: %w", decl.Meta.Common().Name, err)
}
elems, err := gen.extractTSIndexVeneerElements(decl, f)
if err != nil {
return nil, fmt.Errorf("%s: %w", decl.Meta.Common().Name, err)
}
tsf.Nodes = append(tsf.Nodes, elems...)
}
return codejen.NewFile(filepath.Join(gen.dir, "index.gen.ts"), []byte(tsf.String()), gen), nil
}
func (gen *genTSVeneerIndex) extractTSIndexVeneerElements(decl *DeclForGen, tf *ast.File) ([]ast.Decl, error) {
lin := decl.Lineage()
sch := thema.SchemaP(lin, thema.LatestVersion(lin))
comm := decl.Meta.Common()
// Check the root, then walk the tree
rootv := sch.UnwrapCUE()
var raw, custom, rawD, customD ast.Idents
var terr errors.Error
visit := func(p cue.Path, wv cue.Value) bool {
var name string
sels := p.Selectors()
switch len(sels) {
case 0:
name = strings.Title(lin.Name())
fallthrough
case 1:
// Only deal with subpaths that are definitions, for now
// TODO incorporate smarts about grouped lineages here
if name == "" {
if !sels[0].IsDefinition() {
return false
}
// It might seem to make sense that we'd strip replaceout the leading # here for
// definitions. However, cuetsy's tsast actually has the # still present in its
// Ident types, stripping it replaceout on the fly when stringifying.
name = sels[0].String()
}
// Search the generated TS AST for the type and default decl nodes
pair := findDeclNode(name, tf)
if pair.T == nil {
// No generated type for this item, skip it
return false
}
cust, perr := getCustomVeneerAttr(wv)
if perr != nil {
terr = errors.Append(terr, errors.Promote(perr, fmt.Sprintf("%s: ", p.String())))
}
var has bool
for _, tgt := range cust {
has = has || tgt.target == "type"
}
if has {
custom = append(custom, *pair.T)
if pair.D != nil {
customD = append(customD, *pair.D)
}
} else {
raw = append(raw, *pair.T)
if pair.D != nil {
rawD = append(rawD, *pair.D)
}
}
}
return true
}
walk(rootv, visit, nil)
if len(errors.Errors(terr)) != 0 {
return nil, terr
}
vpath := fmt.Sprintf("v%v", thema.LatestVersion(lin)[0])
if decl.Meta.Common().Maturity.Less(kindsys.MaturityStable) {
vpath = "x"
}
ret := make([]ast.Decl, 0)
if len(raw) > 0 {
ret = append(ret, ast.ExportSet{
CommentList: []ast.Comment{ts.CommentFromString(fmt.Sprintf("Raw generated types from %s kind.", comm.Name), 80, false)},
TypeOnly: true,
Exports: raw,
From: ast.Str{Value: fmt.Sprintf("./raw/%s/%s/%s_types.gen", comm.MachineName, vpath, comm.MachineName)},
})
}
if len(rawD) > 0 {
ret = append(ret, ast.ExportSet{
CommentList: []ast.Comment{ts.CommentFromString(fmt.Sprintf("Raw generated default consts from %s kind.", lin.Name()), 80, false)},
TypeOnly: false,
Exports: rawD,
From: ast.Str{Value: fmt.Sprintf("./raw/%s/%s/%s_types.gen", comm.MachineName, vpath, comm.MachineName)},
})
}
vtfile := fmt.Sprintf("./veneer/%s.types", lin.Name())
customstr := fmt.Sprintf(`// The following exported declarations correspond to types in the %s@%s kind's
// schema with attribute @grafana(TSVeneer="type").
//
// The handwritten file for these type and default veneers is expected to be at
// %s.ts.
// This re-export declaration enforces that the handwritten veneer file exists,
// and exports all the symbols in the list.
//
// TODO generate code such that tsc enforces type compatibility between raw and veneer decls`,
lin.Name(), thema.LatestVersion(lin), filepath.ToSlash(filepath.Join(gen.dir, vtfile)))
customComments := []ast.Comment{{Text: customstr}}
if len(custom) > 0 {
ret = append(ret, ast.ExportSet{
CommentList: customComments,
TypeOnly: true,
Exports: custom,
From: ast.Str{Value: vtfile},
})
}
if len(customD) > 0 {
ret = append(ret, ast.ExportSet{
CommentList: customComments,
TypeOnly: false,
Exports: customD,
From: ast.Str{Value: vtfile},
})
}
// TODO emit a decl in the index.gen.ts that ensures any custom veneer types are "compatible" with current version raw types
return ret, nil
}
type declPair struct {
T, D *ast.Ident
}
type tsVeneerAttr struct {
target string
}
func findDeclNode(name string, tf *ast.File) declPair {
var p declPair
for _, decl := range tf.Nodes {
// Peer through export keywords
if ex, is := decl.(ast.ExportKeyword); is {
decl = ex.Decl
}
switch x := decl.(type) {
case ast.TypeDecl:
if x.Name.Name == name {
p.T = &x.Name
}
case ast.VarDecl:
if x.Names.Idents[0].Name == "default"+name {
p.D = &x.Names.Idents[0]
}
}
}
return p
}
func walk(v cue.Value, before func(cue.Path, cue.Value) bool, after func(cue.Path, cue.Value)) {
innerWalk(cue.MakePath(), v, before, after)
}
func innerWalk(p cue.Path, v cue.Value, before func(cue.Path, cue.Value) bool, after func(cue.Path, cue.Value)) {
switch v.Kind() {
default:
if before != nil && !before(p, v) {
return
}
case cue.StructKind:
if before != nil && !before(p, v) {
return
}
iter, err := v.Fields(cue.All())
if err != nil {
panic(err)
}
for iter.Next() {
innerWalk(appendPath(p, iter.Selector()), iter.Value(), before, after)
}
if lv := v.LookupPath(cue.MakePath(cue.AnyString)); lv.Exists() {
innerWalk(appendPath(p, cue.AnyString), lv, before, after)
}
case cue.ListKind:
if before != nil && !before(p, v) {
return
}
list, err := v.List()
if err != nil {
panic(err)
}
for i := 0; list.Next(); i++ {
innerWalk(appendPath(p, cue.Index(i)), list.Value(), before, after)
}
if lv := v.LookupPath(cue.MakePath(cue.AnyIndex)); lv.Exists() {
innerWalk(appendPath(p, cue.AnyString), lv, before, after)
}
}
if after != nil {
after(p, v)
}
}
func appendPath(p cue.Path, sel cue.Selector) cue.Path {
return cue.MakePath(append(p.Selectors(), sel)...)
}
func getCustomVeneerAttr(v cue.Value) ([]tsVeneerAttr, error) {
var attrs []tsVeneerAttr
for _, a := range v.Attributes(cue.ValueAttr) {
if a.Name() != "grafana" {
continue
}
for i := 0; i < a.NumArgs(); i++ {
key, av := a.Arg(i)
if key != "TSVeneer" {
return nil, valError(v, "attribute 'grafana' only allows the arg 'TSVeneer'")
}
aterr := valError(v, "@grafana(TSVeneer=\"x\") requires one or more of the following separated veneer types for x: %s", allowedTSVeneersString())
var some bool
for _, tgt := range strings.Split(av, "|") {
some = true
if !allowedTSVeneers[tgt] {
return nil, aterr
}
attrs = append(attrs, tsVeneerAttr{
target: tgt,
})
}
if !some {
return nil, aterr
}
}
}
sort.Slice(attrs, func(i, j int) bool {
return attrs[i].target < attrs[j].target
})
return attrs, nil
}
var allowedTSVeneers = map[string]bool{
"type": true,
}
func allowedTSVeneersString() string {
var list []string
for tgt := range allowedTSVeneers {
list = append(list, tgt)
}
sort.Strings(list)
return strings.Join(list, "|")
}
func valError(v cue.Value, format string, args ...interface{}) error {
s := v.Source()
if s == nil {
return fmt.Errorf(format, args...)
}
return errors.Newf(s.Pos(), format, args...)
}
+5 -5
View File
@@ -209,7 +209,7 @@ func (pt *PluginTree) GenerateGo(path string, cfg GoGenConfig) (WriteDiffer, err
for subpath, plug := range all {
fullp := filepath.Join(path, subpath)
if cfg.Types {
gwd, err := genGoTypes(plug, path, subpath, cfg.DocPathPrefix)
gwd, err := pgenGoTypes(plug, path, subpath, cfg.DocPathPrefix)
if err != nil {
return nil, fmt.Errorf("error generating go types for %s: %w", fullp, err)
}
@@ -218,7 +218,7 @@ func (pt *PluginTree) GenerateGo(path string, cfg GoGenConfig) (WriteDiffer, err
}
}
if cfg.ThemaBindings {
twd, err := genThemaBindings(plug, path, subpath, cfg.DocPathPrefix)
twd, err := pgenThemaBindings(plug, path, subpath, cfg.DocPathPrefix)
if err != nil {
return nil, fmt.Errorf("error generating thema bindings for %s: %w", fullp, err)
}
@@ -231,7 +231,7 @@ func (pt *PluginTree) GenerateGo(path string, cfg GoGenConfig) (WriteDiffer, err
return wd, nil
}
func genGoTypes(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer, error) {
func pgenGoTypes(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer, error) {
wd := NewWriteDiffer()
for slotname, lin := range plug.SlotImplementations() {
lowslot := strings.ToLower(slotname)
@@ -287,7 +287,7 @@ func genGoTypes(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer,
finalpath := filepath.Join(path, subpath, fmt.Sprintf("types_%s_gen.go", lowslot))
byt, err := postprocessGoFile(genGoFile{
path: finalpath,
walker: makePrefixDropper(strings.Title(lin.Name()), slotname),
walker: PrefixDropper(strings.Title(lin.Name())),
in: buf.Bytes(),
})
if err != nil {
@@ -300,7 +300,7 @@ func genGoTypes(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer,
return wd, nil
}
func genThemaBindings(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer, error) {
func pgenThemaBindings(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer, error) {
wd := NewWriteDiffer()
bindings := make([]tvars_plugin_lineage_binding, 0)
for slotname, lin := range plug.SlotImplementations() {
+6 -3
View File
@@ -29,9 +29,12 @@ type (
LineageCUEPath string
GenLicense bool
}
tvars_coremodel_registry struct {
Header tvars_autogen_header
Coremodels []tplVars
tvars_kind_registry struct {
// Header tvars_autogen_header
NumRaw, NumStructured int
PackageName string
KindPackagePrefix string
Kinds []*DeclForGen
}
tvars_coremodel_imports struct {
PackageName string
-58
View File
@@ -1,58 +0,0 @@
{{ template "autogen_header.tmpl" .Header }}
package registry
import (
"fmt"
"sync"
"github.com/google/wire"
{{range .Coremodels }}
"{{ .PkgPath }}"{{end}}
"github.com/grafana/grafana/pkg/cuectx"
"github.com/grafana/grafana/pkg/framework/coremodel"
"github.com/grafana/thema"
)
// Base is a registry of coremodel.Interface. It provides two modes for accessing
// coremodels: individually via literal named methods, or as a slice returned from All().
//
// Prefer the individual named methods for use cases where the particular coremodel(s) that
// are needed are known to the caller. For example, a dashboard linter can know that it
// specifically wants the dashboard coremodel.
//
// Prefer All() when performing operations generically across all coremodels. For example,
// a validation HTTP middleware for any coremodel-schematized object type.
type Base struct {
all []coremodel.Interface
{{- range .Coremodels }}
{{ .Name }} *{{ .Name }}.Coremodel{{end}}
}
// type guards
var (
{{- range .Coremodels }}
_ coremodel.Interface = &{{ .Name }}.Coremodel{}{{end}}
)
{{range .Coremodels }}
// {{ .TitleName }} returns the {{ .Name }} coremodel. The return value is guaranteed to
// implement coremodel.Interface.
func (b *Base) {{ .TitleName }}() *{{ .Name }}.Coremodel {
return b.{{ .Name }}
}
{{end}}
func doProvideBase(rt *thema.Runtime) *Base {
var err error
reg := &Base{}
{{range .Coremodels }}
reg.{{ .Name }}, err = {{ .Name }}.New(rt)
if err != nil {
panic(fmt.Sprintf("error while initializing {{ .Name }} coremodel: %s", err))
}
reg.all = append(reg.all, reg.{{ .Name }})
{{end}}
return reg
}
+95
View File
@@ -0,0 +1,95 @@
package {{ .Meta.MachineName }}
import (
"github.com/grafana/grafana/pkg/kindsys"
"github.com/grafana/thema"
"github.com/grafana/thema/vmux"
)
// rootrel is the relative path from the grafana repository root to the
// directory containing the .cue files in which this kind is declared. Necessary
// for runtime errors related to the declaration and/or lineage to provide
// a real path to the correct .cue file.
const rootrel string = "kinds/structured/{{ .Meta.MachineName }}"
// TODO standard generated docs
type Kind struct {
lin thema.ConvergentLineage[*{{ .Meta.Name }}]
jendec vmux.Endec
valmux vmux.ValueMux[*{{ .Meta.Name }}]
decl kindsys.Decl[kindsys.CoreStructuredMeta]
}
// type guard
var _ kindsys.Structured = &Kind{}
// TODO standard generated docs
func NewKind(rt *thema.Runtime, opts ...thema.BindOption) (*Kind, error) {
decl, err := kindsys.LoadCoreKind[kindsys.CoreStructuredMeta](rootrel, rt.Context(), nil)
if err != nil {
return nil, err
}
k := &Kind{
decl: *decl,
}
lin, err := decl.Some().BindKindLineage(rt, opts...)
if err != nil {
return nil, err
}
// Get the thema.Schema that the meta says is in the current version (which
// codegen ensures is always the latest)
cursch := thema.SchemaP(lin, k.decl.Meta.CurrentVersion)
tsch, err := thema.BindType[*{{ .Meta.Name }}](cursch, &{{ .Meta.Name }}{})
if err != nil {
// Should be unreachable, modulo bugs in the Thema->Go code generator
return nil, err
}
k.jendec = vmux.NewJSONEndec("{{ .Meta.MachineName }}.json")
k.lin = tsch.ConvergentLineage()
k.valmux = vmux.NewValueMux(k.lin.TypedSchema(), k.jendec)
return k, nil
}
// TODO standard generated docs
func (k *Kind) Name() string {
return "{{ .Meta.MachineName }}"
}
// TODO standard generated docs
func (k *Kind) MachineName() string {
return "{{ .Meta.MachineName }}"
}
// TODO standard generated docs
func (k *Kind) Lineage() thema.Lineage {
return k.lin
}
// TODO standard generated docs
func (k *Kind) ConvergentLineage() thema.ConvergentLineage[*{{ .Meta.Name }}] {
return k.lin
}
// JSONValueMux is a version multiplexer that maps a []byte containing JSON data
// at any schematized dashboard version to an instance of {{ .Meta.Name }}.
//
// Validation and translation errors emitted from this func will identify the
// input bytes as "dashboard.json".
//
// This is a thin wrapper around Thema's [vmux.ValueMux].
func (k *Kind) JSONValueMux(b []byte) (*{{ .Meta.Name }}, thema.TranslationLacunas, error) {
return k.valmux(b)
}
// TODO standard generated docs
func (k *Kind) Maturity() kindsys.Maturity {
return k.decl.Meta.Maturity
}
// TODO standard generated docs
func (k *Kind) Meta() kindsys.CoreStructuredMeta {
return k.decl.Meta
}
+47
View File
@@ -0,0 +1,47 @@
package {{ .Meta.MachineName }}
import (
"github.com/grafana/grafana/pkg/kindsys"
"github.com/grafana/thema"
"github.com/grafana/thema/vmux"
)
// TODO standard generated docs
type Kind struct {
decl kindsys.Decl[kindsys.RawMeta]
}
// type guard
var _ kindsys.Raw = &Kind{}
// TODO standard generated docs
func NewKind() (*Kind, error) {
decl, err := kindsys.LoadCoreKind[kindsys.RawMeta]("kinds/raw/{{ .Meta.MachineName }}", nil, nil)
if err != nil {
return nil, err
}
return &Kind{
decl: *decl,
}, nil
}
// TODO standard generated docs
func (k *Kind) Name() string {
return "{{ .Meta.Name }}"
}
// TODO standard generated docs
func (k *Kind) MachineName() string {
return "{{ .Meta.MachineName }}"
}
// TODO standard generated docs
func (k *Kind) Maturity() kindsys.Maturity {
return k.decl.Meta.Maturity
}
// TODO standard generated docs
func (k *Kind) Meta() kindsys.RawMeta {
return k.decl.Meta
}
+60
View File
@@ -0,0 +1,60 @@
package {{ .PackageName }}
import (
"fmt"
"sync"
{{range .Kinds }}
"{{ $.KindPackagePrefix }}/{{ .Meta.MachineName }}"{{end}}
"github.com/grafana/grafana/pkg/cuectx"
"github.com/grafana/grafana/pkg/kindsys"
"github.com/grafana/thema"
)
// Base is a registry of kindsys.Interface. It provides two modes for accessing
// kinds: individually via literal named methods, or as a slice returned from
// an All*() method.
//
// Prefer the individual named methods for use cases where the particular kind(s) that
// are needed are known to the caller. For example, a dashboard linter can know that it
// specifically wants the dashboard kind.
//
// Prefer All*() methods when performing operations generically across all kinds.
// For example, a validation HTTP middleware for any kind-schematized object type.
type Base struct {
all []kindsys.Interface
numRaw, numStructured int
{{- range .Kinds }}
{{ .Meta.MachineName }} *{{ .Meta.MachineName }}.Kind{{end}}
}
// type guards
var (
{{- range .Kinds }}
_ kindsys.{{ if .IsRaw }}Raw{{ else }}Structured{{ end }} = &{{ .Meta.MachineName }}.Kind{}{{end}}
)
{{range .Kinds }}
// {{ .Meta.Name }} returns the [kindsys.Interface] implementation for the {{ .Meta.MachineName }} kind.
func (b *Base) {{ .Meta.Name }}() *{{ .Meta.MachineName }}.Kind {
return b.{{ .Meta.MachineName }}
}
{{end}}
func doNewBase(rt *thema.Runtime) *Base {
var err error
reg := &Base{
numRaw: {{ .NumRaw }},
numStructured: {{ .NumStructured }},
}
{{range .Kinds }}
reg.{{ .Meta.MachineName }}, err = {{ .Meta.MachineName }}.NewKind({{ if .IsCoreStructured }}rt{{ end }})
if err != nil {
panic(fmt.Sprintf("error while initializing the {{ .Meta.MachineName }} Kind: %s", err))
}
reg.all = append(reg.all, reg.{{ .Meta.MachineName }})
{{end}}
return reg
}
+83
View File
@@ -3,11 +3,13 @@ package codegen
import (
"bytes"
"fmt"
"go/ast"
"go/format"
"go/parser"
"go/token"
"os"
"path/filepath"
"regexp"
"strings"
"golang.org/x/tools/go/ast/astutil"
@@ -65,3 +67,84 @@ func postprocessGoFile(cfg genGoFile) ([]byte, error) {
return byt, nil
}
type prefixmod struct {
str string
base string
rxp *regexp.Regexp
rxpsuff *regexp.Regexp
}
// PrefixDropper returns an astutil.ApplyFunc that removes the provided prefix
// string when it appears as a leading sequence in type names, var names, and
// comments in a generated Go file.
func PrefixDropper(prefix string) astutil.ApplyFunc {
return (&prefixmod{
str: prefix,
rxpsuff: regexp.MustCompile(fmt.Sprintf(`%s([a-zA-Z_]+)`, prefix)),
rxp: regexp.MustCompile(fmt.Sprintf(`%s([\s.,;-])`, prefix)),
}).applyfunc
}
func depoint(e ast.Expr) ast.Expr {
if star, is := e.(*ast.StarExpr); is {
return star.X
}
return e
}
func (d prefixmod) applyfunc(c *astutil.Cursor) bool {
n := c.Node()
switch x := n.(type) {
case *ast.ValueSpec:
d.handleExpr(x.Type)
for _, id := range x.Names {
d.do(id)
}
case *ast.TypeSpec:
// Always do typespecs
d.do(x.Name)
case *ast.Field:
// Don't rename struct fields. We just want to rename type declarations, and
// field value specifications that reference those types.
d.handleExpr(x.Type)
case *ast.CommentGroup:
for _, c := range x.List {
c.Text = d.rxpsuff.ReplaceAllString(c.Text, "$1")
if d.base != "" {
c.Text = d.rxp.ReplaceAllString(c.Text, d.base+"$1")
}
}
}
return true
}
func (d prefixmod) handleExpr(e ast.Expr) {
// Deref a StarExpr, if there is one
expr := depoint(e)
switch x := expr.(type) {
case *ast.Ident:
d.do(x)
case *ast.ArrayType:
if id, is := depoint(x.Elt).(*ast.Ident); is {
d.do(id)
}
case *ast.MapType:
if id, is := depoint(x.Key).(*ast.Ident); is {
d.do(id)
}
if id, is := depoint(x.Value).(*ast.Ident); is {
d.do(id)
}
}
}
func (d prefixmod) do(n *ast.Ident) {
if n.Name != d.str {
n.Name = strings.TrimPrefix(n.Name, d.str)
} else if d.base != "" {
n.Name = d.base
}
}