plugins: New static scanner and validator, with Thema slot support (#53754)
* coremodels: Convert plugin-metadata schema to a coremodel * Newer cuetsy; try quoting field name * Add slot definitions * Start sketching out pfs package * Rerun codegen with fixes, new cuetsy * Catch up dashboard with new cuetsy * Update to go1.18 * Use new vmuxers in thema * Add slot system in Go * Draft finished implementation of pfs * Collapse slot pkg into coremodel dir; add PluginInfo * Add the mux type on top of kernel * Refactor plugin generator for extensibility * Change models.cue package, numerous debugs * Bring new output to parity with old * Remove old plugin generation logic * Misc tweaking * Reintroduce generation of shared schemas * Drop back to go1.17 * Add globbing to tsconfig exclude * Introduce pfs test on existing testdata * Make most existing testdata tests pass with pfs * coremodels: Convert plugin-metadata schema to a coremodel * Newer cuetsy; try quoting field name * Add APIType control concept, regen pluginmeta * Use proper numeric types for schema fields * Make pluginmeta schema follow Go type breakdown * More decomposition into distinct types * Add test case for no plugin.json file * Fix missing ref to #Dependencies * Remove generated TS for pluginmeta * Update dependencies, rearrange go.mod * Regenerate without Model prefix * Use updated thema loader; this is now runnable * Skip app plugin with weird include * Make plugin tree extractor reusable * Split out slot lineage load/validate logic * Add myriad tests for new plugin validation failures * Add test for zip fixtures * One last run of codegen * Proper delinting * Ensure validation order is deterministic * Let there actually be sorting * Undo reliance on builtIn field (#54009) * undo builtIn reliance * fix tests Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>
This commit is contained in:
sam boyer
Will Browne
@@ -12,6 +12,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"cuelang.org/go/cue/cuecontext"
|
||||
"cuelang.org/go/cue/load"
|
||||
"github.com/grafana/cuetsy"
|
||||
gcgen "github.com/grafana/grafana/pkg/codegen"
|
||||
"github.com/grafana/thema"
|
||||
)
|
||||
@@ -52,7 +54,7 @@ func main() {
|
||||
if item.IsDir() {
|
||||
lin, err := gcgen.ExtractLineage(filepath.Join(cmroot, item.Name(), "coremodel.cue"), lib)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not process coremodel dir %s: %s\n", cmroot, err)
|
||||
fmt.Fprintf(os.Stderr, "could not process coremodel dir %s: %s\n", filepath.Join(cmroot, item.Name()), err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -90,6 +92,14 @@ func main() {
|
||||
}
|
||||
wd.Merge(regfiles)
|
||||
|
||||
// TODO generating these is here temporarily until we make a more permanent home
|
||||
wdsh, err := genSharedSchemas(groot)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "TS gen error for shared schemas in %s: %w", filepath.Join(groot, "packages", "grafana-schema", "src", "schema"), err)
|
||||
os.Exit(1)
|
||||
}
|
||||
wd.Merge(wdsh)
|
||||
|
||||
if _, set := os.LookupEnv("CODEGEN_VERIFY"); set {
|
||||
err = wd.Verify()
|
||||
if err != nil {
|
||||
@@ -104,3 +114,36 @@ func main() {
|
||||
}
|
||||
}
|
||||
}
|
||||
func genSharedSchemas(groot string) (gcgen.WriteDiffer, error) {
|
||||
abspath := filepath.Join(groot, "packages", "grafana-schema", "src", "schema")
|
||||
cfg := &load.Config{
|
||||
ModuleRoot: groot,
|
||||
Module: "github.com/grafana/grafana",
|
||||
Dir: abspath,
|
||||
}
|
||||
|
||||
bi := load.Instances(nil, cfg)
|
||||
if len(bi) > 1 {
|
||||
return nil, fmt.Errorf("loading CUE files in %s resulted in more than one instance", abspath)
|
||||
}
|
||||
|
||||
ctx := cuecontext.New()
|
||||
v := ctx.BuildInstance(bi[0])
|
||||
if v.Err() != nil {
|
||||
return nil, fmt.Errorf("errors while building CUE in %s: %s", abspath, v.Err())
|
||||
}
|
||||
|
||||
b, err := cuetsy.Generate(v, cuetsy.Config{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate TS: %w", err)
|
||||
}
|
||||
|
||||
wd := gcgen.NewWriteDiffer()
|
||||
wd[filepath.Join(abspath, "mudball.gen.ts")] = append([]byte(`//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
// This file is autogenerated. DO NOT EDIT.
|
||||
//
|
||||
// To regenerate, run "make gen-cue" from the repository root.
|
||||
//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
`), b...)
|
||||
return wd, nil
|
||||
}
|
||||
|
||||
@@ -1 +1,176 @@
|
||||
package coremodel
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"testing/fstest"
|
||||
|
||||
"cuelang.org/go/cue"
|
||||
"cuelang.org/go/cue/load"
|
||||
"github.com/grafana/grafana/pkg/cuectx"
|
||||
"github.com/grafana/thema/kernel"
|
||||
tload "github.com/grafana/thema/load"
|
||||
)
|
||||
|
||||
// Embed for all framework-related CUE files in this directory
|
||||
//
|
||||
//go:embed *.cue
|
||||
var cueFS embed.FS
|
||||
|
||||
var defaultFramework cue.Value
|
||||
|
||||
func init() {
|
||||
var err error
|
||||
defaultFramework, err = doLoadFrameworkCUE(cuectx.ProvideCUEContext())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
var prefix = filepath.Join("/pkg", "framework", "coremodel")
|
||||
|
||||
//nolint:nakedret
|
||||
func doLoadFrameworkCUE(ctx *cue.Context) (v cue.Value, err error) {
|
||||
m := make(fstest.MapFS)
|
||||
|
||||
err = fs.WalkDir(cueFS, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
b, err := fs.ReadFile(cueFS, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m[path] = &fstest.MapFile{Data: b}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
over := make(map[string]load.Source)
|
||||
err = tload.ToOverlay(prefix, m, over)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
bi := load.Instances(nil, &load.Config{
|
||||
Dir: prefix,
|
||||
Package: "coremodel",
|
||||
Overlay: over,
|
||||
})
|
||||
v = ctx.BuildInstance(bi[0])
|
||||
|
||||
if v.Err() != nil {
|
||||
return cue.Value{}, fmt.Errorf("coremodel framework loaded cue.Value has err: %w", v.Err())
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// CUEFramework returns a cue.Value representing all the coremodel framework
|
||||
// raw CUE files.
|
||||
//
|
||||
// For low-level use in constructing other types and APIs, while still letting
|
||||
// us declare all the frameworky CUE bits in a single package. Other types and
|
||||
// subpackages make the constructs in this value easy to use.
|
||||
//
|
||||
// The returned cue.Value is built from Grafana's standard central CUE context,
|
||||
// ["github.com/grafana/grafana/pkg/cuectx".ProvideCueContext].
|
||||
func CUEFramework() cue.Value {
|
||||
return defaultFramework
|
||||
}
|
||||
|
||||
// CUEFrameworkWithContext is the same as CUEFramework, but allows control over
|
||||
// the cue.Context that's used.
|
||||
//
|
||||
// Prefer CUEFramework unless you understand cue.Context, and absolutely need
|
||||
// this control.
|
||||
func CUEFrameworkWithContext(ctx *cue.Context) cue.Value {
|
||||
// Error guaranteed to be nil here because erroring would have caused init() to panic
|
||||
v, _ := doLoadFrameworkCUE(ctx) // nolint:errcheck
|
||||
return v
|
||||
}
|
||||
|
||||
// Mux takes a coremodel and returns a Thema version muxer that, given a byte
|
||||
// slice containing any version of schema for that coremodel, will translate it
|
||||
// to the Interface.CurrentSchema() version, and optionally decode it onto the
|
||||
// Interface.GoType().
|
||||
//
|
||||
// By default, JSON decoding will be used, and the filename given to any input
|
||||
// bytes (shown in errors, which may be user-facing) will be
|
||||
// "<name>.<encoding>", e.g. dashboard.json.
|
||||
func Mux(cm Interface, opts ...MuxOption) kernel.InputKernel {
|
||||
c := &muxConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(c)
|
||||
}
|
||||
|
||||
cfg := kernel.InputKernelConfig{
|
||||
Typ: cm.GoType(),
|
||||
Lineage: cm.Lineage(),
|
||||
To: cm.CurrentSchema().Version(),
|
||||
}
|
||||
|
||||
switch c.decodetyp {
|
||||
case "", "json": // json by default
|
||||
if c.filename == "" {
|
||||
c.filename = fmt.Sprintf("%s.json", cm.Lineage().Name())
|
||||
}
|
||||
cfg.Loader = kernel.NewJSONDecoder(c.filename)
|
||||
case "yaml":
|
||||
if c.filename == "" {
|
||||
c.filename = fmt.Sprintf("%s.yaml", cm.Lineage().Name())
|
||||
}
|
||||
cfg.Loader = kernel.NewYAMLDecoder(c.filename)
|
||||
default:
|
||||
panic("")
|
||||
}
|
||||
|
||||
mux, err := kernel.NewInputKernel(cfg)
|
||||
if err != nil {
|
||||
// Barring a fundamental bug in Thema's schema->Go type assignability checker or
|
||||
// a direct attempt by a Grafana dev to get around the invariants of coremodel codegen,
|
||||
// this should be unreachable. (And even the latter case should be caught elsewhere
|
||||
// by tests).
|
||||
panic(err)
|
||||
}
|
||||
return mux
|
||||
}
|
||||
|
||||
// A MuxOption defines options that may be specified only at initial
|
||||
// construction of a Lineage via BindLineage.
|
||||
type MuxOption muxOption
|
||||
|
||||
// Internal representation of MuxOption.
|
||||
type muxOption func(c *muxConfig)
|
||||
|
||||
type muxConfig struct {
|
||||
filename string
|
||||
decodetyp string
|
||||
}
|
||||
|
||||
// YAML indicates that the resulting Mux should look for YAML in input bytes,
|
||||
// rather than the default JSON.
|
||||
func YAML() MuxOption {
|
||||
return func(c *muxConfig) {
|
||||
c.decodetyp = "yaml"
|
||||
}
|
||||
}
|
||||
|
||||
// Filename specifies the filename that is given to input bytes passing through
|
||||
// the mux.
|
||||
//
|
||||
// The filename has no impact on mux behavior, but is used in user-facing error
|
||||
// output, such as schema validation failures. Thus, it is recommended to pick a
|
||||
// name that will make sense in the context a user is expected to see the error.
|
||||
func Filename(name string) MuxOption {
|
||||
return func(c *muxConfig) {
|
||||
c.filename = name
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package coremodel
|
||||
|
||||
import (
|
||||
"cuelang.org/go/cue"
|
||||
)
|
||||
|
||||
// Slot represents one of Grafana's named Thema composition slot definitions.
|
||||
type Slot struct {
|
||||
name string
|
||||
raw cue.Value
|
||||
plugins map[string]bool
|
||||
}
|
||||
|
||||
// Name returns the name of the Slot. The name is also used as the path at which
|
||||
// a Slot lineage is defined in a plugin models.cue file.
|
||||
func (s Slot) Name() string {
|
||||
return s.name
|
||||
}
|
||||
|
||||
// MetaSchema returns the meta-schema that is the contract between coremodels
|
||||
// that compose the Slot, and plugins that implement it.
|
||||
func (s Slot) MetaSchema() cue.Value {
|
||||
return s.raw
|
||||
}
|
||||
|
||||
// ForPluginType indicates whether for this Slot, plugins of the given type may
|
||||
// provide a slot implementation (first return value), and for those types that
|
||||
// may, whether they must produce one (second return value).
|
||||
//
|
||||
// Expected values here are those in the set of
|
||||
// ["github.com/grafana/grafana/pkg/coremodel/pluginmeta".Type], though passing
|
||||
// a string not in that set will harmlessly return {false, false}. That type is
|
||||
// not used here to avoid import cycles.
|
||||
//
|
||||
// Note that, at least for now, plugins are not required to provide any slot
|
||||
// implementations, and do so by simply not containing a models.cue file.
|
||||
// Consequently, the "must" return value here is best understood as, "IF a
|
||||
// plugin provides a models.cue file, it MUST contain an implementation of this
|
||||
// slot."
|
||||
func (s Slot) ForPluginType(plugintype string) (may, must bool) {
|
||||
must, may = s.plugins[plugintype]
|
||||
return
|
||||
}
|
||||
|
||||
func AllSlots() map[string]*Slot {
|
||||
fw := CUEFramework()
|
||||
slots := make(map[string]*Slot)
|
||||
|
||||
// Ignore err, can only happen if we change structure of fw files, and all we'd
|
||||
// do is panic and that's what the next line will do anyway. Same for similar ignored
|
||||
// errors later in this func
|
||||
iter, _ := fw.LookupPath(cue.ParsePath("pluginTypeMetaSchema")).Fields(cue.Optional(true))
|
||||
type nameopt struct {
|
||||
name string
|
||||
req bool
|
||||
}
|
||||
plugslots := make(map[string][]nameopt)
|
||||
for iter.Next() {
|
||||
plugin := iter.Selector().String()
|
||||
iiter, _ := iter.Value().Fields(cue.Optional(true))
|
||||
for iiter.Next() {
|
||||
slotname := iiter.Selector().String()
|
||||
plugslots[slotname] = append(plugslots[slotname], nameopt{
|
||||
name: plugin,
|
||||
req: !iiter.IsOptional(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
iter, _ = fw.LookupPath(cue.ParsePath("slots")).Fields(cue.Optional(true))
|
||||
for iter.Next() {
|
||||
n := iter.Selector().String()
|
||||
sl := Slot{
|
||||
name: n,
|
||||
raw: iter.Value(),
|
||||
plugins: make(map[string]bool),
|
||||
}
|
||||
|
||||
for _, no := range plugslots[n] {
|
||||
sl.plugins[no.name] = no.req
|
||||
}
|
||||
|
||||
slots[n] = &sl
|
||||
}
|
||||
|
||||
return slots
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
// Package Slot exposes Grafana's coremodel composition Slot definitions for use in Go.
|
||||
package slot
|
||||
@@ -0,0 +1,71 @@
|
||||
package coremodel
|
||||
|
||||
// The slots named and specified in this file are meta-schemas that act as a
|
||||
// shared contract between Grafana plugins (producers) and coremodel types
|
||||
// (consumers).
|
||||
//
|
||||
// On the consumer side, any coremodel Thema lineage can choose to define a
|
||||
// standard Thema composition slot that specifies one of these named slots as
|
||||
// its meta-schema. Such a specification entails that all schemas in any lineage
|
||||
// placed into that composition slot must adhere to the meta-schema.
|
||||
//
|
||||
// On the producer side, Grafana's plugin system enforces that certain plugin
|
||||
// types are expected to provide Thema lineages for these named slots which
|
||||
// adhere to the slot meta-schema.
|
||||
//
|
||||
// For example, the Panel slot is consumed by the dashboard coremodel, and is
|
||||
// expected to be produced by panel plugins.
|
||||
//
|
||||
// The name given to each slot in this file must be used as the name of the
|
||||
// slot in the coremodel, and the name of the field under which the lineage
|
||||
// is provided in a plugin's models.cue file.
|
||||
//
|
||||
// Conformance to meta-schema is achieved by Thema's native lineage joinSchema,
|
||||
// which Thema internals automatically enforce across all schemas in a lineage.
|
||||
|
||||
// Meta-schema for the Panel slot, as implemented in Grafana panel plugins.
|
||||
//
|
||||
// This is a grouped meta-schema, intended solely for use in composition. Object
|
||||
// literals conforming to it are not expected to exist.
|
||||
slots: Panel: {
|
||||
// Defines plugin-specific options for a panel that should be persisted. Required,
|
||||
// though a panel without any options may specify an empty struct.
|
||||
//
|
||||
// Currently mapped to #Panel.options within the dashboard schema.
|
||||
PanelOptions: {...}
|
||||
// Plugin-specific custom field properties. Optional.
|
||||
//
|
||||
// Currently mapped to #Panel.fieldConfig.defaults.custom within the dashboard schema.
|
||||
PanelFieldConfig?: {...}
|
||||
}
|
||||
|
||||
// Meta-schema for the Query slot, as implemented in Grafana datasource plugins.
|
||||
slots: Query: {...}
|
||||
|
||||
// Meta-schema for the DSOptions slot, as implemented in Grafana datasource plugins.
|
||||
//
|
||||
// This is a grouped meta-schema, intended solely for use in composition. Object
|
||||
// literals conforming to it are not expected to exist.
|
||||
slots: DSOptions: {
|
||||
// Normal datasource configuration options.
|
||||
Options: {...}
|
||||
// Sensitive datasource configuration options that require encryption.
|
||||
SecureOptions: {...}
|
||||
}
|
||||
|
||||
// pluginTypeMetaSchema defines which plugin types should use which metaschemas
|
||||
// as joinSchema for the lineages declared at which paths.
|
||||
pluginTypeMetaSchema: [string]: {...}
|
||||
pluginTypeMetaSchema: {
|
||||
// Panel plugins are expected to provide a lineage at path Panel conforming to
|
||||
// the Panel joinSchema.
|
||||
panel: {
|
||||
Panel: slots.Panel
|
||||
}
|
||||
// Datasource plugins are expected to provide lineages at paths Query and
|
||||
// DSOptions, conforming to those joinSchemas respectively.
|
||||
datasource: {
|
||||
Query: slots.Query
|
||||
DSOptions: slots.DSOptions
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user