Codegen: Isolate schema generation code (#98742)

* Create go.mod files for plugins and core kinds

* Update go work and main go.mod dependencies

* Update pfs import

* Missing update of pfs dependency

* Remove fixed cuelang dependency

* Update codeowners

* Update workspace

* Update Dockerfile

* Missing go.mod codeowner

* Use intermediate kin-openapi dependency to make all workspaces to work

pkg/plugins/codegen/pfs/decl.go

@@ -0,0 +1,26 @@
+package pfs
+
+import (
+	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/ast"
+)
+
+type PluginDecl struct {
+	SchemaInterface SchemaInterface
+	CueFile         cue.Value
+	Imports         []*ast.ImportSpec
+	PluginPath      string
+	PluginMeta      Metadata
+}
+
+type SchemaInterface struct {
+	Name    string
+	IsGroup bool
+}
+
+type Metadata struct {
+	Id      string
+	Name    string
+	Backend *bool
+	Version *string
+}

pkg/plugins/codegen/pfs/decl_parser.go

@@ -0,0 +1,63 @@
+package pfs
+
+import (
+	"fmt"
+	"io/fs"
+	"path/filepath"
+	"sort"
+
+	"cuelang.org/go/cue/cuecontext"
+)
+
+type DeclParser struct {
+	skip map[string]bool
+}
+
+func NewDeclParser(skip map[string]bool) *DeclParser {
+	return &DeclParser{
+		skip: skip,
+	}
+}
+
+// TODO convert this to be the new parser for Tree
+func (psr *DeclParser) Parse(root fs.FS) ([]*PluginDecl, error) {
+	ctx := cuecontext.New()
+	// TODO remove hardcoded tree structure assumption, work from root of provided fs
+	plugins, err := fs.Glob(root, "**/**/plugin.json")
+	if err != nil {
+		return nil, fmt.Errorf("error finding plugin dirs: %w", err)
+	}
+
+	decls := make([]*PluginDecl, 0)
+	for _, plugin := range plugins {
+		path := filepath.ToSlash(filepath.Dir(plugin))
+		base := filepath.Base(path)
+		if skip, ok := psr.skip[base]; ok && skip {
+			continue
+		}
+
+		dir, _ := fs.Sub(root, path)
+		pp, err := ParsePluginFS(ctx, dir, path)
+		if err != nil {
+			return nil, fmt.Errorf("parsing plugin failed for %s: %s", dir, err)
+		}
+
+		if !pp.CueFile.Exists() {
+			continue
+		}
+
+		decls = append(decls, &PluginDecl{
+			SchemaInterface: pp.Variant,
+			CueFile:         pp.CueFile,
+			Imports:         pp.CUEImports,
+			PluginMeta:      pp.Properties,
+			PluginPath:      path,
+		})
+	}
+
+	sort.Slice(decls, func(i, j int) bool {
+		return decls[i].PluginPath < decls[j].PluginPath
+	})
+
+	return decls, nil
+}

pkg/plugins/codegen/pfs/doc.go

@@ -0,0 +1,3 @@
+// Package pfs ("ParsedPlugin FS") defines a virtual filesystem representation of Grafana plugins.
+
+package pfs

pkg/plugins/codegen/pfs/errors.go

@@ -0,0 +1,26 @@
+package pfs
+
+import "errors"
+
+// ErrEmptyFS indicates that the fs.FS provided to ParsePluginFS was empty.
+var ErrEmptyFS = errors.New("provided fs.FS is empty")
+
+// ErrNoRootFile indicates that no root plugin.json file exists.
+var ErrNoRootFile = errors.New("no plugin.json at root of fs.fS")
+
+// ErrInvalidRootFile indicates that the root plugin.json file is invalid.
+var ErrInvalidRootFile = errors.New("plugin.json is invalid")
+
+// ErrInvalidGrafanaPluginInstance indicates a plugin's set of .cue
+// grafanaplugin package files are invalid with respect to the GrafanaPlugin
+// spec.
+var ErrInvalidGrafanaPluginInstance = errors.New("grafanaplugin cue instance is invalid")
+
+// ErrInvalidLineage indicates that the plugin contains an invalid lineage
+// declaration, according to Thema's validation rules in
+// ["github.com/grafana/thema".BindLineage].
+var ErrInvalidLineage = errors.New("invalid lineage")
+
+// ErrDisallowedCUEImport indicates that a plugin's grafanaplugin cue package
+// contains that are not on the allowlist.
+var ErrDisallowedCUEImport = errors.New("CUE import is not allowed")

pkg/plugins/codegen/pfs/pfs.go

@@ -0,0 +1,205 @@
+package pfs
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"strings"
+
+	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/errors"
+	"cuelang.org/go/cue/load"
+	"github.com/grafana/grafana/pkg/codegen"
+)
+
+// PackageName is the name of the CUE package that Grafana will load when
+// looking for a Grafana plugin's kind declarations.
+const PackageName = "grafanaplugin"
+
+var schemaInterface = map[string]SchemaInterface{
+	"DataQuery": {
+		Name:    "DataQuery",
+		IsGroup: false,
+	},
+	"PanelCfg": {
+		Name:    "PanelCfg",
+		IsGroup: true,
+	},
+}
+
+// PermittedCUEImports returns the list of import paths that may be used in a
+// plugin's grafanaplugin cue package.
+var PermittedCUEImports = codegen.PermittedCUEImports
+
+func importAllowed(path string) bool {
+	for _, p := range PermittedCUEImports() {
+		if p == path {
+			return true
+		}
+	}
+	return false
+}
+
+var allowedImportsStr string
+
+func init() {
+	all := make([]string, 0, len(PermittedCUEImports()))
+	for _, im := range PermittedCUEImports() {
+		all = append(all, fmt.Sprintf("\t%s", im))
+	}
+	allowedImportsStr = strings.Join(all, "\n")
+}
+
+// ParsePluginFS takes a virtual filesystem and checks that it contains a valid
+// set of files that statically define a Grafana plugin.
+//
+// The fsys must contain a plugin.json at the root, which must be valid
+// according to the [plugindef] schema. If any .cue files exist in the
+// grafanaplugin package, these will also be loaded and validated according to
+// the [GrafanaPlugin] specification. This includes the validation of any custom
+// or composable kinds and their contained lineages, via [thema.BindLineage].
+//
+// This function parses exactly one plugin. It does not descend into
+// subdirectories to search for additional plugin.json or .cue files.
+//
+// [GrafanaPlugin]: https://github.com/grafana/grafana/blob/main/pkg/plugins/pfs/grafanaplugin.cue
+func ParsePluginFS(ctx *cue.Context, fsys fs.FS, dir string) (ParsedPlugin, error) {
+	if fsys == nil {
+		return ParsedPlugin{}, ErrEmptyFS
+	}
+
+	cuefiles, err := fs.Glob(fsys, "*.cue")
+	if err != nil {
+		return ParsedPlugin{}, fmt.Errorf("error globbing for cue files in fsys: %w", err)
+	} else if len(cuefiles) == 0 {
+		return ParsedPlugin{}, nil
+	}
+
+	metadata, err := getPluginMetadata(fsys)
+	if err != nil {
+		return ParsedPlugin{}, err
+	}
+
+	pp := ParsedPlugin{
+		Properties: metadata,
+	}
+
+	if err != nil {
+		return ParsedPlugin{}, err
+	}
+
+	bi := load.Instances(cuefiles, &load.Config{
+		Package: PackageName,
+		Dir:     dir,
+	})[0]
+	if bi.Err != nil {
+		return ParsedPlugin{}, bi.Err
+	}
+
+	for _, f := range bi.Files {
+		for _, im := range f.Imports {
+			ip := strings.Trim(im.Path.Value, "\"")
+			if !importAllowed(ip) {
+				return ParsedPlugin{}, errors.Wrap(errors.Newf(im.Pos(),
+					"import of %q in grafanaplugin cue package not allowed, plugins may only import from:\n%s\n", ip, allowedImportsStr),
+					ErrDisallowedCUEImport)
+			}
+			pp.CUEImports = append(pp.CUEImports, im)
+		}
+	}
+
+	// build.Instance.Files has a comment indicating the CUE authors want to change
+	// its behavior. This is a tripwire to tell us if/when they do that - otherwise, if
+	// the change they make ends up making bi.Files empty, the above loop will silently
+	// become a no-op, and we'd lose enforcement of import restrictions in plugins without
+	// realizing it.
+	if len(bi.Files) != len(bi.BuildFiles) {
+		panic("Refactor required - upstream CUE implementation changed, bi.Files is no longer populated")
+	}
+
+	gpi := ctx.BuildInstance(bi)
+	if gpi.Err() != nil {
+		return ParsedPlugin{}, errors.Wrap(errors.Promote(ErrInvalidGrafanaPluginInstance, pp.Properties.Id), gpi.Err())
+	}
+
+	for name, si := range schemaInterface {
+		iv := gpi.LookupPath(cue.MakePath(cue.Str("composableKinds"), cue.Str(name)))
+		if !iv.Exists() {
+			continue
+		}
+
+		iv = iv.FillPath(cue.MakePath(cue.Str("schemaInterface")), name)
+		iv = iv.FillPath(cue.MakePath(cue.Str("name")), derivePascalName(pp.Properties.Id, pp.Properties.Name)+name)
+		lineageNamePath := iv.LookupPath(cue.MakePath(cue.Str("lineage"), cue.Str("name")))
+		if !lineageNamePath.Exists() {
+			iv = iv.FillPath(cue.MakePath(cue.Str("lineage"), cue.Str("name")), derivePascalName(pp.Properties.Id, pp.Properties.Name)+name)
+		}
+
+		validSchema := iv.LookupPath(cue.ParsePath("lineage.schemas[0].schema"))
+		if !validSchema.Exists() {
+			return ParsedPlugin{}, errors.Wrap(errors.Promote(ErrInvalidGrafanaPluginInstance, pp.Properties.Id), validSchema.Err())
+		}
+		pp.Variant = si
+		pp.CueFile = iv
+	}
+
+	return pp, nil
+}
+
+func getPluginMetadata(fsys fs.FS) (Metadata, error) {
+	b, err := fs.ReadFile(fsys, "plugin.json")
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return Metadata{}, ErrNoRootFile
+		}
+		return Metadata{}, fmt.Errorf("error reading plugin.json: %w", err)
+	}
+
+	var metadata PluginDef
+	if err := json.Unmarshal(b, &metadata); err != nil {
+		return Metadata{}, fmt.Errorf("error unmarshalling plugin.json: %s", err)
+	}
+
+	if err := metadata.Validate(); err != nil {
+		return Metadata{}, err
+	}
+
+	return Metadata{
+		Id:      metadata.Id,
+		Name:    metadata.Name,
+		Backend: metadata.Backend,
+		Version: metadata.Info.Version,
+	}, nil
+}
+
+func derivePascalName(id string, name string) string {
+	sani := func(s string) string {
+		ret := strings.Title(strings.Map(func(r rune) rune {
+			switch {
+			case r >= 'a' && r <= 'z':
+				return r
+			case r >= 'A' && r <= 'Z':
+				return r
+			default:
+				return -1
+			}
+		}, strings.Title(strings.Map(func(r rune) rune {
+			switch r {
+			case '-', '_':
+				return ' '
+			default:
+				return r
+			}
+		}, s))))
+		if len(ret) > 63 {
+			return ret[:63]
+		}
+		return ret
+	}
+
+	fromname := sani(name)
+	if len(fromname) != 0 {
+		return fromname
+	}
+	return sani(strings.Split(id, "-")[1])
+}

pkg/plugins/codegen/pfs/plugin.go

@@ -0,0 +1,22 @@
+package pfs
+
+import (
+	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/ast"
+)
+
+// ParsedPlugin represents everything knowable about a single plugin from static
+// analysis of its filesystem tree contents, as performed by [ParsePluginFS].
+//
+// Guarantees described in the below comments only exist for instances of this
+// struct returned from [ParsePluginFS].
+type ParsedPlugin struct {
+	// Properties contains the plugin's definition, as declared in plugin.json.
+	Properties Metadata
+	CueFile    cue.Value
+	Variant    SchemaInterface
+
+	// CUEImports lists the CUE import statements in the plugin's grafanaplugin CUE
+	// package, if any.
+	CUEImports []*ast.ImportSpec
+}

pkg/plugins/codegen/pfs/plugindef_types.go

@@ -0,0 +1,42 @@
+package pfs
+
+type Type string
+
+// Defines values for Type.
+const (
+	TypeApp            Type = "app"
+	TypeDatasource     Type = "datasource"
+	TypePanel          Type = "panel"
+	TypeRenderer       Type = "renderer"
+	TypeSecretsmanager Type = "secretsmanager"
+)
+
+type PluginDef struct {
+	Id      string
+	Name    string
+	Backend *bool
+	Type    Type
+	Info    Info
+	IAM     IAM
+}
+
+type Info struct {
+	Version *string
+}
+
+type IAM struct {
+	Permissions []Permission `json:"permissions,omitempty"`
+}
+
+type Permission struct {
+	Action string  `json:"action"`
+	Scope  *string `json:"scope,omitempty"`
+}
+
+func (pd PluginDef) Validate() error {
+	if pd.Id == "" || pd.Name == "" || pd.Type == "" {
+		return ErrInvalidRootFile
+	}
+
+	return nil
+}