plugins: New static scanner and validator, with Thema slot support (#53754)

* coremodels: Convert plugin-metadata schema to a coremodel

* Newer cuetsy; try quoting field name

* Add slot definitions

* Start sketching out pfs package

* Rerun codegen with fixes, new cuetsy

* Catch up dashboard with new cuetsy

* Update to go1.18

* Use new vmuxers in thema

* Add slot system in Go

* Draft finished implementation of pfs

* Collapse slot pkg into coremodel dir; add PluginInfo

* Add the mux type on top of kernel

* Refactor plugin generator for extensibility

* Change models.cue package, numerous debugs

* Bring new output to parity with old

* Remove old plugin generation logic

* Misc tweaking

* Reintroduce generation of shared schemas

* Drop back to go1.17

* Add globbing to tsconfig exclude

* Introduce pfs test on existing testdata

* Make most existing testdata tests pass with pfs

* coremodels: Convert plugin-metadata schema to a coremodel

* Newer cuetsy; try quoting field name

* Add APIType control concept, regen pluginmeta

* Use proper numeric types for schema fields

* Make pluginmeta schema follow Go type breakdown

* More decomposition into distinct types

* Add test case for no plugin.json file

* Fix missing ref to #Dependencies

* Remove generated TS for pluginmeta

* Update dependencies, rearrange go.mod

* Regenerate without Model prefix

* Use updated thema loader; this is now runnable

* Skip app plugin with weird include

* Make plugin tree extractor reusable

* Split out slot lineage load/validate logic

* Add myriad tests for new plugin validation failures

* Add test for zip fixtures

* One last run of codegen

* Proper delinting

* Ensure validation order is deterministic

* Let there actually be sorting

* Undo reliance on builtIn field (#54009)

* undo builtIn reliance

* fix tests

Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>

embed.go

@@ -6,5 +6,5 @@ import (
 
 // CueSchemaFS embeds all schema-related CUE files in the Grafana project.
 //
-//go:embed cue.mod/module.cue packages/grafana-schema/src/schema/*.cue public/app/plugins/*/*/*.cue public/app/plugins/*/*/plugin.json
+//go:embed cue.mod/module.cue packages/grafana-schema/src/schema/*.cue public/app/plugins/*/*/*.cue public/app/plugins/*/*/plugin.json pkg/framework/coremodel/*.cue
 var CueSchemaFS embed.FS

go.mod

@@ -55,6 +55,7 @@ require (
 	github.com/grafana/grafana-aws-sdk v0.10.8
 	github.com/grafana/grafana-azure-sdk-go v1.3.0
 	github.com/grafana/grafana-plugin-sdk-go v0.139.0
+	github.com/grafana/thema v0.0.0-20220817114012-ebeee841c104
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/hashicorp/go-hclog v1.0.0
 	github.com/hashicorp/go-plugin v1.4.3
@@ -93,6 +94,7 @@ require (
 	github.com/vectordotdev/go-datemath v0.1.1-0.20220323213446-f3954d0b18ae
 	github.com/weaveworks/common v0.0.0-20210913144402-035033b78a78 // indirect
 	github.com/xorcare/pointer v1.1.0
+	github.com/yalue/merged_fs v1.2.2
 	github.com/yudai/gojsondiff v1.0.0
 	go.opentelemetry.io/collector v0.31.0
 	go.opentelemetry.io/collector/model v0.31.0
@@ -247,7 +249,6 @@ require (
 	github.com/golang-migrate/migrate/v4 v4.7.0
 	github.com/google/go-github/v45 v45.2.0
 	github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f
-	github.com/grafana/thema v0.0.0-20220816214754-af057f99a2dd
 	github.com/jmoiron/sqlx v1.3.5
 	go.etcd.io/etcd/api/v3 v3.5.4
 	go.opentelemetry.io/contrib/propagators/jaeger v1.6.0

go.sum

@@ -1358,6 +1358,10 @@ github.com/grafana/saml v0.4.9-0.20220727151557-61cd9c9353fc h1:1PY8n+rXuBNr3r1J
 github.com/grafana/saml v0.4.9-0.20220727151557-61cd9c9353fc/go.mod h1:9Zh6dWPtB3MSzTRt8fIFH60Z351QQ+s7hCU3J/tTlA4=
 github.com/grafana/thema v0.0.0-20220816214754-af057f99a2dd h1:OukQ1Nu4PSreZTAaOfXyYhM9jYBs4UflVfOSAIG8JzM=
 github.com/grafana/thema v0.0.0-20220816214754-af057f99a2dd/go.mod h1:fCV1rqv6XRQg2GfIQ7pU9zdxd5fLRcEBCnrDVwlK+ZY=
+github.com/grafana/thema v0.0.0-20220816215847-acc0b0aca0f0 h1:jp0SAT7/Xo9NMND8zEwPo7urvSx0EhgNKXOQ0x4s2PE=
+github.com/grafana/thema v0.0.0-20220816215847-acc0b0aca0f0/go.mod h1:fCV1rqv6XRQg2GfIQ7pU9zdxd5fLRcEBCnrDVwlK+ZY=
+github.com/grafana/thema v0.0.0-20220817114012-ebeee841c104 h1:dYpwFYIChrMfpq3wDa/ZBxAbUGSW5NYmYBeSezhaoao=
+github.com/grafana/thema v0.0.0-20220817114012-ebeee841c104/go.mod h1:fCV1rqv6XRQg2GfIQ7pU9zdxd5fLRcEBCnrDVwlK+ZY=
 github.com/grafana/xorm v0.8.3-0.20220614223926-2fcda7565af6 h1:I9dh1MXGX0wGyxdV/Sl7+ugnki4Dfsy8lv2s5Yf887o=
 github.com/grafana/xorm v0.8.3-0.20220614223926-2fcda7565af6/go.mod h1:ZkJLEYLoVyg7amJK/5r779bHyzs2AU8f8VMiP6BM7uY=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -2483,6 +2487,8 @@ github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd
 github.com/xorcare/pointer v1.1.0 h1:sFwXOhRF8QZ0tyVZrtxWGIoVZNEmRzBCaFWdONPQIUM=
 github.com/xorcare/pointer v1.1.0/go.mod h1:6KLhkOh6YbuvZkT4YbxIbR/wzLBjyMxOiNzZhJTor2Y=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yalue/merged_fs v1.2.2 h1:vXHTpJBluJryju7BBpytr3PDIkzsPMpiEknxVGPhN/I=
+github.com/yalue/merged_fs v1.2.2/go.mod h1:WqqchfVYQyclV2tnR7wtRhBddzBvLVR83Cjw9BKQw0M=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA=
 github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=

packages/grafana-schema/src/schema/mudball.gen.ts

@@ -4,8 +4,6 @@
 // To regenerate, run "make gen-cue" from the repository root.
 //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-
-
 export enum AxisPlacement {
   Auto = 'auto',
   Bottom = 'bottom',

packages/grafana-schema/tsconfig.json

@@ -6,7 +6,7 @@
     "rootDirs": ["."]
   },
   // dashboard_experimental.gen.ts needs ignoring as isolatedModules requires it to contain an import or export statement.
-  "exclude": ["dist/**/*", "src/schema/dashboard/dashboard_experimental.gen.ts"],
+  "exclude": ["dist/**/*", "src/schema/*/*_experimental.gen.ts"],
   "extends": "@grafana/tsconfig",
   "include": ["src/**/*.ts*"]
 }

pkg/codegen/pluggen.go

@@ -2,186 +2,172 @@ package codegen
 
 import (
 	"bytes"
-	gerrors "errors"
 	"fmt"
-	"io"
 	"io/fs"
 	"path/filepath"
+	"sort"
 	"strings"
 	"text/template"
 
-	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/ast"
-	"cuelang.org/go/cue/build"
-	"cuelang.org/go/cue/cuecontext"
-	"cuelang.org/go/cue/errors"
-	"cuelang.org/go/cue/load"
-	"cuelang.org/go/cue/parser"
 	"github.com/grafana/cuetsy"
-	"github.com/grafana/grafana"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
 	"github.com/grafana/thema"
-	tload "github.com/grafana/thema/load"
 )
 
-// The only import statement we currently allow in any models.cue file
-const schemasPath = "github.com/grafana/grafana/packages/grafana-schema/src/schema"
-
 // CUE import paths, mapped to corresponding TS import paths. An empty value
 // indicates the import path should be dropped in the conversion to TS. Imports
 // not present in the list are not not allowed, and code generation will fail.
 var importMap = map[string]string{
-	"github.com/grafana/thema": "",
+	"github.com/grafana/thema":                                      "",
-	schemasPath:                "@grafana/schema",
+	"github.com/grafana/grafana/packages/grafana-schema/src/schema": "@grafana/schema",
 }
 
-// Hard-coded list of paths to skip. Remove a particular file as we're ready
+func init() {
-// to rely on the TypeScript auto-generated by cuetsy for that particular file.
+	allow := pfs.PermittedCUEImports()
-var skipPaths = []string{
+	strsl := make([]string, 0, len(importMap))
-	"public/app/plugins/panel/canvas/models.cue",
+	for p := range importMap {
-	"public/app/plugins/panel/heatmap/models.cue",
+		strsl = append(strsl, p)
-	"public/app/plugins/panel/heatmap-old/models.cue",
-	"public/app/plugins/panel/candlestick/models.cue",
-	"public/app/plugins/panel/state-timeline/models.cue",
-	"public/app/plugins/panel/status-history/models.cue",
-	"public/app/plugins/panel/table/models.cue",
-	"public/app/plugins/panel/timeseries/models.cue",
-}
-
-const prefix = "/"
-
-// CuetsifyPlugins runs cuetsy against plugins' models.cue files.
-func CuetsifyPlugins(ctx *cue.Context, root string) (WriteDiffer, error) {
-	lib := thema.NewLibrary(ctx)
-	// TODO this whole func has a lot of old, crufty behavior from the scuemata era; needs TLC
-	overlay := make(map[string]load.Source)
-	err := toOverlay(prefix, grafana.CueSchemaFS, overlay)
-	// err := tload.ToOverlay(prefix, grafana.CueSchemaFS, overlay)
-	if err != nil {
-		return nil, err
 	}
 
-	exclude := func(path string) bool {
+	sort.Strings(strsl)
-		for _, p := range skipPaths {
+	sort.Strings(allow)
-			if path == p {
+	if strings.Join(strsl, "") != strings.Join(allow, "") {
-				return true
+		panic("CUE import map is not the same as permitted CUE import list - these files must be kept in sync!")
-			}
+	}
+}
+
+// MapCUEImportToTS maps the provided CUE import path to the corresponding
+// TypeScript import path in generated code.
+//
+// Providing an import path that is not allowed results in an error. If a nil
+// error and empty string are returned, the import path should be dropped in
+// generated code.
+func MapCUEImportToTS(path string) (string, error) {
+	i, has := importMap[path]
+	if !has {
+		return "", fmt.Errorf("import %q in models.cue is not allowed", path)
+	}
+	return i, nil
+}
+
+// ExtractPluginTrees attempts to create a *pfs.Tree for each of the top-level child
+// directories in the provided fs.FS.
+//
+// Errors returned from [pfs.ParsePluginFS] are placed in the option map. Only
+// filesystem traversal and read errors will result in a non-nil second return
+// value.
+func ExtractPluginTrees(parent fs.FS, lib thema.Library) (map[string]PluginTreeOrErr, error) {
+	ents, err := fs.ReadDir(parent, ".")
+	if err != nil {
+		return nil, fmt.Errorf("error reading fs root directory: %w", err)
+	}
+
+	ptrees := make(map[string]PluginTreeOrErr)
+	for _, plugdir := range ents {
+		subpath := plugdir.Name()
+		sub, err := fs.Sub(parent, subpath)
+		if err != nil {
+			return nil, fmt.Errorf("error creating subfs for path %s: %w", subpath, err)
 		}
 
-		return filepath.Dir(path) == "cue.mod"
+		var either PluginTreeOrErr
+		if ptree, err := pfs.ParsePluginFS(sub, lib); err == nil {
+			either.Tree = (*PluginTree)(ptree)
+		} else {
+			either.Err = err
+		}
+		ptrees[subpath] = either
 	}
 
-	// Prep the cue load config
+	return ptrees, nil
-	clcfg := &load.Config{
-		Overlay: overlay,
-		// FIXME these module paths won't work for things not under our cue.mod - AKA third-party plugins
-		ModuleRoot: prefix,
-		Module:     "github.com/grafana/grafana",
-	}
-
-	outfiles := NewWriteDiffer()
-
-	cuetsify := func(in fs.FS) error {
-		seen := make(map[string]bool)
-		return fs.WalkDir(in, ".", func(path string, d fs.DirEntry, err error) error {
-			if err != nil {
-				return err
-			}
-			dir := filepath.Dir(path)
-
-			if d.IsDir() || filepath.Ext(d.Name()) != ".cue" || seen[dir] || exclude(path) {
-				return nil
-			}
-			seen[dir] = true
-			clcfg.Dir = filepath.Join(root, dir)
-
-			var b []byte
-			f := &tsFile{}
-
-			switch {
-			default:
-				insts := load.Instances(nil, clcfg)
-				if len(insts) > 1 {
-					return fmt.Errorf("%s: resulted in more than one instance", path)
-				}
-				v := ctx.BuildInstance(insts[0])
-
-				b, err = cuetsy.Generate(v, cuetsy.Config{})
-				if err != nil {
-					return err
-				}
-
-			case strings.Contains(path, "public/app/plugins"): // panel plugin models.cue files
-				// The simple - and preferable - thing would be to have plugins use the same
-				// package name for their models.cue as their containing dir. That's not
-				// possible, though, because we allow dashes in plugin names, but CUE does not
-				// allow them in package names. Yuck.
-				inst, err := loadInstancesWithThema(in, dir, "grafanaschema")
-				if err != nil {
-					return fmt.Errorf("could not load CUE instance for %s: %w", dir, err)
-				}
-
-				// Also parse file directly to extract imports.
-				// NOTE this will need refactoring to support working with more than one file at a time
-				of, _ := in.Open(path)
-				pf, _ := parser.ParseFile(filepath.Base(path), of, parser.ParseComments)
-
-				iseen := make(map[string]bool)
-				for _, im := range pf.Imports {
-					ip := strings.Trim(im.Path.Value, "\"")
-					mappath, has := importMap[ip]
-					if !has {
-						// TODO make a specific error type for this
-						var all []string
-						for im := range importMap {
-							all = append(all, fmt.Sprintf("\t%s", im))
-						}
-						return errors.Newf(im.Pos(), "%s: import %q not allowed, panel plugins may only import from:\n%s\n", path, ip, strings.Join(all, "\n"))
-					}
-					// TODO this approach will silently swallow the unfixable
-					// error case where multiple files in the same dir import
-					// the same package to a different ident
-					if mappath != "" && !iseen[ip] {
-						iseen[ip] = true
-						f.Imports = append(f.Imports, convertImport(im))
-					}
-				}
-
-				v := ctx.BuildInstance(inst)
-
-				lin, err := thema.BindLineage(v.LookupPath(cue.ParsePath("Panel")), lib)
-				if err != nil {
-					return fmt.Errorf("%s: failed to bind lineage: %w", path, err)
-				}
-				f.V = thema.LatestVersion(lin)
-				f.WriteModelVersion = true
-
-				b, err = cuetsy.Generate(thema.SchemaP(lin, f.V).UnwrapCUE(), cuetsy.Config{})
-				if err != nil {
-					return err
-				}
-			}
-
-			f.Body = string(b)
-
-			var buf bytes.Buffer
-			err = tsTemplate.Execute(&buf, f)
-			outfiles[filepath.Join(root, strings.Replace(path, ".cue", ".gen.ts", -1))] = buf.Bytes()
-			return err
-		})
-	}
-
-	err = cuetsify(grafana.CueSchemaFS)
-	if err != nil {
-		return nil, gerrors.New(errors.Details(err, nil))
-	}
-
-	return outfiles, nil
 }
 
-func convertImport(im *ast.ImportSpec) *tsImport {
+// PluginTreeOrErr represents either a *pfs.Tree, or the error that occurred
-	tsim := &tsImport{
+// while trying to create one.
-		Pkg: importMap[schemasPath],
+// TODO replace with generic option type after go 1.18
+type PluginTreeOrErr struct {
+	Err  error
+	Tree *PluginTree
+}
+
+// PluginTree is a pfs.Tree. It exists so we can add methods for code generation to it.
+type PluginTree pfs.Tree
+
+func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
+	t := (*pfs.Tree)(pt)
+
+	// TODO replace with cuetsy's TS AST
+	f := &tsFile{}
+
+	pi := t.RootPlugin()
+	slotimps := pi.SlotImplementations()
+	if len(slotimps) == 0 {
+		return nil, nil
 	}
+	for _, im := range pi.CUEImports() {
+		if tsim := convertImport(im); tsim != nil {
+			f.Imports = append(f.Imports, tsim)
+		}
+	}
+
+	for slotname, lin := range slotimps {
+		v := thema.LatestVersion(lin)
+		sch := thema.SchemaP(lin, v)
+		// TODO need call expressions in cuetsy tsast to be able to do these
+		sec := tsSection{
+			V:         v,
+			ModelName: slotname,
+		}
+
+		// TODO this is hardcoded for now, but should ultimately be a property of
+		// whether the slot is a grouped lineage:
+		// https://github.com/grafana/thema/issues/62
+		switch slotname {
+		case "Panel", "DSConfig":
+			b, err := cuetsy.Generate(sch.UnwrapCUE(), cuetsy.Config{})
+			if err != nil {
+				return nil, fmt.Errorf("%s: error translating %s lineage to TypeScript: %w", path, slotname, err)
+			}
+			sec.Body = string(b)
+		case "Query":
+			a, err := cuetsy.GenerateSingleAST(strings.Title(lin.Name()), sch.UnwrapCUE(), cuetsy.TypeInterface)
+			if err != nil {
+				return nil, fmt.Errorf("%s: error translating %s lineage to TypeScript: %w", path, slotname, err)
+			}
+			sec.Body = fmt.Sprint(a)
+		default:
+			panic("unrecognized slot name: " + slotname)
+		}
+
+		f.Sections = append(f.Sections, sec)
+	}
+
+	wd := NewWriteDiffer()
+	var buf bytes.Buffer
+	err := tsSectionTemplate.Execute(&buf, f)
+	if err != nil {
+		return nil, fmt.Errorf("%s: error executing plugin TS generator template: %w", path, err)
+	}
+	wd[filepath.Join(path, "models.gen.ts")] = buf.Bytes()
+	return wd, nil
+}
+
+// TODO convert this to use cuetsy ts types, once import * form is supported
+func convertImport(im *ast.ImportSpec) *tsImport {
+	var err error
+	tsim := &tsImport{}
+	tsim.Pkg, err = MapCUEImportToTS(strings.Trim(im.Path.Value, "\""))
+	if err != nil {
+		// should be unreachable if paths has been verified already
+		panic(err)
+	}
+
+	if tsim.Pkg == "" {
+		// Empty string mapping means skip it
+		return nil
+	}
+
 	if im.Name != nil && im.Name.String() != "" {
 		tsim.Ident = im.Name.String()
 	} else {
@@ -196,145 +182,15 @@ func convertImport(im *ast.ImportSpec) *tsImport {
 	return tsim
 }
 
-var themamodpath string = filepath.Join("cue.mod", "pkg", "github.com", "grafana", "thema")
-
-// all copied and hacked up from Thema's LoadInstancesWithThema, simply to allow setting the
-// package name
-func loadInstancesWithThema(modFS fs.FS, dir string, pkgname string) (*build.Instance, error) {
-	var modname string
-	err := fs.WalkDir(modFS, "cue.mod", func(path string, d fs.DirEntry, err error) error {
-		// fs.FS implementations tend to not use path separators as expected. Use a
-		// normalized one for comparisons, but retain the original for calls back into modFS.
-		normpath := filepath.FromSlash(path)
-		if err != nil {
-			return err
-		}
-
-		if d.IsDir() {
-			switch normpath {
-			case filepath.Join("cue.mod", "gen"), filepath.Join("cue.mod", "usr"):
-				return fs.SkipDir
-			case themamodpath:
-				return fmt.Errorf("path %q already exists in modFS passed to InstancesWithThema, must be absent for dynamic dependency injection", themamodpath)
-			}
-			return nil
-		} else if normpath == filepath.Join("cue.mod", "module.cue") {
-			modf, err := modFS.Open(path)
-			if err != nil {
-				return err
-			}
-			defer modf.Close() // nolint: errcheck
-
-			b, err := io.ReadAll(modf)
-			if err != nil {
-				return err
-			}
-
-			modname, err = cuecontext.New().CompileBytes(b).LookupPath(cue.MakePath(cue.Str("module"))).String()
-			if err != nil {
-				return err
-			}
-			if modname == "" {
-				return fmt.Errorf("InstancesWithThema requires non-empty module name in modFS' cue.mod/module.cue")
-			}
-		}
-
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	if modname == "" {
-		return nil, errors.New("cue.mod/module.cue did not exist")
-	}
-
-	modroot := filepath.FromSlash(filepath.Join("/", modname))
-	overlay := make(map[string]load.Source)
-	if err := tload.ToOverlay(modroot, modFS, overlay); err != nil {
-		return nil, err
-	}
-
-	// Special case for when we're calling this loader with paths inside the thema module
-	if modname == "github.com/grafana/thema" {
-		if err := tload.ToOverlay(modroot, thema.CueJointFS, overlay); err != nil {
-			return nil, err
-		}
-	} else {
-		if err := tload.ToOverlay(filepath.Join(modroot, themamodpath), thema.CueFS, overlay); err != nil {
-			return nil, err
-		}
-	}
-
-	if dir == "" {
-		dir = "."
-	}
-
-	cfg := &load.Config{
-		Overlay:    overlay,
-		ModuleRoot: modroot,
-		Module:     modname,
-		Dir:        filepath.Join(modroot, dir),
-		Package:    pkgname,
-	}
-	if dir == "." {
-		cfg.Package = filepath.Base(modroot)
-		cfg.Dir = modroot
-	}
-
-	inst := load.Instances(nil, cfg)[0]
-	if inst.Err != nil {
-		return nil, inst.Err
-	}
-
-	return inst, nil
-}
-
-func toOverlay(prefix string, vfs fs.FS, overlay map[string]load.Source) error {
-	if !filepath.IsAbs(prefix) {
-		return fmt.Errorf("must provide absolute path prefix when generating cue overlay, got %q", prefix)
-	}
-	err := fs.WalkDir(vfs, ".", func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if d.IsDir() {
-			return nil
-		}
-
-		f, err := vfs.Open(path)
-		if err != nil {
-			return err
-		}
-		defer func(f fs.File) {
-			err := f.Close()
-			if err != nil {
-				return
-			}
-		}(f)
-
-		b, err := io.ReadAll(f)
-		if err != nil {
-			return err
-		}
-
-		overlay[filepath.Join(prefix, path)] = load.FromBytes(b)
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 type tsFile struct {
-	V                 thema.SyntacticVersion
+	Imports  []*tsImport
-	WriteModelVersion bool
+	Sections []tsSection
-	Imports           []*tsImport
+}
-	Body              string
+
+type tsSection struct {
+	V         thema.SyntacticVersion
+	ModelName string
+	Body      string
 }
 
 type tsImport struct {
@@ -342,14 +198,14 @@ type tsImport struct {
 	Pkg   string
 }
 
-var tsTemplate = template.Must(template.New("cuetsygen").Parse(`//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+var tsSectionTemplate = template.Must(template.New("cuetsymulti").Parse(`//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // This file is autogenerated. DO NOT EDIT.
 //
 // To regenerate, run "make gen-cue" from the repository root.
 //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 {{range .Imports}}
 import * as {{.Ident}} from '{{.Pkg}}';{{end}}
-{{if .WriteModelVersion }}
+{{range .Sections}}{{if ne .ModelName "" }}
-export const modelVersion = Object.freeze([{{index .V 0}}, {{index .V 1}}]);
+export const {{.ModelName}}ModelVersion = Object.freeze([{{index .V 0}}, {{index .V 1}}]);
 {{end}}
-{{.Body}}`))
+{{.Body}}{{end}}`))

pkg/coremodel/pluginmeta/coremodel.cue

@@ -1,6 +1,9 @@
 package pluginmeta
 
-import "github.com/grafana/thema"
+import (
+	"github.com/grafana/thema"
+	"strings"
+)
 
 thema.#Lineage
 name: "pluginmeta"
@@ -11,7 +14,8 @@ seqs: [
 				// Unique name of the plugin. If the plugin is published on
 				// grafana.com, then the plugin id has to follow the naming
 				// conventions.
-				id: =~"^[0-9a-z]+\\-([0-9a-z]+\\-)?(app|panel|datasource)$"
+				id: string & strings.MinRunes(1)
+				id: =~"^([0-9a-z]+\\-([0-9a-z]+\\-)?(app|panel|datasource))|(alertGroups|alertlist|annolist|barchart|bargauge|candlestick|canvas|dashlist|debug|gauge|geomap|gettingstarted|graph|heatmap|heatmap-old|histogram|icon|live|logs|news|nodeGraph|piechart|pluginlist|stat|state-timeline|status-history|table|table-old|text|timeseries|traces|welcome|xychart|alertmanager|cloudwatch|dashboard|elasticsearch|grafana|grafana-azure-monitor-datasource|graphite|influxdb|jaeger|loki|mixed|mssql|mysql|opentsdb|postgres|prometheus|stackdriver|tempo|testdata|zipkin)$"
 
 				// type indicates which type of Grafana plugin this is, of the defined
 				// set of Grafana plugin types.
@@ -41,6 +45,14 @@ seqs: [
 				// If the plugin has a backend component.
 				backend?: bool
 
+				// builtin indicates whether the plugin is developed and shipped as part
+				// of Grafana. Also known as a "core plugin."
+				builtIn: bool | *false
+
+				// hideFromList excludes the plugin from listings in Grafana's UI. Only
+				// allowed for builtin plugins.
+				hideFromList: bool | *false
+
 				// The first part of the file name of the backend component
 				// executable. There can be multiple executables built for
 				// different operating system and architecture. Grafana will
@@ -58,7 +70,7 @@ seqs: [
 				state?: #ReleaseState
 
 				// ReleaseState indicates release maturity state of a plugin.
-				#ReleaseState: "alpha" | "beta" | *"stable"
+				#ReleaseState: "alpha" | "beta" | "deprecated" | *"stable"
 
 				// Resources to include in plugin.
 				includes?: [...#Include]
@@ -185,7 +197,7 @@ seqs: [
 					}]
 
 					// SVG images that are used as plugin icons.
-					logos: {
+					logos?: {
 						// Link to the "small" version of the plugin logo, which must be
 						// an SVG image. "Large" and "small" logos can be the same image.
 						small: string
@@ -203,10 +215,10 @@ seqs: [
 					}]
 
 					// Date when this plugin was built.
-					updated: =~"^(\\d{4}-\\d{2}-\\d{2}|\\%TODAY\\%)$"
+					updated?: =~"^(\\d{4}-\\d{2}-\\d{2}|\\%TODAY\\%)$"
 
 					// Project version of this commit, e.g. `6.7.x`.
-					version: =~"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*$|\\%VERSION\\%)"
+					version?: =~"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*$|\\%VERSION\\%)"
 				}
 
 				#BuildInfo: {

pkg/coremodel/pluginmeta/pluginmeta_gen.go

@@ -86,6 +86,8 @@ const (
 
 	ReleaseStateBeta ReleaseState = "beta"
 
+	ReleaseStateDeprecated ReleaseState = "deprecated"
+
 	ReleaseStateStable ReleaseState = "stable"
 )
 
@@ -108,6 +110,10 @@ type Model struct {
 	// If the plugin has a backend component.
 	Backend *bool `json:"backend,omitempty"`
 
+	// builtin indicates whether the plugin is developed and shipped as part
+	// of Grafana. Also known as a "core plugin."
+	BuiltIn bool `json:"builtIn"`
+
 	// Plugin category used on the Add data source page.
 	Category *Category `json:"category,omitempty"`
 
@@ -146,6 +152,10 @@ type Model struct {
 	// request.
 	HiddenQueries *bool `json:"hiddenQueries,omitempty"`
 
+	// hideFromList excludes the plugin from listings in Grafana's UI. Only
+	// allowed for builtin plugins.
+	HideFromList bool `json:"hideFromList"`
+
 	// Unique name of the plugin. If the plugin is published on
 	// grafana.com, then the plugin id has to follow the naming
 	// conventions.
@@ -185,7 +195,7 @@ type Model struct {
 		} `json:"links,omitempty"`
 
 		// SVG images that are used as plugin icons.
-		Logos struct {
+		Logos *struct {
 			// Link to the "large" version of the plugin logo, which must be
 			// an SVG image. "Large" and "small" logos can be the same image.
 			Large string `json:"large"`
@@ -193,7 +203,7 @@ type Model struct {
 			// Link to the "small" version of the plugin logo, which must be
 			// an SVG image. "Large" and "small" logos can be the same image.
 			Small string `json:"small"`
-		} `json:"logos"`
+		} `json:"logos,omitempty"`
 
 		// An array of screenshot objects in the form `{name: 'bar', path:
 		// 'img/screenshot.png'}`
@@ -203,10 +213,10 @@ type Model struct {
 		} `json:"screenshots,omitempty"`
 
 		// Date when this plugin was built.
-		Updated string `json:"updated"`
+		Updated *string `json:"updated,omitempty"`
 
 		// Project version of this commit, e.g. `6.7.x`.
-		Version string `json:"version"`
+		Version *string `json:"version,omitempty"`
 	} `json:"info"`
 
 	// For data source plugins, if the plugin supports logs.
@@ -420,7 +430,7 @@ type Info struct {
 	} `json:"links,omitempty"`
 
 	// SVG images that are used as plugin icons.
-	Logos struct {
+	Logos *struct {
 		// Link to the "large" version of the plugin logo, which must be
 		// an SVG image. "Large" and "small" logos can be the same image.
 		Large string `json:"large"`
@@ -428,7 +438,7 @@ type Info struct {
 		// Link to the "small" version of the plugin logo, which must be
 		// an SVG image. "Large" and "small" logos can be the same image.
 		Small string `json:"small"`
-	} `json:"logos"`
+	} `json:"logos,omitempty"`
 
 	// An array of screenshot objects in the form `{name: 'bar', path:
 	// 'img/screenshot.png'}`
@@ -438,10 +448,10 @@ type Info struct {
 	} `json:"screenshots,omitempty"`
 
 	// Date when this plugin was built.
-	Updated string `json:"updated"`
+	Updated *string `json:"updated,omitempty"`
 
 	// Project version of this commit, e.g. `6.7.x`.
-	Version string `json:"version"`
+	Version *string `json:"version,omitempty"`
 }
 
 // TODO docs

pkg/cuectx/ctx.go

@@ -5,7 +5,6 @@
 package cuectx
 
 import (
-	"io"
 	"io/fs"
 	"path/filepath"
 	"testing/fstest"
@@ -53,12 +52,9 @@ func JSONtoCUE(path string, b []byte) (cue.Value, error) {
 // lineage.cue file must be the sole contents of the provided fs.FS.
 //
 // More details on underlying behavior can be found in the docs for github.com/grafana/thema/load.InstancesWithThema.
-func LoadGrafanaInstancesWithThema(
+//
-	path string,
+// TODO this approach is complicated and confusing, refactor to something understandable
-	cueFS fs.FS,
+func LoadGrafanaInstancesWithThema(path string, cueFS fs.FS, lib thema.Library, opts ...thema.BindOption) (thema.Lineage, error) {
-	lib thema.Library,
-	opts ...thema.BindOption,
-) (thema.Lineage, error) {
 	prefix := filepath.FromSlash(path)
 	fs, err := prefixWithGrafanaCUE(prefix, cueFS)
 	if err != nil {
@@ -104,13 +100,7 @@ func prefixWithGrafanaCUE(prefix string, inputfs fs.FS) (fs.FS, error) {
 			return nil
 		}
 
-		f, err := inputfs.Open(path)
+		b, err := fs.ReadFile(inputfs, path)
-		if err != nil {
-			return err
-		}
-		defer f.Close() // nolint: errcheck
-
-		b, err := io.ReadAll(f)
 		if err != nil {
 			return err
 		}

pkg/framework/coremodel/gen.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 
 	"cuelang.org/go/cue/cuecontext"
+	"cuelang.org/go/cue/load"
+	"github.com/grafana/cuetsy"
 	gcgen "github.com/grafana/grafana/pkg/codegen"
 	"github.com/grafana/thema"
 )
@@ -52,7 +54,7 @@ func main() {
 		if item.IsDir() {
 			lin, err := gcgen.ExtractLineage(filepath.Join(cmroot, item.Name(), "coremodel.cue"), lib)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "could not process coremodel dir %s: %s\n", cmroot, err)
+				fmt.Fprintf(os.Stderr, "could not process coremodel dir %s: %s\n", filepath.Join(cmroot, item.Name()), err)
 				os.Exit(1)
 			}
 
@@ -90,6 +92,14 @@ func main() {
 	}
 	wd.Merge(regfiles)
 
+	// TODO generating these is here temporarily until we make a more permanent home
+	wdsh, err := genSharedSchemas(groot)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "TS gen error for shared schemas in %s: %w", filepath.Join(groot, "packages", "grafana-schema", "src", "schema"), err)
+		os.Exit(1)
+	}
+	wd.Merge(wdsh)
+
 	if _, set := os.LookupEnv("CODEGEN_VERIFY"); set {
 		err = wd.Verify()
 		if err != nil {
@@ -104,3 +114,36 @@ func main() {
 		}
 	}
 }
+func genSharedSchemas(groot string) (gcgen.WriteDiffer, error) {
+	abspath := filepath.Join(groot, "packages", "grafana-schema", "src", "schema")
+	cfg := &load.Config{
+		ModuleRoot: groot,
+		Module:     "github.com/grafana/grafana",
+		Dir:        abspath,
+	}
+
+	bi := load.Instances(nil, cfg)
+	if len(bi) > 1 {
+		return nil, fmt.Errorf("loading CUE files in %s resulted in more than one instance", abspath)
+	}
+
+	ctx := cuecontext.New()
+	v := ctx.BuildInstance(bi[0])
+	if v.Err() != nil {
+		return nil, fmt.Errorf("errors while building CUE in %s: %s", abspath, v.Err())
+	}
+
+	b, err := cuetsy.Generate(v, cuetsy.Config{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate TS: %w", err)
+	}
+
+	wd := gcgen.NewWriteDiffer()
+	wd[filepath.Join(abspath, "mudball.gen.ts")] = append([]byte(`//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+// This file is autogenerated. DO NOT EDIT.
+//
+// To regenerate, run "make gen-cue" from the repository root.
+//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+`), b...)
+	return wd, nil
+}

pkg/framework/coremodel/helpers.go

@@ -1 +1,176 @@
 package coremodel
+
+import (
+	"embed"
+	"fmt"
+	"io/fs"
+	"path/filepath"
+	"testing/fstest"
+
+	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/load"
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/grafana/thema/kernel"
+	tload "github.com/grafana/thema/load"
+)
+
+// Embed for all framework-related CUE files in this directory
+//
+//go:embed *.cue
+var cueFS embed.FS
+
+var defaultFramework cue.Value
+
+func init() {
+	var err error
+	defaultFramework, err = doLoadFrameworkCUE(cuectx.ProvideCUEContext())
+	if err != nil {
+		panic(err)
+	}
+}
+
+var prefix = filepath.Join("/pkg", "framework", "coremodel")
+
+//nolint:nakedret
+func doLoadFrameworkCUE(ctx *cue.Context) (v cue.Value, err error) {
+	m := make(fstest.MapFS)
+
+	err = fs.WalkDir(cueFS, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		b, err := fs.ReadFile(cueFS, path)
+		if err != nil {
+			return err
+		}
+		m[path] = &fstest.MapFile{Data: b}
+		return nil
+	})
+	if err != nil {
+		return
+	}
+
+	over := make(map[string]load.Source)
+	err = tload.ToOverlay(prefix, m, over)
+	if err != nil {
+		return
+	}
+
+	bi := load.Instances(nil, &load.Config{
+		Dir:     prefix,
+		Package: "coremodel",
+		Overlay: over,
+	})
+	v = ctx.BuildInstance(bi[0])
+
+	if v.Err() != nil {
+		return cue.Value{}, fmt.Errorf("coremodel framework loaded cue.Value has err: %w", v.Err())
+	}
+
+	return
+}
+
+// CUEFramework returns a cue.Value representing all the coremodel framework
+// raw CUE files.
+//
+// For low-level use in constructing other types and APIs, while still letting
+// us declare all the frameworky CUE bits in a single package. Other types and
+// subpackages make the constructs in this value easy to use.
+//
+// The returned cue.Value is built from Grafana's standard central CUE context,
+// ["github.com/grafana/grafana/pkg/cuectx".ProvideCueContext].
+func CUEFramework() cue.Value {
+	return defaultFramework
+}
+
+// CUEFrameworkWithContext is the same as CUEFramework, but allows control over
+// the cue.Context that's used.
+//
+// Prefer CUEFramework unless you understand cue.Context, and absolutely need
+// this control.
+func CUEFrameworkWithContext(ctx *cue.Context) cue.Value {
+	// Error guaranteed to be nil here because erroring would have caused init() to panic
+	v, _ := doLoadFrameworkCUE(ctx) // nolint:errcheck
+	return v
+}
+
+// Mux takes a coremodel and returns a Thema version muxer that, given a byte
+// slice containing any version of schema for that coremodel, will translate it
+// to the Interface.CurrentSchema() version, and optionally decode it onto the
+// Interface.GoType().
+//
+// By default, JSON decoding will be used, and the filename given to any input
+// bytes (shown in errors, which may be user-facing) will be
+// "<name>.<encoding>", e.g. dashboard.json.
+func Mux(cm Interface, opts ...MuxOption) kernel.InputKernel {
+	c := &muxConfig{}
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	cfg := kernel.InputKernelConfig{
+		Typ:     cm.GoType(),
+		Lineage: cm.Lineage(),
+		To:      cm.CurrentSchema().Version(),
+	}
+
+	switch c.decodetyp {
+	case "", "json": // json by default
+		if c.filename == "" {
+			c.filename = fmt.Sprintf("%s.json", cm.Lineage().Name())
+		}
+		cfg.Loader = kernel.NewJSONDecoder(c.filename)
+	case "yaml":
+		if c.filename == "" {
+			c.filename = fmt.Sprintf("%s.yaml", cm.Lineage().Name())
+		}
+		cfg.Loader = kernel.NewYAMLDecoder(c.filename)
+	default:
+		panic("")
+	}
+
+	mux, err := kernel.NewInputKernel(cfg)
+	if err != nil {
+		// Barring a fundamental bug in Thema's schema->Go type assignability checker or
+		// a direct attempt by a Grafana dev to get around the invariants of coremodel codegen,
+		// this should be unreachable. (And even the latter case should be caught elsewhere
+		// by tests).
+		panic(err)
+	}
+	return mux
+}
+
+// A MuxOption defines options that may be specified only at initial
+// construction of a Lineage via BindLineage.
+type MuxOption muxOption
+
+// Internal representation of MuxOption.
+type muxOption func(c *muxConfig)
+
+type muxConfig struct {
+	filename  string
+	decodetyp string
+}
+
+// YAML indicates that the resulting Mux should look for YAML in input bytes,
+// rather than the default JSON.
+func YAML() MuxOption {
+	return func(c *muxConfig) {
+		c.decodetyp = "yaml"
+	}
+}
+
+// Filename specifies the filename that is given to input bytes passing through
+// the mux.
+//
+// The filename has no impact on mux behavior, but is used in user-facing error
+// output, such as schema validation failures. Thus, it is recommended to pick a
+// name that will make sense in the context a user is expected to see the error.
+func Filename(name string) MuxOption {
+	return func(c *muxConfig) {
+		c.filename = name
+	}
+}

pkg/framework/coremodel/slot.go

@@ -0,0 +1,87 @@
+package coremodel
+
+import (
+	"cuelang.org/go/cue"
+)
+
+// Slot represents one of Grafana's named Thema composition slot definitions.
+type Slot struct {
+	name    string
+	raw     cue.Value
+	plugins map[string]bool
+}
+
+// Name returns the name of the Slot. The name is also used as the path at which
+// a Slot lineage is defined in a plugin models.cue file.
+func (s Slot) Name() string {
+	return s.name
+}
+
+// MetaSchema returns the meta-schema that is the contract between coremodels
+// that compose the Slot, and plugins that implement it.
+func (s Slot) MetaSchema() cue.Value {
+	return s.raw
+}
+
+// ForPluginType indicates whether for this Slot, plugins of the given type may
+// provide a slot implementation (first return value), and for those types that
+// may, whether they must produce one (second return value).
+//
+// Expected values here are those in the set of
+// ["github.com/grafana/grafana/pkg/coremodel/pluginmeta".Type], though passing
+// a string not in that set will harmlessly return {false, false}. That type is
+// not used here to avoid import cycles.
+//
+// Note that, at least for now, plugins are not required to provide any slot
+// implementations, and do so by simply not containing a models.cue file.
+// Consequently, the "must" return value here is best understood as, "IF a
+// plugin provides a models.cue file, it MUST contain an implementation of this
+// slot."
+func (s Slot) ForPluginType(plugintype string) (may, must bool) {
+	must, may = s.plugins[plugintype]
+	return
+}
+
+func AllSlots() map[string]*Slot {
+	fw := CUEFramework()
+	slots := make(map[string]*Slot)
+
+	// Ignore err, can only happen if we change structure of fw files, and all we'd
+	// do is panic and that's what the next line will do anyway. Same for similar ignored
+	// errors later in this func
+	iter, _ := fw.LookupPath(cue.ParsePath("pluginTypeMetaSchema")).Fields(cue.Optional(true))
+	type nameopt struct {
+		name string
+		req  bool
+	}
+	plugslots := make(map[string][]nameopt)
+	for iter.Next() {
+		plugin := iter.Selector().String()
+		iiter, _ := iter.Value().Fields(cue.Optional(true))
+		for iiter.Next() {
+			slotname := iiter.Selector().String()
+			plugslots[slotname] = append(plugslots[slotname], nameopt{
+				name: plugin,
+				req:  !iiter.IsOptional(),
+			})
+		}
+	}
+
+	iter, _ = fw.LookupPath(cue.ParsePath("slots")).Fields(cue.Optional(true))
+	for iter.Next() {
+		n := iter.Selector().String()
+		sl := Slot{
+			name:    n,
+			raw:     iter.Value(),
+			plugins: make(map[string]bool),
+		}
+
+		for _, no := range plugslots[n] {
+			sl.plugins[no.name] = no.req
+		}
+
+		slots[n] = &sl
+	}
+
+	return slots
+}

pkg/framework/coremodel/slot/doc.go

@@ -0,0 +1,2 @@
+// Package Slot exposes Grafana's coremodel composition Slot definitions for use in Go.
+package slot

pkg/framework/coremodel/slots.cue

@@ -0,0 +1,71 @@
+package coremodel
+
+// The slots named and specified in this file are meta-schemas that act as a
+// shared contract between Grafana plugins (producers) and coremodel types
+// (consumers).
+//
+// On the consumer side, any coremodel Thema lineage can choose to define a
+// standard Thema composition slot that specifies one of these named slots as
+// its meta-schema. Such a specification entails that all schemas in any lineage
+// placed into that composition slot must adhere to the meta-schema.
+//
+// On the producer side, Grafana's plugin system enforces that certain plugin
+// types are expected to provide Thema lineages for these named slots which
+// adhere to the slot meta-schema.
+//
+// For example, the Panel slot is consumed by the dashboard coremodel, and is
+// expected to be produced by panel plugins.
+//
+// The name given to each slot in this file must be used as the name of the
+// slot in the coremodel, and the name of the field under which the lineage
+// is provided in a plugin's models.cue file.
+//
+// Conformance to meta-schema is achieved by Thema's native lineage joinSchema,
+// which Thema internals automatically enforce across all schemas in a lineage.
+
+// Meta-schema for the Panel slot, as implemented in Grafana panel plugins.
+//
+// This is a grouped meta-schema, intended solely for use in composition. Object
+// literals conforming to it are not expected to exist.
+slots: Panel: {
+  // Defines plugin-specific options for a panel that should be persisted. Required,
+  // though a panel without any options may specify an empty struct.
+  //
+  // Currently mapped to #Panel.options within the dashboard schema.
+  PanelOptions: {...}
+  // Plugin-specific custom field properties. Optional.
+  //
+  // Currently mapped to #Panel.fieldConfig.defaults.custom within the dashboard schema.
+  PanelFieldConfig?: {...}
+}
+
+// Meta-schema for the Query slot, as implemented in Grafana datasource plugins.
+slots: Query: {...}
+
+// Meta-schema for the DSOptions slot, as implemented in Grafana datasource plugins.
+//
+// This is a grouped meta-schema, intended solely for use in composition. Object
+// literals conforming to it are not expected to exist.
+slots: DSOptions: {
+  // Normal datasource configuration options.
+  Options: {...}
+  // Sensitive datasource configuration options that require encryption.
+  SecureOptions: {...}
+}
+
+// pluginTypeMetaSchema defines which plugin types should use which metaschemas
+// as joinSchema for the lineages declared at which paths.
+pluginTypeMetaSchema: [string]: {...}
+pluginTypeMetaSchema: {
+	// Panel plugins are expected to provide a lineage at path Panel conforming to
+	// the Panel joinSchema.
+	panel: {
+		Panel: slots.Panel
+	}
+	// Datasource plugins are expected to provide lineages at paths Query and
+	// DSOptions, conforming to those joinSchemas respectively.
+	datasource: {
+		Query: slots.Query
+		DSOptions: slots.DSOptions
+	}
+}

pkg/plugins/manager/loader/loader_test.go

@@ -108,7 +108,7 @@ func TestLoader_Load(t *testing.T) {
 			want: []*plugins.Plugin{
 				{
 					JSONData: plugins.JSONData{
-						ID:   "test",
+						ID:   "test-datasource",
 						Type: "datasource",
 						Name: "Test",
 						Info: plugins.Info{
@@ -131,8 +131,8 @@ func TestLoader_Load(t *testing.T) {
 						Backend:    true,
 						State:      "alpha",
 					},
-					Module:        "plugins/test/module",
+					Module:        "plugins/test-datasource/module",
-					BaseURL:       "public/plugins/test",
+					BaseURL:       "public/plugins/test-datasource",
 					PluginDir:     filepath.Join(parentDir, "testdata/valid-v2-signature/plugin/"),
 					Signature:     "valid",
 					SignatureType: plugins.GrafanaSignature,
@@ -229,7 +229,7 @@ func TestLoader_Load(t *testing.T) {
 			want: []*plugins.Plugin{
 				{
 					JSONData: plugins.JSONData{
-						ID:   "test",
+						ID:   "test-datasource",
 						Type: "datasource",
 						Name: "Test",
 						Info: plugins.Info{
@@ -251,8 +251,8 @@ func TestLoader_Load(t *testing.T) {
 						State:   plugins.AlphaRelease,
 					},
 					Class:     plugins.External,
-					Module:    "plugins/test/module",
+					Module:    "plugins/test-datasource/module",
-					BaseURL:   "public/plugins/test",
+					BaseURL:   "public/plugins/test-datasource",
 					PluginDir: filepath.Join(parentDir, "testdata/unsigned-datasource/plugin"),
 					Signature: "unsigned",
 				},
@@ -266,8 +266,8 @@ func TestLoader_Load(t *testing.T) {
 			pluginPaths: []string{"../testdata/unsigned-datasource"},
 			want:        []*plugins.Plugin{},
 			pluginErrors: map[string]*plugins.Error{
-				"test": {
+				"test-datasource": {
-					PluginID:  "test",
+					PluginID:  "test-datasource",
 					ErrorCode: "signatureMissing",
 				},
 			},
@@ -277,13 +277,13 @@ func TestLoader_Load(t *testing.T) {
 			class: plugins.External,
 			cfg: &plugins.Cfg{
 				PluginsPath:          filepath.Join(parentDir),
-				PluginsAllowUnsigned: []string{"test"},
+				PluginsAllowUnsigned: []string{"test-datasource"},
 			},
 			pluginPaths: []string{"../testdata/unsigned-datasource"},
 			want: []*plugins.Plugin{
 				{
 					JSONData: plugins.JSONData{
-						ID:   "test",
+						ID:   "test-datasource",
 						Type: "datasource",
 						Name: "Test",
 						Info: plugins.Info{
@@ -305,8 +305,8 @@ func TestLoader_Load(t *testing.T) {
 						State:   plugins.AlphaRelease,
 					},
 					Class:     plugins.External,
-					Module:    "plugins/test/module",
+					Module:    "plugins/test-datasource/module",
-					BaseURL:   "public/plugins/test",
+					BaseURL:   "public/plugins/test-datasource",
 					PluginDir: filepath.Join(parentDir, "testdata/unsigned-datasource/plugin"),
 					Signature: plugins.SignatureUnsigned,
 				},
@@ -321,8 +321,8 @@ func TestLoader_Load(t *testing.T) {
 			pluginPaths: []string{"../testdata/lacking-files"},
 			want:        []*plugins.Plugin{},
 			pluginErrors: map[string]*plugins.Error{
-				"test": {
+				"test-datasource": {
-					PluginID:  "test",
+					PluginID:  "test-datasource",
 					ErrorCode: "signatureModified",
 				},
 			},
@@ -332,13 +332,13 @@ func TestLoader_Load(t *testing.T) {
 			class: plugins.External,
 			cfg: &plugins.Cfg{
 				PluginsPath:          filepath.Join(parentDir),
-				PluginsAllowUnsigned: []string{"test"},
+				PluginsAllowUnsigned: []string{"test-datasource"},
 			},
 			pluginPaths: []string{"../testdata/lacking-files"},
 			want:        []*plugins.Plugin{},
 			pluginErrors: map[string]*plugins.Error{
-				"test": {
+				"test-datasource": {
-					PluginID:  "test",
+					PluginID:  "test-datasource",
 					ErrorCode: "signatureModified",
 				},
 			},
@@ -348,13 +348,13 @@ func TestLoader_Load(t *testing.T) {
 			class: plugins.External,
 			cfg: &plugins.Cfg{
 				PluginsPath:          filepath.Join(parentDir),
-				PluginsAllowUnsigned: []string{"test"},
+				PluginsAllowUnsigned: []string{"test-datasource"},
 			},
 			pluginPaths: []string{"../testdata/invalid-v2-missing-file"},
 			want:        []*plugins.Plugin{},
 			pluginErrors: map[string]*plugins.Error{
-				"test": {
+				"test-datasource": {
-					PluginID:  "test",
+					PluginID:  "test-datasource",
 					ErrorCode: "signatureModified",
 				},
 			},
@@ -364,13 +364,13 @@ func TestLoader_Load(t *testing.T) {
 			class: plugins.External,
 			cfg: &plugins.Cfg{
 				PluginsPath:          filepath.Join(parentDir),
-				PluginsAllowUnsigned: []string{"test"},
+				PluginsAllowUnsigned: []string{"test-datasource"},
 			},
 			pluginPaths: []string{"../testdata/invalid-v2-extra-file"},
 			want:        []*plugins.Plugin{},
 			pluginErrors: map[string]*plugins.Error{
-				"test": {
+				"test-datasource": {
-					PluginID:  "test",
+					PluginID:  "test-datasource",
 					ErrorCode: "signatureModified",
 				},
 			},
@@ -530,7 +530,7 @@ func TestLoader_Load_MultiplePlugins(t *testing.T) {
 				want: []*plugins.Plugin{
 					{
 						JSONData: plugins.JSONData{
-							ID:   "test",
+							ID:   "test-datasource",
 							Type: "datasource",
 							Name: "Test",
 							Info: plugins.Info{
@@ -554,8 +554,8 @@ func TestLoader_Load_MultiplePlugins(t *testing.T) {
 							State:      plugins.AlphaRelease,
 						},
 						Class:         plugins.External,
-						Module:        "plugins/test/module",
+						Module:        "plugins/test-datasource/module",
-						BaseURL:       "public/plugins/test",
+						BaseURL:       "public/plugins/test-datasource",
 						PluginDir:     filepath.Join(parentDir, "testdata/valid-v2-pvt-signature/plugin"),
 						Signature:     "valid",
 						SignatureType: plugins.PrivateSignature,
@@ -621,7 +621,7 @@ func TestLoader_Signature_RootURL(t *testing.T) {
 		expected := []*plugins.Plugin{
 			{
 				JSONData: plugins.JSONData{
-					ID:   "test",
+					ID:   "test-datasource",
 					Type: "datasource",
 					Name: "Test",
 					Info: plugins.Info{
@@ -643,8 +643,8 @@ func TestLoader_Signature_RootURL(t *testing.T) {
 				Signature:     plugins.SignatureValid,
 				SignatureType: plugins.PrivateSignature,
 				SignatureOrg:  "Will Browne",
-				Module:        "plugins/test/module",
+				Module:        "plugins/test-datasource/module",
-				BaseURL:       "public/plugins/test",
+				BaseURL:       "public/plugins/test-datasource",
 			},
 		}
 
@@ -738,7 +738,7 @@ func TestLoader_loadNestedPlugins(t *testing.T) {
 	}
 	parent := &plugins.Plugin{
 		JSONData: plugins.JSONData{
-			ID:   "test-ds",
+			ID:   "test-datasource",
 			Type: "datasource",
 			Name: "Parent",
 			Info: plugins.Info{
@@ -760,8 +760,8 @@ func TestLoader_loadNestedPlugins(t *testing.T) {
 			},
 			Backend: true,
 		},
-		Module:        "plugins/test-ds/module",
+		Module:        "plugins/test-datasource/module",
-		BaseURL:       "public/plugins/test-ds",
+		BaseURL:       "public/plugins/test-datasource",
 		PluginDir:     filepath.Join(rootDir, "testdata/nested-plugins/parent"),
 		Signature:     plugins.SignatureValid,
 		SignatureType: plugins.GrafanaSignature,
@@ -1149,23 +1149,23 @@ func Test_validatePluginJSON(t *testing.T) {
 func Test_setPathsBasedOnApp(t *testing.T) {
 	t.Run("When setting paths based on core plugin on Windows", func(t *testing.T) {
 		child := &plugins.Plugin{
-			PluginDir: "c:\\grafana\\public\\app\\plugins\\app\\testdata\\datasources\\datasource",
+			PluginDir: "c:\\grafana\\public\\app\\plugins\\app\\testdata-app\\datasources\\datasource",
 		}
 		parent := &plugins.Plugin{
 			JSONData: plugins.JSONData{
 				Type: plugins.App,
-				ID:   "testdata",
+				ID:   "testdata-app",
 			},
 			Class:     plugins.Core,
-			PluginDir: "c:\\grafana\\public\\app\\plugins\\app\\testdata",
+			PluginDir: "c:\\grafana\\public\\app\\plugins\\app\\testdata-app",
-			BaseURL:   "public/app/plugins/app/testdata",
+			BaseURL:   "public/app/plugins/app/testdata-app",
 		}
 
 		configureAppChildOPlugin(parent, child)
 
-		assert.Equal(t, "app/plugins/app/testdata/datasources/datasource/module", child.Module)
+		assert.Equal(t, "app/plugins/app/testdata-app/datasources/datasource/module", child.Module)
-		assert.Equal(t, "testdata", child.IncludedInAppID)
+		assert.Equal(t, "testdata-app", child.IncludedInAppID)
-		assert.Equal(t, "public/app/plugins/app/testdata", child.BaseURL)
+		assert.Equal(t, "public/app/plugins/app/testdata-app", child.BaseURL)
 	})
 }
 

pkg/plugins/manager/signature/manifest_test.go

@@ -153,7 +153,7 @@ func TestCalculate(t *testing.T) {
 
 			sig, err := Calculate(log.NewNopLogger(), &plugins.Plugin{
 				JSONData: plugins.JSONData{
-					ID: "test",
+					ID: "test-datasource",
 					Info: plugins.Info{
 						Version: "1.0.0",
 					},

pkg/plugins/manager/testdata/disallowed-cue-import/models.cue

@@ -0,0 +1,23 @@
+package grafanaplugin
+
+import (
+	"github.com/grafana/thema"
+	"github.com/grafana/grafana/pkg/framework/coremodel"
+)
+
+_dummy: coremodel.slots
+
+Panel: thema.#Lineage & {
+	name: "disallowed_cue_import"
+	seqs: [
+		{
+			schemas: [
+				{
+					PanelOptions: {
+						foo: string
+					} @cuetsy(kind="interface")
+				},
+			]
+		},
+	]
+}

pkg/plugins/manager/testdata/disallowed-cue-import/plugin.json

@@ -0,0 +1,14 @@
+{
+  "type": "panel",
+  "name": "Disallowed CUE import",
+  "id": "disallowed-import-panel",
+  "backend": true,
+  "state": "alpha",
+  "info": {
+    "description": "Test",
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    }
+  }
+}

pkg/plugins/manager/testdata/installer/plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "type": "datasource",
   "name": "Test",
-  "id": "test",
+  "id": "test-datasource",
   "backend": true,
   "executable": "test",
   "state": "alpha",

pkg/plugins/manager/testdata/invalid-v1-signature/plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "type": "datasource",
   "name": "Test",
-  "id": "test",
+  "id": "test-datasource",
   "backend": true,
   "state": "alpha",
   "info": {

pkg/plugins/manager/testdata/invalid-v2-extra-file/plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "type": "datasource",
   "name": "Test",
-  "id": "test",
+  "id": "test-datasource",
   "backend": true,
   "executable": "test",
   "state": "alpha",

pkg/plugins/manager/testdata/invalid-v2-missing-file/plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "type": "datasource",
   "name": "Test",
-  "id": "test",
+  "id": "test-datasource",
   "backend": true,
   "executable": "test",
   "state": "alpha",

pkg/plugins/manager/testdata/lacking-files/plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "type": "datasource",
   "name": "Test",
-  "id": "test",
+  "id": "test-datasource",
   "backend": true,
   "info": {
     "description": "Test",

pkg/plugins/manager/testdata/mismatch/models.cue

@@ -0,0 +1,18 @@
+package grafanaplugin
+
+import "github.com/grafana/thema"
+
+Panel: thema.#Lineage & {
+	name: "doesnamatch"
+	seqs: [
+		{
+			schemas: [
+				{
+					PanelOptions: {
+						foo: string
+					} @cuetsy(kind="interface")
+				},
+			]
+		},
+	]
+}

pkg/plugins/manager/testdata/mismatch/plugin.json

@@ -0,0 +1,14 @@
+{
+  "type": "panel",
+  "name": "Slot impl testing",
+  "id": "mismatch-panel",
+  "backend": true,
+  "state": "alpha",
+  "info": {
+    "description": "Test",
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    }
+  }
+}

pkg/plugins/manager/testdata/missing-slot-impl/models.cue

@@ -0,0 +1,16 @@
+package grafanaplugin
+
+import "github.com/grafana/thema"
+
+Query: thema.#Lineage & {
+	name: "missing_slot_impl"
+	seqs: [
+		{
+			schemas: [
+				{
+					foo: string
+				},
+			]
+		},
+	]
+}

pkg/plugins/manager/testdata/missing-slot-impl/plugin.json

@@ -0,0 +1,14 @@
+{
+  "type": "datasource",
+  "name": "Missing slot impl",
+  "id": "missing-slot-datasource",
+  "backend": true,
+  "state": "alpha",
+  "info": {
+    "description": "Test",
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    }
+  }
+}

pkg/plugins/manager/testdata/name-id-mismatch/models.cue

@@ -0,0 +1,18 @@
+package grafanaplugin
+
+import "github.com/grafana/thema"
+
+Panel: thema.#Lineage & {
+	name: "mismatch"
+	seqs: [
+		{
+			schemas: [
+				{
+					PanelOptions: {
+						foo: string
+					} @cuetsy(kind="interface")
+				},
+			]
+		},
+	]
+}

pkg/plugins/manager/testdata/name-id-mismatch/plugin.json

@@ -0,0 +1,14 @@
+{
+  "type": "panel",
+  "name": "ID/Name mismatch",
+  "id": "name-mismatch-panel",
+  "backend": true,
+  "state": "alpha",
+  "info": {
+    "description": "Test",
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    }
+  }
+}

pkg/plugins/manager/testdata/nested-plugins/parent/MANIFEST.txt

@@ -7,22 +7,23 @@ Hash: SHA512
   "signatureType": "grafana",
   "signedByOrg": "grafana",
   "signedByOrgName": "Grafana Labs",
-  "plugin": "test-ds",
+  "plugin": "test-datasource",
   "version": "1.0.0",
-  "time": 1629461930434,
+  "time": 1661172777367,
   "keyId": "7e4d0c6a708866e7",
   "files": {
-    "plugin.json": "64e98031f30cfada473e0ad4b989ac10cd0c86844aab8c0d3fc36d8a9537a0b8",
+    "plugin.json": "a029469ace740e9502bfb0d40924d1cccae73d0b18adcd8f1ceb7f17bf36beb8",
     "nested/plugin.json": "e64abd35cd211e0e4682974ad5cdd1be7a0b7cd24951d302a16d9e2cb6cefea4"
   }
 }
 -----BEGIN PGP SIGNATURE-----
-Version: OpenPGP.js v4.10.1
+Version: OpenPGP.js v4.10.10
 Comment: https://openpgpjs.org
 
-wqIEARMKAAYFAmEfnaoACgkQfk0ManCIZufwYgIJAZULZ72BKYehVw362aOJ
+wrgEARMKAAYFAmMDfCkAIQkQfk0ManCIZucWIQTzOyW2kQdOhGNlcPN+TQxq
-IkUhCaIceQT6rSmWw60Ksxs8xkeCebMPfuxm6xqpvoquVmD2zIirCFUXE41M
+cIhm56w5AgkBeX3H13KSFfSs6i6aJLOIPyqYICT9EQWKxmZIz4vlgnOBOvdA
-SQBys7/aAgkBaaVZvVPLUMYHIGNQXQ0wJ0j6JGn5Mn25GH4lH4vttaCFpQmx
+cf5jtG/CFYikBAHN6PAH6/Jir+4017w1JNHNtxICBj5xERqPkjb3GqT1sNb3
-zwV8J/s7Ho612fU1ijH/nFM97I4nfxonQUEyEbA=
+MJizG0LSveo6dRaap8uC4VPbubiUa7qGu6LTEi/8kpOemMNOLHBI+2/GlY3B
-=7sr3
+i8zqeBLU
+=lRFr
 -----END PGP SIGNATURE-----

pkg/plugins/manager/testdata/nested-plugins/parent/plugin.json

@@ -1,7 +1,7 @@
 {
   "type": "datasource",
   "name": "Parent",
-  "id": "test-ds",
+  "id": "test-datasource",
   "backend": true,
   "info": {
     "description": "Parent plugin",

pkg/plugins/manager/testdata/non-pvt-with-root-url/plugin/MANIFEST.txt

@@ -10,22 +10,22 @@ Hash: SHA512
   "rootUrls": [
     "https://dev.grafana.com/"
   ],
-  "plugin": "test",
+  "plugin": "test-datasource",
   "version": "1.0.0",
-  "time": 1657888677250,
+  "time": 1661173657946,
   "keyId": "7e4d0c6a708866e7",
   "files": {
-    "plugin.json": "2bb467c0bfd6c454551419efe475b8bf8573734e73c7bab52b14842adb62886f"
+    "plugin.json": "203ef4a613c5693c437a665cd67f95e2756a0f71b336b2ffb265db7c180d0b19"
   }
 }
 -----BEGIN PGP SIGNATURE-----
 Version: OpenPGP.js v4.10.10
 Comment: https://openpgpjs.org
 
-wrgEARMKAAYFAmLRX6UAIQkQfk0ManCIZucWIQTzOyW2kQdOhGNlcPN+TQxq
+wrgEARMKAAYFAmMDf5oAIQkQfk0ManCIZucWIQTzOyW2kQdOhGNlcPN+TQxq
-cIhm5wu9Agjhh5II2OyqsYDUqajO9KtwMzAnEMwaT5Kj0oCOsjJruoT/jLz6
+cIhm54/fAgkBVr9FXILsku+PsG86pZbxSbB/5/OeDsoqq9vJ30R3yaBYJC0N
-HO7ioenfCwqNxaJswuFkvpN+5BnrrbIwXDo1mgIJARFtKuRg1t4TK2DPcMiQ
+tcS1PtWPzc3yMqJY1zi5pem0WfmYdH3j++NqB3QCCIUz1eAjgbilvIvoyj/j
-IiEWNrFGK0jCFaofroH1sGnhjNqUy6JAIUQlUn17BHwiJdBqpsihW1HvPhMa
+Ia9Vcje1c3xApMFAeD4DdUBgFljAUFzz48IjZacjSNFm+gaNPhWJzYmo83wz
-8KOdLWED
+VqEbGL1A
-=D70r
+=SzNa
 -----END PGP SIGNATURE-----

pkg/plugins/manager/testdata/non-pvt-with-root-url/plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "type": "datasource",
   "name": "Test",
-  "id": "test",
+  "id": "test-datasource",
   "backend": true,
   "executable": "test",
   "state": "alpha",

pkg/plugins/manager/testdata/panel-conflicting-joinschema/models.cue

@@ -0,0 +1,23 @@
+package grafanaplugin
+
+import "github.com/grafana/thema"
+
+Panel: thema.#Lineage & {
+	joinSchema: {
+		PanelOptions: {...}
+		PanelFieldConfig: string
+	}
+	name: "panel_conflicting_joinschema"
+	seqs: [
+		{
+			schemas: [
+				{
+					PanelOptions: {
+						foo: string
+					} @cuetsy(kind="interface")
+					PanelFieldConfig: string
+				},
+			]
+		},
+	]
+}

pkg/plugins/manager/testdata/panel-conflicting-joinschema/plugin.json

@@ -0,0 +1,14 @@
+{
+  "type": "panel",
+  "name": "Slot impl testing",
+  "id": "panel-conflicting-joinschema",
+  "backend": true,
+  "state": "alpha",
+  "info": {
+    "description": "Test",
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    }
+  }
+}

pkg/plugins/manager/testdata/panel-does-not-follow-slot-joinschema/models.cue

@@ -0,0 +1,19 @@
+package grafanaplugin
+
+import "github.com/grafana/thema"
+
+Panel: thema.#Lineage & {
+	name: "panel_does_not_follow_slot_joinschema"
+	seqs: [
+		{
+			schemas: [
+				{
+					PanelOptions: {
+						foo: string
+					} @cuetsy(kind="interface")
+					PanelFieldConfig: string
+				},
+			]
+		},
+	]
+}

pkg/plugins/manager/testdata/panel-does-not-follow-slot-joinschema/plugin.json

@@ -0,0 +1,14 @@
+{
+  "type": "panel",
+  "name": "Slot impl testing",
+  "id": "panel-does-not-follow-slot-joinschema",
+  "backend": true,
+  "state": "alpha",
+  "info": {
+    "description": "Test",
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    }
+  }
+}

pkg/plugins/manager/testdata/unsigned-datasource/plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "type": "datasource",
   "name": "Test",
-  "id": "test",
+  "id": "test-datasource",
   "backend": true,
   "state": "alpha",
   "info": {

pkg/plugins/manager/testdata/valid-model-datasource/models.cue

@@ -0,0 +1,34 @@
+package grafanaplugin
+
+import "github.com/grafana/thema"
+
+Query: thema.#Lineage & {
+	name: "valid_model_datasource"
+	seqs: [
+		{
+			schemas: [
+				{
+					foo: string
+				},
+			]
+		},
+	]
+}
+
+DSOptions: thema.#Lineage & {
+	name: "valid_model_datasource"
+	seqs: [
+		{
+			schemas: [
+				{
+					Options: {
+						foo: string
+					}
+					SecureOptions: {
+						bar: string
+					}
+				},
+			]
+		},
+	]
+}

pkg/plugins/manager/testdata/valid-model-datasource/plugin.json

@@ -0,0 +1,14 @@
+{
+  "type": "datasource",
+  "name": "Datasource models valid",
+  "id": "valid-model-datasource",
+  "backend": true,
+  "state": "alpha",
+  "info": {
+    "description": "Test",
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    }
+  }
+}

pkg/plugins/manager/testdata/valid-model-panel/models.cue

@@ -0,0 +1,18 @@
+package grafanaplugin
+
+import "github.com/grafana/thema"
+
+Panel: thema.#Lineage & {
+	name: "valid_model_panel"
+	seqs: [
+		{
+			schemas: [
+				{
+					PanelOptions: {
+						foo: string
+					} @cuetsy(kind="interface")
+				},
+			]
+		},
+	]
+}

pkg/plugins/manager/testdata/valid-model-panel/plugin.json

@@ -0,0 +1,14 @@
+{
+  "type": "panel",
+  "name": "Panel models valid",
+  "id": "valid-model-panel",
+  "backend": true,
+  "state": "alpha",
+  "info": {
+    "description": "Test",
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    }
+  }
+}

pkg/plugins/manager/testdata/valid-v2-pvt-signature-root-url-uri/plugin/MANIFEST.txt

@@ -8,23 +8,24 @@ Hash: SHA512
   "signedByOrg": "willbrowne",
   "signedByOrgName": "Will Browne",
   "rootUrls": [
-    "http://localhost:3000/grafana/"
+    "http://localhost:3000/grafana"
   ],
-  "plugin": "test",
+  "plugin": "test-datasource",
   "version": "1.0.0",
-  "time": 1623165794939,
+  "time": 1661171981629,
   "keyId": "7e4d0c6a708866e7",
   "files": {
-    "plugin.json": "2bb467c0bfd6c454551419efe475b8bf8573734e73c7bab52b14842adb62886f"
+    "plugin.json": "203ef4a613c5693c437a665cd67f95e2756a0f71b336b2ffb265db7c180d0b19"
   }
 }
 -----BEGIN PGP SIGNATURE-----
-Version: OpenPGP.js v4.10.1
+Version: OpenPGP.js v4.10.10
 Comment: https://openpgpjs.org
 
-wqEEARMKAAYFAmC/i2MACgkQfk0ManCIZudCEgII80waYmySwVuB2cdeU3Vy
+wrcEARMKAAYFAmMDeQ0AIQkQfk0ManCIZucWIQTzOyW2kQdOhGNlcPN+TQxq
-FvYrhViYYimvTy5EQbDfC955UpHphcr4V5S+09se7D2bK8XZ/MYufnUp9QIU
+cIhm5ygmAgiUFIfZrpxCa5VajERXgejFRwGrWYILWXmmXWC4vqHiQaFEE1Ef
-gOxCDrkCCQHTQ/aWxt8JAHGG/eoydKQEeAc9aFJyphdX57qXHVkAjvLzY5aO
+DtLz0JcdEMhvhydD+efJbWuUcv7fEMWMv6k0YAIGLG4xVsef4OhnfMYKjRBf
-y9UltPQKOAN/soDra2m39VUf6DBi9K/sXfjwaA==
+Obc4/RuzqbjLg04Z9XDq6gAY06NESYscSj+Vy3rKNo0IiVnjrm9qGvZmSqRx
-=cd6n
+sLyae5M=
+=ZAe8
 -----END PGP SIGNATURE-----

pkg/plugins/manager/testdata/valid-v2-pvt-signature-root-url-uri/plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "type": "datasource",
   "name": "Test",
-  "id": "test",
+  "id": "test-datasource",
   "backend": true,
   "executable": "test",
   "state": "alpha",

pkg/plugins/manager/testdata/valid-v2-pvt-signature/plugin/MANIFEST.txt

@@ -10,21 +10,22 @@ Hash: SHA512
   "rootUrls": [
     "http://localhost:3000/"
   ],
-  "plugin": "test",
+  "plugin": "test-datasource",
   "version": "1.0.0",
-  "time": 1605807018050,
+  "time": 1661171417046,
   "keyId": "7e4d0c6a708866e7",
   "files": {
-    "plugin.json": "2bb467c0bfd6c454551419efe475b8bf8573734e73c7bab52b14842adb62886f"
+    "plugin.json": "203ef4a613c5693c437a665cd67f95e2756a0f71b336b2ffb265db7c180d0b19"
   }
 }
 -----BEGIN PGP SIGNATURE-----
-Version: OpenPGP.js v4.10.1
+Version: OpenPGP.js v4.10.10
 Comment: https://openpgpjs.org
 
-wqIEARMKAAYFAl+2q6oACgkQfk0ManCIZudmzwIJAXWz58cd/91rTXszKPnE
+wrgEARMKAAYFAmMDdtkAIQkQfk0ManCIZucWIQTzOyW2kQdOhGNlcPN+TQxq
-xbVEvERCbjKTtPBQBNQyqEvV+Ig3MuBSNOVy2SOGrMsdbS6lONgvgt4Cm+iS
+cIhm577/AgkBnbauM7s/8jLrdJvr+b9B2ZK7EipwI9GFClBdGfxhBzw/QcHS
-wV+vYifkAgkBJtg/9DMB7/iX5O0h49CtSltcpfBFXlGqIeOwRac/yENzRzAA
+ete9DAB0j9V5ilShlg3O4gmbiFUFUKGWByHt/VUCB3TXblS7cf5kJFjB9v0r
-khdr/tZ1PDgRxMqB/u+Vtbpl0xSxgblnrDOYMSI=
+fv5a8NfV8x8ao/WoKTmXRUB7HSScOvb/3KmkNqzcHtZPQS1T0P6l9EUA1QT1
-=rLIE
+l+GB3Wdq
+=pe3h
 -----END PGP SIGNATURE-----

pkg/plugins/manager/testdata/valid-v2-pvt-signature/plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "type": "datasource",
   "name": "Test",
-  "id": "test",
+  "id": "test-datasource",
   "backend": true,
   "executable": "test",
   "state": "alpha",

pkg/plugins/manager/testdata/valid-v2-signature/plugin/MANIFEST.txt

@@ -7,21 +7,22 @@ Hash: SHA512
   "signatureType": "grafana",
   "signedByOrg": "grafana",
   "signedByOrgName": "Grafana Labs",
-  "plugin": "test",
+  "plugin": "test-datasource",
   "version": "1.0.0",
-  "time": 1605807330546,
+  "time": 1661171059101,
   "keyId": "7e4d0c6a708866e7",
   "files": {
-    "plugin.json": "2bb467c0bfd6c454551419efe475b8bf8573734e73c7bab52b14842adb62886f"
+    "plugin.json": "203ef4a613c5693c437a665cd67f95e2756a0f71b336b2ffb265db7c180d0b19"
   }
 }
 -----BEGIN PGP SIGNATURE-----
-Version: OpenPGP.js v4.10.1
+Version: OpenPGP.js v4.10.10
 Comment: https://openpgpjs.org
 
-wqEEARMKAAYFAl+2rOIACgkQfk0ManCIZudNOwIJAT8FTzwnRFCSLTOaR3F3
+wrgEARMKAAYFAmMDdXMAIQkQfk0ManCIZucWIQTzOyW2kQdOhGNlcPN+TQxq
-2Fh96eRbghokXcQG9WqpQAg8ZiVfGXeWWRNtV+nuQ9VOZOTO0BovWLuMkym2
+cIhm54zLAgdfVimeut6Gw9MrIACBZUSH0ht9p9j+iG6MDjpmEFIpqVJrem6f
-ci8ABpWOAgd46LkGn3Dd8XVnGmLI6UPqHAXflItOrCMRiGcYJn5PxP1aCz8h
+8wBv0/kmYU3LV9MWyPuUeRfBdccjQKSjEXlfEAIJAVmut9LcSKIykhWuQA+7
-D0JoNI9TIKrhMtM4voU3Qhf3mIOTHueuDNS48w==
+VMVvJPXzlPkeoYsGYvzAlxh8i2UomCU15UChe62Gzq5V5HgGYkX5layIb5XX
-=mu2j
+y2Pio0lc
+=/TR0
 -----END PGP SIGNATURE-----

pkg/plugins/manager/testdata/valid-v2-signature/plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "type": "datasource",
   "name": "Test",
-  "id": "test",
+  "id": "test-datasource",
   "backend": true,
   "executable": "test",
   "state": "alpha",

pkg/plugins/manager/testdata/wrong-slot-panel/models.cue

@@ -0,0 +1,31 @@
+package grafanaplugin
+
+import "github.com/grafana/thema"
+
+Query: thema.#Lineage & {
+	name: "wrong_slot_panel"
+	seqs: [
+		{
+			schemas: [
+				{
+					foo: string
+				},
+			]
+		},
+	]
+}
+
+Panel: thema.#Lineage & {
+	name: "wrong_slot_panel"
+	seqs: [
+		{
+			schemas: [
+				{
+					PanelOptions: {
+						foo: string
+					} @cuetsy(kind="interface")
+				},
+			]
+		},
+	]
+}

pkg/plugins/manager/testdata/wrong-slot-panel/plugin.json

@@ -0,0 +1,14 @@
+{
+  "type": "panel",
+  "name": "Wrong slot for type",
+  "id": "wrong-slot-panel",
+  "backend": true,
+  "state": "alpha",
+  "info": {
+    "description": "Test",
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    }
+  }
+}

pkg/plugins/pfs/doc.go

@@ -0,0 +1,3 @@
+// Package pfs ("Plugin FS") defines a virtual filesystem representation of Grafana plugins.
+
+package pfs

pkg/plugins/pfs/errors.go

@@ -0,0 +1,30 @@
+package pfs
+
+import "errors"
+
+// ErrEmptyFS indicates that the fs.FS provided to ParsePluginFS was empty.
+var ErrEmptyFS = errors.New("provided fs.FS is empty")
+
+// ErrNoRootFile indicates that no root plugin.json file exists.
+var ErrNoRootFile = errors.New("no plugin.json at root of fs.fS")
+
+// ErrInvalidRootFile indicates that the root plugin.json file is invalid.
+var ErrInvalidRootFile = errors.New("plugin.json is invalid")
+
+// ErrImplementedSlots indicates that a plugin has implemented the wrong set of
+// slots for its type in models.cue. Either:
+//   - A slot is implemented that is not allowed for its type (e.g. datasource plugin implements Panel)
+//   - A required slot for its type is not implemented (e.g. panel plugin does not implemented Panel)
+var ErrImplementedSlots = errors.New("slot implementation not allowed for this plugin type")
+
+// ErrInvalidLineage indicates that the plugin contains an invalid lineage
+// declaration, according to Thema's validation rules in
+// ["github.com/grafana/thema".BindLineage].
+var ErrInvalidLineage = errors.New("invalid lineage")
+
+// ErrLineageNameMismatch indicates a plugin slot lineage name did not match the id of the plugin.
+var ErrLineageNameMismatch = errors.New("lineage name not the same as plugin id")
+
+// ErrDisallowedCUEImport indicates that a plugin's models.cue file imports a
+// CUE package that is not on the whitelist for safe imports.
+var ErrDisallowedCUEImport = errors.New("CUE import is not allowed")

pkg/plugins/pfs/pfs.go

@@ -0,0 +1,336 @@
+package pfs
+
+import (
+	"fmt"
+	"io/fs"
+	"sort"
+	"strings"
+	"sync"
+
+	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/ast"
+	"cuelang.org/go/cue/errors"
+	"cuelang.org/go/cue/parser"
+	"github.com/grafana/grafana"
+	"github.com/grafana/grafana/pkg/coremodel/pluginmeta"
+	"github.com/grafana/grafana/pkg/framework/coremodel"
+	"github.com/grafana/grafana/pkg/framework/coremodel/registry"
+	"github.com/grafana/thema"
+	"github.com/grafana/thema/kernel"
+	"github.com/grafana/thema/load"
+	"github.com/yalue/merged_fs"
+)
+
+// PermittedCUEImports returns the list of packages that may be imported in a
+// plugin models.cue file.
+func PermittedCUEImports() []string {
+	return []string{
+		"github.com/grafana/thema",
+		"github.com/grafana/grafana/packages/grafana-schema/src/schema",
+	}
+}
+
+func importAllowed(path string) bool {
+	for _, p := range PermittedCUEImports() {
+		if p == path {
+			return true
+		}
+	}
+	return false
+}
+
+var allowedImportsStr string
+
+// Name expected to be used for all models.cue files in Grafana plugins
+const pkgname = "grafanaplugin"
+
+type slotandname struct {
+	name string
+	slot *coremodel.Slot
+}
+
+var allslots []slotandname
+
+var plugmux kernel.InputKernel
+
+// TODO re-enable after go1.18
+// var tsch thema.TypedSchema[pluginmeta.Model]
+// var plugmux vmux.ValueMux[pluginmeta.Model]
+
+func init() {
+	var all []string
+	for _, im := range PermittedCUEImports() {
+		all = append(all, fmt.Sprintf("\t%s", im))
+	}
+	allowedImportsStr = strings.Join(all, "\n")
+
+	for n, s := range coremodel.AllSlots() {
+		allslots = append(allslots, slotandname{
+			name: n,
+			slot: s,
+		})
+	}
+
+	sort.Slice(allslots, func(i, j int) bool {
+		return allslots[i].name < allslots[j].name
+	})
+}
+
+var muxonce sync.Once
+
+func loadMux() kernel.InputKernel {
+	muxonce.Do(func() {
+		plugmux = coremodel.Mux(registry.NewBase().Pluginmeta(), coremodel.Filename("plugin.json"))
+	})
+	return plugmux
+}
+
+// This used to be in init(), but that creates a risk for codegen.
+//
+// thema.BindType ensures that Go type and Thema schema are aligned. If we were
+// to call it during init(), then the code generator that fixes misalignments
+// between those two could trigger it if it depends on this package. That would
+// mean that schema changes to pluginmeta get caught in a loop where the codegen
+// process can't heal itself.
+//
+// In theory, that dependency shouldn't exist - this package should only be
+// imported for plugin codegen, which should all happen after coremodel codegen.
+// But in practice, it might exist. And it's really brittle and confusing to
+// fix if that does happen.
+//
+// Better to be resilient to the possibility instead. So, this is a standalone function,
+// called as needed to get our muxer, and internally relies on a sync.Once to avoid
+// repeated processing of thema.BindType.
+// TODO mux loading is easily generalizable in pkg/f/coremodel, shouldn't need one-off
+// TODO switch to this generic signature after go1.18
+// func loadMux() (thema.TypedSchema[pluginmeta.Model], vmux.ValueMux[pluginmeta.Model]) {
+// 	muxonce.Do(func() {
+// 		var err error
+// 		var t pluginmeta.Model
+// 		tsch, err = thema.BindType[pluginmeta.Model](pm.CurrentSchema(), t)
+// 		if err != nil {
+// 			panic(err)
+// 		}
+// 		plugmux = vmux.NewValueMux(tsch, vmux.NewJSONEndec("plugin.json"))
+// 	})
+// 	return tsch, plugmux
+// }
+
+// Tree represents the contents of a plugin filesystem tree.
+type Tree struct {
+	raw      fs.FS
+	rootinfo PluginInfo
+}
+
+func (t *Tree) FS() fs.FS {
+	return t.raw
+}
+
+func (t *Tree) RootPlugin() PluginInfo {
+	return t.rootinfo
+}
+
+// SubPlugins returned a map of the PluginInfos for subplugins
+// within the tree, if any, keyed by subpath.
+func (t *Tree) SubPlugins() map[string]PluginInfo {
+	panic("TODO")
+}
+
+// PluginInfo represents everything knowable about a single plugin from static
+// analysis of its filesystem tree contents.
+type PluginInfo struct {
+	meta      pluginmeta.Model
+	slotimpls map[string]thema.Lineage
+	imports   []*ast.ImportSpec
+}
+
+// CUEImports lists the CUE import statements in the plugin's models.cue file,
+// if any.
+func (pi PluginInfo) CUEImports() []*ast.ImportSpec {
+	return pi.imports
+}
+
+// SlotImplementations returns a map of the plugin's Thema lineages that
+// implement particular slots, keyed by the name of the slot.
+//
+// Returns an empty map if the plugin has not implemented any slots.
+func (pi PluginInfo) SlotImplementations() map[string]thema.Lineage {
+	return pi.slotimpls
+}
+
+// Meta returns the metadata declared in the plugin's plugin.json file.
+func (pi PluginInfo) Meta() pluginmeta.Model {
+	return pi.meta
+}
+
+// ParsePluginFS takes an fs.FS and checks that it represents exactly one valid
+// plugin fs tree, with the fs.FS root as the root of the tree.
+//
+// It does not descend into subdirectories to search for additional
+// plugin.json files.
+// TODO no descent is ok for core plugins, but won't cut it in general
+func ParsePluginFS(f fs.FS, lib thema.Library) (*Tree, error) {
+	if f == nil {
+		return nil, ErrEmptyFS
+	}
+	// _, mux := loadMux()
+	mux := loadMux()
+	ctx := lib.Context()
+
+	b, err := fs.ReadFile(f, "plugin.json")
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, ErrNoRootFile
+		}
+		return nil, fmt.Errorf("error reading plugin.json: %w", err)
+	}
+
+	tree := &Tree{
+		raw: f,
+		rootinfo: PluginInfo{
+			slotimpls: make(map[string]thema.Lineage),
+		},
+	}
+	r := &tree.rootinfo
+
+	// Pass the raw bytes into the muxer, get the populated Model type out that we want.
+	// TODO stop ignoring second return. (for now, lacunas are a WIP and can't occur until there's >1 schema in the pluginmeta lineage)
+	metaany, _, err := mux.Converge(b)
+	if err != nil {
+		// TODO more nuanced error handling by class of Thema failure
+		// return nil, fmt.Errorf("plugin.json was invalid: %w", err)
+		return nil, ewrap(err, ErrInvalidRootFile)
+	}
+	r.meta = *metaany.(*pluginmeta.Model)
+
+	if modbyt, err := fs.ReadFile(f, "models.cue"); err == nil {
+		// TODO introduce layered CUE dependency-injecting loader
+		//
+		// Until CUE has proper dependency management (and possibly even after), loading
+		// CUE files with non-stdlib imports requires injecting the imported packages
+		// into cue.mod/pkg/<import path>, unless the imports are within the same CUE
+		// module. Thema introduced a system for this for its dependers, which we use
+		// here, but we'll need to layer the same on top for importable Grafana packages.
+		// Needing to do this twice strongly suggests it needs a generic, standalone
+		// library.
+
+		mfs := merged_fs.NewMergedFS(f, grafana.CueSchemaFS)
+
+		// Note that this actually will load any .cue files in the fs.FS root dir in the pkgname.
+		// That's...maybe good? But not what it says on the tin
+		bi, err := load.InstancesWithThema(mfs, "", load.Package(pkgname))
+		if err != nil {
+			return nil, fmt.Errorf("loading models.cue failed: %w", err)
+		}
+
+		pf, _ := parser.ParseFile("models.cue", modbyt, parser.ParseComments)
+
+		for _, im := range pf.Imports {
+			ip := strings.Trim(im.Path.Value, "\"")
+			if !importAllowed(ip) {
+				return nil, ewrap(errors.Newf(im.Pos(), "import %q in models.cue not allowed, plugins may only import from:\n%s\n", ip, allowedImportsStr), ErrDisallowedCUEImport)
+			}
+			r.imports = append(r.imports, im)
+		}
+
+		val := ctx.BuildInstance(bi)
+		for _, s := range allslots {
+			iv := val.LookupPath(cue.ParsePath(s.slot.Name()))
+			lin, err := bindSlotLineage(iv, s.slot, r.meta, lib)
+			if lin != nil {
+				r.slotimpls[s.slot.Name()] = lin
+			}
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return tree, nil
+}
+
+func bindSlotLineage(v cue.Value, s *coremodel.Slot, meta pluginmeta.Model, lib thema.Library) (thema.Lineage, error) {
+	accept, required := s.ForPluginType(string(meta.Type))
+	exists := v.Exists()
+
+	if !accept {
+		if exists {
+			// If it's not accepted for the type, but is declared, error out. This keeps a
+			// precise boundary on what's actually expected for plugins to do, which makes
+			// for clearer docs and guarantees for users.
+			return nil, ewrap(fmt.Errorf("%s: %s plugins may not provide a %s slot implementation in models.cue", meta.Id, meta.Type, s.Name()), ErrImplementedSlots)
+		}
+		return nil, nil
+	}
+
+	if !exists && required {
+		return nil, ewrap(fmt.Errorf("%s: %s plugins must provide a %s slot implementation in models.cue", meta.Id, meta.Type, s.Name()), ErrImplementedSlots)
+	}
+
+	// TODO make this opt real in thema, then uncomment to enforce joinSchema
+	// lin, err := thema.BindLineage(iv, lib, thema.SatisfiesJoinSchema(s.MetaSchema()))
+	lin, err := thema.BindLineage(v, lib)
+	if err != nil {
+		return nil, ewrap(fmt.Errorf("%s: invalid thema lineage for slot %s: %w", meta.Id, s.Name(), err), ErrInvalidLineage)
+	}
+
+	sanid := sanitizePluginId(meta.Id)
+	if lin.Name() != sanid {
+		errf := func(format string, args ...interface{}) error {
+			var errin error
+			if n := v.LookupPath(cue.ParsePath("name")).Source(); n != nil {
+				errin = errors.Newf(n.Pos(), format, args...)
+			} else {
+				errin = fmt.Errorf(format, args...)
+			}
+			return ewrap(errin, ErrLineageNameMismatch)
+		}
+		if sanid != meta.Id {
+			return nil, errf("%s: %q slot lineage name must be the sanitized plugin id (%q), got %q", meta.Id, s.Name(), sanid, lin.Name())
+		} else {
+			return nil, errf("%s: %q slot lineage name must be the plugin id, got %q", meta.Id, s.Name(), lin.Name())
+		}
+	}
+	return lin, nil
+}
+
+// Plugin IDs are allowed to contain characters that aren't allowed in thema
+// Lineage names, CUE package names, Go package names, TS or Go type names, etc.
+func sanitizePluginId(s string) string {
+	return strings.Map(func(r rune) rune {
+		switch {
+		case r >= 'a' && r <= 'z':
+			fallthrough
+		case r >= 'A' && r <= 'Z':
+			fallthrough
+		case r >= '0' && r <= '9':
+			fallthrough
+		case r == '_':
+			return r
+		case r == '-':
+			return '_'
+		default:
+			return -1
+		}
+	}, s)
+}
+
+func ewrap(actual, is error) error {
+	return &errPassthrough{
+		actual: actual,
+		is:     is,
+	}
+}
+
+type errPassthrough struct {
+	actual error
+	is     error
+}
+
+func (e *errPassthrough) Is(err error) bool {
+	return errors.Is(err, e.actual) || errors.Is(err, e.is)
+}
+
+func (e *errPassthrough) Error() string {
+	return e.actual.Error()
+}

pkg/plugins/pfs/pfs_test.go

@@ -0,0 +1,283 @@
+package pfs
+
+import (
+	"archive/zip"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseTreeTestdata(t *testing.T) {
+	type tt struct {
+		tfs fs.FS
+		// TODO could remove this by getting rid of inconsistent subdirs
+		subpath string
+		skip    string
+		err     error
+		// TODO could remove this by expecting that dirname == id
+		rootid string
+	}
+	tab := map[string]tt{
+		"app-with-child": {
+			rootid:  "myorgid-simple-app",
+			subpath: "dist",
+			skip:    "schema violation, weirdness in info.version field",
+		},
+		"duplicate-plugins": {
+			rootid:  "test-app",
+			subpath: "nested",
+			skip:    "schema violation, dependencies don't follow naming constraints",
+		},
+		"includes-symlinks": {
+			skip: "schema violation, dependencies don't follow naming constraints",
+		},
+		"installer": {
+			rootid:  "test-datasource",
+			subpath: "plugin",
+		},
+		"invalid-plugin-json": {
+			rootid: "test-app",
+			err:    ErrInvalidRootFile,
+		},
+		"invalid-v1-signature": {
+			rootid:  "test-datasource",
+			subpath: "plugin",
+		},
+		"invalid-v2-extra-file": {
+			rootid:  "test-datasource",
+			subpath: "plugin",
+		},
+		"invalid-v2-missing-file": {
+			rootid:  "test-datasource",
+			subpath: "plugin",
+		},
+		"lacking-files": {
+			rootid:  "test-datasource",
+			subpath: "plugin",
+		},
+		"nested-plugins": {
+			rootid:  "test-datasource",
+			subpath: "parent",
+		},
+		"non-pvt-with-root-url": {
+			rootid:  "test-datasource",
+			subpath: "plugin",
+		},
+		"symbolic-plugin-dirs": {
+			skip: "io/fs-based scanner will not traverse symlinks; caller of ParsePluginFS() must do it",
+		},
+		"test-app": {
+			skip:   "schema violation, dependencies don't follow naming constraints",
+			rootid: "test-app",
+		},
+		"test-app-with-includes": {
+			rootid: "test-app",
+			skip:   "has a 'page'-type include which isn't a known part of spec",
+		},
+		"unsigned-datasource": {
+			rootid:  "test-datasource",
+			subpath: "plugin",
+		},
+		"unsigned-panel": {
+			rootid:  "test-panel",
+			subpath: "plugin",
+		},
+		"valid-v2-pvt-signature": {
+			rootid:  "test-datasource",
+			subpath: "plugin",
+		},
+		"valid-v2-pvt-signature-root-url-uri": {
+			rootid:  "test-datasource",
+			subpath: "plugin",
+		},
+		"valid-v2-signature": {
+			rootid:  "test-datasource",
+			subpath: "plugin",
+		},
+		"no-rootfile": {
+			err: ErrNoRootFile,
+		},
+		"valid-model-panel":      {},
+		"valid-model-datasource": {},
+		"wrong-slot-panel": {
+			err: ErrImplementedSlots,
+		},
+		"missing-slot-impl": {
+			err: ErrImplementedSlots,
+		},
+		"panel-conflicting-joinschema": {
+			err:  ErrInvalidLineage,
+			skip: "TODO implement BindOption in thema, SatisfiesJoinSchema, then use it here",
+		},
+		"panel-does-not-follow-slot-joinschema": {
+			err:  ErrInvalidLineage,
+			skip: "TODO implement BindOption in thema, SatisfiesJoinSchema, then use it here",
+		},
+		"name-id-mismatch": {
+			err: ErrLineageNameMismatch,
+		},
+		"mismatch": {
+			err: ErrLineageNameMismatch,
+		},
+		"disallowed-cue-import": {
+			err: ErrDisallowedCUEImport,
+		},
+	}
+
+	staticRootPath, err := filepath.Abs("../manager/testdata")
+	require.NoError(t, err)
+	dfs := os.DirFS(staticRootPath)
+	ents, err := fs.ReadDir(dfs, ".")
+	require.NoError(t, err)
+
+	// Ensure table test and dir list are ==
+	var dirs, tts []string
+	for k := range tab {
+		tts = append(tts, k)
+	}
+	for _, ent := range ents {
+		dirs = append(dirs, ent.Name())
+	}
+	sort.Strings(tts)
+	sort.Strings(dirs)
+	if !cmp.Equal(tts, dirs) {
+		t.Fatalf("table test map (-) and pkg/plugins/manager/testdata dirs (+) differ: %s", cmp.Diff(tts, dirs))
+	}
+
+	for _, ent := range ents {
+		tst := tab[ent.Name()]
+		tst.tfs, err = fs.Sub(dfs, filepath.Join(ent.Name(), tst.subpath))
+		require.NoError(t, err)
+		tab[ent.Name()] = tst
+	}
+
+	lib := cuectx.ProvideThemaLibrary()
+	for name, otst := range tab {
+		tst := otst // otherwise var is shadowed within func by looping
+		t.Run(name, func(t *testing.T) {
+			if tst.skip != "" {
+				t.Skip(tst.skip)
+			}
+
+			tree, err := ParsePluginFS(tst.tfs, lib)
+			if tst.err == nil {
+				require.NoError(t, err, "unexpected error while parsing plugin tree")
+			} else {
+				require.ErrorIs(t, err, tst.err, "unexpected error type while parsing plugin tree")
+				return
+			}
+
+			if tst.rootid == "" {
+				tst.rootid = name
+			}
+
+			rootp := tree.RootPlugin()
+			require.Equal(t, tst.rootid, rootp.Meta().Id, "expected root plugin id and actual root plugin id differ")
+		})
+	}
+}
+
+func TestParseTreeZips(t *testing.T) {
+	type tt struct {
+		tfs fs.FS
+		// TODO could remove this by getting rid of inconsistent subdirs
+		subpath string
+		skip    string
+		err     error
+		// TODO could remove this by expecting that dirname == id
+		rootid string
+	}
+
+	tab := map[string]tt{
+		"grafana-simple-json-datasource-ec18fa4da8096a952608a7e4c7782b4260b41bcf.zip": {
+			skip: "binary plugin",
+		},
+		"plugin-with-absolute-member.zip": {
+			skip: "not actually a plugin, no plugin.json?",
+		},
+		"plugin-with-absolute-symlink-dir.zip": {
+			skip: "not actually a plugin, no plugin.json?",
+		},
+		"plugin-with-absolute-symlink.zip": {
+			skip: "not actually a plugin, no plugin.json?",
+		},
+		"plugin-with-parent-member.zip": {
+			skip: "not actually a plugin, no plugin.json?",
+		},
+		"plugin-with-symlink-dir.zip": {
+			skip: "not actually a plugin, no plugin.json?",
+		},
+		"plugin-with-symlink.zip": {
+			skip: "not actually a plugin, no plugin.json?",
+		},
+		"plugin-with-symlinks.zip": {
+			subpath: "test-app",
+			rootid:  "test-app",
+		},
+	}
+
+	staticRootPath, err := filepath.Abs("../manager/installer/testdata")
+	require.NoError(t, err)
+	ents, err := os.ReadDir(staticRootPath)
+	require.NoError(t, err)
+
+	// Ensure table test and dir list are ==
+	var dirs, tts []string
+	for k := range tab {
+		tts = append(tts, k)
+	}
+	for _, ent := range ents {
+		dirs = append(dirs, ent.Name())
+	}
+	sort.Strings(tts)
+	sort.Strings(dirs)
+	if !cmp.Equal(tts, dirs) {
+		t.Fatalf("table test map (-) and pkg/plugins/installer/testdata dirs (+) differ: %s", cmp.Diff(tts, dirs))
+	}
+
+	for _, ent := range ents {
+		tst := tab[ent.Name()]
+		r, err := zip.OpenReader(filepath.Join(staticRootPath, ent.Name()))
+		require.NoError(t, err)
+		defer r.Close() //nolint:errcheck
+		if tst.subpath != "" {
+			tst.tfs, err = fs.Sub(r, tst.subpath)
+			require.NoError(t, err)
+		} else {
+			tst.tfs = r
+		}
+
+		tab[ent.Name()] = tst
+	}
+
+	lib := cuectx.ProvideThemaLibrary()
+	for name, otst := range tab {
+		tst := otst // otherwise var is shadowed within func by looping
+		t.Run(name, func(t *testing.T) {
+			if tst.skip != "" {
+				t.Skip(tst.skip)
+			}
+
+			tree, err := ParsePluginFS(tst.tfs, lib)
+			if tst.err == nil {
+				require.NoError(t, err, "unexpected error while parsing plugin tree")
+			} else {
+				require.ErrorIs(t, err, tst.err, "unexpected error type while parsing plugin tree")
+				return
+			}
+
+			if tst.rootid == "" {
+				tst.rootid = name
+			}
+
+			rootp := tree.RootPlugin()
+			require.Equal(t, tst.rootid, rootp.Meta().Id, "expected root plugin id and actual root plugin id differ")
+		})
+	}
+}

public/app/plugins/datasource/dashboard/plugin.json

@@ -2,7 +2,19 @@
   "type": "datasource",
   "name": "-- Dashboard --",
   "id": "dashboard",
-
   "builtIn": true,
+
+  "info": {
+    "description": "TODO",
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    },
+    "logos": {
+      "small": "TODO",
+      "large": "TODO"
+    }
+  },
+
   "metrics": true
 }

public/app/plugins/datasource/grafana/plugin.json

@@ -2,9 +2,20 @@
   "type": "datasource",
   "name": "-- Grafana --",
   "id": "grafana",
-
-  "backend": true,
   "builtIn": true,
+
+  "info": {
+    "description": "TODO",
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    },
+    "logos": {
+      "small": "TODO",
+      "large": "TODO"
+    }
+  },
+  "backend": true,
   "annotations": true,
   "metrics": true
 }

public/app/plugins/gen.go

@@ -1,4 +1,3 @@
-// go:build ignore
 //go:build ignore
 // +build ignore
 
@@ -9,11 +8,27 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 
-	"cuelang.org/go/cue/cuecontext"
 	"github.com/grafana/grafana/pkg/codegen"
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
 )
 
+var skipPlugins = map[string]bool{
+	"canvas":         true,
+	"heatmap":        true,
+	"heatmap-old":    true,
+	"candlestick":    true,
+	"state-timeline": true,
+	"status-history": true,
+	"table":          true,
+	"timeseries":     true,
+	"influxdb":       true, // plugin.json fails validation (defaultMatchFormat)
+	"mixed":          true, // plugin.json fails validation (mixed)
+	"opentsdb":       true, // plugin.json fails validation (defaultMatchFormat)
+}
+
 // Generate TypeScript for all plugin models.cue
 func main() {
 	if len(os.Args) > 1 {
@@ -27,28 +42,53 @@ func main() {
 		os.Exit(1)
 	}
 
-	var find func(path string) (string, error)
+	wd := codegen.NewWriteDiffer()
-	find = func(path string) (string, error) {
+	lib := cuectx.ProvideThemaLibrary()
-		parent := filepath.Dir(path)
+
-		if parent == path {
+	type ptreepath struct {
-			return "", errors.New("grafana root directory could not be found")
+		fullpath string
-		}
+		tree     *codegen.PluginTree
-		fp := filepath.Join(path, "go.mod")
-		if _, err := os.Stat(fp); err == nil {
-			return path, nil
-		}
-		return find(parent)
 	}
-	groot, err := find(cwd)
+	var ptrees []ptreepath
-	if err != nil {
+	for _, typ := range []string{"datasource", "panel"} {
-		fmt.Fprint(os.Stderr, err)
+		dir := filepath.Join(cwd, typ)
-		os.Exit(1)
+		treeor, err := codegen.ExtractPluginTrees(os.DirFS(dir), lib)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "extracting plugin trees failed for %s: %s\n", dir, err)
+			os.Exit(1)
+		}
+
+		for name, option := range treeor {
+			if skipPlugins[name] {
+				continue
+			}
+
+			if option.Tree != nil {
+				ptrees = append(ptrees, ptreepath{
+					fullpath: filepath.Join(typ, name),
+					tree:     option.Tree,
+				})
+			} else if !errors.Is(option.Err, pfs.ErrNoRootFile) {
+				fmt.Fprintf(os.Stderr, "error parsing plugin directory %s: %s\n", filepath.Join(dir, name), option.Err)
+				os.Exit(1)
+			}
+		}
 	}
 
-	wd, err := codegen.CuetsifyPlugins(cuecontext.New(), groot)
+	// Ensure ptrees are sorted, so that visit order is deterministic. Otherwise
-	if err != nil {
+	// having multiple core plugins with errors can cause confusing error
-		fmt.Fprintf(os.Stderr, "error while generating code:\n%s\n", err)
+	// flip-flopping
-		os.Exit(1)
+	sort.Slice(ptrees, func(i, j int) bool {
+		return ptrees[i].fullpath < ptrees[j].fullpath
+	})
+
+	for _, ptp := range ptrees {
+		twd, err := ptp.tree.GenerateTS(ptp.fullpath)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "generating typescript failed for %s: %s\n", ptp.fullpath, err)
+			os.Exit(1)
+		}
+		wd.Merge(twd)
 	}
 
 	if _, set := os.LookupEnv("CODEGEN_VERIFY"); set {

public/app/plugins/panel/annolist/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import "github.com/grafana/thema"
 

public/app/plugins/panel/annolist/models.gen.ts

@@ -5,7 +5,7 @@
 //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 
-export const modelVersion = Object.freeze([0, 0]);
+export const PanelModelVersion = Object.freeze([0, 0]);
 
 
 export interface PanelOptions {

public/app/plugins/panel/barchart/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import (
 	"github.com/grafana/thema"

public/app/plugins/panel/barchart/models.gen.ts

@@ -6,7 +6,7 @@
 
 import * as ui from '@grafana/schema';
 
-export const modelVersion = Object.freeze([0, 0]);
+export const PanelModelVersion = Object.freeze([0, 0]);
 
 
 export interface PanelOptions extends ui.OptionsWithLegend, ui.OptionsWithTooltip, ui.OptionsWithTextFormatting {

public/app/plugins/panel/bargauge/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import (
 	"github.com/grafana/thema"

public/app/plugins/panel/bargauge/models.gen.ts

@@ -6,7 +6,7 @@
 
 import * as ui from '@grafana/schema';
 
-export const modelVersion = Object.freeze([0, 0]);
+export const PanelModelVersion = Object.freeze([0, 0]);
 
 
 export interface PanelOptions extends ui.SingleStatBaseOptions {

public/app/plugins/panel/candlestick/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 Panel: thema.#Lineage & {
 	name: "candlestick"

public/app/plugins/panel/canvas/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import "github.com/grafana/thema"
 

public/app/plugins/panel/dashlist/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import "github.com/grafana/thema"
 

public/app/plugins/panel/dashlist/models.gen.ts

@@ -5,7 +5,7 @@
 //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 
-export const modelVersion = Object.freeze([0, 0]);
+export const PanelModelVersion = Object.freeze([0, 0]);
 
 
 export enum PanelLayout {

public/app/plugins/panel/gauge/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import (
 	"github.com/grafana/thema"

public/app/plugins/panel/gauge/models.gen.ts

@@ -6,7 +6,7 @@
 
 import * as ui from '@grafana/schema';
 
-export const modelVersion = Object.freeze([0, 0]);
+export const PanelModelVersion = Object.freeze([0, 0]);
 
 
 export interface PanelOptions extends ui.SingleStatBaseOptions {

public/app/plugins/panel/heatmap/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import "github.com/grafana/thema"
 

public/app/plugins/panel/histogram/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import (
 	"github.com/grafana/thema"

public/app/plugins/panel/histogram/models.gen.ts

@@ -6,7 +6,7 @@
 
 import * as ui from '@grafana/schema';
 
-export const modelVersion = Object.freeze([0, 0]);
+export const PanelModelVersion = Object.freeze([0, 0]);
 
 
 export interface PanelOptions extends ui.OptionsWithLegend, ui.OptionsWithTooltip {

public/app/plugins/panel/news/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import "github.com/grafana/thema"
 

public/app/plugins/panel/news/models.gen.ts

@@ -5,7 +5,7 @@
 //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 
-export const modelVersion = Object.freeze([0, 0]);
+export const PanelModelVersion = Object.freeze([0, 0]);
 
 
 export interface PanelOptions {

public/app/plugins/panel/piechart/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import (
 	"github.com/grafana/thema"

public/app/plugins/panel/piechart/models.gen.ts

@@ -6,7 +6,7 @@
 
 import * as ui from '@grafana/schema';
 
-export const modelVersion = Object.freeze([0, 0]);
+export const PanelModelVersion = Object.freeze([0, 0]);
 
 
 export enum PieChartType {

public/app/plugins/panel/stat/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import (
 	"github.com/grafana/thema"

public/app/plugins/panel/stat/models.gen.ts

@@ -6,7 +6,7 @@
 
 import * as ui from '@grafana/schema';
 
-export const modelVersion = Object.freeze([0, 0]);
+export const PanelModelVersion = Object.freeze([0, 0]);
 
 
 export interface PanelOptions extends ui.SingleStatBaseOptions {

public/app/plugins/panel/state-timeline/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import (
 	"github.com/grafana/thema"
@@ -21,7 +21,7 @@ import (
 
 Panel: thema.#Lineage & {
 	name: "state-timeline"
-	lineages: [
+	seqs: [
 		{
 			schemas: [
 				{

public/app/plugins/panel/status-history/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import (
 	"github.com/grafana/thema"

public/app/plugins/panel/table/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import (
 	"github.com/grafana/thema"

public/app/plugins/panel/text/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import "github.com/grafana/thema"
 

public/app/plugins/panel/text/models.gen.ts

@@ -5,7 +5,7 @@
 //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 
-export const modelVersion = Object.freeze([0, 0]);
+export const PanelModelVersion = Object.freeze([0, 0]);
 
 
 export enum TextMode {

public/app/plugins/panel/timeseries/models.cue

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package grafanaschema
+package grafanaplugin
 
 import (
 	"github.com/grafana/thema"