Reconcile coremodels, entities, objects under new kind framework (#56492)

* Update thema to latest

* Deal with s/Library/*Runtime/

* Commit new, working results of codegen

* We like pointers now

* Always take runtime arg for NewBase()

* Sketchy handwavy pass at entity meta framework

* Little nibbles

* Update pkg/framework/coremodel/entityframework.cue

Co-authored-by: Artur Wierzbicki <wierzbicki.artur.94@gmail.com>

* Move file into new framework location

* Introduce loaders, Go code

* Complete rename to kind

* Flesh out framework, add svg/dashboard examples

* Cruft removal

* Remove generated kind go files from gitignore

* Refine maturity concept, add SlotKind

* Update embed and go deps

* Export PrefixWithGrafanaCUE

* Make the loader actually work, holy crap

* Many small tweaks to type.cue

* Add Apache 2 licensing exceptions for kinds

* Add new kinds dir, start of generator

* Roll back to earlier oapi-codegen

* Introduce new grafana-specific CUE loaders

* Introduce new tidy code generators framework

* Catch up kind framework with tinkering

* Add slices for the generators

* Add write/verify step to main generator

* Many renames

* Split up kind framework cue files

* Use kind.Decl within generated kinds

* Create kind.SomeDecl wrapper type to cache lineages

* Better names again

* Get one generated implemented, hopefully

* Copy dashboard schema into new kind.cue

* Small fixes to make the initial gen work

* Put svg kind in its new home

* Add generated Go dashboard type

* More renames and cleanups

* Add base kind registry and generator

* Stop blacklisting *_gen.go files

This is not the Go best practice, anyway. All we actually want to ignore
for enterprise is generated wire files.

* Change codegen output directories

pkg/kind -> pkg/kinds
pkg/registry/kindreg -> pkg/registry/corekind

* Rename pkg/framework/kind to pkg/kindsys

* Add core structured kind generator

* Add plural and machine names to kind spec

* Copy playlist over to kind system

* Consolidate kindsys files

* Add raw kind generator

* Update CODEOWNERS for kind framework

* Touch up comments a bit

* More docs tweaks

* Remove generated types to reduce noise for review

* Split each generator into its own file

* Rename Slot kind to Composable kind

* Add handwavy types for customkind loading

* Guard against init calls to framework loader

* First pass at doc on extending the kind system

* Improve attribute example in docs

* Fix wire imports

* Add basic TS types generator

* Fix composable kind category def

* No need for a separate file with generate directive

* Catch dashboard schema up

* Rename generator types to something saner and generic

* Make version configurable in ts/go generators

* Add CommonMeta to ease property access

* Add kindsys prop indicating whether lineage is group

* Put all kind categories back in a single file

* Finish with kindsys group props

* Refactor maturity progression per discussion

- Replace "committed" with "merged"
- All kindcats can use all maturity levels, at least for now

* Convert ts veneer index generator to modular system

* Move over to new jennywrites framework

* Strip down old coremodel generator

* Use public version of jennywrites

* Pull latest thema

* Commit generated Go types

* Add header injection postprocessor

* Move sdboyer/jennywrites to grafana/codejen

* Tweak header output

* Remove dashboard and playlist coremodels

* Fix up backend dashboards devenv test

* Fix TS import patterns to new gen filename

* Update internal imports, remove coremodel registry

* Fix compilation errors, wire generation

* Export and replace the prefix dropper

* More Go struct and field name changes

* Last name fixes, hopefully

* Fix lint errors

* Last lint error

Co-authored-by: Artur Wierzbicki <wierzbicki.artur.94@gmail.com>

pkg/api/dashboard.go

@@ -16,9 +16,8 @@ import (
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/components/dashdiffs"
 	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/coremodel/dashboard"
-	"github.com/grafana/grafana/pkg/cuectx"
 	"github.com/grafana/grafana/pkg/infra/metrics"
+	"github.com/grafana/grafana/pkg/kinds/dashboard"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/alerting"
@@ -365,21 +364,22 @@ func (hs *HTTPServer) PostDashboard(c *models.ReqContext) response.Response {
 	}
 
 	if hs.Features.IsEnabled(featuremgmt.FlagValidateDashboardsOnSave) {
-		cm := hs.Coremodels.Dashboard()
+		kind := hs.Kinds.Dashboard()
 
+		dashbytes, err := cmd.Dashboard.Bytes()
+		if err != nil {
+			return response.Error(http.StatusBadRequest, "unable to parse dashboard", err)
+		}
 		// Ideally, coremodel validation calls would be integrated into the web
 		// framework. But this does the job for now.
 		schv, err := cmd.Dashboard.Get("schemaVersion").Int()
 
 		// Only try to validate if the schemaVersion is at least the handoff version
 		// (the minimum schemaVersion against which the dashboard schema is known to
-		// work), or if schemaVersion is absent (which will happen once the Thema
-		// schema becomes canonical).
+		// work), or if schemaVersion is absent (which will happen once the kind schema
+		// becomes canonical).
 		if err != nil || schv >= dashboard.HandoffSchemaVersion {
-			// Can't fail, web.Bind() already ensured it's valid JSON
-			b, _ := cmd.Dashboard.Bytes()
-			v, _ := cuectx.JSONtoCUE("dashboard.json", b)
-			if _, err := cm.CurrentSchema().Validate(v); err != nil {
+			if _, _, err := kind.JSONValueMux(dashbytes); err != nil {
 				return response.Error(http.StatusBadRequest, "invalid dashboard json", err)
 			}
 		}
@@ -772,7 +772,7 @@ func (hs *HTTPServer) ValidateDashboard(c *models.ReqContext) response.Response
 		return response.Error(http.StatusBadRequest, "bad request data", err)
 	}
 
-	cm := hs.Coremodels.Dashboard()
+	dk := hs.Kinds.Dashboard()
 	dashboardBytes := []byte(cmd.Dashboard)
 
 	// POST api receives dashboard as a string of json (so line numbers for errors stay consistent),
@@ -793,8 +793,7 @@ func (hs *HTTPServer) ValidateDashboard(c *models.ReqContext) response.Response
 	// work), or if schemaVersion is absent (which will happen once the Thema
 	// schema becomes canonical).
 	if err != nil || schemaVersion >= dashboard.HandoffSchemaVersion {
-		v, _ := cuectx.JSONtoCUE("dashboard.json", dashboardBytes)
-		_, validationErr := cm.CurrentSchema().Validate(v)
+		_, _, validationErr := dk.JSONValueMux(dashboardBytes)
 
 		if validationErr == nil {
 			isValid = true

pkg/api/dashboard_test.go

@@ -17,11 +17,11 @@ import (
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/api/routing"
 	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/framework/coremodel/registry"
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/usagestats"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/registry/corekind"
 	accesscontrolmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
 	"github.com/grafana/grafana/pkg/services/alerting"
 	"github.com/grafana/grafana/pkg/services/annotations/annotationstest"
@@ -65,7 +65,7 @@ func TestGetHomeDashboard(t *testing.T) {
 		SQLStore:                mockstore.NewSQLStoreMock(),
 		preferenceService:       prefService,
 		dashboardVersionService: dashboardVersionService,
-		Coremodels:              registry.NewBase(nil),
+		Kinds:                   corekind.NewBase(nil),
 	}
 
 	tests := []struct {
@@ -149,7 +149,7 @@ func TestDashboardAPIEndpoint(t *testing.T) {
 			Features:                featuremgmt.WithFeatures(),
 			DashboardService:        dashboardService,
 			dashboardVersionService: fakeDashboardVersionService,
-			Coremodels:              registry.NewBase(nil),
+			Kinds:                   corekind.NewBase(nil),
 		}
 
 		setUp := func() {
@@ -271,7 +271,7 @@ func TestDashboardAPIEndpoint(t *testing.T) {
 			DashboardService:        dashboardService,
 			dashboardVersionService: fakeDashboardVersionService,
 			Features:                featuremgmt.WithFeatures(),
-			Coremodels:              registry.NewBase(nil),
+			Kinds:                   corekind.NewBase(nil),
 		}
 
 		setUp := func() {
@@ -968,7 +968,7 @@ func TestDashboardAPIEndpoint(t *testing.T) {
 				AccessControl:                accesscontrolmock.New(),
 				DashboardService:             dashboardService,
 				Features:                     featuremgmt.WithFeatures(),
-				Coremodels:                   registry.NewBase(nil),
+				Kinds:                        corekind.NewBase(nil),
 			}
 			hs.callGetDashboard(sc)
 
@@ -1023,7 +1023,7 @@ func getDashboardShouldReturn200WithConfig(t *testing.T, sc *scenarioContext, pr
 		),
 		DashboardService: dashboardService,
 		Features:         featuremgmt.WithFeatures(),
-		Coremodels:       registry.NewBase(nil),
+		Kinds:            corekind.NewBase(nil),
 	}
 
 	hs.callGetDashboard(sc)
@@ -1089,7 +1089,7 @@ func postDashboardScenario(t *testing.T, desc string, url string, routePattern s
 			DashboardService:      dashboardService,
 			folderService:         folderService,
 			Features:              featuremgmt.WithFeatures(),
-			Coremodels:            registry.NewBase(nil),
+			Kinds:                 corekind.NewBase(nil),
 		}
 
 		sc := setupScenarioContext(t, url)
@@ -1121,7 +1121,7 @@ func postValidateScenario(t *testing.T, desc string, url string, routePattern st
 			LibraryElementService: &mockLibraryElementService{},
 			SQLStore:              sqlmock,
 			Features:              featuremgmt.WithFeatures(),
-			Coremodels:            registry.NewBase(nil),
+			Kinds:                 corekind.NewBase(nil),
 		}
 
 		sc := setupScenarioContext(t, url)
@@ -1158,7 +1158,7 @@ func postDiffScenario(t *testing.T, desc string, url string, routePattern string
 			SQLStore:                sqlmock,
 			dashboardVersionService: fakeDashboardVersionService,
 			Features:                featuremgmt.WithFeatures(),
-			Coremodels:              registry.NewBase(nil),
+			Kinds:                   corekind.NewBase(nil),
 		}
 
 		sc := setupScenarioContext(t, url)
@@ -1197,7 +1197,7 @@ func restoreDashboardVersionScenario(t *testing.T, desc string, url string, rout
 			SQLStore:                sqlStore,
 			Features:                featuremgmt.WithFeatures(),
 			dashboardVersionService: fakeDashboardVersionService,
-			Coremodels:              registry.NewBase(nil),
+			Kinds:                   corekind.NewBase(nil),
 		}
 
 		sc := setupScenarioContext(t, url)

pkg/api/http_server.go

@@ -29,7 +29,6 @@ import (
 	"github.com/grafana/grafana/pkg/api/routing"
 	httpstatic "github.com/grafana/grafana/pkg/api/static"
 	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/framework/coremodel/registry"
 	"github.com/grafana/grafana/pkg/infra/kvstore"
 	"github.com/grafana/grafana/pkg/infra/localcache"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -41,6 +40,7 @@ import (
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/plugins/plugincontext"
+	"github.com/grafana/grafana/pkg/registry/corekind"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/alerting"
 	"github.com/grafana/grafana/pkg/services/annotations"
@@ -192,7 +192,7 @@ type HTTPServer struct {
 	dashboardVersionService      dashver.Service
 	PublicDashboardsApi          *publicdashboardsApi.Api
 	starService                  star.Service
-	Coremodels                   *registry.Base
+	Kinds                        *corekind.Base
 	playlistService              playlist.Service
 	apiKeyService                apikey.Service
 	kvStore                      kvstore.KVStore
@@ -241,7 +241,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	avatarCacheServer *avatar.AvatarCacheServer, preferenceService pref.Service,
 	teamsPermissionsService accesscontrol.TeamPermissionsService, folderPermissionsService accesscontrol.FolderPermissionsService,
 	dashboardPermissionsService accesscontrol.DashboardPermissionsService, dashboardVersionService dashver.Service,
-	starService star.Service, csrfService csrf.Service, coremodels *registry.Base,
+	starService star.Service, csrfService csrf.Service, basekinds *corekind.Base,
 	playlistService playlist.Service, apiKeyService apikey.Service, kvStore kvstore.KVStore,
 	secretsMigrator secrets.Migrator, secretsPluginManager plugins.SecretsPluginManager, secretsService secrets.Service,
 	secretsPluginMigrator spm.SecretMigrationProvider, secretsStore secretsKV.SecretsKVStore,
@@ -337,7 +337,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 		dashboardPermissionsService:  dashboardPermissionsService,
 		dashboardVersionService:      dashboardVersionService,
 		starService:                  starService,
-		Coremodels:                   coremodels,
+		Kinds:                        basekinds,
 		playlistService:              playlistService,
 		apiKeyService:                apiKeyService,
 		kvStore:                      kvStore,