K8s: Add App SDK installer (#107773)

pkg/services/apiserver/appinstaller/README.md

@@ -0,0 +1,80 @@
+# AppInstaller
+
+An `AppInstaller` (`appsdkapiserver.AppInstaller`) is a new concept introduced in v0.40.0 of the Grafana App SDK. It is an interface responsible for installing an app into the Grafana API server. Each app provides an `AppInstaller` implementation, which is then used by the API server to manage the app.
+
+## Architectural Changes
+
+### New App Installer Framework
+
+A new framework for installing and managing apps has been introduced in `pkg/services/apiserver/appinstaller`. This framework is responsible for:
+
+-   **Schema Registration:** Registering the Kubernetes-style API schemas defined by an app.
+-   **Admission Plugins:** Registering admission plugins for apps.
+-   **OpenAPI Definitions:** Aggregating OpenAPI definitions from all installed apps.
+-   **API Installation:** Installing the API groups and resources defined by an app.
+-   **Lifecycle Management:** Managing the lifecycle of an app, including initialization and startup.
+
+### API Server Integration
+
+Grafana's core API server (`pkg/services/apiserver/service.go`) has been significantly updated to integrate with the new app installer framework. During startup, the API server now performs the following steps:
+
+1.  Collects all `AppInstaller` instances provided by the apps.
+2.  Uses the `appinstaller` framework to register schemas, admission plugins, and OpenAPI definitions.
+3.  Installs the APIs for each app.
+4.  Initializes and starts each app.
+
+This ensures that apps are seamlessly integrated into Grafana's API server and that their resources are available through the Grafana API.
+
+### App Registration
+
+The method for registering apps has been updated:
+
+-   The old way of registering apps using `ProvideBuilderRunners` in `pkg/registry/apps/apps.go` is now deprecated.
+-   The new, preferred way is to use the `ProvideAppInstallers` function, which returns a list of `AppInstaller` instances.
+
+This change streamlines the app registration process and makes it more consistent.
+
+## Migration Guide
+
+To migrate an existing app to the new SDK, you need to:
+
+1.  Implement the `appsdkapiserver.AppInstaller` interface for your app.
+2.  Update your app's registration to use the `ProvideAppInstallers` function.
+
+### Example: Playlist App
+
+The Playlist app has been migrated to the new App SDK. Let's look at how it was done.
+
+Previously, the Playlist app was registered using `ProvideBuilderRunners`:
+
+```go
+// pkg/registry/apps/apps.go (before)
+func ProvideBuilderRunners(
+    // ...
+    playlistAppProvider *playlist.PlaylistAppProvider,
+    // ...
+) (*Service, error) {
+    // ...
+    providers := []app.Provider{playlistAppProvider}
+    // ...
+}
+```
+
+Now, the Playlist app provides an `AppInstaller` and is registered through `ProvideAppInstallers`:
+
+```go
+// pkg/registry/apps/apps.go (after)
+func ProvideAppInstallers(
+	playlistAppInstaller *playlist.PlaylistAppInstaller,
+) []appsdkapiserver.AppInstaller {
+	return []appsdkapiserver.AppInstaller{playlistAppInstaller}
+}
+```
+
+The implementation of the `PlaylistAppInstaller` can be found in `pkg/registry/apps/playlist/register.go`. This file is a good reference for how to implement an `AppInstaller` for your own app.
+
+The `pkg/registry/apps/playlist/register.go` file does the following:
+- It defines `PlaylistAppInstaller`, which embeds `appsdkapiserver.AppInstaller`.
+- It implements `appinstaller.LegacyStorageProvider` to bridge with Grafana's existing playlist service.
+- The `RegisterAppInstaller` function initializes the installer, providing the app's manifest, specific configuration, and associating Go types with API kinds.
+- It shows how to provide a custom table converter for `kubectl get` style output. 

pkg/services/apiserver/appinstaller/installer.go

@@ -0,0 +1,178 @@
+package appinstaller
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"time"
+
+	appsdkapiserver "github.com/grafana/grafana-app-sdk/k8s/apiserver"
+	"github.com/grafana/grafana-app-sdk/logging"
+	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
+	"github.com/grafana/grafana/pkg/services/apiserver/builder"
+	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
+	grafanaapiserveroptions "github.com/grafana/grafana/pkg/services/apiserver/options"
+	"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/registry/generic"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/kube-openapi/pkg/common"
+)
+
+type LegacyStorageGetterFunc func(schema.GroupVersionResource) grafanarest.Storage
+
+type LegacyStorageProvider interface {
+	GetLegacyStorage(schema.GroupVersionResource) grafanarest.Storage
+}
+
+type AuthorizerProvider interface {
+	GetAuthorizer() authorizer.Authorizer
+}
+
+type APIEnablementProvider interface {
+	// Do not implement this unless you have special circumstances! This is a list of resources that are allowed to be accessed in v0alpha1,
+	// to prevent accidental exposure of experimental APIs. While developing, use the feature flag `grafanaAPIServerWithExperimentalAPIs`.
+	// And then, when you're ready to expose this to the end user, go to v1beta1 instead.
+	GetAllowedV0Alpha1Resources() []string
+}
+
+type AppInstallerConfig struct {
+	CustomConfig             any
+	AllowedV0Alpha1Resources []string
+}
+
+// serverLock interface defines a lock mechanism for executing actions with a timeout
+type serverLock interface {
+	LockExecuteAndRelease(ctx context.Context, actionName string, maxInterval time.Duration, fn func(ctx context.Context)) error
+}
+
+// AddToScheme adds app installer schemas to the runtime scheme
+func AddToScheme(
+	appInstallers []appsdkapiserver.AppInstaller,
+	scheme *runtime.Scheme,
+) ([]schema.GroupVersion, error) {
+	var additionalGroupVersions []schema.GroupVersion
+	for _, installer := range appInstallers {
+		if err := installer.AddToScheme(scheme); err != nil {
+			return nil, fmt.Errorf("failed to add app installer scheme: %w", err)
+		}
+		additionalGroupVersions = append(additionalGroupVersions, installer.GroupVersions()...)
+	}
+	return additionalGroupVersions, nil
+}
+
+// RegisterAdmissionPlugins registers admission plugins for app installers
+func RegisterAdmissionPlugins(
+	ctx context.Context,
+	appInstallers []appsdkapiserver.AppInstaller,
+	options *grafanaapiserveroptions.Options,
+) error {
+	logger := logging.FromContext(ctx)
+
+	for _, installer := range appInstallers {
+		plugin := installer.AdmissionPlugin()
+		if plugin != nil {
+			md := installer.ManifestData()
+			if md == nil {
+				return fmt.Errorf("manifest is not initialized for installer for GroupVersions %v", installer.GroupVersions())
+			}
+			pluginName := md.AppName + " admission"
+			options.RecommendedOptions.Admission.Plugins.Register(pluginName, plugin)
+			logger.Info("Registered admission plugin", "app", md.AppName)
+		}
+	}
+	return nil
+}
+
+func BuildOpenAPIDefGetter(
+	appInstallers []appsdkapiserver.AppInstaller,
+) func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition {
+	return func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition {
+		defs := make(map[string]common.OpenAPIDefinition)
+		maps.Copy(defs, appsdkapiserver.GetCommonOpenAPIDefinitions(ref))
+		for _, installer := range appInstallers {
+			maps.Copy(defs, installer.GetOpenAPIDefinitions(ref))
+		}
+		return defs
+	}
+}
+
+func InstallAPIs(
+	ctx context.Context,
+	appInstallers []appsdkapiserver.AppInstaller,
+	server *genericapiserver.GenericAPIServer,
+	restOpsGetter generic.RESTOptionsGetter,
+	storageOpts *grafanaapiserveroptions.StorageOptions,
+	kvStore grafanarest.NamespacedKVStore,
+	lock serverLock,
+	namespaceMapper request.NamespaceMapper,
+	dualWriteService dualwrite.Service,
+	dualWriterMetrics *grafanarest.DualWriterMetrics,
+	builderMetrics *builder.BuilderMetrics,
+) error {
+	logger := logging.FromContext(ctx)
+
+	for _, installer := range appInstallers {
+		logger.Debug("Installing APIs for app installer", "app", installer.ManifestData().AppName)
+		wrapper := &serverWrapper{
+			ctx:               ctx,
+			GenericAPIServer:  server,
+			installer:         installer,
+			storageOpts:       storageOpts,
+			restOptionsGetter: restOpsGetter,
+			kvStore:           kvStore,
+			lock:              lock,
+			namespaceMapper:   namespaceMapper,
+			dualWriteService:  dualWriteService,
+			dualWriterMetrics: dualWriterMetrics,
+			builderMetrics:    builderMetrics,
+		}
+		if err := installer.InstallAPIs(wrapper, restOpsGetter); err != nil {
+			return fmt.Errorf("failed to install APIs for app %s: %w", installer.ManifestData().AppName, err)
+		}
+		logger.Info("Installed APIs for app", "app", installer.ManifestData().AppName)
+	}
+	return nil
+}
+
+// RegisterPostStartHooks registers individual post start hooks for each app installer
+func RegisterPostStartHooks(
+	appInstallers []appsdkapiserver.AppInstaller,
+	serverConfig *genericapiserver.RecommendedConfig,
+) error {
+	for _, installer := range appInstallers {
+		md := installer.ManifestData()
+		if md == nil {
+			return fmt.Errorf("app installer has nil manifest data: %T", installer)
+		}
+		hook := createPostStartHook(installer)
+		if err := serverConfig.AddPostStartHook(md.AppName, hook); err != nil {
+			return fmt.Errorf("failed to register post start hook for app %s: %w", md.AppName, err)
+		}
+	}
+	return nil
+}
+
+func createPostStartHook(
+	installer appsdkapiserver.AppInstaller,
+) genericapiserver.PostStartHookFunc {
+	return func(hookContext genericapiserver.PostStartHookContext) error {
+		logger := logging.FromContext(hookContext.Context)
+		logger.Debug("Initializing app", "app", installer.ManifestData().AppName)
+
+		if err := installer.InitializeApp(*hookContext.LoopbackClientConfig); err != nil {
+			logger.Error("Failed to initialize app", "app", installer.ManifestData().AppName, "error", err)
+			return fmt.Errorf("failed to initialize app %s: %w", installer.ManifestData().AppName, err)
+		}
+
+		logger.Info("App initialized", "app", installer.ManifestData().AppName)
+		app, err := installer.App()
+		if err != nil {
+			logger.Error("Failed to initialize app", "app", installer.ManifestData().AppName, "error", err)
+			return fmt.Errorf("failed to get app from installer %s: %w", installer.ManifestData().AppName, err)
+		}
+		return app.Runner().Run(hookContext.Context)
+	}
+}

pkg/services/apiserver/appinstaller/server.go

@@ -0,0 +1,95 @@
+package appinstaller
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/registry/generic"
+	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+
+	appsdkapiserver "github.com/grafana/grafana-app-sdk/k8s/apiserver"
+	"github.com/grafana/grafana-app-sdk/logging"
+	grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic"
+	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
+	"github.com/grafana/grafana/pkg/services/apiserver/builder"
+	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
+	grafanaapiserveroptions "github.com/grafana/grafana/pkg/services/apiserver/options"
+	"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
+)
+
+var _ appsdkapiserver.GenericAPIServer = (*serverWrapper)(nil)
+
+type serverWrapper struct {
+	ctx context.Context
+	appsdkapiserver.GenericAPIServer
+	installer         appsdkapiserver.AppInstaller
+	restOptionsGetter generic.RESTOptionsGetter
+	storageOpts       *grafanaapiserveroptions.StorageOptions
+	kvStore           grafanarest.NamespacedKVStore
+	lock              serverLock
+	namespaceMapper   request.NamespaceMapper
+	dualWriteService  dualwrite.Service
+	dualWriterMetrics *grafanarest.DualWriterMetrics
+	builderMetrics    *builder.BuilderMetrics
+}
+
+func (s *serverWrapper) InstallAPIGroup(apiGroupInfo *genericapiserver.APIGroupInfo) error {
+	log := logging.FromContext(s.ctx)
+	legacyProvider, ok := s.installer.(LegacyStorageProvider)
+	if !ok {
+		return s.GenericAPIServer.InstallAPIGroup(apiGroupInfo)
+	}
+	for v, storageMap := range apiGroupInfo.VersionedResourcesStorageMap {
+		for storagePath, restStorage := range storageMap {
+			genericStorage, ok := restStorage.(*genericregistry.Store)
+			if !ok {
+				log.Error("Expected generic registry store", "storagePath", storagePath, "version", v)
+				continue
+			}
+			resource, err := getResourceFromStoragePath(storagePath)
+			if err != nil {
+				return err
+			}
+			gr := schema.GroupResource{
+				Group:    s.installer.ManifestData().Group,
+				Resource: resource,
+			}
+			genericStorage.KeyRootFunc = grafanaregistry.KeyRootFunc(gr)
+			genericStorage.KeyFunc = grafanaregistry.NamespaceKeyFunc(gr)
+			genericStorage.UpdateStrategy = &updateStrategyWrapper{
+				RESTUpdateStrategy: genericStorage.UpdateStrategy,
+			}
+
+			dw, err := NewDualWriter(
+				s.ctx,
+				gr,
+				s.storageOpts,
+				legacyProvider.GetLegacyStorage(gr.WithVersion(v)),
+				grafanarest.Storage(genericStorage),
+				s.kvStore,
+				s.lock,
+				s.namespaceMapper,
+				s.dualWriteService,
+				s.dualWriterMetrics,
+				s.builderMetrics,
+			)
+			if err != nil {
+				return err
+			}
+			apiGroupInfo.VersionedResourcesStorageMap[v][storagePath] = dw
+		}
+	}
+
+	return s.GenericAPIServer.InstallAPIGroup(apiGroupInfo)
+}
+
+func getResourceFromStoragePath(storagePath string) (string, error) {
+	parts := strings.Split(storagePath, "/")
+	if len(parts) < 1 {
+		return "", fmt.Errorf("invalid storage path: %s", storagePath)
+	}
+	return parts[0], nil
+}

pkg/services/apiserver/appinstaller/storage.go

@@ -0,0 +1,119 @@
+package appinstaller
+
+import (
+	"context"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	k8srequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/klog/v2"
+
+	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
+	"github.com/grafana/grafana/pkg/services/apiserver/builder"
+	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
+	"github.com/grafana/grafana/pkg/services/apiserver/options"
+	"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
+)
+
+// NewDualWriter creates a dual writer for the given group resource using the provided configuration
+func NewDualWriter(
+	_ context.Context,
+	gr schema.GroupResource,
+	storageOpts *options.StorageOptions,
+	legacy grafanarest.Storage,
+	storage grafanarest.Storage,
+	kvStore grafanarest.NamespacedKVStore,
+	lock serverLock,
+	namespaceMapper request.NamespaceMapper,
+	dualWriteService dualwrite.Service,
+	dualWriterMetrics *grafanarest.DualWriterMetrics,
+	builderMetrics *builder.BuilderMetrics,
+) (grafanarest.Storage, error) {
+	// Dashboards + Folders may be managed (depends on feature toggles and database state)
+	if dualWriteService != nil && dualWriteService.ShouldManage(gr) {
+		return dualWriteService.NewStorage(gr, legacy, storage) // eventually this can replace this whole function
+	}
+
+	key := gr.String() // ${resource}.{group} eg playlists.playlist.grafana.app
+
+	// Get the option from custom.ini/command line
+	// when missing this will default to mode zero (legacy only)
+	var mode = grafanarest.DualWriterMode(0)
+
+	var (
+		dualWriterPeriodicDataSyncJobEnabled bool
+		dualWriterMigrationDataSyncDisabled  bool
+		dataSyncerInterval                   = time.Hour
+		dataSyncerRecordsLimit               = 1000
+	)
+
+	resourceConfig, resourceExists := storageOpts.UnifiedStorageConfig[key]
+	if resourceExists {
+		mode = resourceConfig.DualWriterMode
+		dualWriterPeriodicDataSyncJobEnabled = resourceConfig.DualWriterPeriodicDataSyncJobEnabled
+		dualWriterMigrationDataSyncDisabled = resourceConfig.DualWriterMigrationDataSyncDisabled
+		dataSyncerInterval = resourceConfig.DataSyncerInterval
+		dataSyncerRecordsLimit = resourceConfig.DataSyncerRecordsLimit
+	}
+
+	// Force using storage only -- regardless of internal synchronization state
+	if mode == grafanarest.Mode5 {
+		return storage, nil
+	}
+
+	// Moving from one version to the next can only happen after the previous step has
+	// successfully synchronized.
+	requestInfo := getRequestInfo(gr, namespaceMapper)
+
+	syncerCfg := &grafanarest.SyncerConfig{
+		Kind:                   key,
+		RequestInfo:            requestInfo,
+		Mode:                   mode,
+		SkipDataSync:           dualWriterMigrationDataSyncDisabled,
+		LegacyStorage:          legacy,
+		Storage:                storage,
+		ServerLockService:      lock,
+		DataSyncerInterval:     dataSyncerInterval,
+		DataSyncerRecordsLimit: dataSyncerRecordsLimit,
+	}
+
+	ctx := context.Background()
+	// This also sets the currentMode on the syncer config.
+	currentMode, err := grafanarest.SetDualWritingMode(ctx, kvStore, syncerCfg, dualWriterMetrics)
+	if err != nil {
+		return nil, err
+	}
+
+	builderMetrics.RecordDualWriterModes(gr.Resource, gr.Group, mode, currentMode)
+
+	switch currentMode {
+	case grafanarest.Mode0:
+		return legacy, nil
+	case grafanarest.Mode4, grafanarest.Mode5:
+		return storage, nil
+	default:
+	}
+
+	if dualWriterPeriodicDataSyncJobEnabled {
+		// The mode might have changed in SetDualWritingMode, so apply current mode first.
+		syncerCfg.Mode = currentMode
+		if err := grafanarest.StartPeriodicDataSyncer(ctx, syncerCfg, dualWriterMetrics); err != nil {
+			return nil, err
+		}
+	}
+
+	// when unable to use
+	if currentMode != mode {
+		klog.Warningf("Requested DualWrite mode: %d, but using %d for %+v", mode, currentMode, gr)
+	}
+	return dualwrite.NewDualWriter(gr, currentMode, legacy, storage)
+}
+
+func getRequestInfo(gr schema.GroupResource, namespaceMapper request.NamespaceMapper) *k8srequest.RequestInfo {
+	return &k8srequest.RequestInfo{
+		APIGroup:  gr.Group,
+		Resource:  gr.Resource,
+		Name:      "",
+		Namespace: namespaceMapper(int64(1)),
+	}
+}

pkg/services/apiserver/appinstaller/strategy.go

@@ -0,0 +1,19 @@
+package appinstaller
+
+import (
+	genericrest "k8s.io/apiserver/pkg/registry/rest"
+)
+
+type updateStrategyWrapper struct {
+	genericrest.RESTUpdateStrategy
+}
+
+func (s *updateStrategyWrapper) AllowCreateOnUpdate() bool {
+	// needed for dual write to work correctly
+	return true
+}
+
+func (s *updateStrategyWrapper) AllowUnconditionalUpdate() bool {
+	// needed for dual write to work correctly
+	return true
+}

pkg/services/apiserver/builder/helper.go

@@ -107,9 +107,11 @@ func SetupConfig(
 	buildCommit string,
 	buildBranch string,
 	buildHandlerChainFuncFromBuilders BuildHandlerChainFuncFromBuilders,
+	gvs []schema.GroupVersion,
+	additionalOpenAPIDefGetters []common.GetOpenAPIDefinitions,
 ) error {
 	serverConfig.AdmissionControl = NewAdmissionFromBuilders(builders)
-	defsGetter := GetOpenAPIDefinitions(builders)
+	defsGetter := GetOpenAPIDefinitions(builders, additionalOpenAPIDefGetters...)
 	serverConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(
 		openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(defsGetter),
 		openapinamer.NewDefinitionNamer(scheme, k8sscheme.Scheme))
@@ -119,7 +121,7 @@ func SetupConfig(
 		openapinamer.NewDefinitionNamer(scheme, k8sscheme.Scheme))
 
 	// Add the custom routes to service discovery
-	serverConfig.OpenAPIV3Config.PostProcessSpec = getOpenAPIPostProcessor(buildVersion, builders)
+	serverConfig.OpenAPIV3Config.PostProcessSpec = getOpenAPIPostProcessor(buildVersion, builders, gvs)
 	serverConfig.OpenAPIV3Config.GetOperationIDAndTagsFromRoute = func(r common.Route) (string, []string, error) {
 		meta := r.Metadata()
 		kind := ""
@@ -226,6 +228,7 @@ func SetupConfig(
 	// Set the swagger build versions
 	serverConfig.OpenAPIConfig.Info.Title = "Grafana API Server"
 	serverConfig.OpenAPIConfig.Info.Version = buildVersion
+	serverConfig.OpenAPIV3Config.Info.Title = "Grafana API Server"
 	serverConfig.OpenAPIV3Config.Info.Version = buildVersion
 
 	serverConfig.SkipOpenAPIInstallation = false
@@ -278,13 +281,13 @@ func InstallAPIs(
 	dualWriteService dualwrite.Service,
 	optsregister apistore.StorageOptionsRegister,
 	features featuremgmt.FeatureToggles,
+	dualWriterMetrics *grafanarest.DualWriterMetrics,
+	builderMetrics *BuilderMetrics,
 ) error {
 	// dual writing is only enabled when the storage type is not legacy.
 	// this is needed to support setting a default RESTOptionsGetter for new APIs that don't
 	// support the legacy storage type.
 	var dualWrite grafanarest.DualWriteBuilder
-	metrics := newBuilderMetrics(reg)
-	dualWriterMetrics := grafanarest.NewDualWriterMetrics(reg)
 
 	// nolint:staticcheck
 	if storageOpts.StorageType != options.StorageTypeLegacy {
@@ -346,7 +349,7 @@ func InstallAPIs(
 				return nil, err
 			}
 
-			metrics.recordDualWriterModes(gr.Resource, gr.Group, mode, currentMode)
+			builderMetrics.RecordDualWriterModes(gr.Resource, gr.Group, mode, currentMode)
 
 			switch currentMode {
 			case grafanarest.Mode0:

pkg/services/apiserver/builder/metrics.go

@@ -6,13 +6,13 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
-type builderMetrics struct {
+type BuilderMetrics struct {
 	dualWriterTargetMode  *prometheus.GaugeVec
 	dualWriterCurrentMode *prometheus.GaugeVec
 }
 
-func newBuilderMetrics(reg prometheus.Registerer) *builderMetrics {
-	return &builderMetrics{
+func ProvideBuilderMetrics(reg prometheus.Registerer) *BuilderMetrics {
+	return &BuilderMetrics{
 		dualWriterTargetMode: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
 			Name: "unified_storage_dual_writer_target_mode",
 			Help: "Unified Storage dual writer target mode",
@@ -24,7 +24,11 @@ func newBuilderMetrics(reg prometheus.Registerer) *builderMetrics {
 	}
 }
 
-func (m *builderMetrics) recordDualWriterModes(resource, group string, targetMode, currentMode grafanarest.DualWriterMode) {
+func (m *BuilderMetrics) RecordDualWriterModes(resource, group string, targetMode, currentMode grafanarest.DualWriterMode) {
 	m.dualWriterTargetMode.WithLabelValues(resource, group).Set(float64(targetMode))
 	m.dualWriterCurrentMode.WithLabelValues(resource, group).Set(float64(currentMode))
 }
+
+func ProvideDualWriterMetrics(reg prometheus.Registerer) *grafanarest.DualWriterMetrics {
+	return grafanarest.NewDualWriterMetrics(reg)
+}

pkg/services/apiserver/builder/openapi.go

@@ -4,6 +4,7 @@ import (
 	"maps"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	openapi "k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/spec3"
 	spec "k8s.io/kube-openapi/pkg/validation/spec"
@@ -14,11 +15,18 @@ import (
 )
 
 // This should eventually live in grafana-app-sdk
-func GetOpenAPIDefinitions(builders []APIGroupBuilder) openapi.GetOpenAPIDefinitions {
+func GetOpenAPIDefinitions(builders []APIGroupBuilder, additionalGetters ...openapi.GetOpenAPIDefinitions) openapi.GetOpenAPIDefinitions {
 	return func(ref openapi.ReferenceCallback) map[string]openapi.OpenAPIDefinition {
 		defs := common.GetOpenAPIDefinitions(ref) // common grafana apis
 		maps.Copy(defs, data.GetOpenAPIDefinitions(ref))
 		maps.Copy(defs, secret.GetOpenAPIDefinitions(ref)) // Expose secret reference to all resources
+
+		for _, getter := range additionalGetters {
+			if getter != nil {
+				maps.Copy(defs, getter(ref))
+			}
+		}
+
 		// TODO: add timerange to upstream SDK setup
 		maps.Copy(defs, map[string]openapi.OpenAPIDefinition{
 			"github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1.TimeRange": {
@@ -41,157 +49,165 @@ func GetOpenAPIDefinitions(builders []APIGroupBuilder) openapi.GetOpenAPIDefinit
 	}
 }
 
+func addBuilderRoutes(
+	targetGroupVersion schema.GroupVersion,
+	openAPISpec *spec3.OpenAPI,
+	apiGroupBuilders []APIGroupBuilder,
+) (*spec3.OpenAPI, error) {
+	for _, apiGroupBuilder := range apiGroupBuilders {
+		// Optionally include raw http handlers for all builders
+		for _, gv := range GetGroupVersions(apiGroupBuilder) {
+			if gv != targetGroupVersion {
+				continue // Only add routes for the target group version
+			}
+			provider, ok := apiGroupBuilder.(APIGroupRouteProvider)
+			if ok && provider != nil {
+				routes := provider.GetAPIRoutes(gv)
+				if routes != nil {
+					for _, route := range routes.Root {
+						openAPISpec.Paths.Paths["/apis/"+gv.String()+"/"+route.Path] = &spec3.Path{
+							PathProps: *route.Spec,
+						}
+					}
+
+					for _, route := range routes.Namespace {
+						openAPISpec.Paths.Paths["/apis/"+gv.String()+"/namespaces/{namespace}/"+route.Path] = &spec3.Path{
+							PathProps: *route.Spec,
+						}
+					}
+				}
+			}
+			// Support direct manipulation of API results
+			processor, ok := apiGroupBuilder.(OpenAPIPostProcessor)
+			if ok {
+				return processor.PostProcessOpenAPI(openAPISpec)
+			}
+		}
+	}
+	return openAPISpec, nil
+}
+
 // Modify the OpenAPI spec to include the additional routes.
-// Currently this requires: https://github.com/kubernetes/kube-openapi/pull/420
-// In future k8s release, the hook will use Config3 rather than the same hook for both v2 and v3
 // nolint:gocyclo
-func getOpenAPIPostProcessor(version string, builders []APIGroupBuilder) func(*spec3.OpenAPI) (*spec3.OpenAPI, error) {
+func getOpenAPIPostProcessor(version string, builders []APIGroupBuilder, gvs []schema.GroupVersion) func(*spec3.OpenAPI) (*spec3.OpenAPI, error) {
 	return func(s *spec3.OpenAPI) (*spec3.OpenAPI, error) {
 		if s.Paths == nil {
 			return s, nil
 		}
 
-		for _, b := range builders {
-			for _, gv := range GetGroupVersions(b) {
-				prefix := "/apis/" + gv.String() + "/"
-				if s.Paths.Paths[prefix] != nil {
-					copy := spec3.OpenAPI{
-						Version: s.Version,
-						Info: &spec.Info{
-							InfoProps: spec.InfoProps{
-								Title:   gv.String(),
-								Version: version,
-							},
+		for _, gv := range gvs {
+			prefix := "/apis/" + gv.String() + "/"
+			if s.Paths.Paths[prefix] != nil {
+				copy := spec3.OpenAPI{
+					Version: s.Version,
+					Info: &spec.Info{
+						InfoProps: spec.InfoProps{
+							Title:   gv.String(),
+							Version: version,
 						},
-						Components:   s.Components,
-						ExternalDocs: s.ExternalDocs,
-						Servers:      s.Servers,
-						Paths:        s.Paths,
-					}
-
-					for k, v := range copy.Paths.Paths {
-						if k == prefix {
-							continue // API discovery
-						}
-
-						// Remove the deprecated watch URL -- can use list with ?watch=true
-						if strings.HasPrefix(k, prefix+"watch/") {
-							delete(copy.Paths.Paths, k)
-							continue
-						}
-
-						// Remove the "for all namespaces" global routes from OpenAPI (v3)
-						if !strings.HasPrefix(k, prefix+"namespaces/") {
-							delete(copy.Paths.Paths, k)
-							continue
-						}
-
-						// Delete has all parameters in the query string already
-						if v.Delete != nil {
-							action, ok := v.Delete.Extensions.GetString("x-kubernetes-action")
-							if ok && (action == "deletecollection" || action == "delete") {
-								v.Delete.RequestBody = nil // duplicates all the parameters
-							}
-						}
-
-						// Replace any */* media types with json+yaml (protobuf?)
-						ops := []*spec3.Operation{v.Delete, v.Put, v.Post}
-						for _, op := range ops {
-							if op == nil || op.RequestBody == nil || len(op.RequestBody.Content) != 1 {
-								continue
-							}
-							content, ok := op.RequestBody.Content["*/*"]
-							if ok {
-								op.RequestBody.Content = map[string]*spec3.MediaType{
-									"application/json":                    content,
-									"application/yaml":                    content,
-									"application/vnd.kubernetes.protobuf": content,
-								}
-							}
-						}
-					}
-
-					sub := copy.Paths.Paths[prefix]
-					if sub != nil && sub.Get != nil {
-						sub.Get.Tags = []string{"API Discovery"}
-						sub.Get.Description = "Describe the available kubernetes resources"
-					}
-
-					// Remove the growing list of kinds
-					for k, v := range copy.Components.Schemas {
-						if v.Extensions == nil {
-							continue
-						}
-						if strings.HasPrefix(k, "io.k8s.apimachinery.pkg.apis.meta.v1") {
-							delete(v.Extensions, "x-kubernetes-group-version-kind") // a growing list of everything
-							continue
-						}
-
-						// Remove the internal annotations
-						val, ok := v.Extensions["x-kubernetes-group-version-kind"]
-						if ok {
-							gvks, ok := val.([]any)
-							if ok {
-								keep := make([]map[string]any, 0, len(gvks))
-								for _, val := range gvks {
-									gvk, ok := val.(map[string]any)
-									if ok && gvk["version"] == "__internal" {
-										continue
-									}
-									keep = append(keep, gvk)
-								}
-								v.Extensions["x-kubernetes-group-version-kind"] = keep
-							}
-						}
-					}
-
-					// Optionally include raw http handlers
-					provider, ok := b.(APIGroupRouteProvider)
-					if ok && provider != nil {
-						routes := provider.GetAPIRoutes(gv)
-						if routes != nil {
-							for _, route := range routes.Root {
-								copy.Paths.Paths[prefix+route.Path] = &spec3.Path{
-									PathProps: *route.Spec,
-								}
-							}
-
-							for _, route := range routes.Namespace {
-								copy.Paths.Paths[prefix+"namespaces/{namespace}/"+route.Path] = &spec3.Path{
-									PathProps: *route.Spec,
-								}
-							}
-						}
-					}
-
-					// Make the sub-resources (connect) share the same tags as the main resource
-					for path, spec := range copy.Paths.Paths {
-						idx := strings.LastIndex(path, "{name}/")
-						if idx > 0 {
-							parent := copy.Paths.Paths[path[:idx+6]]
-							if parent != nil && parent.Get != nil {
-								for _, op := range GetPathOperations(spec) {
-									if op != nil && op.Extensions != nil {
-										action, ok := op.Extensions.GetString("x-kubernetes-action")
-										if ok && action == "connect" {
-											op.Tags = parent.Get.Tags
-										}
-									}
-								}
-							}
-						}
-					}
-
-					// Support direct manipulation of API results
-					processor, ok := b.(OpenAPIPostProcessor)
-					if ok {
-						return processor.PostProcessOpenAPI(&copy)
-					}
-					return &copy, nil
+					},
+					Components:   s.Components,
+					ExternalDocs: s.ExternalDocs,
+					Servers:      s.Servers,
+					Paths:        s.Paths,
 				}
+
+				for k, v := range copy.Paths.Paths {
+					if k == prefix {
+						continue // API discovery
+					}
+
+					// Remove the deprecated watch URL -- can use list with ?watch=true
+					if strings.HasPrefix(k, prefix+"watch/") {
+						delete(copy.Paths.Paths, k)
+						continue
+					}
+
+					// Remove the "for all namespaces" global routes from OpenAPI (v3)
+					if !strings.HasPrefix(k, prefix+"namespaces/") {
+						delete(copy.Paths.Paths, k)
+						continue
+					}
+
+					// Delete has all parameters in the query string already
+					if v.Delete != nil {
+						action, ok := v.Delete.Extensions.GetString("x-kubernetes-action")
+						if ok && (action == "deletecollection" || action == "delete") {
+							v.Delete.RequestBody = nil // duplicates all the parameters
+						}
+					}
+
+					// Replace any */* media types with json+yaml (protobuf?)
+					ops := []*spec3.Operation{v.Delete, v.Put, v.Post}
+					for _, op := range ops {
+						if op == nil || op.RequestBody == nil || len(op.RequestBody.Content) != 1 {
+							continue
+						}
+						content, ok := op.RequestBody.Content["*/*"]
+						if ok {
+							op.RequestBody.Content = map[string]*spec3.MediaType{
+								"application/json":                    content,
+								"application/yaml":                    content,
+								"application/vnd.kubernetes.protobuf": content,
+							}
+						}
+					}
+				}
+
+				sub := copy.Paths.Paths[prefix]
+				if sub != nil && sub.Get != nil {
+					sub.Get.Tags = []string{"API Discovery"}
+					sub.Get.Description = "Describe the available kubernetes resources"
+				}
+
+				// Remove the growing list of kinds
+				for k, v := range copy.Components.Schemas {
+					if v.Extensions == nil {
+						continue
+					}
+					if strings.HasPrefix(k, "io.k8s.apimachinery.pkg.apis.meta.v1") {
+						delete(v.Extensions, "x-kubernetes-group-version-kind") // a growing list of everything
+						continue
+					}
+
+					// Remove the internal annotations
+					val, ok := v.Extensions["x-kubernetes-group-version-kind"]
+					if ok {
+						gvks, ok := val.([]any)
+						if ok {
+							keep := make([]map[string]any, 0, len(gvks))
+							for _, val := range gvks {
+								gvk, ok := val.(map[string]any)
+								if ok && gvk["version"] == "__internal" {
+									continue
+								}
+								keep = append(keep, gvk)
+							}
+							v.Extensions["x-kubernetes-group-version-kind"] = keep
+						}
+					}
+				}
+
+				// Make the sub-resources (connect) share the same tags as the main resource
+				for path, spec := range copy.Paths.Paths {
+					idx := strings.LastIndex(path, "{name}/")
+					if idx > 0 {
+						parent := copy.Paths.Paths[path[:idx+6]]
+						if parent != nil && parent.Get != nil {
+							for _, op := range GetPathOperations(spec) {
+								if op != nil && op.Extensions != nil {
+									action, ok := op.Extensions.GetString("x-kubernetes-action")
+									if ok && action == "connect" {
+										op.Tags = parent.Get.Tags
+									}
+								}
+							}
+						}
+					}
+				}
+				return addBuilderRoutes(gv, &copy, builders)
 			}
 		}
-
 		return s, nil
 	}
 }

pkg/services/apiserver/service.go

@@ -17,14 +17,17 @@ import (
 	"k8s.io/apiserver/pkg/util/notfoundhandler"
 	clientrest "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kube-openapi/pkg/common"
 
 	"github.com/grafana/authlib/types"
 	"github.com/grafana/dskit/services"
+	appsdkapiserver "github.com/grafana/grafana-app-sdk/k8s/apiserver"
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	dataplaneaggregator "github.com/grafana/grafana/pkg/aggregator/apiserver"
 	"github.com/grafana/grafana/pkg/api/routing"
 	"github.com/grafana/grafana/pkg/apimachinery/identity"
 	grafanaresponsewriter "github.com/grafana/grafana/pkg/apiserver/endpoints/responsewriter"
+	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/kvstore"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -37,6 +40,7 @@ import (
 	"github.com/grafana/grafana/pkg/registry"
 	"github.com/grafana/grafana/pkg/registry/apis/datasource"
 	"github.com/grafana/grafana/pkg/services/apiserver/aggregatorrunner"
+	"github.com/grafana/grafana/pkg/services/apiserver/appinstaller"
 	"github.com/grafana/grafana/pkg/services/apiserver/auth/authenticator"
 	"github.com/grafana/grafana/pkg/services/apiserver/auth/authorizer"
 	"github.com/grafana/grafana/pkg/services/apiserver/builder"
@@ -105,6 +109,9 @@ type service struct {
 
 	buildHandlerChainFuncFromBuilders builder.BuildHandlerChainFuncFromBuilders
 	aggregatorRunner                  aggregatorrunner.AggregatorRunner
+	appInstallers                     []appsdkapiserver.AppInstaller
+	builderMetrics                    *builder.BuilderMetrics
+	dualWriterMetrics                 *grafanarest.DualWriterMetrics
 }
 
 func ProvideService(
@@ -126,6 +133,8 @@ func ProvideService(
 	eventualRestConfigProvider *eventualRestConfigProvider,
 	reg prometheus.Registerer,
 	aggregatorRunner aggregatorrunner.AggregatorRunner,
+	appInstallers []appsdkapiserver.AppInstaller,
+	builderMetrics *builder.BuilderMetrics,
 ) (*service, error) {
 	scheme := builder.ProvideScheme()
 	codecs := builder.ProvideCodecFactory(scheme)
@@ -153,6 +162,9 @@ func ProvideService(
 		restConfigProvider:                restConfigProvider,
 		buildHandlerChainFuncFromBuilders: buildHandlerChainFuncFromBuilders,
 		aggregatorRunner:                  aggregatorRunner,
+		appInstallers:                     appInstallers,
+		builderMetrics:                    builderMetrics,
+		dualWriterMetrics:                 grafanarest.NewDualWriterMetrics(reg),
 	}
 	// This will be used when running as a dskit service
 	service := services.NewBasicService(s.start, s.running, nil).WithName(modules.GrafanaAPIServer)
@@ -244,7 +256,7 @@ func (s *service) start(ctx context.Context) error {
 	builders := s.builders
 	groupVersions := make([]schema.GroupVersion, 0, len(builders))
 
-	// Install schemas
+	// Install schemas for existing builders
 	for _, b := range builders {
 		gvs := builder.GetGroupVersions(b)
 		groupVersions = append(groupVersions, gvs...)
@@ -266,14 +278,26 @@ func (s *service) start(ctx context.Context) error {
 		}
 	}
 
+	// Add schemas from app installers to the scheme before creating options
+	additionalGroupVersions, err := appinstaller.AddToScheme(s.appInstallers, s.scheme)
+	if err != nil {
+		return err
+	}
+	groupVersions = append(groupVersions, additionalGroupVersions...)
+
 	o := grafanaapiserveroptions.NewOptions(s.codecs.LegacyCodec(groupVersions...))
-	err := applyGrafanaConfig(s.cfg, s.features, o)
+
+	// Register admission plugins from app installers after options are created
+	if err := appinstaller.RegisterAdmissionPlugins(ctx, s.appInstallers, o); err != nil {
+		return err
+	}
+
+	err = applyGrafanaConfig(s.cfg, s.features, o)
 	if err != nil {
 		return err
 	}
 
 	if errs := o.Validate(); len(errs) != 0 {
-		// TODO: handle multiple errors
 		return errs[0]
 	}
 
@@ -281,6 +305,7 @@ func (s *service) start(ctx context.Context) error {
 	if err := o.ApplyTo(serverConfig); err != nil {
 		return err
 	}
+
 	serverConfig.Authorization.Authorizer = s.authorizer
 	serverConfig.Authentication.Authenticator = authenticator.NewAuthenticator(serverConfig.Authentication.Authenticator)
 	serverConfig.TracerProvider = s.tracing.GetTracerProvider()
@@ -303,12 +328,14 @@ func (s *service) start(ctx context.Context) error {
 	} else {
 		getter := apistore.NewRESTOptionsGetterForClient(s.unified, o.RecommendedOptions.Etcd.StorageConfig, s.restConfigProvider)
 		optsregister = getter.RegisterOptions
-
-		// Use unified storage client
 		serverConfig.RESTOptionsGetter = getter
 	}
 
-	// Add OpenAPI specs for each group+version
+	defGetters := []common.GetOpenAPIDefinitions{
+		appinstaller.BuildOpenAPIDefGetter(s.appInstallers),
+	}
+
+	// Add OpenAPI specs for each group+version (existing builders)
 	err = builder.SetupConfig(
 		s.scheme,
 		serverConfig,
@@ -318,6 +345,8 @@ func (s *service) start(ctx context.Context) error {
 		s.cfg.BuildCommit,
 		s.cfg.BuildBranch,
 		s.buildHandlerChainFuncFromBuilders,
+		groupVersions,
+		defGetters,
 	)
 	if err != nil {
 		return err
@@ -325,27 +354,48 @@ func (s *service) start(ctx context.Context) error {
 
 	notFoundHandler := notfoundhandler.New(s.codecs, genericapifilters.NoMuxAndDiscoveryIncompleteKey)
 
+	if err := appinstaller.RegisterPostStartHooks(s.appInstallers, serverConfig); err != nil {
+		return fmt.Errorf("failed to register post start hooks for app installers: %w", err)
+	}
+
 	// Create the server
 	server, err := serverConfig.Complete().New("grafana-apiserver", genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler))
 	if err != nil {
 		return err
 	}
 
-	// Install the API group+version
+	// Install the API group+version for existing builders
 	err = builder.InstallAPIs(s.scheme, s.codecs, server, serverConfig.RESTOptionsGetter, builders, o.StorageOptions,
-		// Required for the dual writer initialization
 		s.metrics,
 		request.GetNamespaceMapper(s.cfg),
-		kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"), // NOTE: will be removed and replaced with the dual writer utility
+		kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"),
 		s.serverLockService,
 		s.storageStatus,
 		optsregister,
 		s.features,
+		s.dualWriterMetrics,
+		s.builderMetrics,
 	)
 	if err != nil {
 		return err
 	}
 
+	if err := appinstaller.InstallAPIs(
+		ctx,
+		s.appInstallers,
+		server,
+		serverConfig.RESTOptionsGetter,
+		o.StorageOptions,
+		kvstore.WithNamespace(s.kvStore, 0, "storage.dualwriting"),
+		s.serverLockService,
+		request.GetNamespaceMapper(s.cfg),
+		s.storageStatus,
+		s.dualWriterMetrics,
+		s.builderMetrics,
+	); err != nil {
+		return err
+	}
+
 	// stash the options for later use
 	s.options = o
 

pkg/services/apiserver/wireset.go

@@ -7,6 +7,8 @@ import (
 )
 
 var WireSet = wire.NewSet(
+	builder.ProvideDualWriterMetrics,
+	builder.ProvideBuilderMetrics,
 	ProvideEventualRestConfigProvider,
 	wire.Bind(new(RestConfigProvider), new(*eventualRestConfigProvider)),
 	wire.Bind(new(DirectRestConfigProvider), new(*eventualRestConfigProvider)),