K8s: Refactor config/options for aggregation (#81739)

pkg/services/apiserver/aggregator/README.md

@@ -0,0 +1,47 @@
+# aggregator
+
+This is a package that is intended to power the aggregation of microservices within Grafana. The concept
+as well as implementation is largely borrowed from [kube-aggregator](https://github.com/kubernetes/kube-aggregator).
+
+## Why aggregate services?
+
+Grafana's future architecture will entail the same API Server design as that of Kubernetes API Servers. API Servers
+provide a standard way of stitching together API Groups through discovery and shared routing patterns that allows
+them to aggregate to a parent API Server in a seamless manner. Since we desire to break Grafana monolith up into
+more functionally divided microservices, aggregation does the job of still being able to provide these services
+under a single address. Other benefits of aggregation include free health checks and being able to independently 
+roll out features for each service without downtime.
+
+To read more about the concept, see 
+[here](https://kubernetes.io/docs/tasks/extend-kubernetes/setup-extension-api-server/).
+
+Note that, this aggregation will be a totally internal detail to Grafana. External fully functional APIServers that
+may themselves act as parent API Servers to Grafana will never be made aware of them. Any of the `APIService` 
+related to Grafana Groups registered in a real K8s environment will take the address of Grafana's 
+parent server (which will bundle grafana-aggregator).
+
+### kube-aggregator versus grafana-aggregator
+
+The `grafana-aggregator` component will work similarly to how `kube-aggregator` works for `kube-apiserver`, the major
+difference being that it doesn't require core V1 APIs such as `Service`. Early on, we decided to not have core V1
+APIs in the root Grafana API Server. In order to still be able to implement aggregation, we do the following in this Go
+package:
+
+1. We do not start the core shared informer factories as well as any default controllers that utilize them. 
+This is achieved using `DisabledPostStartHooks` facility under the GenericAPIServer's RecommendedConfig.
+2. We provide an `externalname` Kind API implementation under `service.grafana.app` group which works functionally 
+equivalent to the idea with the same name under `core/v1/Service`.
+3. Lastly, we swap the default available condition controller with the custom one written by us. This one is based on
+our `externalname` (`service.grafana.app`) implementation. We register separate `PostStartHooks` 
+using `AddPostStartHookOrDie` on the GenericAPIServer to start the corresponding custom controller as well as 
+requisite informer factories for our own `externalname` Kind.
+4. For now, we bundle apiextensions-apiserver under our aggregator component. This is slightly different from K8s
+where kube-apiserver is called the top-level component and controlplane, aggregator and apiextensions-apiserver
+live under that instead.
+
+### Gotchas (Pay Attention)
+
+1. `grafana-aggregator` uses file storage under `data/grafana-aggregator` (`apiregistration.k8s.io`,
+`service.grafana.app`) and `data/grafana-apiextensions` (`apiextensions.k8s.io`).
+2. Since `grafana-aggregator` outputs configuration (TLS and kubeconfig) that is used in the invocation of aggregated
+  servers, ensure you start the aggregated service after launching the aggregator during local development.

pkg/services/apiserver/aggregator/aggregator.go

@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/aggregator.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: The Kubernetes Authors.
+// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: The Kubernetes Authors.
+// Provenance-includes-location: https://github.com/kubernetes/kubernetes/blob/master/pkg/controlplane/apiserver/apiextensions.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: The Kubernetes Authors.
+
+package aggregator
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apimachinery/pkg/util/sets"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog/v2"
+	v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"
+	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
+	apiregistrationclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1"
+	apiregistrationInformers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1"
+	"k8s.io/kube-aggregator/pkg/controllers/autoregister"
+
+	serviceclientset "github.com/grafana/grafana/pkg/generated/clientset/versioned"
+	informersv0alpha1 "github.com/grafana/grafana/pkg/generated/informers/externalversions"
+	"github.com/grafana/grafana/pkg/services/apiserver/options"
+)
+
+func CreateAggregatorConfig(commandOptions *options.Options, sharedConfig genericapiserver.RecommendedConfig) (*aggregatorapiserver.Config, informersv0alpha1.SharedInformerFactory, error) {
+	// Create a fake clientset and informers for the k8s v1 API group.
+	// These are not used in grafana's aggregator because v1 APIs are not available.
+	fakev1Informers := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 10*time.Minute)
+
+	serviceClient, err := serviceclientset.NewForConfig(sharedConfig.LoopbackClientConfig)
+	if err != nil {
+		return nil, nil, err
+	}
+	sharedInformerFactory := informersv0alpha1.NewSharedInformerFactory(
+		serviceClient,
+		5*time.Minute, // this is effectively used as a refresh interval right now.  Might want to do something nicer later on.
+	)
+	serviceResolver := NewExternalNameResolver(sharedInformerFactory.Service().V0alpha1().ExternalNames().Lister())
+
+	aggregatorConfig := &aggregatorapiserver.Config{
+		GenericConfig: &genericapiserver.RecommendedConfig{
+			Config:                sharedConfig.Config,
+			SharedInformerFactory: fakev1Informers,
+			ClientConfig:          sharedConfig.LoopbackClientConfig,
+		},
+		ExtraConfig: aggregatorapiserver.ExtraConfig{
+			ProxyClientCertFile: commandOptions.AggregatorOptions.ProxyClientCertFile,
+			ProxyClientKeyFile:  commandOptions.AggregatorOptions.ProxyClientKeyFile,
+			// NOTE: while ProxyTransport can be skipped in the configuration, it allows honoring
+			// DISABLE_HTTP2, HTTPS_PROXY and NO_PROXY env vars as needed
+			ProxyTransport:  createProxyTransport(),
+			ServiceResolver: serviceResolver,
+		},
+	}
+
+	if err := commandOptions.AggregatorOptions.ApplyTo(aggregatorConfig, commandOptions.RecommendedOptions.Etcd, commandOptions.StorageOptions.DataPath); err != nil {
+		return nil, nil, err
+	}
+
+	return aggregatorConfig, sharedInformerFactory, nil
+}
+
+func CreateAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, sharedInformerFactory informersv0alpha1.SharedInformerFactory, delegateAPIServer genericapiserver.DelegationTarget) (*aggregatorapiserver.APIAggregator, error) {
+	completedConfig := aggregatorConfig.Complete()
+	aggregatorServer, err := completedConfig.NewWithDelegate(delegateAPIServer)
+	if err != nil {
+		return nil, err
+	}
+
+	// create controllers for auto-registration
+	apiRegistrationClient, err := apiregistrationclient.NewForConfig(completedConfig.GenericConfig.LoopbackClientConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	autoRegistrationController := autoregister.NewAutoRegisterController(aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), apiRegistrationClient)
+	apiServices := apiServicesToRegister(delegateAPIServer, autoRegistrationController)
+
+	// Imbue all builtin group-priorities onto the aggregated discovery
+	if completedConfig.GenericConfig.AggregatedDiscoveryGroupManager != nil {
+		for gv, entry := range APIVersionPriorities {
+			completedConfig.GenericConfig.AggregatedDiscoveryGroupManager.SetGroupVersionPriority(metav1.GroupVersion(gv), int(entry.Group), int(entry.Version))
+		}
+	}
+
+	err = aggregatorServer.GenericAPIServer.AddPostStartHook("grafana-apiserver-autoregistration", func(context genericapiserver.PostStartHookContext) error {
+		go func() {
+			autoRegistrationController.Run(5, context.StopCh)
+		}()
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = aggregatorServer.GenericAPIServer.AddBootSequenceHealthChecks(
+		makeAPIServiceAvailableHealthCheck(
+			"autoregister-completion",
+			apiServices,
+			aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(),
+		),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	apiregistrationClient, err := apiregistrationclientset.NewForConfig(completedConfig.GenericConfig.LoopbackClientConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	availableController, err := NewAvailableConditionController(
+		aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(),
+		sharedInformerFactory.Service().V0alpha1().ExternalNames(),
+		apiregistrationClient.ApiregistrationV1(),
+		nil,
+		(func() ([]byte, []byte))(nil),
+		completedConfig.ExtraConfig.ServiceResolver,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	aggregatorServer.GenericAPIServer.AddPostStartHookOrDie("apiservice-status-override-available-controller", func(context genericapiserver.PostStartHookContext) error {
+		// if we end up blocking for long periods of time, we may need to increase workers.
+		go availableController.Run(5, context.StopCh)
+		return nil
+	})
+
+	aggregatorServer.GenericAPIServer.AddPostStartHookOrDie("start-grafana-aggregator-informers", func(context genericapiserver.PostStartHookContext) error {
+		sharedInformerFactory.Start(context.StopCh)
+		aggregatorServer.APIRegistrationInformers.Start(context.StopCh)
+		return nil
+	})
+
+	return aggregatorServer, nil
+}
+
+func makeAPIService(gv schema.GroupVersion) *v1.APIService {
+	apiServicePriority, ok := APIVersionPriorities[gv]
+	if !ok {
+		// if we aren't found, then we shouldn't register ourselves because it could result in a CRD group version
+		// being permanently stuck in the APIServices list.
+		klog.Infof("Skipping APIService creation for %v", gv)
+		return nil
+	}
+	return &v1.APIService{
+		ObjectMeta: metav1.ObjectMeta{Name: gv.Version + "." + gv.Group},
+		Spec: v1.APIServiceSpec{
+			Group:                gv.Group,
+			Version:              gv.Version,
+			GroupPriorityMinimum: apiServicePriority.Group,
+			VersionPriority:      apiServicePriority.Version,
+		},
+	}
+}
+
+// makeAPIServiceAvailableHealthCheck returns a healthz check that returns healthy
+// once all of the specified services have been observed to be available at least once.
+func makeAPIServiceAvailableHealthCheck(name string, apiServices []*v1.APIService, apiServiceInformer apiregistrationInformers.APIServiceInformer) healthz.HealthChecker {
+	// Track the auto-registered API services that have not been observed to be available yet
+	pendingServiceNamesLock := &sync.RWMutex{}
+	pendingServiceNames := sets.NewString()
+	for _, service := range apiServices {
+		pendingServiceNames.Insert(service.Name)
+	}
+
+	// When an APIService in the list is seen as available, remove it from the pending list
+	handleAPIServiceChange := func(service *v1.APIService) {
+		pendingServiceNamesLock.Lock()
+		defer pendingServiceNamesLock.Unlock()
+		if !pendingServiceNames.Has(service.Name) {
+			return
+		}
+		if v1helper.IsAPIServiceConditionTrue(service, v1.Available) {
+			pendingServiceNames.Delete(service.Name)
+		}
+	}
+
+	// Watch add/update events for APIServices
+	_, _ = apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    func(obj interface{}) { handleAPIServiceChange(obj.(*v1.APIService)) },
+		UpdateFunc: func(old, new interface{}) { handleAPIServiceChange(new.(*v1.APIService)) },
+	})
+
+	// Don't return healthy until the pending list is empty
+	return healthz.NamedCheck(name, func(r *http.Request) error {
+		pendingServiceNamesLock.RLock()
+		defer pendingServiceNamesLock.RUnlock()
+		if pendingServiceNames.Len() > 0 {
+			return fmt.Errorf("missing APIService: %v", pendingServiceNames.List())
+		}
+		return nil
+	})
+}
+
+// Priority defines group Priority that is used in discovery. This controls
+// group position in the kubectl output.
+type Priority struct {
+	// Group indicates the order of the Group relative to other groups.
+	Group int32
+	// Version indicates the relative order of the Version inside of its group.
+	Version int32
+}
+
+// APIVersionPriorities are the proper way to resolve this letting the aggregator know the desired group and version-within-group order of the underlying servers
+// is to refactor the genericapiserver.DelegationTarget to include a list of priorities based on which APIs were installed.
+// This requires the APIGroupInfo struct to evolve and include the concept of priorities and to avoid mistakes, the core storage map there needs to be updated.
+// That ripples out every bit as far as you'd expect, so for 1.7 we'll include the list here instead of being built up during storage.
+var APIVersionPriorities = map[schema.GroupVersion]Priority{
+	{Group: "", Version: "v1"}: {Group: 18000, Version: 1},
+	// to my knowledge, nothing below here collides
+	{Group: "admissionregistration.k8s.io", Version: "v1"}:       {Group: 16700, Version: 15},
+	{Group: "admissionregistration.k8s.io", Version: "v1beta1"}:  {Group: 16700, Version: 12},
+	{Group: "admissionregistration.k8s.io", Version: "v1alpha1"}: {Group: 16700, Version: 9},
+	// Append a new group to the end of the list if unsure.
+	// You can use min(existing group)-100 as the initial value for a group.
+	// Version can be set to 9 (to have space around) for a new group.
+}
+
+func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*v1.APIService {
+	apiServices := []*v1.APIService{}
+
+	for _, curr := range delegateAPIServer.ListedPaths() {
+		if curr == "/api/v1" {
+			apiService := makeAPIService(schema.GroupVersion{Group: "", Version: "v1"})
+			registration.AddAPIServiceToSyncOnStart(apiService)
+			apiServices = append(apiServices, apiService)
+			continue
+		}
+
+		if !strings.HasPrefix(curr, "/apis/") {
+			continue
+		}
+		// this comes back in a list that looks like /apis/rbac.authorization.k8s.io/v1alpha1
+		tokens := strings.Split(curr, "/")
+		if len(tokens) != 4 {
+			continue
+		}
+
+		apiService := makeAPIService(schema.GroupVersion{Group: tokens[2], Version: tokens[3]})
+		if apiService == nil {
+			continue
+		}
+		registration.AddAPIServiceToSyncOnStart(apiService)
+		apiServices = append(apiServices, apiService)
+	}
+
+	return apiServices
+}
+
+// NOTE: below function imported from https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-apiserver/app/server.go#L197
+// createProxyTransport creates the dialer infrastructure to connect to the api servers.
+func createProxyTransport() *http.Transport {
+	// NOTE: We don't set proxyDialerFn but the below SetTransportDefaults will
+	// See https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/net/http.go#L109
+	var proxyDialerFn utilnet.DialFunc
+	// Proxying to services is IP-based... don't expect to be able to verify the hostname
+	proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
+	proxyTransport := utilnet.SetTransportDefaults(&http.Transport{
+		DialContext:     proxyDialerFn,
+		TLSClientConfig: proxyTLSClientConfig,
+	})
+	return proxyTransport
+}

pkg/services/apiserver/aggregator/availableController.go

@@ -0,0 +1,466 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Provenance-includes-location: https://github.com/kubernetes/kube-aggregator/blob/master/pkg/controllers/status/available_controller.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: The Kubernetes Authors.
+
+package aggregator
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"reflect"
+	"sync"
+	"time"
+
+	"github.com/grafana/grafana/pkg/apis/service/v0alpha1"
+	informersservicev0alpha1 "github.com/grafana/grafana/pkg/generated/informers/externalversions/service/v0alpha1"
+	listersservicev0alpha1 "github.com/grafana/grafana/pkg/generated/listers/service/v0alpha1"
+
+	"k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/transport"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	apiregistrationv1apihelper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"
+	apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1"
+	informers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1"
+	listers "k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1"
+	"k8s.io/kube-aggregator/pkg/controllers"
+)
+
+type certKeyFunc func() ([]byte, []byte)
+
+// ServiceResolver knows how to convert a service reference into an actual location.
+type ServiceResolver interface {
+	ResolveEndpoint(namespace, name string, port int32) (*url.URL, error)
+}
+
+// AvailableConditionController handles checking the availability of registered API services.
+type AvailableConditionController struct {
+	apiServiceClient apiregistrationclient.APIServicesGetter
+
+	apiServiceLister listers.APIServiceLister
+	apiServiceSynced cache.InformerSynced
+
+	// externalNameLister is used to get the IP to create the transport for
+	externalNameLister listersservicev0alpha1.ExternalNameLister
+	servicesSynced     cache.InformerSynced
+
+	// proxyTransportDial specifies the dial function for creating unencrypted TCP connections.
+	proxyTransportDial         *transport.DialHolder
+	proxyCurrentCertKeyContent certKeyFunc
+	serviceResolver            ServiceResolver
+
+	// To allow injection for testing.
+	syncFn func(key string) error
+
+	queue workqueue.RateLimitingInterface
+	// map from service-namespace -> service-name -> apiservice names
+	cache map[string]map[string][]string
+	// this lock protects operations on the above cache
+	cacheLock sync.RWMutex
+}
+
+// NewAvailableConditionController returns a new AvailableConditionController.
+func NewAvailableConditionController(
+	apiServiceInformer informers.APIServiceInformer,
+	externalNameInformer informersservicev0alpha1.ExternalNameInformer,
+	apiServiceClient apiregistrationclient.APIServicesGetter,
+	proxyTransportDial *transport.DialHolder,
+	proxyCurrentCertKeyContent certKeyFunc,
+	serviceResolver ServiceResolver,
+) (*AvailableConditionController, error) {
+	c := &AvailableConditionController{
+		apiServiceClient:   apiServiceClient,
+		apiServiceLister:   apiServiceInformer.Lister(),
+		externalNameLister: externalNameInformer.Lister(),
+		serviceResolver:    serviceResolver,
+		queue: workqueue.NewNamedRateLimitingQueue(
+			// We want a fairly tight requeue time.  The controller listens to the API, but because it relies on the routability of the
+			// service network, it is possible for an external, non-watchable factor to affect availability.  This keeps
+			// the maximum disruption time to a minimum, but it does prevent hot loops.
+			workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 30*time.Second),
+			"AvailableConditionController"),
+		proxyTransportDial:         proxyTransportDial,
+		proxyCurrentCertKeyContent: proxyCurrentCertKeyContent,
+	}
+
+	// resync on this one because it is low cardinality and rechecking the actual discovery
+	// allows us to detect health in a more timely fashion when network connectivity to
+	// nodes is snipped, but the network still attempts to route there.  See
+	// https://github.com/openshift/origin/issues/17159#issuecomment-341798063
+	apiServiceHandler, _ := apiServiceInformer.Informer().AddEventHandlerWithResyncPeriod(
+		cache.ResourceEventHandlerFuncs{
+			AddFunc:    c.addAPIService,
+			UpdateFunc: c.updateAPIService,
+			DeleteFunc: c.deleteAPIService,
+		},
+		30*time.Second)
+	c.apiServiceSynced = apiServiceHandler.HasSynced
+
+	serviceHandler, _ := externalNameInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    c.addService,
+		UpdateFunc: c.updateService,
+		DeleteFunc: c.deleteService,
+	})
+	c.servicesSynced = serviceHandler.HasSynced
+
+	c.syncFn = c.sync
+
+	return c, nil
+}
+
+func (c *AvailableConditionController) sync(key string) error {
+	originalAPIService, err := c.apiServiceLister.Get(key)
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	// if a particular transport was specified, use that otherwise build one
+	// construct an http client that will ignore TLS verification (if someone owns the network and messes with your status
+	// that's not so bad) and sets a very short timeout.  This is a best effort GET that provides no additional information
+	transportConfig := &transport.Config{
+		TLS: transport.TLSConfig{
+			Insecure: true,
+		},
+		DialHolder: c.proxyTransportDial,
+	}
+
+	if c.proxyCurrentCertKeyContent != nil {
+		proxyClientCert, proxyClientKey := c.proxyCurrentCertKeyContent()
+
+		transportConfig.TLS.CertData = proxyClientCert
+		transportConfig.TLS.KeyData = proxyClientKey
+	}
+	restTransport, err := transport.New(transportConfig)
+	if err != nil {
+		return err
+	}
+	discoveryClient := &http.Client{
+		Transport: restTransport,
+		// the request should happen quickly.
+		Timeout: 5 * time.Second,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	apiService := originalAPIService.DeepCopy()
+
+	availableCondition := apiregistrationv1.APIServiceCondition{
+		Type:               apiregistrationv1.Available,
+		Status:             apiregistrationv1.ConditionTrue,
+		LastTransitionTime: metav1.Now(),
+	}
+
+	// local API services are always considered available
+	if apiService.Spec.Service == nil {
+		apiregistrationv1apihelper.SetAPIServiceCondition(apiService, apiregistrationv1apihelper.NewLocalAvailableAPIServiceCondition())
+		_, err := c.updateAPIServiceStatus(originalAPIService, apiService)
+		return err
+	}
+
+	_, err = c.externalNameLister.ExternalNames(apiService.Spec.Service.Namespace).Get(apiService.Spec.Service.Name)
+	if apierrors.IsNotFound(err) {
+		availableCondition.Status = apiregistrationv1.ConditionFalse
+		availableCondition.Reason = "ServiceNotFound"
+		availableCondition.Message = fmt.Sprintf("service/%s in %q is not present", apiService.Spec.Service.Name, apiService.Spec.Service.Namespace)
+		apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+		_, err := c.updateAPIServiceStatus(originalAPIService, apiService)
+		return err
+	} else if err != nil {
+		availableCondition.Status = apiregistrationv1.ConditionUnknown
+		availableCondition.Reason = "ServiceAccessError"
+		availableCondition.Message = fmt.Sprintf("service/%s in %q cannot be checked due to: %v", apiService.Spec.Service.Name, apiService.Spec.Service.Namespace, err)
+		apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+		_, err := c.updateAPIServiceStatus(originalAPIService, apiService)
+		return err
+	}
+
+	// actually try to hit the discovery endpoint when it isn't local and when we're routing as a service.
+	if apiService.Spec.Service != nil && c.serviceResolver != nil {
+		attempts := 5
+		results := make(chan error, attempts)
+		for i := 0; i < attempts; i++ {
+			go func() {
+				discoveryURL, err := c.serviceResolver.ResolveEndpoint(apiService.Spec.Service.Namespace, apiService.Spec.Service.Name, *apiService.Spec.Service.Port)
+				if err != nil {
+					results <- err
+					return
+				}
+				// render legacyAPIService health check path when it is delegated to a service
+				if apiService.Name == "v1." {
+					discoveryURL.Path = "/api/" + apiService.Spec.Version
+				} else {
+					discoveryURL.Path = "/apis/" + apiService.Spec.Group + "/" + apiService.Spec.Version
+				}
+
+				errCh := make(chan error, 1)
+				go func() {
+					// be sure to check a URL that the aggregated API server is required to serve
+					newReq, err := http.NewRequest("GET", discoveryURL.String(), nil)
+					if err != nil {
+						errCh <- err
+						return
+					}
+
+					// setting the system-masters identity ensures that we will always have access rights
+					transport.SetAuthProxyHeaders(newReq, "system:kube-aggregator", []string{"system:masters"}, nil)
+					resp, err := discoveryClient.Do(newReq)
+					if resp != nil {
+						_ = resp.Body.Close()
+						// we should always been in the 200s or 300s
+						if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+							errCh <- fmt.Errorf("bad status from %v: %v", discoveryURL, resp.StatusCode)
+							return
+						}
+					}
+
+					errCh <- err
+				}()
+
+				select {
+				case err = <-errCh:
+					if err != nil {
+						results <- fmt.Errorf("failing or missing response from %v: %v", discoveryURL, err)
+						return
+					}
+
+					// we had trouble with slow dial and DNS responses causing us to wait too long.
+					// we added this as insurance
+				case <-time.After(6 * time.Second):
+					results <- fmt.Errorf("timed out waiting for %v", discoveryURL)
+					return
+				}
+
+				results <- nil
+			}()
+		}
+
+		var lastError error
+		for i := 0; i < attempts; i++ {
+			lastError = <-results
+			// if we had at least one success, we are successful overall and we can return now
+			if lastError == nil {
+				break
+			}
+		}
+
+		if lastError != nil {
+			availableCondition.Status = apiregistrationv1.ConditionFalse
+			availableCondition.Reason = "FailedDiscoveryCheck"
+			availableCondition.Message = lastError.Error()
+			apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+			_, updateErr := c.updateAPIServiceStatus(originalAPIService, apiService)
+			if updateErr != nil {
+				return updateErr
+			}
+			// force a requeue to make it very obvious that this will be retried at some point in the future
+			// along with other requeues done via service change, endpoint change, and resync
+			return lastError
+		}
+	}
+
+	availableCondition.Reason = "Passed"
+	availableCondition.Message = "all checks passed"
+	apiregistrationv1apihelper.SetAPIServiceCondition(apiService, availableCondition)
+	_, err = c.updateAPIServiceStatus(originalAPIService, apiService)
+	return err
+}
+
+// updateAPIServiceStatus only issues an update if a change is detected.  We have a tight resync loop to quickly detect dead
+// apiservices. Doing that means we don't want to quickly issue no-op updates.
+func (c *AvailableConditionController) updateAPIServiceStatus(originalAPIService, newAPIService *apiregistrationv1.APIService) (*apiregistrationv1.APIService, error) {
+	if equality.Semantic.DeepEqual(originalAPIService.Status, newAPIService.Status) {
+		return newAPIService, nil
+	}
+
+	orig := apiregistrationv1apihelper.GetAPIServiceConditionByType(originalAPIService, apiregistrationv1.Available)
+	now := apiregistrationv1apihelper.GetAPIServiceConditionByType(newAPIService, apiregistrationv1.Available)
+	unknown := apiregistrationv1.APIServiceCondition{
+		Type:   apiregistrationv1.Available,
+		Status: apiregistrationv1.ConditionUnknown,
+	}
+	if orig == nil {
+		orig = &unknown
+	}
+	if now == nil {
+		now = &unknown
+	}
+	if *orig != *now {
+		klog.V(2).InfoS("changing APIService availability", "name", newAPIService.Name, "oldStatus", orig.Status, "newStatus", now.Status, "message", now.Message, "reason", now.Reason)
+	}
+
+	newAPIService, err := c.apiServiceClient.APIServices().UpdateStatus(context.TODO(), newAPIService, metav1.UpdateOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return newAPIService, nil
+}
+
+// Run starts the AvailableConditionController loop which manages the availability condition of API services.
+func (c *AvailableConditionController) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Info("Starting AvailableConditionController")
+	defer klog.Info("Shutting down AvailableConditionController")
+
+	// This waits not just for the informers to sync, but for our handlers
+	// to be called; since the handlers are three different ways of
+	// enqueueing the same thing, waiting for this permits the queue to
+	// maximally de-duplicate the entries.
+	if !controllers.WaitForCacheSync("AvailableConditionCOverrideController", stopCh, c.apiServiceSynced, c.servicesSynced) {
+		return
+	}
+
+	for i := 0; i < workers; i++ {
+		go wait.Until(c.runWorker, time.Second, stopCh)
+	}
+
+	<-stopCh
+}
+
+func (c *AvailableConditionController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
+func (c *AvailableConditionController) processNextWorkItem() bool {
+	key, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(key)
+
+	err := c.syncFn(key.(string))
+	if err == nil {
+		c.queue.Forget(key)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with: %v", key, err))
+	c.queue.AddRateLimited(key)
+
+	return true
+}
+
+func (c *AvailableConditionController) addAPIService(obj interface{}) {
+	castObj := obj.(*apiregistrationv1.APIService)
+	klog.V(4).Infof("Adding %s", castObj.Name)
+	if castObj.Spec.Service != nil {
+		c.rebuildAPIServiceCache()
+	}
+	c.queue.Add(castObj.Name)
+}
+
+func (c *AvailableConditionController) updateAPIService(oldObj, newObj interface{}) {
+	castObj := newObj.(*apiregistrationv1.APIService)
+	oldCastObj := oldObj.(*apiregistrationv1.APIService)
+	klog.V(4).Infof("Updating %s", oldCastObj.Name)
+	if !reflect.DeepEqual(castObj.Spec.Service, oldCastObj.Spec.Service) {
+		c.rebuildAPIServiceCache()
+	}
+	c.queue.Add(oldCastObj.Name)
+}
+
+func (c *AvailableConditionController) deleteAPIService(obj interface{}) {
+	castObj, ok := obj.(*apiregistrationv1.APIService)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			klog.Errorf("Couldn't get object from tombstone %#v", obj)
+			return
+		}
+		castObj, ok = tombstone.Obj.(*apiregistrationv1.APIService)
+		if !ok {
+			klog.Errorf("Tombstone contained object that is not expected %#v", obj)
+			return
+		}
+	}
+	klog.V(4).Infof("Deleting %q", castObj.Name)
+	if castObj.Spec.Service != nil {
+		c.rebuildAPIServiceCache()
+	}
+	c.queue.Add(castObj.Name)
+}
+
+func (c *AvailableConditionController) getAPIServicesFor(obj runtime.Object) []string {
+	metadata, err := meta.Accessor(obj)
+	if err != nil {
+		utilruntime.HandleError(err)
+		return nil
+	}
+	c.cacheLock.RLock()
+	defer c.cacheLock.RUnlock()
+	return c.cache[metadata.GetNamespace()][metadata.GetName()]
+}
+
+// if the service/endpoint handler wins the race against the cache rebuilding, it may queue a no-longer-relevant apiservice
+// (which will get processed an extra time - this doesn't matter),
+// and miss a newly relevant apiservice (which will get queued by the apiservice handler)
+func (c *AvailableConditionController) rebuildAPIServiceCache() {
+	apiServiceList, _ := c.apiServiceLister.List(labels.Everything())
+	newCache := map[string]map[string][]string{}
+	for _, apiService := range apiServiceList {
+		if apiService.Spec.Service == nil {
+			continue
+		}
+		if newCache[apiService.Spec.Service.Namespace] == nil {
+			newCache[apiService.Spec.Service.Namespace] = map[string][]string{}
+		}
+		newCache[apiService.Spec.Service.Namespace][apiService.Spec.Service.Name] = append(newCache[apiService.Spec.Service.Namespace][apiService.Spec.Service.Name], apiService.Name)
+	}
+
+	c.cacheLock.Lock()
+	defer c.cacheLock.Unlock()
+	c.cache = newCache
+}
+
+// TODO, think of a way to avoid checking on every service manipulation
+
+func (c *AvailableConditionController) addService(obj interface{}) {
+	for _, apiService := range c.getAPIServicesFor(obj.(*v0alpha1.ExternalName)) {
+		c.queue.Add(apiService)
+	}
+}
+
+func (c *AvailableConditionController) updateService(obj, _ interface{}) {
+	for _, apiService := range c.getAPIServicesFor(obj.(*v0alpha1.ExternalName)) {
+		c.queue.Add(apiService)
+	}
+}
+
+func (c *AvailableConditionController) deleteService(obj interface{}) {
+	castObj, ok := obj.(*v0alpha1.ExternalName)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			klog.Errorf("Couldn't get object from tombstone %#v", obj)
+			return
+		}
+		castObj, ok = tombstone.Obj.(*v0alpha1.ExternalName)
+		if !ok {
+			klog.Errorf("Tombstone contained object that is not expected %#v", obj)
+			return
+		}
+	}
+	for _, apiService := range c.getAPIServicesFor(castObj) {
+		c.queue.Add(apiService)
+	}
+}

pkg/services/apiserver/aggregator/resolver.go

@@ -0,0 +1,32 @@
+package aggregator
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+
+	"k8s.io/kube-aggregator/pkg/apiserver"
+
+	servicelistersv0alpha1 "github.com/grafana/grafana/pkg/generated/listers/service/v0alpha1"
+)
+
+func NewExternalNameResolver(externalNames servicelistersv0alpha1.ExternalNameLister) apiserver.ServiceResolver {
+	return &externalNameResolver{
+		externalNames: externalNames,
+	}
+}
+
+type externalNameResolver struct {
+	externalNames servicelistersv0alpha1.ExternalNameLister
+}
+
+func (r *externalNameResolver) ResolveEndpoint(namespace, name string, port int32) (*url.URL, error) {
+	extName, err := r.externalNames.ExternalNames(namespace).Get(name)
+	if err != nil {
+		return nil, err
+	}
+	return &url.URL{
+		Scheme: "https",
+		Host:   net.JoinHostPort(extName.Spec.Host, fmt.Sprintf("%d", port)),
+	}, nil
+}

pkg/services/apiserver/auth/authorizer/org_id.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/appcontext"
 	"github.com/grafana/grafana/pkg/infra/log"
-	grafanarequest "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request"
+	grafanarequest "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
 	"github.com/grafana/grafana/pkg/services/org"
 )
 

pkg/services/apiserver/auth/authorizer/stack_id.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/appcontext"
 	"github.com/grafana/grafana/pkg/infra/log"
-	grafanarequest "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request"
+	grafanarequest "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
 	"github.com/grafana/grafana/pkg/setting"
 )
 

pkg/services/apiserver/builder/common.go

@@ -1,4 +1,4 @@
-package grafanaapiserver
+package builder
 
 import (
 	"net/http"
@@ -27,6 +27,7 @@ type APIGroupBuilder interface {
 		scheme *runtime.Scheme,
 		codecs serializer.CodecFactory,
 		optsGetter generic.RESTOptionsGetter,
+		dualWrite bool,
 	) (*genericapiserver.APIGroupInfo, error)
 
 	// Get OpenAPI definitions
@@ -57,3 +58,7 @@ type APIRoutes struct {
 	// Namespace handlers are mounted under the namespace
 	Namespace []APIRouteHandler
 }
+
+type APIRegistrar interface {
+	RegisterAPI(builder APIGroupBuilder)
+}

pkg/services/apiserver/builder/helper.go

@@ -1,34 +1,45 @@
-package grafanaapiserver
+package builder
 
 import (
 	"fmt"
 	"net/http"
 	goruntime "runtime"
+	"runtime/debug"
+	"strconv"
 	"strings"
 	"time"
 
+	"golang.org/x/mod/semver"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/version"
 	openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
 	"k8s.io/apiserver/pkg/registry/generic"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/util/openapi"
-	"k8s.io/client-go/kubernetes/scheme"
+	k8sscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/kube-openapi/pkg/common"
 
 	"github.com/grafana/grafana/pkg/setting"
 )
 
-func SetupConfig(serverConfig *genericapiserver.RecommendedConfig, builders []APIGroupBuilder) error {
+func SetupConfig(
+	scheme *runtime.Scheme,
+	serverConfig *genericapiserver.RecommendedConfig,
+	builders []APIGroupBuilder,
+) error {
 	defsGetter := GetOpenAPIDefinitions(builders)
 	serverConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(
 		openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(defsGetter),
-		openapinamer.NewDefinitionNamer(Scheme, scheme.Scheme))
+		openapinamer.NewDefinitionNamer(scheme, k8sscheme.Scheme))
 
 	serverConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(
 		openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(defsGetter),
-		openapinamer.NewDefinitionNamer(Scheme, scheme.Scheme))
+		openapinamer.NewDefinitionNamer(scheme, k8sscheme.Scheme))
 
+	// Add the custom routes to service discovery
+	serverConfig.OpenAPIV3Config.PostProcessSpec = getOpenAPIPostProcessor(builders)
 	serverConfig.OpenAPIV3Config.GetOperationIDAndTagsFromRoute = func(r common.Route) (string, []string, error) {
 		tags := []string{}
 		prop, ok := r.Metadata()["x-kubernetes-group-version-kind"]
@@ -41,9 +52,6 @@ func SetupConfig(serverConfig *genericapiserver.RecommendedConfig, builders []AP
 		return r.OperationName(), tags, nil
 	}
 
-	// Add the custom routes to service discovery
-	serverConfig.OpenAPIV3Config.PostProcessSpec = getOpenAPIPostProcessor(builders)
-
 	// Set the swagger build versions
 	serverConfig.OpenAPIConfig.Info.Version = setting.BuildVersion
 	serverConfig.OpenAPIV3Config.Info.Version = setting.BuildVersion
@@ -82,12 +90,16 @@ func SetupConfig(serverConfig *genericapiserver.RecommendedConfig, builders []AP
 	return nil
 }
 
-func InstallAPIs(server *genericapiserver.GenericAPIServer,
+func InstallAPIs(
+	scheme *runtime.Scheme,
+	codecs serializer.CodecFactory,
+	server *genericapiserver.GenericAPIServer,
 	optsGetter generic.RESTOptionsGetter,
 	builders []APIGroupBuilder,
+	dualWrite bool,
 ) error {
 	for _, b := range builders {
-		g, err := b.GetAPIGroupInfo(Scheme, Codecs, optsGetter)
+		g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, dualWrite)
 		if err != nil {
 			return err
 		}
@@ -101,3 +113,33 @@ func InstallAPIs(server *genericapiserver.GenericAPIServer,
 	}
 	return nil
 }
+
+// find the k8s version according to build info
+func getK8sApiserverVersion() (string, error) {
+	bi, ok := debug.ReadBuildInfo()
+	if !ok {
+		return "", fmt.Errorf("debug.ReadBuildInfo() failed")
+	}
+
+	if len(bi.Deps) == 0 {
+		return "v?.?", nil // this is normal while debugging
+	}
+
+	for _, dep := range bi.Deps {
+		if dep.Path == "k8s.io/apiserver" {
+			if !semver.IsValid(dep.Version) {
+				return "", fmt.Errorf("invalid semantic version for k8s.io/apiserver")
+			}
+			// v0 => v1
+			majorVersion := strings.TrimPrefix(semver.Major(dep.Version), "v")
+			majorInt, err := strconv.Atoi(majorVersion)
+			if err != nil {
+				return "", fmt.Errorf("could not convert majorVersion to int. majorVersion: %s", majorVersion)
+			}
+			newMajor := fmt.Sprintf("v%d", majorInt+1)
+			return strings.Replace(dep.Version, semver.Major(dep.Version), newMajor, 1), nil
+		}
+	}
+
+	return "", fmt.Errorf("could not find k8s.io/apiserver in build info")
+}

pkg/services/apiserver/builder/openapi.go

@@ -1,4 +1,4 @@
-package grafanaapiserver
+package builder
 
 import (
 	"maps"
@@ -15,8 +15,8 @@ func GetOpenAPIDefinitions(builders []APIGroupBuilder) common.GetOpenAPIDefiniti
 	return func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition {
 		defs := getStandardOpenAPIDefinitions(ref)
 		maps.Copy(defs, v0alpha1.GetOpenAPIDefinitions(ref)) // common grafana apis
-		for _, builder := range builders {
-			g := builder.GetOpenAPIDefinitions()
+		for _, b := range builders {
+			g := b.GetOpenAPIDefinitions()
 			if g != nil {
 				out := g(ref)
 				maps.Copy(defs, out)

pkg/services/apiserver/builder/request_handler.go

@@ -1,4 +1,4 @@
-package grafanaapiserver
+package builder
 
 import (
 	"fmt"
@@ -125,9 +125,9 @@ func getOpenAPIPostProcessor(builders []APIGroupBuilder) func(*spec3.OpenAPI) (*
 		if s.Paths == nil {
 			return s, nil
 		}
-		for _, builder := range builders {
-			routes := builder.GetAPIRoutes()
-			gv := builder.GetGroupVersion()
+		for _, b := range builders {
+			routes := b.GetAPIRoutes()
+			gv := b.GetGroupVersion()
 			prefix := "/apis/" + gv.String() + "/"
 			if s.Paths.Paths[prefix] != nil {
 				copy := spec3.OpenAPI{

pkg/services/apiserver/config.go

@@ -0,0 +1,48 @@
+package apiserver
+
+import (
+	"fmt"
+	"net"
+	"path/filepath"
+	"strconv"
+
+	"github.com/grafana/grafana/pkg/services/apiserver/options"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+func applyGrafanaConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles, o *options.Options) {
+	defaultLogLevel := 0
+	ip := net.ParseIP(cfg.HTTPAddr)
+	apiURL := cfg.AppURL
+	port, err := strconv.Atoi(cfg.HTTPPort)
+	if err != nil {
+		port = 3000
+	}
+
+	if cfg.Env == setting.Dev {
+		defaultLogLevel = 10
+		port = 6443
+		ip = net.ParseIP("127.0.0.1")
+		apiURL = fmt.Sprintf("https://%s:%d", ip, port)
+	}
+
+	host := fmt.Sprintf("%s:%d", ip, port)
+
+	o.RecommendedOptions.Etcd.StorageConfig.Transport.ServerList = cfg.SectionWithEnvOverrides("grafana-apiserver").Key("etcd_servers").Strings(",")
+
+	o.RecommendedOptions.SecureServing.BindAddress = ip
+	o.RecommendedOptions.SecureServing.BindPort = port
+	o.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true
+	o.RecommendedOptions.Authorization.RemoteKubeConfigFileOptional = true
+
+	o.RecommendedOptions.Admission = nil
+	o.RecommendedOptions.CoreAPI = nil
+
+	o.StorageOptions.StorageType = options.StorageType(cfg.SectionWithEnvOverrides("grafana-apiserver").Key("storage_type").MustString(string(options.StorageTypeLegacy)))
+	o.StorageOptions.DataPath = filepath.Join(cfg.DataPath, "grafana-apiserver")
+	o.ExtraOptions.DevMode = features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerEnsureKubectlAccess)
+	o.ExtraOptions.ExternalAddress = host
+	o.ExtraOptions.APIURL = apiURL
+	o.ExtraOptions.Verbosity = defaultLogLevel
+}

pkg/services/apiserver/endpoints/request/namespace_test.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request"
+	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
 	"github.com/grafana/grafana/pkg/setting"
 )
 

pkg/services/apiserver/options/aggregator.go

@@ -0,0 +1,112 @@
+package options
+
+import (
+	"github.com/spf13/pflag"
+	v1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
+	genericfeatures "k8s.io/apiserver/pkg/features"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/options"
+	"k8s.io/apiserver/pkg/server/resourceconfig"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
+	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
+	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
+	aggregatoropenapi "k8s.io/kube-aggregator/pkg/generated/openapi"
+	"k8s.io/kube-openapi/pkg/common"
+
+	servicev0alpha1 "github.com/grafana/grafana/pkg/apis/service/v0alpha1"
+	filestorage "github.com/grafana/grafana/pkg/services/apiserver/storage/file"
+)
+
+// AggregatorServerOptions contains the state for the aggregator apiserver
+type AggregatorServerOptions struct {
+	AlternateDNS        []string
+	ProxyClientCertFile string
+	ProxyClientKeyFile  string
+}
+
+func NewAggregatorServerOptions() *AggregatorServerOptions {
+	return &AggregatorServerOptions{}
+}
+
+func (o *AggregatorServerOptions) getMergedOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition {
+	aggregatorAPIs := aggregatoropenapi.GetOpenAPIDefinitions(ref)
+	return aggregatorAPIs
+}
+
+func (o *AggregatorServerOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.ProxyClientCertFile, "proxy-client-cert-file", o.ProxyClientCertFile,
+		"path to proxy client cert file")
+
+	fs.StringVar(&o.ProxyClientKeyFile, "proxy-client-key-file", o.ProxyClientKeyFile,
+		"path to proxy client cert file")
+}
+
+func (o *AggregatorServerOptions) Validate() []error {
+	if o == nil {
+		return nil
+	}
+
+	// TODO: do we need to validate anything here?
+	return nil
+}
+
+func (o *AggregatorServerOptions) ApplyTo(aggregatorConfig *aggregatorapiserver.Config, etcdOpts *options.EtcdOptions, dataPath string) error {
+	genericConfig := aggregatorConfig.GenericConfig
+
+	genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{}
+	genericConfig.RESTOptionsGetter = nil
+
+	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) &&
+		utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
+		// Add StorageVersionPrecondition handler to aggregator-apiserver.
+		// The handler will block write requests to built-in resources until the
+		// target resources' storage versions are up-to-date.
+		genericConfig.BuildHandlerChainFunc = genericapiserver.BuildHandlerChainWithStorageVersionPrecondition
+	}
+
+	// copy the etcd options so we don't mutate originals.
+	// we assume that the etcd options have been completed already.  avoid messing with anything outside
+	// of changes to StorageConfig as that may lead to unexpected behavior when the options are applied.
+	etcdOptions := *etcdOpts
+	etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion,
+		apiregistrationv1beta1.SchemeGroupVersion,
+		servicev0alpha1.SchemeGroupVersion)
+	etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1.SchemeGroupVersion,
+		schema.GroupKind{Group: apiregistrationv1beta1.GroupName},
+		schema.GroupKind{Group: servicev0alpha1.GROUP})
+	etcdOptions.SkipHealthEndpoints = true // avoid double wiring of health checks
+	if err := etcdOptions.ApplyTo(&genericConfig.Config); err != nil {
+		return err
+	}
+	// override the RESTOptionsGetter to use the file storage options getter
+	aggregatorConfig.GenericConfig.RESTOptionsGetter = filestorage.NewRESTOptionsGetter(dataPath, etcdOptions.StorageConfig)
+
+	// prevent generic API server from installing the OpenAPI handler. Aggregator server has its own customized OpenAPI handler.
+	genericConfig.SkipOpenAPIInstallation = true
+	mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(aggregatorapiserver.DefaultAPIResourceConfigSource(), nil, aggregatorscheme.Scheme)
+	if err != nil {
+		return err
+	}
+	genericConfig.MergedResourceConfig = mergedResourceConfig
+
+	namer := openapinamer.NewDefinitionNamer(aggregatorscheme.Scheme)
+	genericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(o.getMergedOpenAPIDefinitions, namer)
+	genericConfig.OpenAPIV3Config.Info.Title = "Kubernetes"
+	genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(o.getMergedOpenAPIDefinitions, namer)
+	genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"
+	genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{}
+
+	// These hooks use v1 informers, which are not available in the grafana aggregator.
+	genericConfig.DisabledPostStartHooks = genericConfig.DisabledPostStartHooks.Insert("apiservice-status-available-controller")
+	genericConfig.DisabledPostStartHooks = genericConfig.DisabledPostStartHooks.Insert("start-kube-aggregator-informers")
+
+	return nil
+}

pkg/services/apiserver/options/extra.go

@@ -0,0 +1,46 @@
+package options
+
+import (
+	"strconv"
+
+	"github.com/go-logr/logr"
+	"github.com/spf13/pflag"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/component-base/logs"
+	"k8s.io/klog/v2"
+)
+
+type ExtraOptions struct {
+	DevMode         bool
+	ExternalAddress string
+	APIURL          string
+	Verbosity       int
+}
+
+func NewExtraOptions() *ExtraOptions {
+	return &ExtraOptions{
+		DevMode:   false,
+		Verbosity: 0,
+	}
+}
+
+func (o *ExtraOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.BoolVar(&o.DevMode, "grafana-apiserver-dev-mode", o.DevMode, "Enable dev mode")
+	fs.StringVar(&o.ExternalAddress, "grafana-apiserver-host", o.ExternalAddress, "Host")
+	fs.StringVar(&o.APIURL, "grafana-apiserver-api-url", o.APIURL, "API URL")
+	fs.IntVar(&o.Verbosity, "verbosity", o.Verbosity, "Verbosity")
+}
+
+func (o *ExtraOptions) Validate() []error {
+	return nil
+}
+
+func (o *ExtraOptions) ApplyTo(c *genericapiserver.RecommendedConfig) error {
+	logger := logr.New(newLogAdapter(o.Verbosity))
+	klog.SetLoggerWithOptions(logger, klog.ContextualLogger(true))
+	if _, err := logs.GlogSetter(strconv.Itoa(o.Verbosity)); err != nil {
+		logger.Error(err, "failed to set log level")
+	}
+	c.ExternalAddress = o.ExternalAddress
+	return nil
+}

pkg/services/apiserver/options/log.go

@@ -1,4 +1,4 @@
-package grafanaapiserver
+package options
 
 import (
 	"strings"

pkg/services/apiserver/options/options.go

@@ -0,0 +1,126 @@
+package options
+
+import (
+	"net"
+
+	"github.com/spf13/pflag"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	genericoptions "k8s.io/apiserver/pkg/server/options"
+)
+
+const defaultEtcdPathPrefix = "/registry/grafana.app"
+
+type Options struct {
+	RecommendedOptions *genericoptions.RecommendedOptions
+	AggregatorOptions  *AggregatorServerOptions
+	StorageOptions     *StorageOptions
+	ExtraOptions       *ExtraOptions
+}
+
+func NewOptions(codec runtime.Codec) *Options {
+	return &Options{
+		RecommendedOptions: genericoptions.NewRecommendedOptions(
+			defaultEtcdPathPrefix,
+			codec,
+		),
+		AggregatorOptions: NewAggregatorServerOptions(),
+		StorageOptions:    NewStorageOptions(),
+		ExtraOptions:      NewExtraOptions(),
+	}
+}
+
+func (o *Options) AddFlags(fs *pflag.FlagSet) {
+	o.RecommendedOptions.AddFlags(fs)
+	o.AggregatorOptions.AddFlags(fs)
+	o.StorageOptions.AddFlags(fs)
+	o.ExtraOptions.AddFlags(fs)
+}
+
+func (o *Options) Validate() []error {
+	if errs := o.ExtraOptions.Validate(); len(errs) != 0 {
+		return errs
+	}
+
+	if errs := o.StorageOptions.Validate(); len(errs) != 0 {
+		return errs
+	}
+
+	if errs := o.AggregatorOptions.Validate(); len(errs) != 0 {
+		return errs
+	}
+
+	if errs := o.RecommendedOptions.SecureServing.Validate(); len(errs) != 0 {
+		return errs
+	}
+
+	if errs := o.RecommendedOptions.Authentication.Validate(); len(errs) != 0 {
+		return errs
+	}
+
+	if o.StorageOptions.StorageType == StorageTypeEtcd {
+		if errs := o.RecommendedOptions.Etcd.Validate(); len(errs) != 0 {
+			return errs
+		}
+	}
+
+	return nil
+}
+
+func (o *Options) ApplyTo(serverConfig *genericapiserver.RecommendedConfig) error {
+	serverConfig.AggregatedDiscoveryGroupManager = aggregated.NewResourceManager("apis")
+
+	if err := o.ExtraOptions.ApplyTo(serverConfig); err != nil {
+		return err
+	}
+
+	if !o.ExtraOptions.DevMode {
+		o.RecommendedOptions.SecureServing.Listener = newFakeListener()
+	}
+
+	if err := o.RecommendedOptions.SecureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil {
+		return err
+	}
+
+	if err := o.RecommendedOptions.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, serverConfig.OpenAPIConfig); err != nil {
+		return err
+	}
+
+	if !o.ExtraOptions.DevMode {
+		if err := serverConfig.SecureServing.Listener.Close(); err != nil {
+			return err
+		}
+		serverConfig.SecureServing = nil
+	}
+
+	return nil
+}
+
+type fakeListener struct {
+	server net.Conn
+	client net.Conn
+}
+
+func newFakeListener() *fakeListener {
+	server, client := net.Pipe()
+	return &fakeListener{
+		server: server,
+		client: client,
+	}
+}
+
+func (f *fakeListener) Accept() (net.Conn, error) {
+	return f.server, nil
+}
+
+func (f *fakeListener) Close() error {
+	if err := f.client.Close(); err != nil {
+		return err
+	}
+	return f.server.Close()
+}
+
+func (f *fakeListener) Addr() net.Addr {
+	return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 3000, Zone: ""}
+}

pkg/services/apiserver/options/storage.go

@@ -0,0 +1,51 @@
+package options
+
+import (
+	"fmt"
+
+	"github.com/spf13/pflag"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/options"
+)
+
+type StorageType string
+
+const (
+	StorageTypeFile        StorageType = "file"
+	StorageTypeEtcd        StorageType = "etcd"
+	StorageTypeLegacy      StorageType = "legacy"
+	StorageTypeUnified     StorageType = "unified"
+	StorageTypeUnifiedGrpc StorageType = "unified-grpc"
+)
+
+type StorageOptions struct {
+	StorageType StorageType
+	DataPath    string
+}
+
+func NewStorageOptions() *StorageOptions {
+	return &StorageOptions{
+		StorageType: StorageTypeLegacy,
+	}
+}
+
+func (o *StorageOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar((*string)(&o.StorageType), "grafana-apiserver-storage-type", string(o.StorageType), "Storage type")
+	fs.StringVar((*string)(&o.StorageType), "grafana-apiserver-storage-path", string(o.StorageType), "Storage path for file storage")
+}
+
+func (o *StorageOptions) Validate() []error {
+	errs := []error{}
+	switch o.StorageType {
+	case StorageTypeFile, StorageTypeEtcd, StorageTypeLegacy, StorageTypeUnified, StorageTypeUnifiedGrpc:
+		// no-op
+	default:
+		errs = append(errs, fmt.Errorf("--grafana-apiserver-storage-type must be one of %s, %s, %s, %s, %s", StorageTypeFile, StorageTypeEtcd, StorageTypeLegacy, StorageTypeUnified, StorageTypeUnifiedGrpc))
+	}
+	return errs
+}
+
+func (o *StorageOptions) ApplyTo(serverConfig *genericapiserver.RecommendedConfig, etcdOptions *options.EtcdOptions) error {
+	// TODO: move storage setup here
+	return nil
+}

pkg/services/apiserver/service.go

@@ -1,4 +1,4 @@
-package grafanaapiserver
+package apiserver
 
 import (
 	"context"
@@ -6,13 +6,8 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"path"
-	"runtime/debug"
-	"strconv"
-	"strings"
 
-	"github.com/go-logr/logr"
 	"github.com/grafana/dskit/services"
-	"golang.org/x/mod/semver"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -21,15 +16,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apiserver/pkg/endpoints/responsewriter"
 	genericapiserver "k8s.io/apiserver/pkg/server"
-	"k8s.io/apiserver/pkg/server/options"
 	clientrest "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/component-base/logs"
-	"k8s.io/klog/v2"
-
-	"github.com/grafana/grafana/pkg/services/grafana-apiserver/auth/authorizer"
-	"github.com/grafana/grafana/pkg/services/grafana-apiserver/utils"
-	"github.com/grafana/grafana/pkg/services/org"
 
 	"github.com/grafana/grafana/pkg/api/routing"
 	"github.com/grafana/grafana/pkg/infra/appcontext"
@@ -38,26 +26,21 @@ import (
 	"github.com/grafana/grafana/pkg/middleware"
 	"github.com/grafana/grafana/pkg/modules"
 	"github.com/grafana/grafana/pkg/registry"
+	"github.com/grafana/grafana/pkg/services/apiserver/auth/authorizer"
+	"github.com/grafana/grafana/pkg/services/apiserver/builder"
+	grafanaapiserveroptions "github.com/grafana/grafana/pkg/services/apiserver/options"
+	entitystorage "github.com/grafana/grafana/pkg/services/apiserver/storage/entity"
+	filestorage "github.com/grafana/grafana/pkg/services/apiserver/storage/file"
+	"github.com/grafana/grafana/pkg/services/apiserver/utils"
 	contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
-	entitystorage "github.com/grafana/grafana/pkg/services/grafana-apiserver/storage/entity"
-	filestorage "github.com/grafana/grafana/pkg/services/grafana-apiserver/storage/file"
+	"github.com/grafana/grafana/pkg/services/org"
 	"github.com/grafana/grafana/pkg/services/store/entity"
 	"github.com/grafana/grafana/pkg/services/store/entity/db/dbimpl"
 	"github.com/grafana/grafana/pkg/services/store/entity/sqlstash"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
-type StorageType string
-
-const (
-	StorageTypeFile        StorageType = "file"
-	StorageTypeEtcd        StorageType = "etcd"
-	StorageTypeLegacy      StorageType = "legacy"
-	StorageTypeUnified     StorageType = "unified"
-	StorageTypeUnifiedGrpc StorageType = "unified-grpc"
-)
-
 var (
 	_ Service                    = (*service)(nil)
 	_ RestConfigProvider         = (*service)(nil)
@@ -90,10 +73,6 @@ type Service interface {
 	registry.CanBeDisabled
 }
 
-type APIRegistrar interface {
-	RegisterAPI(builder APIGroupBuilder)
-}
-
 type RestConfigProvider interface {
 	GetRestConfig() *clientrest.Config
 }
@@ -111,7 +90,7 @@ type DirectRestConfigProvider interface {
 type service struct {
 	*services.BasicService
 
-	config     *config
+	options    *grafanaapiserveroptions.Options
 	restConfig *clientrest.Config
 
 	cfg      *setting.Cfg
@@ -123,7 +102,7 @@ type service struct {
 	db       db.DB
 	rr       routing.RouteRegister
 	handler  http.Handler
-	builders []APIGroupBuilder
+	builders []builder.APIGroupBuilder
 
 	tracing *tracing.TracingService
 
@@ -139,12 +118,11 @@ func ProvideService(
 	db db.DB,
 ) (*service, error) {
 	s := &service{
-		config:     newConfig(cfg, features),
 		cfg:        cfg,
 		features:   features,
 		rr:         rr,
 		stopCh:     make(chan struct{}),
-		builders:   []APIGroupBuilder{},
+		builders:   []builder.APIGroupBuilder{},
 		authorizer: authorizer.NewGrafanaAuthorizer(cfg, orgService),
 		tracing:    tracing,
 		db:         db, // For Unified storage
@@ -182,7 +160,9 @@ func ProvideService(
 	}
 
 	s.rr.Group("/apis", proxyHandler)
-	s.rr.Group("/apiserver-metrics", proxyHandler)
+	s.rr.Group("/livez", proxyHandler)
+	s.rr.Group("/readyz", proxyHandler)
+	s.rr.Group("/healthz", proxyHandler)
 	s.rr.Group("/openapi", proxyHandler)
 
 	return s, nil
@@ -193,7 +173,7 @@ func (s *service) GetRestConfig() *clientrest.Config {
 }
 
 func (s *service) IsDisabled() bool {
-	return !s.config.enabled
+	return false
 }
 
 // Run is an adapter for the BackgroundService interface.
@@ -204,17 +184,11 @@ func (s *service) Run(ctx context.Context) error {
 	return s.running(ctx)
 }
 
-func (s *service) RegisterAPI(builder APIGroupBuilder) {
-	s.builders = append(s.builders, builder)
+func (s *service) RegisterAPI(b builder.APIGroupBuilder) {
+	s.builders = append(s.builders, b)
 }
 
 func (s *service) start(ctx context.Context) error {
-	logger := logr.New(newLogAdapter(s.config.logLevel))
-	klog.SetLoggerWithOptions(logger, klog.ContextualLogger(true))
-	if _, err := logs.GlogSetter(strconv.Itoa(s.config.logLevel)); err != nil {
-		logger.Error(err, "failed to set log level")
-	}
-
 	// Get the list of groups the server will support
 	builders := s.builders
 
@@ -226,59 +200,42 @@ func (s *service) start(ctx context.Context) error {
 			return err
 		}
 
-		// Optionally register a custom authorizer
 		auth := b.GetAuthorizer()
 		if auth != nil {
 			s.authorizer.Register(b.GetGroupVersion(), auth)
 		}
 	}
 
-	o := options.NewRecommendedOptions("/registry/grafana.app", Codecs.LegacyCodec(groupVersions...))
-	o.SecureServing.BindAddress = s.config.ip
-	o.SecureServing.BindPort = s.config.port
-	o.Authentication.RemoteKubeConfigFileOptional = true
-	o.Authorization.RemoteKubeConfigFileOptional = true
+	o := grafanaapiserveroptions.NewOptions(Codecs.LegacyCodec(groupVersions...))
+	applyGrafanaConfig(s.cfg, s.features, o)
 
-	o.Admission = nil
-	o.CoreAPI = nil
-
-	serverConfig := genericapiserver.NewRecommendedConfig(Codecs)
-	serverConfig.ExternalAddress = s.config.host
-
-	if s.config.devMode {
-		// SecureServingOptions is used when the apiserver needs it's own listener.
-		// this is not needed in production, but it's useful for development kubectl access.
-		if err := o.SecureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil {
-			return err
-		}
-		// AuthenticationOptions is needed to authenticate requests from kubectl in dev mode.
-		if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, serverConfig.OpenAPIConfig); err != nil {
-			return err
-		}
-	} else {
-		// In production mode, override ExternalAddress and LoopbackClientConfig.
-		// In dev mode we want to use the loopback client config
-		// and address provided by SecureServingOptions.
-		serverConfig.ExternalAddress = s.config.host
-		serverConfig.LoopbackClientConfig = &clientrest.Config{
-			Host: s.config.apiURL,
-			TLSClientConfig: clientrest.TLSClientConfig{
-				Insecure: true,
-			},
-		}
+	if errs := o.Validate(); len(errs) != 0 {
+		// TODO: handle multiple errors
+		return errs[0]
 	}
 
-	switch s.config.storageType {
-	case StorageTypeEtcd:
-		o.Etcd.StorageConfig.Transport.ServerList = s.config.etcdServers
-		if err := o.Etcd.Validate(); len(err) > 0 {
+	serverConfig := genericapiserver.NewRecommendedConfig(Codecs)
+	if err := o.ApplyTo(serverConfig); err != nil {
+		return err
+	}
+	serverConfig.Authorization.Authorizer = s.authorizer
+	serverConfig.TracerProvider = s.tracing.GetTracerProvider()
+
+	// setup loopback transport
+	transport := &roundTripperFunc{ready: make(chan struct{})}
+	serverConfig.LoopbackClientConfig.Transport = transport
+	serverConfig.LoopbackClientConfig.TLSClientConfig = clientrest.TLSClientConfig{}
+
+	switch o.StorageOptions.StorageType {
+	case grafanaapiserveroptions.StorageTypeEtcd:
+		if err := o.RecommendedOptions.Etcd.Validate(); len(err) > 0 {
 			return err[0]
 		}
-		if err := o.Etcd.ApplyTo(&serverConfig.Config); err != nil {
+		if err := o.RecommendedOptions.Etcd.ApplyTo(&serverConfig.Config); err != nil {
 			return err
 		}
 
-	case StorageTypeUnified:
+	case grafanaapiserveroptions.StorageTypeUnified:
 		if !s.features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorage) {
 			return fmt.Errorf("unified storage requires the unifiedStorage feature flag (and app_mode = development)")
 		}
@@ -293,9 +250,9 @@ func (s *service) start(ctx context.Context) error {
 			return err
 		}
 
-		serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.Etcd.StorageConfig.Codec)
+		serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.RecommendedOptions.Etcd.StorageConfig.Codec)
 
-	case StorageTypeUnifiedGrpc:
+	case grafanaapiserveroptions.StorageTypeUnifiedGrpc:
 		// Create a connection to the gRPC server
 		// TODO: support configuring the gRPC server address
 		conn, err := grpc.Dial("localhost:10000", grpc.WithTransportCredentials(insecure.NewCredentials()))
@@ -309,20 +266,16 @@ func (s *service) start(ctx context.Context) error {
 		// Create a client instance
 		store := entity.NewEntityStoreClientWrapper(conn)
 
-		serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.Etcd.StorageConfig.Codec)
+		serverConfig.Config.RESTOptionsGetter = entitystorage.NewRESTOptionsGetter(s.cfg, store, o.RecommendedOptions.Etcd.StorageConfig.Codec)
 
-	case StorageTypeFile:
-		serverConfig.RESTOptionsGetter = filestorage.NewRESTOptionsGetter(s.config.dataPath, o.Etcd.StorageConfig)
-
-	case StorageTypeLegacy:
-		// do nothing?
+	case grafanaapiserveroptions.StorageTypeLegacy:
+		fallthrough
+	case grafanaapiserveroptions.StorageTypeFile:
+		serverConfig.RESTOptionsGetter = filestorage.NewRESTOptionsGetter(o.StorageOptions.DataPath, o.RecommendedOptions.Etcd.StorageConfig)
 	}
 
-	serverConfig.Authorization.Authorizer = s.authorizer
-	serverConfig.TracerProvider = s.tracing.GetTracerProvider()
-
 	// Add OpenAPI specs for each group+version
-	err := SetupConfig(serverConfig, builders)
+	err := builder.SetupConfig(Scheme, serverConfig, builders)
 	if err != nil {
 		return err
 	}
@@ -339,28 +292,37 @@ func (s *service) start(ctx context.Context) error {
 		return err
 	}
 
+	dualWriteEnabled := o.StorageOptions.StorageType != grafanaapiserveroptions.StorageTypeLegacy
+
 	// Install the API Group+version
-	err = InstallAPIs(server, serverConfig.RESTOptionsGetter, builders)
+	err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, dualWriteEnabled)
 	if err != nil {
 		return err
 	}
 
+	// set the transport function and signal that it's ready
+	transport.fn = func(req *http.Request) (*http.Response, error) {
+		w := newWrappedResponseWriter()
+		resp := responsewriter.WrapForHTTP1Or2(w)
+		server.Handler.ServeHTTP(resp, req)
+		return w.Result(), nil
+	}
+	close(transport.ready)
+
+	// only write kubeconfig in dev mode
+	if o.ExtraOptions.DevMode {
+		if err := ensureKubeConfig(server.LoopbackClientConfig, o.StorageOptions.DataPath); err != nil {
+			return err
+		}
+	}
+
 	// Used by the proxy wrapper registered in ProvideService
 	s.handler = server.Handler
 	s.restConfig = server.LoopbackClientConfig
+	s.options = o
 
 	prepared := server.PrepareRun()
 
-	// When running in production, do not start a standalone https server
-	if !s.config.devMode {
-		return nil
-	}
-
-	// only write kubeconfig in dev mode
-	if err := s.ensureKubeConfig(); err != nil {
-		return err
-	}
-
 	go func() {
 		s.stoppedCh <- prepared.Run(s.stopCh)
 	}()
@@ -385,12 +347,6 @@ func (s *service) DirectlyServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *service) running(ctx context.Context) error {
-	// skip waiting for the server in prod mode
-	if !s.config.devMode {
-		<-ctx.Done()
-		return nil
-	}
-
 	select {
 	case err := <-s.stoppedCh:
 		if err != nil {
@@ -402,47 +358,42 @@ func (s *service) running(ctx context.Context) error {
 	return nil
 }
 
-func (s *service) ensureKubeConfig() error {
+func ensureKubeConfig(restConfig *clientrest.Config, dir string) error {
 	return clientcmd.WriteToFile(
-		utils.FormatKubeConfig(s.restConfig),
-		path.Join(s.config.dataPath, "grafana.kubeconfig"),
+		utils.FormatKubeConfig(restConfig),
+		path.Join(dir, "grafana.kubeconfig"),
 	)
 }
 
 type roundTripperFunc struct {
-	fn func(req *http.Request) (*http.Response, error)
+	ready chan struct{}
+	fn    func(req *http.Request) (*http.Response, error)
 }
 
 func (f *roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	if f.fn == nil {
+		<-f.ready
+	}
 	return f.fn(req)
 }
 
-// find the k8s version according to build info
-func getK8sApiserverVersion() (string, error) {
-	bi, ok := debug.ReadBuildInfo()
-	if !ok {
-		return "", fmt.Errorf("debug.ReadBuildInfo() failed")
-	}
+var _ http.ResponseWriter = (*wrappedResponseWriter)(nil)
+var _ responsewriter.UserProvidedDecorator = (*wrappedResponseWriter)(nil)
 
-	if len(bi.Deps) == 0 {
-		return "v?.?", nil // this is normal while debugging
-	}
+type wrappedResponseWriter struct {
+	*httptest.ResponseRecorder
+}
 
-	for _, dep := range bi.Deps {
-		if dep.Path == "k8s.io/apiserver" {
-			if !semver.IsValid(dep.Version) {
-				return "", fmt.Errorf("invalid semantic version for k8s.io/apiserver")
-			}
-			// v0 => v1
-			majorVersion := strings.TrimPrefix(semver.Major(dep.Version), "v")
-			majorInt, err := strconv.Atoi(majorVersion)
-			if err != nil {
-				return "", fmt.Errorf("could not convert majorVersion to int. majorVersion: %s", majorVersion)
-			}
-			newMajor := fmt.Sprintf("v%d", majorInt+1)
-			return strings.Replace(dep.Version, semver.Major(dep.Version), newMajor, 1), nil
-		}
-	}
+func newWrappedResponseWriter() *wrappedResponseWriter {
+	w := httptest.NewRecorder()
+	return &wrappedResponseWriter{w}
+}
 
-	return "", fmt.Errorf("could not find k8s.io/apiserver in build info")
+func (w *wrappedResponseWriter) Unwrap() http.ResponseWriter {
+	return w.ResponseRecorder
+}
+
+func (w *wrappedResponseWriter) CloseNotify() <-chan bool {
+	// TODO: this is probably not the right thing to do here
+	return make(<-chan bool)
 }

pkg/services/apiserver/storage/entity/utils.go

@@ -15,7 +15,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apiserver/pkg/endpoints/request"
 
-	"github.com/grafana/grafana/pkg/services/grafana-apiserver/utils"
+	"github.com/grafana/grafana/pkg/services/apiserver/utils"
 	entityStore "github.com/grafana/grafana/pkg/services/store/entity"
 )
 

pkg/services/apiserver/utils/meta_test.go

@@ -8,7 +8,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 
-	"github.com/grafana/grafana/pkg/services/grafana-apiserver/utils"
+	"github.com/grafana/grafana/pkg/services/apiserver/utils"
 )
 
 type TestResource struct {

pkg/services/apiserver/utils/tableConverter_test.go

@@ -11,7 +11,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
-	"github.com/grafana/grafana/pkg/services/grafana-apiserver/utils"
+	"github.com/grafana/grafana/pkg/services/apiserver/utils"
 )
 
 func TestTableConverter(t *testing.T) {

pkg/services/apiserver/wireset.go

@@ -1,13 +1,15 @@
-package grafanaapiserver
+package apiserver
 
 import (
 	"github.com/google/wire"
+
+	"github.com/grafana/grafana/pkg/services/apiserver/builder"
 )
 
 var WireSet = wire.NewSet(
 	ProvideService,
 	wire.Bind(new(RestConfigProvider), new(*service)),
 	wire.Bind(new(Service), new(*service)),
-	wire.Bind(new(APIRegistrar), new(*service)),
 	wire.Bind(new(DirectRestConfigProvider), new(*service)),
+	wire.Bind(new(builder.APIRegistrar), new(*service)),
 )

pkg/services/grafana-apiserver/config.go

@@ -1,60 +0,0 @@
-package grafanaapiserver
-
-import (
-	"fmt"
-	"net"
-	"path/filepath"
-	"strconv"
-
-	"github.com/grafana/grafana/pkg/services/featuremgmt"
-	"github.com/grafana/grafana/pkg/setting"
-)
-
-type config struct {
-	enabled bool
-	devMode bool
-
-	ip     net.IP
-	port   int
-	host   string
-	apiURL string
-
-	storageType StorageType
-
-	etcdServers []string
-	dataPath    string
-
-	logLevel int
-}
-
-func newConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles) *config {
-	defaultLogLevel := 0
-	ip := net.ParseIP(cfg.HTTPAddr)
-	apiURL := cfg.AppURL
-	port, err := strconv.Atoi(cfg.HTTPPort)
-	if err != nil {
-		port = 3000
-	}
-
-	if cfg.Env == setting.Dev {
-		defaultLogLevel = 10
-		port = 6443
-		ip = net.ParseIP("127.0.0.1")
-		apiURL = fmt.Sprintf("https://%s:%d", ip, port)
-	}
-
-	host := fmt.Sprintf("%s:%d", ip, port)
-
-	return &config{
-		enabled:     true,
-		devMode:     features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerEnsureKubectlAccess),
-		dataPath:    filepath.Join(cfg.DataPath, "grafana-apiserver"),
-		ip:          ip,
-		port:        port,
-		host:        host,
-		logLevel:    cfg.SectionWithEnvOverrides("grafana-apiserver").Key("log_level").MustInt(defaultLogLevel),
-		etcdServers: cfg.SectionWithEnvOverrides("grafana-apiserver").Key("etcd_servers").Strings(","),
-		storageType: StorageType(cfg.SectionWithEnvOverrides("grafana-apiserver").Key("storage_type").MustString(string(StorageTypeLegacy))),
-		apiURL:      apiURL,
-	}
-}

pkg/services/grafana-apiserver/config_test.go

@@ -1,40 +0,0 @@
-package grafanaapiserver
-
-import (
-	"net"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/grafana/grafana/pkg/services/featuremgmt"
-	"github.com/grafana/grafana/pkg/setting"
-)
-
-func TestNewConfig(t *testing.T) {
-	cfg := setting.NewCfg()
-	cfg.Env = setting.Prod
-	cfg.DataPath = "/tmp/grafana"
-	cfg.HTTPAddr = "10.0.0.1"
-	cfg.HTTPPort = "4000"
-	cfg.AppURL = "http://test:4000"
-
-	section := cfg.Raw.Section("grafana-apiserver")
-	section.Key("log_level").SetValue("5")
-	section.Key("etcd_servers").SetValue("http://localhost:2379")
-
-	actual := newConfig(cfg, featuremgmt.WithFeatures())
-
-	expected := &config{
-		enabled:     true,
-		devMode:     false,
-		storageType: StorageTypeLegacy,
-		etcdServers: []string{"http://localhost:2379"},
-		apiURL:      "http://test:4000",
-		ip:          net.ParseIP("10.0.0.1"),
-		port:        4000,
-		host:        "10.0.0.1:4000",
-		dataPath:    "/tmp/grafana/grafana-apiserver",
-		logLevel:    5,
-	}
-	require.Equal(t, expected, actual)
-}