K8s: Add storage dual writer (#75403)

.github/CODEOWNERS

@@ -177,6 +177,7 @@
 /devenv/docker/blocks/alert_webhook_listener/ @grafana/alerting-backend-product
 /devenv/docker/blocks/clickhouse/ @grafana/partner-datasources
 /devenv/docker/blocks/collectd/ @grafana/observability-metrics
+/devenv/docker/blocks/etcd @grafana/grafana-app-platform-squad
 /devenv/docker/blocks/grafana/ @grafana/grafana-as-code
 /devenv/docker/blocks/graphite/ @grafana/observability-metrics
 /devenv/docker/blocks/graphite09/ @grafana/observability-metrics

devenv/docker/blocks/etcd/docker-compose.yaml

@@ -0,0 +1,11 @@
+  
+  etcd:
+    image: bitnami/etcd:latest
+    restart: always
+    container_name: etcd
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    ports:
+      - 2379:2379
+      - 2380:2380

pkg/apis/playlist/v0alpha1/doc.go

@@ -2,4 +2,4 @@
 // +k8s:openapi-gen=true
 // +groupName=playlist.grafana.io
 
-package v1 // import "github.com/grafana/grafana/pkg/apis/playlist/v1"
+package v0alpha1 // import "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1"

pkg/apis/playlist/v0alpha1/legacy_storage.go

@@ -0,0 +1,128 @@
+package v0alpha1
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/apis/meta/internalversion"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/registry/rest"
+
+	grafanarequest "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request"
+	"github.com/grafana/grafana/pkg/services/playlist"
+)
+
+var (
+	_ rest.Scoper               = (*legacyStorage)(nil)
+	_ rest.SingularNameProvider = (*legacyStorage)(nil)
+	_ rest.Getter               = (*legacyStorage)(nil)
+	_ rest.Lister               = (*legacyStorage)(nil)
+	_ rest.Storage              = (*legacyStorage)(nil)
+)
+
+type legacyStorage struct {
+	service playlist.Service
+}
+
+func newLegacyStorage(s playlist.Service) *legacyStorage {
+	return &legacyStorage{
+		service: s,
+	}
+}
+
+func (s *legacyStorage) New() runtime.Object {
+	return &Playlist{}
+}
+
+func (s *legacyStorage) Destroy() {}
+
+func (s *legacyStorage) NamespaceScoped() bool {
+	return true // namespace == org
+}
+
+func (s *legacyStorage) GetSingularName() string {
+	return "playlist"
+}
+
+func (s *legacyStorage) NewList() runtime.Object {
+	return &PlaylistList{}
+}
+
+func (s *legacyStorage) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
+	return rest.NewDefaultTableConvertor(Resource("playlists")).ConvertToTable(ctx, object, tableOptions)
+}
+
+func (s *legacyStorage) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) {
+	// TODO: handle fetching all available orgs when no namespace is specified
+	// To test: kubectl get playlists --all-namespaces
+	orgId, ok := grafanarequest.OrgIDFrom(ctx)
+	if !ok {
+		orgId = 1 // TODO: default org ID 1 for now
+	}
+
+	limit := 100
+	if options.Limit > 0 {
+		limit = int(options.Limit)
+	}
+	res, err := s.service.Search(ctx, &playlist.GetPlaylistsQuery{
+		OrgId: orgId,
+		Limit: limit,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	list := &PlaylistList{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "PlaylistList",
+			APIVersion: APIVersion,
+		},
+	}
+	for _, v := range res {
+		p := Playlist{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "Playlist",
+				APIVersion: APIVersion,
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name: v.UID,
+			},
+		}
+		p.Name = v.Name + " // " + v.Interval
+		list.Items = append(list.Items, p)
+	}
+	if len(list.Items) == limit {
+		list.Continue = "<more>" // TODO?
+	}
+	return list, nil
+}
+
+func (s *legacyStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
+	orgId, ok := grafanarequest.OrgIDFrom(ctx)
+	if !ok {
+		orgId = 1 // TODO: default org ID 1 for now
+	}
+
+	p, err := s.service.Get(ctx, &playlist.GetPlaylistByUidQuery{
+		UID:   name,
+		OrgId: orgId,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if p == nil {
+		return nil, fmt.Errorf("not found?")
+	}
+
+	return &Playlist{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Playlist",
+			APIVersion: APIVersion,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: p.Uid,
+		},
+		Name: p.Name + "//" + p.Interval,
+	}, nil
+}

pkg/apis/playlist/v0alpha1/openapi.go

@@ -1,4 +1,4 @@
-package v1
+package v0alpha1
 
 import (
 	common "k8s.io/kube-openapi/pkg/common"
@@ -6,7 +6,7 @@ import (
 )
 
 // NOTE: this must match the golang fully qualifid name!
-const kindKey = "github.com/grafana/grafana/pkg/apis/playlist/v1.Playlist"
+const kindKey = "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1.Playlist"
 
 func getOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition {
 	return map[string]common.OpenAPIDefinition{

pkg/apis/playlist/v0alpha1/register.go

@@ -1,22 +1,24 @@
-package v1
+package v0alpha1
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/registry/rest"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	common "k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/spec3"
 
 	grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver"
+	grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest"
 	"github.com/grafana/grafana/pkg/services/playlist"
 )
 
 // GroupName is the group name for this API.
 const GroupName = "playlist.x.grafana.com"
-const VersionID = "v0-alpha" //
+const VersionID = "v0alpha1" //
 const APIVersion = GroupName + "/" + VersionID
 
 var _ grafanaapiserver.APIGroupBuilder = (*PlaylistAPIBuilder)(nil)
@@ -45,15 +47,25 @@ func (b *PlaylistAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
 func (b *PlaylistAPIBuilder) GetAPIGroupInfo(
 	scheme *runtime.Scheme,
 	codecs serializer.CodecFactory, // pointer?
-) *genericapiserver.APIGroupInfo {
+	optsGetter generic.RESTOptionsGetter,
+) (*genericapiserver.APIGroupInfo, error) {
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(GroupName, scheme, metav1.ParameterCodec, codecs)
 	storage := map[string]rest.Storage{}
-	storage["playlists"] = &handler{
+
-		service: b.service,
+	legacyStore := newLegacyStorage(b.service)
+	storage["playlists"] = legacyStore
+
+	// enable dual writes if a RESTOptionsGetter is provided
+	if optsGetter != nil {
+		store, err := newStorage(scheme, optsGetter)
+		if err != nil {
+			return nil, err
+		}
+		storage["playlists"] = grafanarest.NewDualWriter(legacyStore, store)
 	}
 
 	apiGroupInfo.VersionedResourcesStorageMap[VersionID] = storage
-	return &apiGroupInfo
+	return &apiGroupInfo, nil
 }
 
 func (b *PlaylistAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions {

pkg/apis/playlist/v0alpha1/storage.go

@@ -0,0 +1,40 @@
+package v0alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/registry/generic"
+	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
+	"k8s.io/apiserver/pkg/registry/rest"
+
+	grafanaregistry "github.com/grafana/grafana/pkg/services/grafana-apiserver/registry/generic"
+	grafanarest "github.com/grafana/grafana/pkg/services/grafana-apiserver/rest"
+)
+
+var _ grafanarest.Storage = (*storage)(nil)
+
+type storage struct {
+	*genericregistry.Store
+}
+
+func newStorage(scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter) (*storage, error) {
+	strategy := grafanaregistry.NewStrategy(scheme)
+
+	store := &genericregistry.Store{
+		NewFunc:                   func() runtime.Object { return &Playlist{} },
+		NewListFunc:               func() runtime.Object { return &PlaylistList{} },
+		PredicateFunc:             grafanaregistry.Matcher,
+		DefaultQualifiedResource:  Resource("playlists"),
+		SingularQualifiedResource: Resource("playlist"),
+
+		CreateStrategy: strategy,
+		UpdateStrategy: strategy,
+		DeleteStrategy: strategy,
+
+		TableConvertor: rest.NewDefaultTableConvertor(Resource("playlists")),
+	}
+	options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: grafanaregistry.GetAttrs}
+	if err := store.CompleteWithOptions(options); err != nil {
+		return nil, err
+	}
+	return &storage{Store: store}, nil
+}

pkg/apis/playlist/v0alpha1/types.go

@@ -1,4 +1,4 @@
-package v1
+package v0alpha1
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

pkg/apis/playlist/v0alpha1/zz_generated.deepcopy.go

@@ -1,11 +1,9 @@
 //go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
-// generated by scripts/k8s/update-codegen.sh
-
 // Code generated by deepcopy-gen. DO NOT EDIT.
 
-package v1
+package v0alpha1
 
 import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
@@ -71,17 +69,17 @@ func (in *PlaylistList) DeepCopyObject() runtime.Object {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *handler) DeepCopyInto(out *handler) {
+func (in *legacyStorage) DeepCopyInto(out *legacyStorage) {
 	*out = *in
 	return
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Storage.
-func (in *handler) DeepCopy() *handler {
+func (in *legacyStorage) DeepCopy() *legacyStorage {
 	if in == nil {
 		return nil
 	}
-	out := new(handler)
+	out := new(legacyStorage)
 	in.DeepCopyInto(out)
 	return out
 }

pkg/apis/playlist/v1/handler.go

@@ -1,130 +0,0 @@
-package v1
-
-import (
-	"context"
-	"fmt"
-
-	"k8s.io/apimachinery/pkg/apis/meta/internalversion"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apiserver/pkg/endpoints/request"
-	"k8s.io/apiserver/pkg/registry/rest"
-
-	grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver"
-	"github.com/grafana/grafana/pkg/services/playlist"
-)
-
-var _ rest.Scoper = (*handler)(nil)
-var _ rest.SingularNameProvider = (*handler)(nil)
-var _ rest.Getter = (*handler)(nil)
-var _ rest.Lister = (*handler)(nil)
-var _ rest.Storage = (*handler)(nil)
-
-type handler struct {
-	service playlist.Service
-}
-
-func (r *handler) New() runtime.Object {
-	return &Playlist{}
-}
-
-func (r *handler) Destroy() {}
-
-func (r *handler) NamespaceScoped() bool {
-	return true // namespace == org
-}
-
-func (r *handler) GetSingularName() string {
-	return "playlist"
-}
-
-func (r *handler) NewList() runtime.Object {
-	return &PlaylistList{}
-}
-
-func (r *handler) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
-	return rest.NewDefaultTableConvertor(Resource("playlists")).ConvertToTable(ctx, object, tableOptions)
-}
-
-func (r *handler) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) {
-	ns, ok := request.NamespaceFrom(ctx)
-	if !ok || ns == "" {
-		return nil, fmt.Errorf("namespace required")
-	}
-
-	orgId, err := grafanaapiserver.NamespaceToOrgID(ns)
-	if err != nil {
-		return nil, err
-	}
-
-	limit := 100
-	if options.Limit > 0 {
-		limit = int(options.Limit)
-	}
-	res, err := r.service.Search(ctx, &playlist.GetPlaylistsQuery{
-		OrgId: orgId,
-		Limit: limit,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	list := &PlaylistList{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "PlaylistList",
-			APIVersion: APIVersion,
-		},
-	}
-	for _, v := range res {
-		p := Playlist{
-			TypeMeta: metav1.TypeMeta{
-				Kind:       "Playlist",
-				APIVersion: APIVersion,
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name: v.UID,
-			},
-		}
-		p.Name = v.Name + " // " + v.Interval
-		list.Items = append(list.Items, p)
-		// TODO?? if table... we don't need the body of each, otherwise full lookup!
-	}
-	if len(list.Items) == limit {
-		list.Continue = "<more>" // TODO?
-	}
-	return list, nil
-}
-
-func (r *handler) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
-	ns, ok := request.NamespaceFrom(ctx)
-	if !ok || ns == "" {
-		return nil, fmt.Errorf("namespace required")
-	}
-
-	orgId, err := grafanaapiserver.NamespaceToOrgID(ns)
-	if err != nil {
-		return nil, err
-	}
-
-	p, err := r.service.Get(ctx, &playlist.GetPlaylistByUidQuery{
-		UID:   name,
-		OrgId: orgId,
-	})
-	if err != nil {
-		return nil, err
-	}
-	if p == nil {
-		return nil, fmt.Errorf("not found?")
-	}
-
-	return &Playlist{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Playlist",
-			APIVersion: APIVersion,
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: p.Uid,
-		},
-		Name: p.Name + "//" + p.Interval,
-	}, nil
-}

pkg/apis/wireset.go

@@ -2,9 +2,10 @@ package apis
 
 import (
 	"github.com/google/wire"
-	playlistv1 "github.com/grafana/grafana/pkg/apis/playlist/v1"
+
+	playlistsv0alpha1 "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1"
 )
 
 var WireSet = wire.NewSet(
-	playlistv1.RegisterAPIService,
+	playlistsv0alpha1.RegisterAPIService,
 )

pkg/registry/apis/apis.go

@@ -3,7 +3,7 @@ package apiregistry
 import (
 	"context"
 
-	playlistsv1 "github.com/grafana/grafana/pkg/apis/playlist/v1"
+	playlistsv0alpha1 "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1"
 	"github.com/grafana/grafana/pkg/registry"
 )
 
@@ -14,7 +14,7 @@ var (
 type Service struct{}
 
 func ProvideService(
-	_ *playlistsv1.PlaylistAPIBuilder,
+	_ *playlistsv0alpha1.PlaylistAPIBuilder,
 ) *Service {
 	return &Service{}
 }

pkg/services/grafana-apiserver/README.md

@@ -0,0 +1,43 @@
+# Grafana Kubernetes compatible API Server
+
+## Basic Setup
+
+```ini
+app_mode = development
+
+[feature_toggles]
+grafanaAPIServer = true
+```
+
+Start Grafana:
+
+```bash
+make run
+```
+
+## Enable dual write to `etcd`
+
+Start `etcd`:
+```bash
+make devenv sources=etcd
+```
+
+Enable dual write to `etcd`:
+
+```ini
+[grafana-apiserver]
+etcd_servers = 127.0.0.1:2379
+```
+
+### `kubectl` access
+
+From the root of the repository:
+
+```bash
+export KUBECONFIG=$PWD/data/k8s/grafana.kubeconfig
+kubectl api-resources
+```
+
+### Grafana API Access
+
+The Kubernetes compatible API can be accessed using existing Grafana AuthN at: [http://localhost:3000/k8s/apis/](http://localhost:3000/k8s/apis/).

pkg/services/grafana-apiserver/common.go

@@ -1,12 +1,9 @@
 package grafanaapiserver
 
 import (
-	"fmt"
-	"strconv"
-	"strings"
-
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apiserver/pkg/registry/generic"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/spec3"
@@ -22,7 +19,8 @@ type APIGroupBuilder interface {
 	GetAPIGroupInfo(
 		scheme *runtime.Scheme,
 		codecs serializer.CodecFactory, // pointer?
-	) *genericapiserver.APIGroupInfo
+		optsGetter generic.RESTOptionsGetter,
+	) (*genericapiserver.APIGroupInfo, error)
 
 	// Get OpenAPI definitions
 	GetOpenAPIDefinitions() common.GetOpenAPIDefinitions
@@ -30,34 +28,3 @@ type APIGroupBuilder interface {
 	// Register additional routes with the server
 	GetOpenAPIPostProcessor() func(*spec3.OpenAPI) (*spec3.OpenAPI, error)
 }
-
-func OrgIdToNamespace(orgId int64) string {
-	if orgId > 1 {
-		return fmt.Sprintf("org-%d", orgId)
-	}
-	return "default"
-}
-
-func NamespaceToOrgID(ns string) (int64, error) {
-	parts := strings.Split(ns, "-")
-	switch len(parts) {
-	case 1:
-		if parts[0] == "default" {
-			return 1, nil
-		}
-		if parts[0] == "" {
-			return 0, nil // no orgId, cluster scope
-		}
-		return 0, fmt.Errorf("invalid namespace (expected default)")
-	case 2:
-		if !(parts[0] == "org" || parts[0] == "tenant") {
-			return 0, fmt.Errorf("invalid namespace (org|tenant)")
-		}
-		n, err := strconv.ParseInt(parts[1], 10, 64)
-		if err != nil {
-			return 0, fmt.Errorf("invalid namepscae (%w)", err)
-		}
-		return n, nil
-	}
-	return 0, fmt.Errorf("invalid namespace (%d parts)", len(parts))
-}

pkg/services/grafana-apiserver/endpoints/request/request.go

@@ -0,0 +1,22 @@
+package request
+
+import (
+	"context"
+	"strconv"
+
+	"k8s.io/apiserver/pkg/endpoints/request"
+)
+
+func OrgIDFrom(ctx context.Context) (int64, bool) {
+	ns := request.NamespaceValue(ctx)
+	if len(ns) < 5 || ns[:4] != "org-" {
+		return 0, false
+	}
+
+	orgID, err := strconv.Atoi(ns[4:])
+	if err != nil {
+		return 0, false
+	}
+
+	return int64(orgID), true
+}

pkg/services/grafana-apiserver/endpoints/request/request_test.go

@@ -0,0 +1,62 @@
+package request_test
+
+import (
+	"context"
+	"testing"
+
+	"k8s.io/apiserver/pkg/endpoints/request"
+
+	grafanarequest "github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request"
+)
+
+func TestOrgIDFrom(t *testing.T) {
+	tests := []struct {
+		name     string
+		ctx      context.Context
+		expected int64
+		ok       bool
+	}{
+		{
+			name:     "empty namespace",
+			ctx:      context.Background(),
+			expected: 0,
+			ok:       false,
+		},
+		{
+			name:     "incorrect number of parts",
+			ctx:      request.WithNamespace(context.Background(), "org-123-a"),
+			expected: 0,
+			ok:       false,
+		},
+		{
+			name:     "incorrect prefix",
+			ctx:      request.WithNamespace(context.Background(), "abc-123"),
+			expected: 0,
+			ok:       false,
+		},
+		{
+			name:     "org id not a number",
+			ctx:      request.WithNamespace(context.Background(), "org-invalid"),
+			expected: 0,
+			ok:       false,
+		},
+		{
+			name:     "valid org id",
+			ctx:      request.WithNamespace(context.Background(), "org-123"),
+			expected: 123,
+			ok:       true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual, ok := grafanarequest.OrgIDFrom(tt.ctx)
+			if actual != tt.expected {
+				t.Errorf("OrgIDFrom() returned %d, expected %d", actual, tt.expected)
+			}
+			if ok != tt.ok {
+				t.Errorf("OrgIDFrom() returned %t, expected %t", ok, tt.ok)
+			}
+		})
+	}
+}

pkg/services/grafana-apiserver/registry/generic/strategy.go

@@ -0,0 +1,79 @@
+package generic
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/names"
+)
+
+type genericStrategy struct {
+	runtime.ObjectTyper
+	names.NameGenerator
+}
+
+// NewStrategy creates and returns a genericStrategy instance.
+func NewStrategy(typer runtime.ObjectTyper) genericStrategy {
+	return genericStrategy{typer, names.SimpleNameGenerator}
+}
+
+// NamespaceScoped returns true because all Generic resources must be within a namespace.
+func (genericStrategy) NamespaceScoped() bool {
+	return true
+}
+
+func (genericStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) {}
+
+func (genericStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {}
+
+func (genericStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList {
+	return field.ErrorList{}
+}
+
+// WarningsOnCreate returns warnings for the creation of the given object.
+func (genericStrategy) WarningsOnCreate(ctx context.Context, obj runtime.Object) []string { return nil }
+
+func (genericStrategy) AllowCreateOnUpdate() bool {
+	return false
+}
+
+func (genericStrategy) AllowUnconditionalUpdate() bool {
+	return false
+}
+
+func (genericStrategy) Canonicalize(obj runtime.Object) {}
+
+func (genericStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
+	return field.ErrorList{}
+}
+
+// WarningsOnUpdate returns warnings for the given update.
+func (genericStrategy) WarningsOnUpdate(ctx context.Context, obj, old runtime.Object) []string {
+	return nil
+}
+
+// GetAttrs returns labels and fields of an object.
+func GetAttrs(obj runtime.Object) (labels.Set, fields.Set, error) {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return nil, nil, err
+	}
+	fieldsSet := fields.Set{
+		"metadata.name": accessor.GetName(),
+	}
+	return labels.Set(accessor.GetLabels()), fieldsSet, nil
+}
+
+// Matcher returns a generic.SelectionPredicate that matches on label and field selectors.
+func Matcher(label labels.Selector, field fields.Selector) storage.SelectionPredicate {
+	return storage.SelectionPredicate{
+		Label:    label,
+		Field:    field,
+		GetAttrs: GetAttrs,
+	}
+}

pkg/services/grafana-apiserver/rest/dualwriter.go

@@ -0,0 +1,113 @@
+package rest
+
+import (
+	"context"
+
+	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/registry/rest"
+)
+
+var (
+	_ rest.Storage              = (*DualWriter)(nil)
+	_ rest.Scoper               = (*DualWriter)(nil)
+	_ rest.TableConvertor       = (*DualWriter)(nil)
+	_ rest.CreaterUpdater       = (*DualWriter)(nil)
+	_ rest.CollectionDeleter    = (*DualWriter)(nil)
+	_ rest.GracefulDeleter      = (*DualWriter)(nil)
+	_ rest.SingularNameProvider = (*DualWriter)(nil)
+)
+
+// Storage is a storage implementation that satisfies the same interfaces as genericregistry.Store.
+type Storage interface {
+	rest.Storage
+	rest.StandardStorage
+	rest.Scoper
+	rest.TableConvertor
+	rest.SingularNameProvider
+}
+
+// LegacyStorage is a storage implementation that writes to the Grafana SQL database.
+type LegacyStorage interface {
+	rest.Storage
+	rest.Scoper
+	rest.SingularNameProvider
+	rest.TableConvertor
+}
+
+// DualWriter is a storage implementation that writes first to LegacyStorage and then to Storage.
+// If writing to LegacyStorage fails, the write to Storage is skipped and the error is returned.
+// Storage is used for all read operations.
+//
+// The LegacyStorage implementation must implement the following interfaces:
+// - rest.Storage
+// - rest.TableConvertor
+// - rest.Scoper
+// - rest.SingularNameProvider
+//
+// These interfaces are optional, but they all should be implemented to fully support dual writes:
+// - rest.Creater
+// - rest.Updater
+// - rest.GracefulDeleter
+// - rest.CollectionDeleter
+type DualWriter struct {
+	Storage
+	legacy LegacyStorage
+}
+
+// NewDualWriter returns a new DualWriter.
+func NewDualWriter(legacy LegacyStorage, storage Storage) *DualWriter {
+	return &DualWriter{
+		Storage: storage,
+		legacy:  legacy,
+	}
+}
+
+// Create overrides the default behavior of the Storage and writes to both the LegacyStorage and Storage.
+func (d *DualWriter) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
+	if legacy, ok := d.legacy.(rest.Creater); ok {
+		_, err := legacy.Create(ctx, obj, createValidation, options)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return d.Storage.Create(ctx, obj, createValidation, options)
+}
+
+// Update overrides the default behavior of the Storage and writes to both the LegacyStorage and Storage.
+func (d *DualWriter) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
+	if legacy, ok := d.legacy.(rest.Updater); ok {
+		_, _, err := legacy.Update(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
+		if err != nil {
+			return nil, false, err
+		}
+	}
+
+	return d.Storage.Update(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
+}
+
+// Delete overrides the default behavior of the Storage and delete from both the LegacyStorage and Storage.
+func (d *DualWriter) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
+	if legacy, ok := d.legacy.(rest.GracefulDeleter); ok {
+		_, _, err := legacy.Delete(ctx, name, deleteValidation, options)
+		if err != nil {
+			return nil, false, err
+		}
+	}
+
+	return d.Storage.Delete(ctx, name, deleteValidation, options)
+}
+
+// DeleteCollection overrides the default behavior of the Storage and delete from both the LegacyStorage and Storage.
+func (d *DualWriter) DeleteCollection(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) {
+	if legacy, ok := d.legacy.(rest.CollectionDeleter); ok {
+		_, err := legacy.DeleteCollection(ctx, deleteValidation, options, listOptions)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return d.Storage.DeleteCollection(ctx, deleteValidation, options, listOptions)
+}

pkg/services/grafana-apiserver/service.go

@@ -90,7 +90,8 @@ type RestConfigProvider interface {
 type service struct {
 	*services.BasicService
 
-	restConfig *clientrest.Config
+	restConfig   *clientrest.Config
+	etcd_servers []string
 
 	enabled   bool
 	dataPath  string
@@ -102,15 +103,17 @@ type service struct {
 	builders []APIGroupBuilder
 }
 
-func ProvideService(cfg *setting.Cfg,
+func ProvideService(
+	cfg *setting.Cfg,
 	rr routing.RouteRegister,
 ) (*service, error) {
 	s := &service{
-		enabled:  cfg.IsFeatureToggleEnabled(featuremgmt.FlagGrafanaAPIServer),
+		etcd_servers: cfg.SectionWithEnvOverrides("grafana-apiserver").Key("etcd_servers").Strings(","),
-		rr:       rr,
+		enabled:      cfg.IsFeatureToggleEnabled(featuremgmt.FlagGrafanaAPIServer),
-		dataPath: path.Join(cfg.DataPath, "k8s"),
+		rr:           rr,
-		stopCh:   make(chan struct{}),
+		dataPath:     path.Join(cfg.DataPath, "k8s"),
-		builders: []APIGroupBuilder{},
+		stopCh:       make(chan struct{}),
+		builders:     []APIGroupBuilder{},
 	}
 
 	// This will be used when running as a dskit service
@@ -170,9 +173,13 @@ func (s *service) start(ctx context.Context) error {
 	o.Authorization.RemoteKubeConfigFileOptional = true
 	o.Authorization.AlwaysAllowPaths = []string{"*"}
 	o.Authorization.AlwaysAllowGroups = []string{user.SystemPrivilegedGroup, "grafana"}
-	o.Etcd = nil
+	o.Etcd.StorageConfig.Transport.ServerList = s.etcd_servers
+
 	o.Admission = nil
 	o.CoreAPI = nil
+	if len(o.Etcd.StorageConfig.Transport.ServerList) == 0 {
+		o.Etcd = nil
+	}
 
 	// Get the util to get the paths to pre-generated certs
 	certUtil := certgenerator.CertUtil{
@@ -246,7 +253,11 @@ func (s *service) start(ctx context.Context) error {
 
 	// Install the API Group+version
 	for _, b := range builders {
-		err = server.InstallAPIGroup(b.GetAPIGroupInfo(Scheme, Codecs))
+		g, err := b.GetAPIGroupInfo(Scheme, Codecs, serverConfig.RESTOptionsGetter)
+		if err != nil {
+			return err
+		}
+		err = server.InstallAPIGroup(g)
 		if err != nil {
 			return err
 		}