diff --git a/.gitignore b/.gitignore index f1a90d05693..1adde5e1186 100644 --- a/.gitignore +++ b/.gitignore @@ -134,6 +134,10 @@ profile.cov /pkg/operators/enterprise_* /pkg/operators/**/enterprise_* +# Enterprise apiextensions server +pkg/registry/apis/apiextensions/* +!pkg/registry/apis/apiextensions/register.go + debug.test /examples/*/dist /packaging/**/*.rpm diff --git a/pkg/registry/apis/apiextensions/README.md b/pkg/registry/apis/apiextensions/README.md deleted file mode 100644 index 98a9c17afeb..00000000000 --- a/pkg/registry/apis/apiextensions/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# Grafana CRD Support - POC Testing Guide - -This directory contains example CRD definitions and custom resources for testing the Kubernetes CustomResourceDefinition (CRD) support in Grafana server. - - -## How to run - -To run just compile Grafana with `make build-go` and then run the service with the `apiextensions.ini` provided under the `conf` folder. - -```bash -./bin/darwin-arm64/grafana server --config conf/apiextensions.ini -``` - -To enable this feature we use the `apiExtensions = true` flag and also have unified storage as our storage backend. - -There is no need for US service to run in ST Grafana, and the database will be a SQLite one. - -## Testing Steps - -### Step 1: Create a CustomResourceDefinition - -Create the example CRD that defines a "Widget" resource: - -```bash -kubectl apply -f ./pkg/registry/apis/apiextensions/resources/example-crd.yaml -``` - -Or use curl after you set a Grafana Service account token: -```bash -export AUTH_SVC="Authorization: Bearer glsa_" -``` - -```bash -# From Grafana root -curl -k -X POST https://localhost:1111/apis/apiextensions.k8s.io/v1/customresourcedefinitions \ - -H "$AUTH_SVC" \ - -H "Content-Type: application/yaml" \ - --data-binary @$PWD/pkg/registry/apis/apiextensions/resources/example-crd.yaml -``` - -### Step 2: Verify the CRD was created - -List all CRDs: - -```bash -# Be sure to use the generated kube-config file -KUBECONFIG=$PWD/data/grafana-apiserver/apiserver.kubeconfig \ -kubectl get customresourcedefinitions.apiextensions.k8s.io - -# Or with curl: -curl -k -X GET https://localhost:1111/apis/apiextensions.k8s.io/v1/customresourcedefinitions \ - -H "$AUTH_SVC" -``` - -### Step 3: Create a Custom Resource Instance - -Now that the CRD is registered, create an instance of the Widget resource: - -```bash -kubectl apply -f ./pkg/registry/apis/apiextensions/resources/example-widget.yaml -``` - -Or use curl: - -```bash -curl -k -X POST https://localhost:1111/apis/customcrdtest.grafana.app/v1/namespaces/default/widgets \ - -H "$AUTH_SVC" \ - -H "Content-Type: application/yaml" \ - --data-binary @$PWD/pkg/registry/apis/apiextensions/resources/example-widget.yaml -``` - -### Step 4: Verify the Custom Resource - -> Note: You can see all the resources in SQLite database called `grafana.db` - -List all widgets: - -```bash -KUBECONFIG=$PWD/data/grafana-apiserver/apiserver.kubeconfig \ -kubectl get widgets -n default - -# Or with curl: -curl -k -X GET https://localhost:1111/apis/customcrdtest.grafana.app/v1/namespaces/default/widgets \ - -H "$AUTH_SVC" | jq . -``` - -### Step 5: Update the Custom Resource - -Update the widget's spec: - -```bash -KUBECONFIG=$PWD/data/grafana-apiserver/apiserver.kubeconfig \ -kubectl edit widget my-widget -n default - -# Or with curl (PATCH): -curl -X PATCH https://localhost:1111/apis/customcrdtest.grafana.app/v1/namespaces/default/widgets/my-widget \ - -H "Content-Type: application/merge-patch+json" \ - -H "$AUTH_SVC" \ - -d '{"spec":{"replicas":5}}' -``` - - - -## What is left - -- [ ] Support multiple versions of CRDs (example in `discovery.go`) -- [ ] Watch new CRDs, so we do not require server restart `dynamic_registry.go`. This is needed for horizontal deployments. -- [ ] Update the storage to handle new CRDs even in the same service (related to above). Currently needs service restart. -- [ ] Support `/status` subresource -- [ ] Add tracer and logger and remove `fmt.Print` -- [ ] Implement MT setup -- [ ] Figure out how to modify storage checks for Cluster scoped resources (when we create a new CRD) -- [ ] How to tackle Cluster scoped CRs (Gabriel is helping) -- [X] Use the feature flag to start the `apiextensions` service on-demand diff --git a/pkg/registry/apis/apiextensions/register.go b/pkg/registry/apis/apiextensions/register.go index a9050cf89eb..f6d7ab12040 100644 --- a/pkg/registry/apis/apiextensions/register.go +++ b/pkg/registry/apis/apiextensions/register.go @@ -14,6 +14,7 @@ import ( "k8s.io/kube-openapi/pkg/common" authlib "github.com/grafana/authlib/types" + genericregistry "k8s.io/apiserver/pkg/registry/generic" "github.com/grafana/grafana/pkg/services/apiserver/builder" "github.com/grafana/grafana/pkg/services/featuremgmt" @@ -23,6 +24,29 @@ import ( "github.com/grafana/grafana/pkg/storage/unified/resource" ) +// CRDStorageProvider is an interface for creating CRD REST options getters. +// Enterprise provides the real implementation, OSS returns nil. +type CRDStorageProvider interface { + NewCRDRESTOptionsGetter( + delegate *apistore.RESTOptionsGetter, + unifiedClient resource.ResourceClient, + ) genericregistry.RESTOptionsGetter +} + +// OSSCRDStorageProvider is the OSS implementation that returns nil (feature disabled) +type OSSCRDStorageProvider struct{} + +func ProvideOSSCRDStorageProvider() CRDStorageProvider { + return &OSSCRDStorageProvider{} +} + +func (p *OSSCRDStorageProvider) NewCRDRESTOptionsGetter( + delegate *apistore.RESTOptionsGetter, + unifiedClient resource.ResourceClient, +) genericregistry.RESTOptionsGetter { + return nil +} + var _ builder.APIGroupBuilder = (*Builder)(nil) // Builder implements builder.APIGroupBuilder for CustomResourceDefinitions. @@ -39,6 +63,7 @@ type Builder struct { unifiedClient resource.ResourceClient apiExtensionsServer *apiextensionsapiserver.CustomResourceDefinitions restOptsGetter *apistore.RESTOptionsGetter + storageProvider CRDStorageProvider } // RegisterAPIService registers the apiextensions API group in single-tenant mode @@ -49,15 +74,17 @@ func RegisterAPIService( accessClient authlib.AccessClient, registerer prometheus.Registerer, unified resource.ResourceClient, + storageProvider CRDStorageProvider, ) (*Builder, error) { if !features.IsEnabledGlobally(featuremgmt.FlagApiExtensions) { return nil, nil } b := &Builder{ - features: features, - accessClient: accessClient, - unifiedClient: unified, + features: features, + accessClient: accessClient, + unifiedClient: unified, + storageProvider: storageProvider, } // Register the builder to install the schema @@ -145,7 +172,11 @@ func (b *Builder) CreateAPIExtensionsServer( } // Create the CRD REST options getter that uses unified storage - crdRestOptsGetter := NewCRDRESTOptionsGetter(restOptsGetter, b.unifiedClient) + crdRestOptsGetter := b.storageProvider.NewCRDRESTOptionsGetter(restOptsGetter, b.unifiedClient) + if crdRestOptsGetter == nil { + // Enterprise feature not available + return nil, nil + } // Create a fresh copy of the config for the apiextensions server // We need to clear PostStartHooks to avoid conflicts with hooks diff --git a/pkg/registry/apis/apiextensions/resources/example-crd.yaml b/pkg/registry/apis/apiextensions/resources/example-crd.yaml deleted file mode 100644 index c44494a4954..00000000000 --- a/pkg/registry/apis/apiextensions/resources/example-crd.yaml +++ /dev/null @@ -1,69 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: widgets.customcrdtest.grafana.app -spec: - group: customcrdtest.grafana.app - names: - kind: Widget - listKind: WidgetList - plural: widgets - singular: widget - shortNames: - - wg - scope: Namespaced - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - type: object - properties: - size: - type: string - enum: - - small - - medium - - large - replicas: - type: integer - minimum: 1 - maximum: 10 - required: - - size - status: - type: object - properties: - ready: - type: boolean - message: - type: string - subresources: - status: {} - additionalPrinterColumns: - - name: Size - type: string - description: The size of the widget - jsonPath: .spec.size - - name: Replicas - type: integer - description: Number of replicas - jsonPath: .spec.replicas - - name: Ready - type: boolean - description: Is the widget ready? - jsonPath: .status.ready - - name: Age - type: date - jsonPath: .metadata.creationTimestamp - diff --git a/pkg/registry/apis/apiextensions/resources/example-widget.yaml b/pkg/registry/apis/apiextensions/resources/example-widget.yaml deleted file mode 100644 index a171420c46a..00000000000 --- a/pkg/registry/apis/apiextensions/resources/example-widget.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: customcrdtest.grafana.app/v1 -kind: Widget -metadata: - name: my-widget - namespace: default -spec: - size: medium - replicas: 3 \ No newline at end of file diff --git a/pkg/registry/apis/apiextensions/resources/outage-crd.yaml b/pkg/registry/apis/apiextensions/resources/outage-crd.yaml deleted file mode 100644 index d50cf32b540..00000000000 --- a/pkg/registry/apis/apiextensions/resources/outage-crd.yaml +++ /dev/null @@ -1,111 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: outages.monitoring.grafana.app -spec: - group: monitoring.grafana.app - names: - kind: Outage - listKind: OutageList - plural: outages - singular: outage - shortNames: - - out - scope: Cluster # ← CLUSTER-SCOPED (not namespaced) - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - type: object - properties: - region: - type: string - description: Geographic region affected by the outage - enum: - - us-east-1 - - us-west-2 - - eu-west-1 - - eu-central-1 - - ap-southeast-1 - - ap-northeast-1 - severity: - type: string - description: Severity level of the outage - enum: - - critical - - major - - minor - default: major - affectedServices: - type: array - description: List of services affected - items: - type: string - startTime: - type: string - format: date-time - description: When the outage started - description: - type: string - description: Description of the outage - required: - - region - - severity - - startTime - status: - type: object - properties: - resolved: - type: boolean - description: Whether the outage has been resolved - resolvedAt: - type: string - format: date-time - description: When the outage was resolved - affectedCustomers: - type: integer - description: Number of customers affected - updates: - type: array - description: Status updates - items: - type: object - properties: - timestamp: - type: string - format: date-time - message: - type: string - subresources: - status: {} - additionalPrinterColumns: - - name: Region - type: string - description: Geographic region - jsonPath: .spec.region - - name: Severity - type: string - description: Severity level - jsonPath: .spec.severity - - name: Resolved - type: boolean - description: Resolution status - jsonPath: .status.resolved - - name: Start Time - type: date - description: When the outage started - jsonPath: .spec.startTime - - name: Age - type: date - jsonPath: .metadata.creationTimestamp diff --git a/pkg/registry/apis/apiextensions/resources/outage-instance.yaml b/pkg/registry/apis/apiextensions/resources/outage-instance.yaml deleted file mode 100644 index 18d6ecf099e..00000000000 --- a/pkg/registry/apis/apiextensions/resources/outage-instance.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: monitoring.grafana.app/v1 -kind: Outage -metadata: - name: outage-2025-11-20-us-east - # NO namespace field - this is cluster-scoped! -spec: - region: us-east-1 - severity: critical - affectedServices: - - grafana-cloud-metrics - - grafana-cloud-logs - - grafana-cloud-traces - startTime: "2025-11-20T10:00:00Z" - description: "Database connectivity issues affecting multiple services in US East region" -status: - resolved: false - affectedCustomers: 1247 - updates: - - timestamp: "2025-11-20T10:15:00Z" - message: "Incident detected, investigating database connectivity" - - timestamp: "2025-11-20T10:30:00Z" - message: "Root cause identified, applying fix to primary database cluster" diff --git a/pkg/registry/apis/apiextensions/storage.go b/pkg/registry/apis/apiextensions/storage.go deleted file mode 100644 index 23925a086b6..00000000000 --- a/pkg/registry/apis/apiextensions/storage.go +++ /dev/null @@ -1,142 +0,0 @@ -package apiextensions - -import ( - "strings" - - apiextensionsinternal "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - genericregistry "k8s.io/apiserver/pkg/registry/generic" - "k8s.io/apiserver/pkg/storage" - "k8s.io/apiserver/pkg/storage/storagebackend" - "k8s.io/apiserver/pkg/storage/storagebackend/factory" - "k8s.io/client-go/tools/cache" - - "github.com/grafana/grafana/pkg/storage/unified/apistore" - "github.com/grafana/grafana/pkg/storage/unified/resource" - resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb" -) - -// CRDRESTOptionsGetter wraps the Grafana unified storage RESTOptionsGetter -// to be compatible with the Kubernetes apiextensions-apiserver requirements. -type CRDRESTOptionsGetter struct { - delegate *apistore.RESTOptionsGetter - unifiedClient resource.ResourceClient - scheme *runtime.Scheme - codecs serializer.CodecFactory -} - -// NewCRDRESTOptionsGetter creates a new CRDRESTOptionsGetter -func NewCRDRESTOptionsGetter( - delegate *apistore.RESTOptionsGetter, - unifiedClient resource.ResourceClient, -) *CRDRESTOptionsGetter { - scheme := runtime.NewScheme() - _ = apiextensionsv1.AddToScheme(scheme) - _ = apiextensionsinternal.AddToScheme(scheme) - codecs := serializer.NewCodecFactory(scheme) - - return &CRDRESTOptionsGetter{ - delegate: delegate, - unifiedClient: unifiedClient, - scheme: scheme, - codecs: codecs, - } -} - -// GetRESTOptions returns REST options for a given resource. -func (r *CRDRESTOptionsGetter) GetRESTOptions( - res schema.GroupResource, - example runtime.Object, -) (genericregistry.RESTOptions, error) { - r.delegate.RegisterOptions(res, apistore.StorageOptions{}) - - opts, err := r.delegate.GetRESTOptions(res, example) - if err != nil { - return opts, err - } - - // CRDs use apiextensions codec, Custom Resources use unstructured - if res.Group == apiextensionsv1.SchemeGroupVersion.Group { - opts.StorageConfig.Config.Codec = r.codecs.LegacyCodec(apiextensionsv1.SchemeGroupVersion) - } else { - opts.StorageConfig.Config.Codec = unstructured.UnstructuredJSONScheme - } - - // CRDs are cluster-scoped, Custom Resources are typically namespace-scoped - isClusterScoped := res.Group == apiextensionsv1.SchemeGroupVersion.Group - - opts.Decorator = func( - config *storagebackend.ConfigForResource, - resourcePrefix string, - keyFunc func(obj runtime.Object) (string, error), - newFunc func() runtime.Object, - newListFunc func() runtime.Object, - getAttrsFunc storage.AttrFunc, - trigger storage.IndexerFuncs, - indexers *cache.Indexers, - ) (storage.Interface, factory.DestroyFunc, error) { - // Key parser that handles K8s apiextensions-apiserver key format - // Keys are: /group//resource/[/][/] - keyParser := makeKeyParser(config.GroupResource, isClusterScoped) - - return apistore.NewStorage( - config, - r.unifiedClient, - keyFunc, - keyParser, - newFunc, - newListFunc, - getAttrsFunc, - trigger, - indexers, - nil, - apistore.StorageOptions{}, - ) - } - - return opts, nil -} - -// makeKeyParser creates a key parser for the given resource. -// K8s apiextensions-apiserver generates keys in the format: -// - Cluster-scoped: /group//resource// -// - Namespace-scoped: /group//resource/// -func makeKeyParser(gr schema.GroupResource, isClusterScoped bool) func(key string) (*resourcepb.ResourceKey, error) { - return func(key string) (*resourcepb.ResourceKey, error) { - // Key format: /group//resource/[/][/] - // Strip prefix and parse - parts := strings.Split(strings.TrimPrefix(key, "/"), "/") - - result := &resourcepb.ResourceKey{ - Group: gr.Group, - Resource: gr.Resource, - } - - // Expected: ["group", "", "resource", "", ...] - // Skip the known prefix parts and extract namespace/name - if len(parts) >= 4 { - extra := parts[4:] // Parts after /group/X/resource/Y - - if isClusterScoped { - // Cluster-scoped: extra is [name] or [] - if len(extra) >= 1 && extra[0] != "" { - result.Name = extra[0] - } - } else { - // Namespace-scoped: extra is [namespace] or [namespace, name] - if len(extra) >= 1 && extra[0] != "" { - result.Namespace = extra[0] - } - if len(extra) >= 2 && extra[1] != "" { - result.Name = extra[1] - } - } - } - - return result, nil - } -} diff --git a/pkg/server/wire_gen.go b/pkg/server/wire_gen.go index 3de4c344e44..5a83d5de7d4 100644 --- a/pkg/server/wire_gen.go +++ b/pkg/server/wire_gen.go @@ -862,7 +862,8 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api identitySynchronizer := authnimpl.ProvideIdentitySynchronizer(authnimplService) ldapImpl := service12.ProvideService(cfg, featureToggles, ssosettingsimplService) apiService := api4.ProvideService(cfg, routeRegisterImpl, accessControl, userService, authinfoimplService, ossGroups, identitySynchronizer, orgService, ldapImpl, userAuthTokenService, bundleregistryService) - apiExtensionsBuilder, err := apiextensions.RegisterAPIService(cfg, featureToggles, apiserverService, accessClient, registerer, resourceClient) + crdStorageProvider := apiextensions.ProvideOSSCRDStorageProvider() + apiextensionsBuilder, err := apiextensions.RegisterAPIService(cfg, featureToggles, apiserverService, accessClient, registerer, resourceClient, crdStorageProvider) if err != nil { return nil, err } @@ -1516,7 +1517,8 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac identitySynchronizer := authnimpl.ProvideIdentitySynchronizer(authnimplService) ldapImpl := service12.ProvideService(cfg, featureToggles, ssosettingsimplService) apiService := api4.ProvideService(cfg, routeRegisterImpl, accessControl, userService, authinfoimplService, ossGroups, identitySynchronizer, orgService, ldapImpl, userAuthTokenService, bundleregistryService) - apiExtensionsBuilder, err := apiextensions.RegisterAPIService(cfg, featureToggles, apiserverService, accessClient, registerer, resourceClient) + crdStorageProvider := apiextensions.ProvideOSSCRDStorageProvider() + apiextensionsBuilder, err := apiextensions.RegisterAPIService(cfg, featureToggles, apiserverService, accessClient, registerer, resourceClient, crdStorageProvider) if err != nil { return nil, err } diff --git a/pkg/server/wireexts_oss.go b/pkg/server/wireexts_oss.go index 4d70c3f2f4c..44aefb8d887 100644 --- a/pkg/server/wireexts_oss.go +++ b/pkg/server/wireexts_oss.go @@ -14,6 +14,7 @@ import ( "github.com/grafana/grafana/pkg/plugins/manager" "github.com/grafana/grafana/pkg/registry" apisregistry "github.com/grafana/grafana/pkg/registry/apis" + "github.com/grafana/grafana/pkg/registry/apis/apiextensions" "github.com/grafana/grafana/pkg/registry/apis/provisioning/extras" "github.com/grafana/grafana/pkg/registry/apis/secret" "github.com/grafana/grafana/pkg/registry/apis/secret/contracts" @@ -149,6 +150,7 @@ var wireExtsBasicSet = wire.NewSet( sql.ProvideStorageBackend, builder.ProvideDefaultBuildHandlerChainFuncFromBuilders, aggregatorrunner.ProvideNoopAggregatorConfigurator, + apiextensions.ProvideOSSCRDStorageProvider, apisregistry.WireSetExts, gsmKMSProviders.ProvideOSSKMSProviders, secret.ProvideSecureValueClient,