This commit is contained in:
konsalex
2025-11-26 11:50:14 +01:00
parent 5272cfcc99
commit 3f23ae36c9
10 changed files with 45 additions and 472 deletions
+4
View File
@@ -134,6 +134,10 @@ profile.cov
/pkg/operators/enterprise_*
/pkg/operators/**/enterprise_*
# Enterprise apiextensions server
pkg/registry/apis/apiextensions/*
!pkg/registry/apis/apiextensions/register.go
debug.test
/examples/*/dist
/packaging/**/*.rpm
-114
View File
@@ -1,114 +0,0 @@
# Grafana CRD Support - POC Testing Guide
This directory contains example CRD definitions and custom resources for testing the Kubernetes CustomResourceDefinition (CRD) support in Grafana server.
## How to run
To run just compile Grafana with `make build-go` and then run the service with the `apiextensions.ini` provided under the `conf` folder.
```bash
./bin/darwin-arm64/grafana server --config conf/apiextensions.ini
```
To enable this feature we use the `apiExtensions = true` flag and also have unified storage as our storage backend.
There is no need for US service to run in ST Grafana, and the database will be a SQLite one.
## Testing Steps
### Step 1: Create a CustomResourceDefinition
Create the example CRD that defines a "Widget" resource:
```bash
kubectl apply -f ./pkg/registry/apis/apiextensions/resources/example-crd.yaml
```
Or use curl after you set a Grafana Service account token:
```bash
export AUTH_SVC="Authorization: Bearer glsa_<rest of the token>"
```
```bash
# From Grafana root
curl -k -X POST https://localhost:1111/apis/apiextensions.k8s.io/v1/customresourcedefinitions \
-H "$AUTH_SVC" \
-H "Content-Type: application/yaml" \
--data-binary @$PWD/pkg/registry/apis/apiextensions/resources/example-crd.yaml
```
### Step 2: Verify the CRD was created
List all CRDs:
```bash
# Be sure to use the generated kube-config file
KUBECONFIG=$PWD/data/grafana-apiserver/apiserver.kubeconfig \
kubectl get customresourcedefinitions.apiextensions.k8s.io
# Or with curl:
curl -k -X GET https://localhost:1111/apis/apiextensions.k8s.io/v1/customresourcedefinitions \
-H "$AUTH_SVC"
```
### Step 3: Create a Custom Resource Instance
Now that the CRD is registered, create an instance of the Widget resource:
```bash
kubectl apply -f ./pkg/registry/apis/apiextensions/resources/example-widget.yaml
```
Or use curl:
```bash
curl -k -X POST https://localhost:1111/apis/customcrdtest.grafana.app/v1/namespaces/default/widgets \
-H "$AUTH_SVC" \
-H "Content-Type: application/yaml" \
--data-binary @$PWD/pkg/registry/apis/apiextensions/resources/example-widget.yaml
```
### Step 4: Verify the Custom Resource
> Note: You can see all the resources in SQLite database called `grafana.db`
List all widgets:
```bash
KUBECONFIG=$PWD/data/grafana-apiserver/apiserver.kubeconfig \
kubectl get widgets -n default
# Or with curl:
curl -k -X GET https://localhost:1111/apis/customcrdtest.grafana.app/v1/namespaces/default/widgets \
-H "$AUTH_SVC" | jq .
```
### Step 5: Update the Custom Resource
Update the widget's spec:
```bash
KUBECONFIG=$PWD/data/grafana-apiserver/apiserver.kubeconfig \
kubectl edit widget my-widget -n default
# Or with curl (PATCH):
curl -X PATCH https://localhost:1111/apis/customcrdtest.grafana.app/v1/namespaces/default/widgets/my-widget \
-H "Content-Type: application/merge-patch+json" \
-H "$AUTH_SVC" \
-d '{"spec":{"replicas":5}}'
```
## What is left
- [ ] Support multiple versions of CRDs (example in `discovery.go`)
- [ ] Watch new CRDs, so we do not require server restart `dynamic_registry.go`. This is needed for horizontal deployments.
- [ ] Update the storage to handle new CRDs even in the same service (related to above). Currently needs service restart.
- [ ] Support `/status` subresource
- [ ] Add tracer and logger and remove `fmt.Print`
- [ ] Implement MT setup
- [ ] Figure out how to modify storage checks for Cluster scoped resources (when we create a new CRD)
- [ ] How to tackle Cluster scoped CRs (Gabriel is helping)
- [X] Use the feature flag to start the `apiextensions` service on-demand
+35 -4
View File
@@ -14,6 +14,7 @@ import (
"k8s.io/kube-openapi/pkg/common"
authlib "github.com/grafana/authlib/types"
genericregistry "k8s.io/apiserver/pkg/registry/generic"
"github.com/grafana/grafana/pkg/services/apiserver/builder"
"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -23,6 +24,29 @@ import (
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
// CRDStorageProvider is an interface for creating CRD REST options getters.
// Enterprise provides the real implementation, OSS returns nil.
type CRDStorageProvider interface {
NewCRDRESTOptionsGetter(
delegate *apistore.RESTOptionsGetter,
unifiedClient resource.ResourceClient,
) genericregistry.RESTOptionsGetter
}
// OSSCRDStorageProvider is the OSS implementation that returns nil (feature disabled)
type OSSCRDStorageProvider struct{}
func ProvideOSSCRDStorageProvider() CRDStorageProvider {
return &OSSCRDStorageProvider{}
}
func (p *OSSCRDStorageProvider) NewCRDRESTOptionsGetter(
delegate *apistore.RESTOptionsGetter,
unifiedClient resource.ResourceClient,
) genericregistry.RESTOptionsGetter {
return nil
}
var _ builder.APIGroupBuilder = (*Builder)(nil)
// Builder implements builder.APIGroupBuilder for CustomResourceDefinitions.
@@ -39,6 +63,7 @@ type Builder struct {
unifiedClient resource.ResourceClient
apiExtensionsServer *apiextensionsapiserver.CustomResourceDefinitions
restOptsGetter *apistore.RESTOptionsGetter
storageProvider CRDStorageProvider
}
// RegisterAPIService registers the apiextensions API group in single-tenant mode
@@ -49,15 +74,17 @@ func RegisterAPIService(
accessClient authlib.AccessClient,
registerer prometheus.Registerer,
unified resource.ResourceClient,
storageProvider CRDStorageProvider,
) (*Builder, error) {
if !features.IsEnabledGlobally(featuremgmt.FlagApiExtensions) {
return nil, nil
}
b := &Builder{
features: features,
accessClient: accessClient,
unifiedClient: unified,
features: features,
accessClient: accessClient,
unifiedClient: unified,
storageProvider: storageProvider,
}
// Register the builder to install the schema
@@ -145,7 +172,11 @@ func (b *Builder) CreateAPIExtensionsServer(
}
// Create the CRD REST options getter that uses unified storage
crdRestOptsGetter := NewCRDRESTOptionsGetter(restOptsGetter, b.unifiedClient)
crdRestOptsGetter := b.storageProvider.NewCRDRESTOptionsGetter(restOptsGetter, b.unifiedClient)
if crdRestOptsGetter == nil {
// Enterprise feature not available
return nil, nil
}
// Create a fresh copy of the config for the apiextensions server
// We need to clear PostStartHooks to avoid conflicts with hooks
@@ -1,69 +0,0 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: widgets.customcrdtest.grafana.app
spec:
group: customcrdtest.grafana.app
names:
kind: Widget
listKind: WidgetList
plural: widgets
singular: widget
shortNames:
- wg
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
properties:
apiVersion:
type: string
kind:
type: string
metadata:
type: object
spec:
type: object
properties:
size:
type: string
enum:
- small
- medium
- large
replicas:
type: integer
minimum: 1
maximum: 10
required:
- size
status:
type: object
properties:
ready:
type: boolean
message:
type: string
subresources:
status: {}
additionalPrinterColumns:
- name: Size
type: string
description: The size of the widget
jsonPath: .spec.size
- name: Replicas
type: integer
description: Number of replicas
jsonPath: .spec.replicas
- name: Ready
type: boolean
description: Is the widget ready?
jsonPath: .status.ready
- name: Age
type: date
jsonPath: .metadata.creationTimestamp
@@ -1,8 +0,0 @@
apiVersion: customcrdtest.grafana.app/v1
kind: Widget
metadata:
name: my-widget
namespace: default
spec:
size: medium
replicas: 3
@@ -1,111 +0,0 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: outages.monitoring.grafana.app
spec:
group: monitoring.grafana.app
names:
kind: Outage
listKind: OutageList
plural: outages
singular: outage
shortNames:
- out
scope: Cluster # ← CLUSTER-SCOPED (not namespaced)
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
properties:
apiVersion:
type: string
kind:
type: string
metadata:
type: object
spec:
type: object
properties:
region:
type: string
description: Geographic region affected by the outage
enum:
- us-east-1
- us-west-2
- eu-west-1
- eu-central-1
- ap-southeast-1
- ap-northeast-1
severity:
type: string
description: Severity level of the outage
enum:
- critical
- major
- minor
default: major
affectedServices:
type: array
description: List of services affected
items:
type: string
startTime:
type: string
format: date-time
description: When the outage started
description:
type: string
description: Description of the outage
required:
- region
- severity
- startTime
status:
type: object
properties:
resolved:
type: boolean
description: Whether the outage has been resolved
resolvedAt:
type: string
format: date-time
description: When the outage was resolved
affectedCustomers:
type: integer
description: Number of customers affected
updates:
type: array
description: Status updates
items:
type: object
properties:
timestamp:
type: string
format: date-time
message:
type: string
subresources:
status: {}
additionalPrinterColumns:
- name: Region
type: string
description: Geographic region
jsonPath: .spec.region
- name: Severity
type: string
description: Severity level
jsonPath: .spec.severity
- name: Resolved
type: boolean
description: Resolution status
jsonPath: .status.resolved
- name: Start Time
type: date
description: When the outage started
jsonPath: .spec.startTime
- name: Age
type: date
jsonPath: .metadata.creationTimestamp
@@ -1,22 +0,0 @@
apiVersion: monitoring.grafana.app/v1
kind: Outage
metadata:
name: outage-2025-11-20-us-east
# NO namespace field - this is cluster-scoped!
spec:
region: us-east-1
severity: critical
affectedServices:
- grafana-cloud-metrics
- grafana-cloud-logs
- grafana-cloud-traces
startTime: "2025-11-20T10:00:00Z"
description: "Database connectivity issues affecting multiple services in US East region"
status:
resolved: false
affectedCustomers: 1247
updates:
- timestamp: "2025-11-20T10:15:00Z"
message: "Incident detected, investigating database connectivity"
- timestamp: "2025-11-20T10:30:00Z"
message: "Root cause identified, applying fix to primary database cluster"
-142
View File
@@ -1,142 +0,0 @@
package apiextensions
import (
"strings"
apiextensionsinternal "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
genericregistry "k8s.io/apiserver/pkg/registry/generic"
"k8s.io/apiserver/pkg/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/storage/storagebackend/factory"
"k8s.io/client-go/tools/cache"
"github.com/grafana/grafana/pkg/storage/unified/apistore"
"github.com/grafana/grafana/pkg/storage/unified/resource"
resourcepb "github.com/grafana/grafana/pkg/storage/unified/resourcepb"
)
// CRDRESTOptionsGetter wraps the Grafana unified storage RESTOptionsGetter
// to be compatible with the Kubernetes apiextensions-apiserver requirements.
type CRDRESTOptionsGetter struct {
delegate *apistore.RESTOptionsGetter
unifiedClient resource.ResourceClient
scheme *runtime.Scheme
codecs serializer.CodecFactory
}
// NewCRDRESTOptionsGetter creates a new CRDRESTOptionsGetter
func NewCRDRESTOptionsGetter(
delegate *apistore.RESTOptionsGetter,
unifiedClient resource.ResourceClient,
) *CRDRESTOptionsGetter {
scheme := runtime.NewScheme()
_ = apiextensionsv1.AddToScheme(scheme)
_ = apiextensionsinternal.AddToScheme(scheme)
codecs := serializer.NewCodecFactory(scheme)
return &CRDRESTOptionsGetter{
delegate: delegate,
unifiedClient: unifiedClient,
scheme: scheme,
codecs: codecs,
}
}
// GetRESTOptions returns REST options for a given resource.
func (r *CRDRESTOptionsGetter) GetRESTOptions(
res schema.GroupResource,
example runtime.Object,
) (genericregistry.RESTOptions, error) {
r.delegate.RegisterOptions(res, apistore.StorageOptions{})
opts, err := r.delegate.GetRESTOptions(res, example)
if err != nil {
return opts, err
}
// CRDs use apiextensions codec, Custom Resources use unstructured
if res.Group == apiextensionsv1.SchemeGroupVersion.Group {
opts.StorageConfig.Config.Codec = r.codecs.LegacyCodec(apiextensionsv1.SchemeGroupVersion)
} else {
opts.StorageConfig.Config.Codec = unstructured.UnstructuredJSONScheme
}
// CRDs are cluster-scoped, Custom Resources are typically namespace-scoped
isClusterScoped := res.Group == apiextensionsv1.SchemeGroupVersion.Group
opts.Decorator = func(
config *storagebackend.ConfigForResource,
resourcePrefix string,
keyFunc func(obj runtime.Object) (string, error),
newFunc func() runtime.Object,
newListFunc func() runtime.Object,
getAttrsFunc storage.AttrFunc,
trigger storage.IndexerFuncs,
indexers *cache.Indexers,
) (storage.Interface, factory.DestroyFunc, error) {
// Key parser that handles K8s apiextensions-apiserver key format
// Keys are: /group/<group>/resource/<resource>[/<namespace>][/<name>]
keyParser := makeKeyParser(config.GroupResource, isClusterScoped)
return apistore.NewStorage(
config,
r.unifiedClient,
keyFunc,
keyParser,
newFunc,
newListFunc,
getAttrsFunc,
trigger,
indexers,
nil,
apistore.StorageOptions{},
)
}
return opts, nil
}
// makeKeyParser creates a key parser for the given resource.
// K8s apiextensions-apiserver generates keys in the format:
// - Cluster-scoped: /group/<group>/resource/<resource>/<name>
// - Namespace-scoped: /group/<group>/resource/<resource>/<namespace>/<name>
func makeKeyParser(gr schema.GroupResource, isClusterScoped bool) func(key string) (*resourcepb.ResourceKey, error) {
return func(key string) (*resourcepb.ResourceKey, error) {
// Key format: /group/<group>/resource/<resource>[/<extra1>][/<extra2>]
// Strip prefix and parse
parts := strings.Split(strings.TrimPrefix(key, "/"), "/")
result := &resourcepb.ResourceKey{
Group: gr.Group,
Resource: gr.Resource,
}
// Expected: ["group", "<group>", "resource", "<resource>", ...]
// Skip the known prefix parts and extract namespace/name
if len(parts) >= 4 {
extra := parts[4:] // Parts after /group/X/resource/Y
if isClusterScoped {
// Cluster-scoped: extra is [name] or []
if len(extra) >= 1 && extra[0] != "" {
result.Name = extra[0]
}
} else {
// Namespace-scoped: extra is [namespace] or [namespace, name]
if len(extra) >= 1 && extra[0] != "" {
result.Namespace = extra[0]
}
if len(extra) >= 2 && extra[1] != "" {
result.Name = extra[1]
}
}
}
return result, nil
}
}
+4 -2
View File
@@ -862,7 +862,8 @@ func Initialize(ctx context.Context, cfg *setting.Cfg, opts Options, apiOpts api
identitySynchronizer := authnimpl.ProvideIdentitySynchronizer(authnimplService)
ldapImpl := service12.ProvideService(cfg, featureToggles, ssosettingsimplService)
apiService := api4.ProvideService(cfg, routeRegisterImpl, accessControl, userService, authinfoimplService, ossGroups, identitySynchronizer, orgService, ldapImpl, userAuthTokenService, bundleregistryService)
apiExtensionsBuilder, err := apiextensions.RegisterAPIService(cfg, featureToggles, apiserverService, accessClient, registerer, resourceClient)
crdStorageProvider := apiextensions.ProvideOSSCRDStorageProvider()
apiextensionsBuilder, err := apiextensions.RegisterAPIService(cfg, featureToggles, apiserverService, accessClient, registerer, resourceClient, crdStorageProvider)
if err != nil {
return nil, err
}
@@ -1516,7 +1517,8 @@ func InitializeForTest(ctx context.Context, t sqlutil.ITestDB, testingT interfac
identitySynchronizer := authnimpl.ProvideIdentitySynchronizer(authnimplService)
ldapImpl := service12.ProvideService(cfg, featureToggles, ssosettingsimplService)
apiService := api4.ProvideService(cfg, routeRegisterImpl, accessControl, userService, authinfoimplService, ossGroups, identitySynchronizer, orgService, ldapImpl, userAuthTokenService, bundleregistryService)
apiExtensionsBuilder, err := apiextensions.RegisterAPIService(cfg, featureToggles, apiserverService, accessClient, registerer, resourceClient)
crdStorageProvider := apiextensions.ProvideOSSCRDStorageProvider()
apiextensionsBuilder, err := apiextensions.RegisterAPIService(cfg, featureToggles, apiserverService, accessClient, registerer, resourceClient, crdStorageProvider)
if err != nil {
return nil, err
}
+2
View File
@@ -14,6 +14,7 @@ import (
"github.com/grafana/grafana/pkg/plugins/manager"
"github.com/grafana/grafana/pkg/registry"
apisregistry "github.com/grafana/grafana/pkg/registry/apis"
"github.com/grafana/grafana/pkg/registry/apis/apiextensions"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/extras"
"github.com/grafana/grafana/pkg/registry/apis/secret"
"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
@@ -149,6 +150,7 @@ var wireExtsBasicSet = wire.NewSet(
sql.ProvideStorageBackend,
builder.ProvideDefaultBuildHandlerChainFuncFromBuilders,
aggregatorrunner.ProvideNoopAggregatorConfigurator,
apiextensions.ProvideOSSCRDStorageProvider,
apisregistry.WireSetExts,
gsmKMSProviders.ProvideOSSKMSProviders,
secret.ProvideSecureValueClient,