public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

K8s: Data Plane Aggregator (#91228)

Browse Source
This commit is contained in:
Todd Treece
2024-08-13 22:28:17 +03:00
committed by GitHub
parent 4ce82d3f14
commit 1b0e6b5c09
64 changed files with 5555 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/apiserver/aggregator/aggregator.go
+6 -6
View File
@@ -124,8 +124,8 @@ func CreateAggregatorConfig(commandOptions *options.Options, sharedConfig generi
ClientConfig: sharedConfig.LoopbackClientConfig,
},
ExtraConfig: aggregatorapiserver.ExtraConfig{
ProxyClientCertFile: commandOptions.AggregatorOptions.ProxyClientCertFile,
ProxyClientKeyFile: commandOptions.AggregatorOptions.ProxyClientKeyFile,
ProxyClientCertFile: commandOptions.KubeAggregatorOptions.ProxyClientCertFile,
ProxyClientKeyFile: commandOptions.KubeAggregatorOptions.ProxyClientKeyFile,
// NOTE: while ProxyTransport can be skipped in the configuration, it allows honoring
// DISABLE_HTTP2, HTTPS_PROXY and NO_PROXY env vars as needed
ProxyTransport: createProxyTransport(),
@@ -133,7 +133,7 @@ func CreateAggregatorConfig(commandOptions *options.Options, sharedConfig generi
},
}
if err := commandOptions.AggregatorOptions.ApplyTo(aggregatorConfig, commandOptions.RecommendedOptions.Etcd); err != nil {
if err := commandOptions.KubeAggregatorOptions.ApplyTo(aggregatorConfig, commandOptions.RecommendedOptions.Etcd); err != nil {
return nil, err
}
@@ -144,15 +144,15 @@ func CreateAggregatorConfig(commandOptions *options.Options, sharedConfig generi
APIVersionPriorities[serviceAPIBuilder.GetGroupVersion()] = Priority{Group: 15000, Version: int32(1)}
// Exit early, if no remote services file is configured
if commandOptions.AggregatorOptions.RemoteServicesFile == "" {
if commandOptions.KubeAggregatorOptions.RemoteServicesFile == "" {
return NewConfig(aggregatorConfig, sharedInformerFactory, []builder.APIGroupBuilder{serviceAPIBuilder}, nil), nil
}
_, err = readCABundlePEM(commandOptions.AggregatorOptions.APIServiceCABundleFile, commandOptions.ExtraOptions.DevMode)
_, err = readCABundlePEM(commandOptions.KubeAggregatorOptions.APIServiceCABundleFile, commandOptions.ExtraOptions.DevMode)
if err != nil {
return nil, err
}
remoteServices, err := readRemoteServices(commandOptions.AggregatorOptions.RemoteServicesFile)
remoteServices, err := readRemoteServices(commandOptions.KubeAggregatorOptions.RemoteServicesFile)
if err != nil {
return nil, err
}
pkg/services/apiserver/config.go
+4 -4
View File
@@ -43,11 +43,11 @@ func applyGrafanaConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles, o
o.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true
o.RecommendedOptions.Authorization.RemoteKubeConfigFileOptional = true
o.AggregatorOptions.ProxyClientCertFile = apiserverCfg.Key("proxy_client_cert_file").MustString("")
o.AggregatorOptions.ProxyClientKeyFile = apiserverCfg.Key("proxy_client_key_file").MustString("")
o.KubeAggregatorOptions.ProxyClientCertFile = apiserverCfg.Key("proxy_client_cert_file").MustString("")
o.KubeAggregatorOptions.ProxyClientKeyFile = apiserverCfg.Key("proxy_client_key_file").MustString("")
o.AggregatorOptions.APIServiceCABundleFile = apiserverCfg.Key("apiservice_ca_bundle_file").MustString("")
o.AggregatorOptions.RemoteServicesFile = apiserverCfg.Key("remote_services_file").MustString("")
o.KubeAggregatorOptions.APIServiceCABundleFile = apiserverCfg.Key("apiservice_ca_bundle_file").MustString("")
o.KubeAggregatorOptions.RemoteServicesFile = apiserverCfg.Key("remote_services_file").MustString("")
o.RecommendedOptions.Admission = nil
o.RecommendedOptions.CoreAPI = nil
pkg/services/apiserver/options/grafana-aggregator.go
+94
View File
@@ -0,0 +1,94 @@
package options
import (
"maps"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/endpoints/openapi"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/server/resourceconfig"
"k8s.io/kube-openapi/pkg/common"
"github.com/grafana/grafana/pkg/aggregator/apis/aggregation/v0alpha1"
aggregatorapiserver "github.com/grafana/grafana/pkg/aggregator/apiserver"
aggregatorscheme "github.com/grafana/grafana/pkg/aggregator/apiserver/scheme"
commonv0alpha1 "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
"github.com/grafana/grafana/pkg/storage/unified/apistore"
)
// GrafanaAggregatorOptions contains the state for the aggregator apiserver
type GrafanaAggregatorOptions struct {
}
func NewGrafanaAggregatorOptions() *GrafanaAggregatorOptions {
return &GrafanaAggregatorOptions{}
}
func (o *GrafanaAggregatorOptions) AddFlags(fs *pflag.FlagSet) {
if o == nil {
return
}
// TODO: do we need any CLI flags here?
}
func (o *GrafanaAggregatorOptions) Validate() []error {
if o == nil {
return nil
}
// TODO: do we need to validate anything here?
return nil
}
func (o *GrafanaAggregatorOptions) ApplyTo(aggregatorConfig *aggregatorapiserver.Config, etcdOpts *options.EtcdOptions) error {
genericConfig := aggregatorConfig.GenericConfig
genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{}
genericConfig.RESTOptionsGetter = nil
getOpenAPIDefinitions := func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition {
defs := commonv0alpha1.GetOpenAPIDefinitions(ref)
maps.Copy(defs, v0alpha1.GetOpenAPIDefinitions(ref))
return defs
}
genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(getOpenAPIDefinitions, openapi.NewDefinitionNamer(aggregatorscheme.Scheme))
genericConfig.OpenAPIConfig.Info.Title = "Grafana Aggregator"
genericConfig.OpenAPIConfig.Info.Version = "0.1"
genericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitions, openapi.NewDefinitionNamer(aggregatorscheme.Scheme))
genericConfig.OpenAPIV3Config.Info.Title = "Grafana Aggregator"
genericConfig.OpenAPIV3Config.Info.Version = "0.1"
// copy the etcd options so we don't mutate originals.
// we assume that the etcd options have been completed already. avoid messing with anything outside
// of changes to StorageConfig as that may lead to unexpected behavior when the options are applied.
etcdOptions := *etcdOpts
etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v0alpha1.SchemeGroupVersion)
etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v0alpha1.SchemeGroupVersion, schema.GroupKind{Group: v0alpha1.SchemeGroupVersion.Group})
etcdOptions.SkipHealthEndpoints = true // avoid double wiring of health checks
if err := etcdOptions.ApplyTo(&genericConfig.Config); err != nil {
return err
}
// override the RESTOptionsGetter to use the in memory storage options
restOptionsGetter, err := apistore.NewRESTOptionsGetterMemory(etcdOptions.StorageConfig)
if err != nil {
return err
}
aggregatorConfig.GenericConfig.RESTOptionsGetter = restOptionsGetter
// prevent generic API server from installing the OpenAPI handler. Aggregator server has its own customized OpenAPI handler.
genericConfig.SkipOpenAPIInstallation = true
mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(aggregatorapiserver.DefaultAPIResourceConfigSource(), nil, aggregatorscheme.Scheme)
if err != nil {
return err
}
genericConfig.MergedResourceConfig = mergedResourceConfig
genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{}
return nil
}
pkg/services/apiserver/options/aggregator.go → pkg/services/apiserver/options/kube-aggregator.go
+7 -7
View File
@@ -18,8 +18,8 @@ import (
"github.com/grafana/grafana/pkg/storage/unified/apistore"
)
// AggregatorServerOptions contains the state for the aggregator apiserver
type AggregatorServerOptions struct {
// KubeAggregatorOptions contains the state for the aggregator apiserver
type KubeAggregatorOptions struct {
AlternateDNS []string
ProxyClientCertFile string
ProxyClientKeyFile string
@@ -27,11 +27,11 @@ type AggregatorServerOptions struct {
APIServiceCABundleFile string
}
func NewAggregatorServerOptions() *AggregatorServerOptions {
return &AggregatorServerOptions{}
func NewAggregatorServerOptions() *KubeAggregatorOptions {
return &KubeAggregatorOptions{}
}
func (o *AggregatorServerOptions) AddFlags(fs *pflag.FlagSet) {
func (o *KubeAggregatorOptions) AddFlags(fs *pflag.FlagSet) {
if o == nil {
return
}
@@ -43,7 +43,7 @@ func (o *AggregatorServerOptions) AddFlags(fs *pflag.FlagSet) {
"path to proxy client key file")
}
func (o *AggregatorServerOptions) Validate() []error {
func (o *KubeAggregatorOptions) Validate() []error {
if o == nil {
return nil
}
@@ -52,7 +52,7 @@ func (o *AggregatorServerOptions) Validate() []error {
return nil
}
func (o *AggregatorServerOptions) ApplyTo(aggregatorConfig *aggregatorapiserver.Config, etcdOpts *options.EtcdOptions) error {
func (o *KubeAggregatorOptions) ApplyTo(aggregatorConfig *aggregatorapiserver.Config, etcdOpts *options.EtcdOptions) error {
genericConfig := aggregatorConfig.GenericConfig
genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{}
pkg/services/apiserver/options/options.go
+18 -11
View File
@@ -19,25 +19,28 @@ type OptionsProvider interface {
const defaultEtcdPathPrefix = "/registry/grafana.app"
type Options struct {
RecommendedOptions *genericoptions.RecommendedOptions
AggregatorOptions *AggregatorServerOptions
StorageOptions *StorageOptions
ExtraOptions *ExtraOptions
APIOptions []OptionsProvider
RecommendedOptions *genericoptions.RecommendedOptions
GrafanaAggregatorOptions *GrafanaAggregatorOptions
KubeAggregatorOptions *KubeAggregatorOptions
StorageOptions *StorageOptions
ExtraOptions *ExtraOptions
APIOptions []OptionsProvider
}
func NewOptions(codec runtime.Codec) *Options {
return &Options{
RecommendedOptions: NewRecommendedOptions(codec),
AggregatorOptions: NewAggregatorServerOptions(),
StorageOptions: NewStorageOptions(),
ExtraOptions: NewExtraOptions(),
RecommendedOptions: NewRecommendedOptions(codec),
GrafanaAggregatorOptions: NewGrafanaAggregatorOptions(),
KubeAggregatorOptions: NewAggregatorServerOptions(),
StorageOptions: NewStorageOptions(),
ExtraOptions: NewExtraOptions(),
}
}
func (o *Options) AddFlags(fs *pflag.FlagSet) {
o.RecommendedOptions.AddFlags(fs)
o.AggregatorOptions.AddFlags(fs)
o.GrafanaAggregatorOptions.AddFlags(fs)
o.KubeAggregatorOptions.AddFlags(fs)
o.StorageOptions.AddFlags(fs)
o.ExtraOptions.AddFlags(fs)
@@ -55,7 +58,11 @@ func (o *Options) Validate() []error {
return errs
}
if errs := o.AggregatorOptions.Validate(); len(errs) != 0 {
if errs := o.GrafanaAggregatorOptions.Validate(); len(errs) != 0 {
return errs
}
if errs := o.KubeAggregatorOptions.Validate(); len(errs) != 0 {
return errs
}
pkg/services/apiserver/service.go
+133 -23
View File
@@ -7,6 +7,7 @@ import (
"path"
"github.com/grafana/dskit/services"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"google.golang.org/grpc"
@@ -19,7 +20,9 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
clientrest "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
dataplaneaggregator "github.com/grafana/grafana/pkg/aggregator/apiserver"
"github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/apimachinery/identity"
grafanaresponsewriter "github.com/grafana/grafana/pkg/apiserver/endpoints/responsewriter"
@@ -30,8 +33,10 @@ import (
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/middleware"
"github.com/grafana/grafana/pkg/modules"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/services/apiserver/aggregator"
"github.com/grafana/grafana/pkg/registry/apis/datasource"
kubeaggregator "github.com/grafana/grafana/pkg/services/apiserver/aggregator"
"github.com/grafana/grafana/pkg/services/apiserver/auth/authenticator"
"github.com/grafana/grafana/pkg/services/apiserver/auth/authorizer"
"github.com/grafana/grafana/pkg/services/apiserver/builder"
@@ -41,6 +46,7 @@ import (
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/unified/apistore"
"github.com/grafana/grafana/pkg/storage/unified/resource"
@@ -117,6 +123,11 @@ type service struct {
authorizer *authorizer.GrafanaAuthorizer
serverLockService builder.ServerLockService
kvStore kvstore.KVStore
pluginClient plugins.Client
datasources datasource.ScopedPluginDatasourceProvider
contextProvider datasource.PluginContextWrapper
pluginStore pluginstore.Store
}
func ProvideService(
@@ -128,19 +139,27 @@ func ProvideService(
serverLockService *serverlock.ServerLockService,
db db.DB,
kvStore kvstore.KVStore,
pluginClient plugins.Client,
datasources datasource.ScopedPluginDatasourceProvider,
contextProvider datasource.PluginContextWrapper,
pluginStore pluginstore.Store,
) (*service, error) {
s := &service{
cfg: cfg,
features: features,
rr: rr,
startedCh: make(chan struct{}),
stopCh: make(chan struct{}),
builders: []builder.APIGroupBuilder{},
authorizer: authorizer.NewGrafanaAuthorizer(cfg, orgService),
tracing: tracing,
db: db, // For Unified storage
metrics: metrics.ProvideRegisterer(),
kvStore: kvStore,
cfg: cfg,
features: features,
rr: rr,
startedCh: make(chan struct{}),
stopCh: make(chan struct{}),
builders: []builder.APIGroupBuilder{},
authorizer: authorizer.NewGrafanaAuthorizer(cfg, orgService),
tracing: tracing,
db: db, // For Unified storage
metrics: metrics.ProvideRegisterer(),
kvStore: kvStore,
pluginClient: pluginClient,
datasources: datasources,
contextProvider: contextProvider,
pluginStore: pluginStore,
}
// This will be used when running as a dskit service
@@ -214,7 +233,7 @@ func (s *service) start(ctx context.Context) error {
groupVersions := make([]schema.GroupVersion, 0, len(builders))
// Install schemas
initialSize := len(aggregator.APIVersionPriorities)
initialSize := len(kubeaggregator.APIVersionPriorities)
for i, b := range builders {
groupVersions = append(groupVersions, b.GetGroupVersion())
if err := b.InstallSchema(Scheme); err != nil {
@@ -223,7 +242,7 @@ func (s *service) start(ctx context.Context) error {
if s.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) {
// set the priority for the group+version
aggregator.APIVersionPriorities[b.GetGroupVersion()] = aggregator.Priority{Group: 15000, Version: int32(i + initialSize)}
kubeaggregator.APIVersionPriorities[b.GetGroupVersion()] = kubeaggregator.Priority{Group: 15000, Version: int32(i + initialSize)}
}
auth := b.GetAuthorizer()
@@ -339,9 +358,24 @@ func (s *service) start(ctx context.Context) error {
// stash the options for later use
s.options = o
var runningServer *genericapiserver.GenericAPIServer
delegate := server
var aggregatorServer *aggregatorapiserver.APIAggregator
if s.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) {
runningServer, err = s.startAggregator(ctx, transport, serverConfig, server, s.metrics)
aggregatorServer, err = s.createKubeAggregator(serverConfig, server, s.metrics)
if err != nil {
return err
}
delegate = aggregatorServer.GenericAPIServer
}
var runningServer *genericapiserver.GenericAPIServer
if s.features.IsEnabledGlobally(featuremgmt.FlagDataplaneAggregator) {
runningServer, err = s.startDataplaneAggregator(ctx, transport, serverConfig, delegate)
if err != nil {
return err
}
} else if s.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) {
runningServer, err = s.startKubeAggregator(ctx, transport, aggregatorServer)
if err != nil {
return err
}
@@ -387,25 +421,74 @@ func (s *service) startCoreServer(
return server, nil
}
func (s *service) startAggregator(
func (s *service) startDataplaneAggregator(
ctx context.Context,
transport *roundTripperFunc,
serverConfig *genericapiserver.RecommendedConfig,
delegate *genericapiserver.GenericAPIServer,
) (*genericapiserver.GenericAPIServer, error) {
config := &dataplaneaggregator.Config{
GenericConfig: serverConfig,
ExtraConfig: dataplaneaggregator.ExtraConfig{
PluginClient: s.pluginClient,
PluginContextProvider: &pluginContextProvider{
pluginStore: s.pluginStore,
datasources: s.datasources,
contextProvider: s.contextProvider,
},
},
}
if err := s.options.GrafanaAggregatorOptions.ApplyTo(config, s.options.RecommendedOptions.Etcd); err != nil {
return nil, err
}
completedConfig := config.Complete()
aggregatorServer, err := completedConfig.NewWithDelegate(delegate)
if err != nil {
return nil, err
}
// setup the loopback transport for the aggregator server and signal that it's ready
// ignore the lint error because the response is passed directly to the client,
// so the client will be responsible for closing the response body.
// nolint:bodyclose
transport.fn = grafanaresponsewriter.WrapHandler(aggregatorServer.GenericAPIServer.Handler)
close(transport.ready)
prepared, err := aggregatorServer.PrepareRun()
if err != nil {
return nil, err
}
go func() {
s.stoppedCh <- prepared.RunWithContext(ctx)
}()
return aggregatorServer.GenericAPIServer, nil
}
func (s *service) createKubeAggregator(
serverConfig *genericapiserver.RecommendedConfig,
server *genericapiserver.GenericAPIServer,
reg prometheus.Registerer,
) (*genericapiserver.GenericAPIServer, error) {
) (*aggregatorapiserver.APIAggregator, error) {
namespaceMapper := request.GetNamespaceMapper(s.cfg)
aggregatorConfig, err := aggregator.CreateAggregatorConfig(s.options, *serverConfig, namespaceMapper(1))
aggregatorConfig, err := kubeaggregator.CreateAggregatorConfig(s.options, *serverConfig, namespaceMapper(1))
if err != nil {
return nil, err
}
aggregatorServer, err := aggregator.CreateAggregatorServer(aggregatorConfig, server, reg)
if err != nil {
return nil, err
}
return kubeaggregator.CreateAggregatorServer(aggregatorConfig, server, reg)
}
func (s *service) startKubeAggregator(
ctx context.Context,
transport *roundTripperFunc,
aggregatorServer *aggregatorapiserver.APIAggregator,
) (*genericapiserver.GenericAPIServer, error) {
// setup the loopback transport for the aggregator server and signal that it's ready
// ignore the lint error because the response is passed directly to the client,
// so the client will be responsible for closing the response body.
@@ -473,3 +556,30 @@ func (f *roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error)
}
return f.fn(req)
}
type pluginContextProvider struct {
pluginStore pluginstore.Store
datasources datasource.ScopedPluginDatasourceProvider
contextProvider datasource.PluginContextWrapper
}
func (p *pluginContextProvider) GetPluginContext(ctx context.Context, pluginID string, uid string) (backend.PluginContext, error) {
all := p.pluginStore.Plugins(ctx)
var datasourceProvider datasource.PluginDatasourceProvider
for _, plugin := range all {
if plugin.ID == pluginID {
datasourceProvider = p.datasources.GetDatasourceProvider(plugin.JSONData)
}
}
if datasourceProvider == nil {
return backend.PluginContext{}, fmt.Errorf("plugin not found")
}
s, err := datasourceProvider.GetInstanceSettings(ctx, uid)
if err != nil {
return backend.PluginContext{}, err
}
return p.contextProvider.PluginContextForDataSource(ctx, s)
}