public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

K8s: Refactor config/options for aggregation (#81739)

Browse Source
This commit is contained in:
Todd Treece
2024-02-01 17:27:30 -05:00
committed by GitHub
parent 7a17963ab9
commit 67b6be5515
93 changed files with 1102 additions and 1446 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cmd/grafana/apiserver/aggregator.md
-51
View File
@@ -1,51 +0,0 @@
# grafana aggregator
The `aggregator` command in this binary is our equivalent of what kube-apiserver does for aggregation using
the `kube-aggregator` pkg. Here, we enable only select controllers that are useful for aggregation in a Grafana
cloud context. In future, Grafana microservices (and even plugins) will run as separate API servers
hosting each their own APIs (with specific Group/Versions). The `aggregator` component here shall act similar to what
`kube-apiserver` does: doing healthchecks for `APIService` objects registered against it and acting as a proxy for
the specified `GroupVersion` therein.
## How to get started
1. Generate the PKI using `openssl` (for development purposes, we will use the CN of `system:masters`):
```shell
./hack/make-aggregator-pki.sh
```
2. Start the aggregator:
```shell
# This will generate the kubeconfig which you can use in the extension apiservers for
# enforcing delegate authnz under $PWD/data/grafana-apiserver/aggregator.kubeconfig
go run ./pkg/cmd/grafana aggregator --secure-port 8443 \
--proxy-client-cert-file $PWD/data/grafana-aggregator/client.crt \
--proxy-client-key-file $PWD/data/grafana-aggregator/client.key
```
3. Apply the manifests:
```shell
export KUBECONFIG=$PWD/data/grafana-apiserver/aggregator.kubeconfig
kubectl apply -k ./pkg/cmd/grafana/apiserver/deploy/aggregator-test
# SAMPLE OUTPUT
# apiservice.apiregistration.k8s.io/v0alpha1.example.grafana.app created
# externalname.service.grafana.app/example-apiserver created
kubectl get apiservice
# SAMPLE OUTPUT
# NAME SERVICE AVAILABLE AGE
# v0alpha1.example.grafana.app grafana/example-apiserver False (FailedDiscoveryCheck) 29m
```
4. In another tab, start the example microservice that will be aggregated by the parent apiserver:
```shell
go run ./pkg/cmd/grafana apiserver example.grafana.app \
--kubeconfig $PWD/data/grafana-aggregator/aggregator.kubeconfig \
--secure-port 7443 \
--client-ca-file=$PWD/data/grafana-aggregator/ca.crt
```
5. Check `APIService` again:
```shell
export KUBECONFIG=$PWD/data/grafana-apiserver/aggregator.kubeconfig
kubectl get apiservice
# SAMPLE OUTPUT
# NAME SERVICE AVAILABLE AGE
# v0alpha1.example.grafana.app grafana/example-apiserver True 30m
```
pkg/cmd/grafana/apiserver/cmd.go
+1 -108
View File
@@ -2,26 +2,13 @@ package apiserver
import (
"os"
"path"
"github.com/spf13/cobra"
genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/util/notfoundhandler"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/component-base/cli"
"k8s.io/klog/v2"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
"github.com/grafana/grafana/pkg/aggregator"
grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver"
"github.com/grafana/grafana/pkg/services/grafana-apiserver/utils"
)
const (
aggregatorDataPath = "data"
defaultAggregatorEtcdPathPrefix = "/registry/grafana.aggregator"
grafanaapiserver "github.com/grafana/grafana/pkg/services/apiserver"
)
func newCommandStartExampleAPIServer(o *APIServerOptions, stopCh <-chan struct{}) *cobra.Command {
@@ -74,97 +61,3 @@ func RunCLI() int {
return cli.Run(cmd)
}
func newCommandStartAggregator() *cobra.Command {
devAcknowledgementNotice := "The aggregator command is in heavy development. The entire setup is subject to change without notice"
cwd, err := os.Getwd()
if err != nil {
panic("could not determine current directory")
}
extraConfig := &aggregator.ExtraConfig{
DataPath: path.Join(cwd, aggregatorDataPath),
}
// Register standard k8s flags with the command line
recommendedOptions := options.NewRecommendedOptions(
defaultAggregatorEtcdPathPrefix,
aggregatorscheme.Codecs.LegacyCodec(), // codec is passed to etcd and hence not used
)
cmd := &cobra.Command{
Use: "aggregator",
Short: "Run the grafana aggregator",
Long: "Run a standalone kubernetes based aggregator server. " +
devAcknowledgementNotice,
Example: "grafana aggregator",
RunE: func(c *cobra.Command, args []string) error {
serverOptions, err := aggregator.NewAggregatorServerOptions(os.Stdout, os.Stderr, recommendedOptions, extraConfig)
serverOptions.Config.Complete()
if err != nil {
klog.Errorf("Could not create aggregator server options: %s", err)
os.Exit(1)
}
return run(serverOptions)
},
}
recommendedOptions.AddFlags(cmd.Flags())
extraConfig.AddFlags(cmd.Flags())
return cmd
}
func run(serverOptions *aggregator.AggregatorServerOptions) error {
if err := serverOptions.LoadAPIGroupBuilders(); err != nil {
klog.Errorf("Error loading prerequisite APIs: %s", err)
return err
}
notFoundHandler := notfoundhandler.New(serverOptions.Config.SharedConfig.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey)
apiExtensionsServer, err := serverOptions.Config.ApiExtensionsComplete.New(genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler))
if err != nil {
return err
}
aggregator, err := serverOptions.CreateAggregatorServer(apiExtensionsServer.GenericAPIServer, apiExtensionsServer.Informers)
if err != nil {
klog.Errorf("Error creating aggregator server: %s", err)
return err
}
// Install the API Group+version
err = grafanaapiserver.InstallAPIs(aggregator.GenericAPIServer, serverOptions.Config.Aggregator.GenericConfig.RESTOptionsGetter, serverOptions.Builders)
if err != nil {
klog.Errorf("Error installing apis: %s", err)
return err
}
if err := clientcmd.WriteToFile(
utils.FormatKubeConfig(aggregator.GenericAPIServer.LoopbackClientConfig),
path.Join(aggregatorDataPath, "grafana-aggregator", "aggregator.kubeconfig"),
); err != nil {
klog.Errorf("Error persisting aggregator.kubeconfig: %s", err)
return err
}
prepared, err := aggregator.PrepareRun()
if err != nil {
return err
}
stopCh := genericapiserver.SetupSignalHandler()
if err := prepared.Run(stopCh); err != nil {
return err
}
return nil
}
func RunCobraWrapper() int {
cmd := newCommandStartAggregator()
return cli.Run(cmd)
}
pkg/cmd/grafana/apiserver/server.go
+7 -6
View File
@@ -17,9 +17,10 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/query"
"github.com/grafana/grafana/pkg/registry/apis/query/runner"
"github.com/grafana/grafana/pkg/server"
grafanaAPIServer "github.com/grafana/grafana/pkg/services/apiserver"
"github.com/grafana/grafana/pkg/services/apiserver/builder"
"github.com/grafana/grafana/pkg/services/apiserver/utils"
"github.com/grafana/grafana/pkg/services/featuremgmt"
grafanaAPIServer "github.com/grafana/grafana/pkg/services/grafana-apiserver"
"github.com/grafana/grafana/pkg/services/grafana-apiserver/utils"
"github.com/grafana/grafana/pkg/setting"
)
@@ -30,7 +31,7 @@ const (
// APIServerOptions contains the state for the apiserver
type APIServerOptions struct {
builders []grafanaAPIServer.APIGroupBuilder
builders []builder.APIGroupBuilder
RecommendedOptions *options.RecommendedOptions
AlternateDNS []string
@@ -46,7 +47,7 @@ func newAPIServerOptions(out, errOut io.Writer) *APIServerOptions {
}
func (o *APIServerOptions) loadAPIGroupBuilders(args []string) error {
o.builders = []grafanaAPIServer.APIGroupBuilder{}
o.builders = []builder.APIGroupBuilder{}
for _, g := range args {
switch g {
// No dependencies for testing
@@ -171,7 +172,7 @@ func (o *APIServerOptions) Config() (*genericapiserver.RecommendedConfig, error)
serverConfig.DisabledPostStartHooks = serverConfig.DisabledPostStartHooks.Insert("priority-and-fairness-config-consumer")
// Add OpenAPI specs for each group+version
err := grafanaAPIServer.SetupConfig(serverConfig, o.builders)
err := builder.SetupConfig(grafanaAPIServer.Scheme, serverConfig, o.builders)
return serverConfig, err
}
@@ -199,7 +200,7 @@ func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConf
}
// Install the API Group+version
err = grafanaAPIServer.InstallAPIs(server, config.RESTOptionsGetter, o.builders)
err = builder.InstallAPIs(grafanaAPIServer.Scheme, grafanaAPIServer.Codecs, server, config.RESTOptionsGetter, o.builders, true)
if err != nil {
return err
}
pkg/cmd/grafana/main.go
-12
View File
@@ -46,18 +46,6 @@ func main() {
},
},
gsrv.ServerCommand(version, commit, enterpriseCommit, buildBranch, buildstamp),
{
// The kube-aggregator inspired grafana aggregator
Name: "aggregator",
Usage: "run grafana aggregator (experimental)",
// Skip parsing flags because the command line is actually managed by cobra
SkipFlagParsing: true,
Action: func(context *cli.Context) error {
// exit here because apiserver handles its own error output
os.Exit(apiserver.RunCobraWrapper())
return nil
},
},
},
CommandNotFound: cmdNotFound,
EnableBashCompletion: true,