public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

K8s: fix standalone command and add hack scripts (#79052)

Browse Source 
Co-authored-by: Charandas Batra <charandas.batra@grafana.com>
This commit is contained in:
Ryan McKinley
2023-12-05 14:31:49 -08:00
committed by GitHub
co-authored by Charandas Batra
parent 66df17869d
commit 439edebcd6
21 changed files with 307 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cmd/grafana/apiserver/README.md
+28 -5
View File
@@ -4,22 +4,45 @@ The example-apiserver closely resembles the
[sample-apiserver](https://github.com/kubernetes/sample-apiserver/tree/master) project in code and thus
allows the same
[CLI flags](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) as kube-apiserver.
It is currently used for testing our deployment pipelines for aggregated servers.
It is currently used for testing our deployment pipelines for aggregated servers. You can optionally omit the
aggregation path altogether and just run this example apiserver as a standalone process.
## Prerequisites:
## Standalone Mode
### Usage
```shell
go run ./pkg/cmd/grafana apiserver example.grafana.app \
--secure-port 8443
```
### Verify that all works
```shell
export KUBECONFIG=./example-apiserver/kubeconfig
kubectl api-resources
NAME SHORTNAMES APIVERSION NAMESPACED KIND
dummy example.grafana.app/v0alpha1 true DummyResource
runtime example.grafana.app/v0alpha1 false RuntimeInfo
```
## Aggregated Mode
### Prerequisites:
1. kind: you will need kind (or another local K8s setup) if you want to test aggregation.
```
go install sigs.k8s.io/kind@v0.20.0 && kind create cluster
```
## Usage
### Usage
You can start the example-apiserver with an invocation as shown below. The Authn / Authz flags are set up so that the kind cluster
can be used as a root server for this example-apiserver (in aggregated mode). Here, it's assumed that you have a local
kind cluster and that you can provide its kubeconfig in the parameters to the example-apiserver.
```shell
go run ./pkg/cmd/grafana apiserver example.grafana.app\
go run ./pkg/cmd/grafana apiserver example.grafana.app \
--authentication-kubeconfig ~/.kube/config \
--authorization-kubeconfig ~/.kube/config \
--kubeconfig ~/.kube/config \
@@ -35,7 +58,7 @@ kubectl deploy -k ./deploy/darwin # or /linux
```
## Verify that all works
### Verify that all works
With kubectl configured against `kind-kind` context, you can run the following:
pkg/cmd/grafana/apiserver/cmd.go
+5 -19
View File
@@ -1,7 +1,6 @@
package apiserver
import (
"fmt"
"os"
"github.com/spf13/cobra"
@@ -10,9 +9,7 @@ import (
"k8s.io/component-base/cli"
)
func newCommandStartExampleAPIServer(o *ExampleServerOptions, stopCh <-chan struct{}) *cobra.Command {
// While this exists as an experimental feature, we require adding the scarry looking command line
devAcknowledgementFlag := "grafana-enable-experimental-apiserver"
func newCommandStartExampleAPIServer(o *APIServerOptions, stopCh <-chan struct{}) *cobra.Command {
devAcknowledgementNotice := "The apiserver command is in heavy development. The entire setup is subject to change without notice"
cmd := &cobra.Command{
@@ -20,22 +17,14 @@ func newCommandStartExampleAPIServer(o *ExampleServerOptions, stopCh <-chan stru
Short: "Run the grafana apiserver",
Long: "Run a standalone kubernetes based apiserver that can be aggregated by a root apiserver. " +
devAcknowledgementNotice,
Example: fmt.Sprintf("grafana apiserver example.grafana.app --%s", devAcknowledgementFlag),
PersistentPreRun: func(cmd *cobra.Command, args []string) {
ok, err := cmd.Flags().GetBool(devAcknowledgementFlag)
if !ok || err != nil {
fmt.Printf("requires running with the flag: --%s\n\n%s\n\n",
devAcknowledgementFlag, devAcknowledgementNotice)
os.Exit(1)
}
},
Example: "grafana apiserver example.grafana.app",
RunE: func(c *cobra.Command, args []string) error {
// Load each group from the args
if err := o.LoadAPIGroupBuilders(args[1:]); err != nil {
return err
}
// Finish the config (applies all defaults)
// Finish the config (a noop for now)
if err := o.Complete(); err != nil {
return err
}
@@ -45,16 +34,13 @@ func newCommandStartExampleAPIServer(o *ExampleServerOptions, stopCh <-chan stru
return err
}
if err := o.RunExampleServer(config, stopCh); err != nil {
if err := o.RunAPIServer(config, stopCh); err != nil {
return err
}
return nil
},
}
// Register grafana flags
cmd.PersistentFlags().Bool(devAcknowledgementFlag, false, devAcknowledgementNotice)
// Register standard k8s flags with the command line
o.RecommendedOptions = options.NewRecommendedOptions(
defaultEtcdPathPrefix,
@@ -68,7 +54,7 @@ func newCommandStartExampleAPIServer(o *ExampleServerOptions, stopCh <-chan stru
func RunCLI() int {
stopCh := genericapiserver.SetupSignalHandler()
options := newExampleServerOptions(os.Stdout, os.Stderr)
options := newAPIServerOptions(os.Stdout, os.Stderr)
cmd := newCommandStartExampleAPIServer(options, stopCh)
return cli.Run(cmd)
pkg/cmd/grafana/apiserver/server.go
+101 -19
View File
@@ -4,21 +4,30 @@ import (
"fmt"
"io"
"net"
"github.com/grafana/grafana/pkg/registry/apis/example"
grafanaAPIServer "github.com/grafana/grafana/pkg/services/grafana-apiserver"
"path"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/util/openapi"
"k8s.io/client-go/tools/clientcmd"
netutils "k8s.io/utils/net"
"github.com/grafana/grafana/pkg/services/grafana-apiserver/utils"
"github.com/grafana/grafana/pkg/registry/apis/example"
grafanaAPIServer "github.com/grafana/grafana/pkg/services/grafana-apiserver"
)
const defaultEtcdPathPrefix = "/registry/example.grafana.app"
const (
defaultEtcdPathPrefix = "/registry/grafana.app"
dataPath = "data/grafana-apiserver" // same as grafana core
)
var (
Scheme = runtime.NewScheme()
@@ -41,8 +50,8 @@ func init() {
Scheme.AddUnversionedTypes(unversionedVersion, unversionedTypes...)
}
// ExampleServerOptions contains the state for the apiserver
type ExampleServerOptions struct {
// APIServerOptions contains the state for the apiserver
type APIServerOptions struct {
builders []grafanaAPIServer.APIGroupBuilder
RecommendedOptions *options.RecommendedOptions
AlternateDNS []string
@@ -51,20 +60,20 @@ type ExampleServerOptions struct {
StdErr io.Writer
}
func newExampleServerOptions(out, errOut io.Writer) *ExampleServerOptions {
return &ExampleServerOptions{
func newAPIServerOptions(out, errOut io.Writer) *APIServerOptions {
return &APIServerOptions{
StdOut: out,
StdErr: errOut,
}
}
func (o *ExampleServerOptions) LoadAPIGroupBuilders(args []string) error {
func (o *APIServerOptions) LoadAPIGroupBuilders(args []string) error {
o.builders = []grafanaAPIServer.APIGroupBuilder{}
for _, g := range args {
switch g {
// No dependencies for testing
case "example.grafana.app":
o.builders = append(o.builders, &example.TestingAPIBuilder{})
o.builders = append(o.builders, example.NewTestingAPIBuilder())
default:
return fmt.Errorf("unknown group: %s", g)
}
@@ -83,8 +92,49 @@ func (o *ExampleServerOptions) LoadAPIGroupBuilders(args []string) error {
return nil
}
func (o *ExampleServerOptions) Config() (*genericapiserver.RecommendedConfig, error) {
if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", o.AlternateDNS, []net.IP{netutils.ParseIPSloppy("127.0.0.1")}); err != nil {
// A copy of ApplyTo in recommended.go, but for >= 0.28, server pkg in apiserver does a bit extra causing
// a panic when CoreAPI is set to nil
func (o *APIServerOptions) ModifiedApplyTo(config *genericapiserver.RecommendedConfig) error {
if err := o.RecommendedOptions.Etcd.ApplyTo(&config.Config); err != nil {
return err
}
if err := o.RecommendedOptions.EgressSelector.ApplyTo(&config.Config); err != nil {
return err
}
if err := o.RecommendedOptions.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil {
return err
}
if err := o.RecommendedOptions.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil {
return err
}
if err := o.RecommendedOptions.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil {
return err
}
if err := o.RecommendedOptions.Authorization.ApplyTo(&config.Config.Authorization); err != nil {
return err
}
if err := o.RecommendedOptions.Audit.ApplyTo(&config.Config); err != nil {
return err
}
if err := o.RecommendedOptions.Features.ApplyTo(&config.Config); err != nil {
return err
}
if err := o.RecommendedOptions.CoreAPI.ApplyTo(config); err != nil {
return err
}
_, err := o.RecommendedOptions.ExtraAdmissionInitializers(config)
if err != nil {
return err
}
return nil
}
func (o *APIServerOptions) Config() (*genericapiserver.RecommendedConfig, error) {
if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts(
"localhost", o.AlternateDNS, []net.IP{netutils.ParseIPSloppy("127.0.0.1")},
); err != nil {
return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
}
@@ -92,33 +142,55 @@ func (o *ExampleServerOptions) Config() (*genericapiserver.RecommendedConfig, er
o.RecommendedOptions.Authorization.RemoteKubeConfigFileOptional = true
o.RecommendedOptions.Admission = nil
o.RecommendedOptions.CoreAPI = nil
o.RecommendedOptions.Etcd = nil
if o.RecommendedOptions.CoreAPI.CoreAPIKubeconfigPath == "" {
o.RecommendedOptions.CoreAPI = nil
}
serverConfig := genericapiserver.NewRecommendedConfig(Codecs)
if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
return nil, err
if o.RecommendedOptions.CoreAPI == nil {
if err := o.ModifiedApplyTo(serverConfig); err != nil {
return nil, err
}
} else {
if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
return nil, err
}
}
// Add OpenAPI specs for each group+version
defsGetter := grafanaAPIServer.GetOpenAPIDefinitions(o.builders)
serverConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(
openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(defsGetter),
openapinamer.NewDefinitionNamer(Scheme))
serverConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(
openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(defsGetter),
openapinamer.NewDefinitionNamer(Scheme))
// Add the custom routes to service discovery
serverConfig.OpenAPIV3Config.PostProcessSpec3 = grafanaAPIServer.GetOpenAPIPostProcessor(o.builders)
return serverConfig, nil
}
// Validate validates ExampleServerOptions
// Validate validates APIServerOptions
// NOTE: we don't call validate on the top level recommended options as it doesn't like skipping etcd-servers
// the function is left here for troubleshooting any other config issues
func (o *ExampleServerOptions) Validate(args []string) error {
func (o *APIServerOptions) Validate(args []string) error {
errors := []error{}
errors = append(errors, o.RecommendedOptions.Validate()...)
return utilerrors.NewAggregate(errors)
}
// Complete fills in fields required to have valid data
func (o *ExampleServerOptions) Complete() error {
func (o *APIServerOptions) Complete() error {
return nil
}
func (o *ExampleServerOptions) RunExampleServer(config *genericapiserver.RecommendedConfig, stopCh <-chan struct{}) error {
func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConfig, stopCh <-chan struct{}) error {
delegationTarget := genericapiserver.NewEmptyDelegate()
completedConfig := config.Complete()
server, err := completedConfig.New("example-apiserver", delegationTarget)
@@ -141,5 +213,15 @@ func (o *ExampleServerOptions) RunExampleServer(config *genericapiserver.Recomme
}
}
// in standalone mode, write the local config to disk
if o.RecommendedOptions.CoreAPI == nil {
if err = clientcmd.WriteToFile(
utils.FormatKubeConfig(server.LoopbackClientConfig),
path.Join(dataPath, "grafana.kubeconfig"),
); err != nil {
return err
}
}
return server.PrepareRun().Run(stopCh)
}