From 8a46dc39d0f01d92075e7e20dae503dbeac16e9c Mon Sep 17 00:00:00 2001 From: Charandas Date: Tue, 7 Nov 2023 22:35:15 -0800 Subject: [PATCH] Grafana App Platform: provide an example-apiserver to easily deploy aggregated APIservers (#77826) --- .github/CODEOWNERS | 1 + .gitignore | 3 + Makefile | 4 + pkg/build/cmd.go | 16 ++- pkg/build/grafana/build.go | 2 +- pkg/cmd/grafana-example-apiserver/README.md | 77 +++++++++++ .../deploy/base/apiservice.yaml | 14 ++ .../deploy/base/kustomization.yaml | 3 + .../deploy/base/namespace.yaml | 4 + .../deploy/darwin/kustomization.yaml | 5 + .../deploy/darwin/service.yaml | 10 ++ .../deploy/linux/kustomization.yaml | 5 + .../deploy/linux/service.yaml | 23 ++++ pkg/cmd/grafana-example-apiserver/main.go | 54 ++++++++ pkg/cmd/grafana-example-apiserver/server.go | 129 ++++++++++++++++++ pkg/registry/apis/example/register.go | 3 +- 16 files changed, 347 insertions(+), 6 deletions(-) create mode 100644 pkg/cmd/grafana-example-apiserver/README.md create mode 100644 pkg/cmd/grafana-example-apiserver/deploy/base/apiservice.yaml create mode 100644 pkg/cmd/grafana-example-apiserver/deploy/base/kustomization.yaml create mode 100644 pkg/cmd/grafana-example-apiserver/deploy/base/namespace.yaml create mode 100644 pkg/cmd/grafana-example-apiserver/deploy/darwin/kustomization.yaml create mode 100644 pkg/cmd/grafana-example-apiserver/deploy/darwin/service.yaml create mode 100644 pkg/cmd/grafana-example-apiserver/deploy/linux/kustomization.yaml create mode 100644 pkg/cmd/grafana-example-apiserver/deploy/linux/service.yaml create mode 100644 pkg/cmd/grafana-example-apiserver/main.go create mode 100644 pkg/cmd/grafana-example-apiserver/server.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 97d6b8883f1..22d11ec4328 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -69,6 +69,7 @@ /pkg/apis/ @grafana/grafana-app-platform-squad /pkg/bus/ @grafana/backend-platform /pkg/cmd/ @grafana/backend-platform +/pkg/cmd/grafana-example-apiserver @grafana/grafana-app-platform-squad /pkg/components/apikeygen/ @grafana/identity-access-team /pkg/components/satokengen/ @grafana/identity-access-team /pkg/components/dashdiffs/ @grafana/backend-platform diff --git a/.gitignore b/.gitignore index 431a6300f76..76bd2088512 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,9 @@ public/css/*.min.css /data/* /bin/* +# any certificates generated by grafana-example-apiserver +apiserver.local.config/ + # devenv /devenv/docker-compose.yaml /devenv/docker-compose.override.yaml diff --git a/Makefile b/Makefile index 686e7f55b4d..2c2e52c03d2 100644 --- a/Makefile +++ b/Makefile @@ -132,6 +132,10 @@ build-server: ## Build Grafana server. @echo "build server" $(GO) run build.go $(GO_BUILD_FLAGS) build-server +build-example-apiserver: ## Build Grafana example-apiserver application. + @echo "build grafana-cli" + $(GO) run build.go $(GO_BUILD_FLAGS) build-example-apiserver + build-cli: ## Build Grafana CLI application. @echo "build grafana-cli" $(GO) run build.go $(GO_BUILD_FLAGS) build-cli diff --git a/pkg/build/cmd.go b/pkg/build/cmd.go index fa98316e9c2..34d1e037d9a 100644 --- a/pkg/build/cmd.go +++ b/pkg/build/cmd.go @@ -17,12 +17,13 @@ const ( GoOSWindows = "windows" GoOSLinux = "linux" - BackendBinary = "grafana" - ServerBinary = "grafana-server" - CLIBinary = "grafana-cli" + BackendBinary = "grafana" + ExampleAPIServerBinary = "grafana-example-apiserver" + ServerBinary = "grafana-server" + CLIBinary = "grafana-cli" ) -var binaries = []string{BackendBinary, ServerBinary, CLIBinary} +var binaries = []string{BackendBinary, ExampleAPIServerBinary, ServerBinary, CLIBinary} func logError(message string, err error) int { log.Println(message, err) @@ -85,6 +86,13 @@ func RunCmd() int { return 1 } + case "build-example-apiserver": + clean(opts) + if err := doBuild("grafana-example-apiserver", "./pkg/cmd/grafana-example-apiserver", opts); err != nil { + log.Println(err) + return 1 + } + case "build-cli": clean(opts) if err := doBuild("grafana-cli", "./pkg/cmd/grafana-cli", opts); err != nil { diff --git a/pkg/build/grafana/build.go b/pkg/build/grafana/build.go index 586b2d22839..084b6c5f624 100644 --- a/pkg/build/grafana/build.go +++ b/pkg/build/grafana/build.go @@ -12,7 +12,7 @@ import ( "github.com/grafana/grafana/pkg/build/golangutils" ) -var binaries = []string{"grafana", "grafana-server", "grafana-cli"} +var binaries = []string{"grafana", "grafana-example-apiserver", "grafana-server", "grafana-cli"} const ( SuffixEnterprise2 = "-enterprise2" diff --git a/pkg/cmd/grafana-example-apiserver/README.md b/pkg/cmd/grafana-example-apiserver/README.md new file mode 100644 index 00000000000..685637a27ef --- /dev/null +++ b/pkg/cmd/grafana-example-apiserver/README.md @@ -0,0 +1,77 @@ +# grafana-example-apiserver + +The example-apiserver closely resembles the +[sample-apiserver](https://github.com/kubernetes/sample-apiserver/tree/master) project in code and thus +allows the same +[CLI flags](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) as kube-apiserver +It is currently used for testing our deployment pipelines for aggregated servers. + +## Prerequisites: +1. etcd + ```shell + brew install etcd + ``` +2. kind: you will need kind (or another local K8s setup) if you want to test aggregation. + ``` + go install sigs.k8s.io/kind@v0.20.0 && kind create cluster + ``` + +## Usage + +Have `etcd` running in your environment: + +```shell +etcd & +``` + +With etcd running, you can now start the example-apiserver. The Authn / Authz flags are set up so that the kind cluster +can be used as a root server for this example-apiserver (in aggregated mode). Here, it's assumed that you have a local +kind cluster and that you can provide its kubeconfig in the parameters to the example-apiserver. + +```shell +go run ./pkg/cmd/grafana-example-apiserver \ + --etcd-servers=http://127.0.0.1:2379 \ + --authentication-kubeconfig ~/.kube/config \ + --authorization-kubeconfig ~/.kube/config \ + --kubeconfig ~/.kube/config \ + --secure-port 8443 +``` + +Once, the `example-apiserver` is running, you can configure aggregation against your kind cluster +by applying a `APIService` and it's corresponding `Service` object. Sample kustomizations are provided +for local development on [Linux](./deploy/linux/kustomization.yaml) and [macOS](./deploy/darwin/kustomization.yaml). + +```shell +kubectl deploy -k ./deploy/darwin # or /linux +``` + + +## Verify that all works + +With kubectl configured against `kind-kind` context, you can run the following: + +```shell +kubectl get --raw /apis/example.grafana.app/v0alpha1 | jq -r +{ + "kind": "APIResourceList", + "apiVersion": "v1", + "groupVersion": "example.grafana.app/v0alpha1", + "resources": [ + { + "name": "runtime", + "singularName": "runtime", + "namespaced": false, + "kind": "RuntimeInfo", + "verbs": [ + "list" + ] + } + ] +} +``` + +```shell +kubectl get apiservice v0alpha1.example.grafana.app +NAME SERVICE AVAILABLE AGE +v0alpha1.example.grafana.app grafana/example-apiserver True 4h1m +``` diff --git a/pkg/cmd/grafana-example-apiserver/deploy/base/apiservice.yaml b/pkg/cmd/grafana-example-apiserver/deploy/base/apiservice.yaml new file mode 100644 index 00000000000..c86833954b8 --- /dev/null +++ b/pkg/cmd/grafana-example-apiserver/deploy/base/apiservice.yaml @@ -0,0 +1,14 @@ +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v0alpha1.example.grafana.app +spec: + version: v0alpha1 + insecureSkipTLSVerify: true + group: example.grafana.app + groupPriorityMinimum: 1000 + versionPriority: 15 + service: + name: example-apiserver + namespace: grafana + port: 8443 diff --git a/pkg/cmd/grafana-example-apiserver/deploy/base/kustomization.yaml b/pkg/cmd/grafana-example-apiserver/deploy/base/kustomization.yaml new file mode 100644 index 00000000000..cc578fada2d --- /dev/null +++ b/pkg/cmd/grafana-example-apiserver/deploy/base/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - namespace.yaml + - apiservice.yaml diff --git a/pkg/cmd/grafana-example-apiserver/deploy/base/namespace.yaml b/pkg/cmd/grafana-example-apiserver/deploy/base/namespace.yaml new file mode 100644 index 00000000000..201d7d3e55b --- /dev/null +++ b/pkg/cmd/grafana-example-apiserver/deploy/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: grafana diff --git a/pkg/cmd/grafana-example-apiserver/deploy/darwin/kustomization.yaml b/pkg/cmd/grafana-example-apiserver/deploy/darwin/kustomization.yaml new file mode 100644 index 00000000000..50deda0eb2e --- /dev/null +++ b/pkg/cmd/grafana-example-apiserver/deploy/darwin/kustomization.yaml @@ -0,0 +1,5 @@ +namespace: grafana + +resources: + - ../base + - service.yaml diff --git a/pkg/cmd/grafana-example-apiserver/deploy/darwin/service.yaml b/pkg/cmd/grafana-example-apiserver/deploy/darwin/service.yaml new file mode 100644 index 00000000000..08fad7959f8 --- /dev/null +++ b/pkg/cmd/grafana-example-apiserver/deploy/darwin/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: example-apiserver +spec: + type: ExternalName + externalName: host.docker.internal + ports: + - port: 8443 + name: https diff --git a/pkg/cmd/grafana-example-apiserver/deploy/linux/kustomization.yaml b/pkg/cmd/grafana-example-apiserver/deploy/linux/kustomization.yaml new file mode 100644 index 00000000000..50deda0eb2e --- /dev/null +++ b/pkg/cmd/grafana-example-apiserver/deploy/linux/kustomization.yaml @@ -0,0 +1,5 @@ +namespace: grafana + +resources: + - ../base + - service.yaml diff --git a/pkg/cmd/grafana-example-apiserver/deploy/linux/service.yaml b/pkg/cmd/grafana-example-apiserver/deploy/linux/service.yaml new file mode 100644 index 00000000000..2f3b9b52eed --- /dev/null +++ b/pkg/cmd/grafana-example-apiserver/deploy/linux/service.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: example-apiserver +subsets: + - addresses: + - ip: 172.17.0.1 # this is the gateway IP in the "bridge" docker network + ports: + - appProtocol: https + port: 8443 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: example-apiserver +spec: + ports: + - protocol: TCP + appProtocol: https + port: 8443 + targetPort: 8443 diff --git a/pkg/cmd/grafana-example-apiserver/main.go b/pkg/cmd/grafana-example-apiserver/main.go new file mode 100644 index 00000000000..7ecce09251f --- /dev/null +++ b/pkg/cmd/grafana-example-apiserver/main.go @@ -0,0 +1,54 @@ +package main + +import ( + "os" + + "github.com/spf13/cobra" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/component-base/cli" +) + +func NewCommandStartExampleAPIServer(defaults *ExampleServerOptions, stopCh <-chan struct{}) *cobra.Command { + o := *defaults + cmd := &cobra.Command{ + Short: "Launch the example API server", + Long: "Launch the example API server", + RunE: func(c *cobra.Command, args []string) error { + if err := o.Complete(); err != nil { + return err + } + + if err := o.Validate(args); err != nil { + return err + } + + config, err := o.Config() + if err != nil { + return err + } + + if err := o.RunExampleServer(config, stopCh); err != nil { + return err + } + return nil + }, + } + + flags := cmd.Flags() + o.RecommendedOptions.AddFlags(flags) + + return cmd +} + +func main() { + stopCh := genericapiserver.SetupSignalHandler() + options, err := NewExampleServerOptions(os.Stdout, os.Stderr) + if err != nil { + panic(err) + } + + cmd := NewCommandStartExampleAPIServer(options, stopCh) + + code := cli.Run(cmd) + os.Exit(code) +} diff --git a/pkg/cmd/grafana-example-apiserver/server.go b/pkg/cmd/grafana-example-apiserver/server.go new file mode 100644 index 00000000000..519d620bc58 --- /dev/null +++ b/pkg/cmd/grafana-example-apiserver/server.go @@ -0,0 +1,129 @@ +package main + +import ( + "fmt" + "io" + "net" + + exampleAPI "github.com/grafana/grafana/pkg/registry/apis/example" + grafanaAPIServer "github.com/grafana/grafana/pkg/services/grafana-apiserver" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/options" + netutils "k8s.io/utils/net" +) + +const defaultEtcdPathPrefix = "/registry/example.grafana.app" + +var ( + Scheme = runtime.NewScheme() + Codecs = serializer.NewCodecFactory(Scheme) + + unversionedVersion = schema.GroupVersion{Group: "", Version: "v1"} + unversionedTypes = []runtime.Object{ + &metav1.Status{}, + &metav1.WatchEvent{}, + &metav1.APIVersions{}, + &metav1.APIGroupList{}, + &metav1.APIGroup{}, + &metav1.APIResourceList{}, + } +) + +func init() { + // we need to add the options to empty v1 + metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Group: "", Version: "v1"}) + Scheme.AddUnversionedTypes(unversionedVersion, unversionedTypes...) +} + +// ExampleServerOptions contains the state for the apiserver +type ExampleServerOptions struct { + RecommendedOptions *options.RecommendedOptions + Builders []grafanaAPIServer.APIGroupBuilder + AlternateDNS []string + + StdOut io.Writer + StdErr io.Writer +} + +func NewExampleServerOptions(out, errOut io.Writer) (*ExampleServerOptions, error) { + builder := &exampleAPI.TestingAPIBuilder{} + + // Install schema + if err := builder.InstallSchema(Scheme); err != nil { + return nil, err + } + + return &ExampleServerOptions{ + Builders: []grafanaAPIServer.APIGroupBuilder{builder}, + RecommendedOptions: options.NewRecommendedOptions( + defaultEtcdPathPrefix, + Codecs.LegacyCodec(builder.GetGroupVersion()), + ), + StdOut: out, + StdErr: errOut, + }, nil +} + +func (o ExampleServerOptions) Config() (*genericapiserver.RecommendedConfig, error) { + if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", o.AlternateDNS, []net.IP{netutils.ParseIPSloppy("127.0.0.1")}); err != nil { + return nil, fmt.Errorf("error creating self-signed certificates: %v", err) + } + + o.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true + o.RecommendedOptions.Authorization.RemoteKubeConfigFileOptional = true + + o.RecommendedOptions.Admission = nil + o.RecommendedOptions.CoreAPI = nil + + serverConfig := genericapiserver.NewRecommendedConfig(Codecs) + + if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { + return nil, err + } + + return serverConfig, nil +} + +// Validate validates ExampleServerOptions +func (o ExampleServerOptions) Validate(args []string) error { + errors := []error{} + errors = append(errors, o.RecommendedOptions.Validate()...) + return utilerrors.NewAggregate(errors) +} + +// Complete fills in fields required to have valid data +func (o ExampleServerOptions) Complete() error { + return nil +} + +func (o ExampleServerOptions) RunExampleServer(config *genericapiserver.RecommendedConfig, stopCh <-chan struct{}) error { + delegationTarget := genericapiserver.NewEmptyDelegate() + completedConfig := config.Complete() + server, err := completedConfig.New("example-apiserver", delegationTarget) + if err != nil { + return err + } + + // Install the API Group+version + for _, b := range o.Builders { + g, err := b.GetAPIGroupInfo(Scheme, Codecs, completedConfig.RESTOptionsGetter) + if err != nil { + return err + } + if g == nil || len(g.PrioritizedVersions) < 1 { + continue + } + err = server.InstallAPIGroup(g) + if err != nil { + return err + } + } + + return server.PrepareRun().Run(stopCh) +} diff --git a/pkg/registry/apis/example/register.go b/pkg/registry/apis/example/register.go index 7222d993d34..2e7222f7b10 100644 --- a/pkg/registry/apis/example/register.go +++ b/pkg/registry/apis/example/register.go @@ -192,6 +192,7 @@ func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &example.RuntimeInfo{}, ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) - return nil + return scheme.SetVersionPriority(SchemeGroupVersion) }