From 30ae434a2e0fbd917804d24e34cff22bbdfd0644 Mon Sep 17 00:00:00 2001 From: Todd Treece <360020+toddtreece@users.noreply.github.com> Date: Fri, 14 Feb 2025 12:29:43 -0500 Subject: [PATCH] K8s: Support multiple versions in builder (#100331) --- pkg/services/apiserver/builder/admission.go | 12 +- .../apiserver/builder/admission_test.go | 4 +- pkg/services/apiserver/builder/common.go | 46 +++++- pkg/services/apiserver/builder/helper.go | 5 +- pkg/services/apiserver/builder/openapi.go | 131 +++++++++--------- .../apiserver/builder/request_handler.go | 63 ++++----- pkg/services/apiserver/service.go | 23 +-- 7 files changed, 165 insertions(+), 119 deletions(-) diff --git a/pkg/services/apiserver/builder/admission.go b/pkg/services/apiserver/builder/admission.go index dbd39130572..6c491d28ca5 100644 --- a/pkg/services/apiserver/builder/admission.go +++ b/pkg/services/apiserver/builder/admission.go @@ -37,11 +37,13 @@ func NewAdmissionFromBuilders(builders []APIGroupBuilder) *builderAdmission { mutators := make(map[schema.GroupVersion]APIGroupMutation) validators := make(map[schema.GroupVersion]APIGroupValidation) for _, builder := range builders { - if m, ok := builder.(APIGroupMutation); ok { - mutators[builder.GetGroupVersion()] = m - } - if v, ok := builder.(APIGroupValidation); ok { - validators[builder.GetGroupVersion()] = v + for _, gv := range GetGroupVersions(builder) { + if m, ok := builder.(APIGroupMutation); ok { + mutators[gv] = m + } + if v, ok := builder.(APIGroupValidation); ok { + validators[gv] = v + } } } return NewAdmission(mutators, validators) diff --git a/pkg/services/apiserver/builder/admission_test.go b/pkg/services/apiserver/builder/admission_test.go index 361825fd732..007ff61ea3a 100644 --- a/pkg/services/apiserver/builder/admission_test.go +++ b/pkg/services/apiserver/builder/admission_test.go @@ -207,8 +207,8 @@ type mockBuilder struct { validator builder.APIGroupValidation } -func (m *mockBuilder) GetGroupVersion() schema.GroupVersion { - return m.groupVersion +func (m *mockBuilder) GetGroupVersions() []schema.GroupVersion { + return []schema.GroupVersion{m.groupVersion} } func (m *mockBuilder) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error { diff --git a/pkg/services/apiserver/builder/common.go b/pkg/services/apiserver/builder/common.go index 5761479c092..4eb5f07cffb 100644 --- a/pkg/services/apiserver/builder/common.go +++ b/pkg/services/apiserver/builder/common.go @@ -2,6 +2,7 @@ package builder import ( "context" + "fmt" "net/http" "github.com/prometheus/client_golang/prometheus" @@ -21,9 +22,6 @@ import ( // TODO: this (or something like it) belongs in grafana-app-sdk, // but lets keep it here while we iterate on a few simple examples type APIGroupBuilder interface { - // Get the main group name - GetGroupVersion() schema.GroupVersion - // Add the kinds to the server scheme InstallSchema(scheme *runtime.Scheme) error @@ -38,10 +36,17 @@ type APIGroupBuilder interface { // Get OpenAPI definitions GetOpenAPIDefinitions() common.GetOpenAPIDefinitions +} - // Optionally add an authorization hook - // Standard namespace checking will happen before this is called, specifically - // the namespace must matches an org|stack that the user belongs to +type APIGroupVersionProvider interface { + GetGroupVersion() schema.GroupVersion +} + +type APIGroupVersionsProvider interface { + GetGroupVersions() []schema.GroupVersion +} + +type APIGroupAuthorizer interface { GetAuthorizer() authorizer.Authorizer } @@ -100,3 +105,32 @@ type APIRoutes struct { type APIRegistrar interface { RegisterAPI(builder APIGroupBuilder) } + +func getGroup(builder APIGroupBuilder) (string, error) { + if v, ok := builder.(APIGroupVersionProvider); ok { + return v.GetGroupVersion().Group, nil + } + + if v, ok := builder.(APIGroupVersionsProvider); ok { + if len(v.GetGroupVersions()) == 0 { + return "", fmt.Errorf("unable to get group: builder returned no versions") + } + + return v.GetGroupVersions()[0].Group, nil + } + + return "", fmt.Errorf("unable to get group: builder does not implement APIGroupVersionProvider or APIGroupVersionsProvider") +} + +func GetGroupVersions(builder APIGroupBuilder) []schema.GroupVersion { + if v, ok := builder.(APIGroupVersionProvider); ok { + return []schema.GroupVersion{v.GetGroupVersion()} + } + + if v, ok := builder.(APIGroupVersionsProvider); ok { + return v.GetGroupVersions() + } + + // this should never happen + panic("builder does not implement APIGroupVersionProvider or APIGroupVersionsProvider") +} diff --git a/pkg/services/apiserver/builder/helper.go b/pkg/services/apiserver/builder/helper.go index c1f81ae6a3b..46b8e33aa8b 100644 --- a/pkg/services/apiserver/builder/helper.go +++ b/pkg/services/apiserver/builder/helper.go @@ -360,7 +360,10 @@ func InstallAPIs( // in other places, working with a flat []APIGroupBuilder list is much nicer buildersGroupMap := make(map[string][]APIGroupBuilder, 0) for _, b := range builders { - group := b.GetGroupVersion().Group + group, err := getGroup(b) + if err != nil { + return err + } if _, ok := buildersGroupMap[group]; !ok { buildersGroupMap[group] = make([]APIGroupBuilder, 0) } diff --git a/pkg/services/apiserver/builder/openapi.go b/pkg/services/apiserver/builder/openapi.go index 5ca84226306..2a229cdc85d 100644 --- a/pkg/services/apiserver/builder/openapi.go +++ b/pkg/services/apiserver/builder/openapi.go @@ -49,87 +49,88 @@ func getOpenAPIPostProcessor(version string, builders []APIGroupBuilder) func(*s } for _, b := range builders { - gv := b.GetGroupVersion() - prefix := "/apis/" + gv.String() + "/" - if s.Paths.Paths[prefix] != nil { - copy := spec3.OpenAPI{ - Version: s.Version, - Info: &spec.Info{ - InfoProps: spec.InfoProps{ - Title: gv.String(), - Version: version, + for _, gv := range GetGroupVersions(b) { + prefix := "/apis/" + gv.String() + "/" + if s.Paths.Paths[prefix] != nil { + copy := spec3.OpenAPI{ + Version: s.Version, + Info: &spec.Info{ + InfoProps: spec.InfoProps{ + Title: gv.String(), + Version: version, + }, }, - }, - Components: s.Components, - ExternalDocs: s.ExternalDocs, - Servers: s.Servers, - Paths: s.Paths, - } - - for k := range copy.Paths.Paths { - // Remove the deprecated watch URL -- can use list with ?watch=true - if strings.HasPrefix(k, prefix+"watch/") { - delete(copy.Paths.Paths, k) - continue + Components: s.Components, + ExternalDocs: s.ExternalDocs, + Servers: s.Servers, + Paths: s.Paths, } - } - sub := copy.Paths.Paths[prefix] - if sub != nil && sub.Get != nil { - sub.Get.Tags = []string{"API Discovery"} - sub.Get.Description = "Describe the available kubernetes resources" - } - - // Remove the growing list of kinds - for k, v := range copy.Components.Schemas { - if strings.HasPrefix(k, "io.k8s.apimachinery.pkg.apis.meta.v1") && v.Extensions != nil { - delete(v.Extensions, "x-kubernetes-group-version-kind") // a growing list of everything - } - } - - // Optionally include raw http handlers - provider, ok := b.(APIGroupRouteProvider) - if ok && provider != nil { - routes := provider.GetAPIRoutes() - if routes != nil { - for _, route := range routes.Root { - copy.Paths.Paths[prefix+route.Path] = &spec3.Path{ - PathProps: *route.Spec, - } + for k := range copy.Paths.Paths { + // Remove the deprecated watch URL -- can use list with ?watch=true + if strings.HasPrefix(k, prefix+"watch/") { + delete(copy.Paths.Paths, k) + continue } + } - for _, route := range routes.Namespace { - copy.Paths.Paths[prefix+"namespaces/{namespace}/"+route.Path] = &spec3.Path{ - PathProps: *route.Spec, + sub := copy.Paths.Paths[prefix] + if sub != nil && sub.Get != nil { + sub.Get.Tags = []string{"API Discovery"} + sub.Get.Description = "Describe the available kubernetes resources" + } + + // Remove the growing list of kinds + for k, v := range copy.Components.Schemas { + if strings.HasPrefix(k, "io.k8s.apimachinery.pkg.apis.meta.v1") && v.Extensions != nil { + delete(v.Extensions, "x-kubernetes-group-version-kind") // a growing list of everything + } + } + + // Optionally include raw http handlers + provider, ok := b.(APIGroupRouteProvider) + if ok && provider != nil { + routes := provider.GetAPIRoutes() + if routes != nil { + for _, route := range routes.Root { + copy.Paths.Paths[prefix+route.Path] = &spec3.Path{ + PathProps: *route.Spec, + } + } + + for _, route := range routes.Namespace { + copy.Paths.Paths[prefix+"namespaces/{namespace}/"+route.Path] = &spec3.Path{ + PathProps: *route.Spec, + } } } } - } - // Make the sub-resources (connect) share the same tags as the main resource - for path, spec := range copy.Paths.Paths { - idx := strings.LastIndex(path, "{name}/") - if idx > 0 { - parent := copy.Paths.Paths[path[:idx+6]] - if parent != nil && parent.Get != nil { - for _, op := range GetPathOperations(spec) { - if op != nil && op.Extensions != nil { - action, ok := op.Extensions.GetString("x-kubernetes-action") - if ok && action == "connect" { - op.Tags = parent.Get.Tags + // Make the sub-resources (connect) share the same tags as the main resource + for path, spec := range copy.Paths.Paths { + idx := strings.LastIndex(path, "{name}/") + if idx > 0 { + parent := copy.Paths.Paths[path[:idx+6]] + if parent != nil && parent.Get != nil { + for _, op := range GetPathOperations(spec) { + if op != nil && op.Extensions != nil { + action, ok := op.Extensions.GetString("x-kubernetes-action") + if ok && action == "connect" { + op.Tags = parent.Get.Tags + } } } } } } - } - // Support direct manipulation of API results - processor, ok := b.(OpenAPIPostProcessor) - if ok { - return processor.PostProcessOpenAPI(©) + // Support direct manipulation of API results + processor, ok := b.(OpenAPIPostProcessor) + if ok { + return processor.PostProcessOpenAPI(©) + } + return ©, nil } - return ©, nil } } diff --git a/pkg/services/apiserver/builder/request_handler.go b/pkg/services/apiserver/builder/request_handler.go index 42f84155159..ded0bd4a562 100644 --- a/pkg/services/apiserver/builder/request_handler.go +++ b/pkg/services/apiserver/builder/request_handler.go @@ -28,42 +28,43 @@ func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient continue } - gv := builder.GetGroupVersion() - prefix := "/apis/" + gv.String() + for _, gv := range GetGroupVersions(builder) { + prefix := "/apis/" + gv.String() - // Root handlers - var sub *mux.Router - for _, route := range routes.Root { - if sub == nil { - sub = router.PathPrefix(prefix).Subrouter() - sub.MethodNotAllowedHandler = &methodNotAllowedHandler{} + // Root handlers + var sub *mux.Router + for _, route := range routes.Root { + if sub == nil { + sub = router.PathPrefix(prefix).Subrouter() + sub.MethodNotAllowedHandler = &methodNotAllowedHandler{} + } + + useful = true + methods, err := methodsFromSpec(route.Path, route.Spec) + if err != nil { + return nil, err + } + sub.HandleFunc("/"+route.Path, route.Handler). + Methods(methods...) } - useful = true - methods, err := methodsFromSpec(route.Path, route.Spec) - if err != nil { - return nil, err - } - sub.HandleFunc("/"+route.Path, route.Handler). - Methods(methods...) - } + // Namespace handlers + sub = nil + prefix += "/namespaces/{namespace}" + for _, route := range routes.Namespace { + if sub == nil { + sub = router.PathPrefix(prefix).Subrouter() + sub.MethodNotAllowedHandler = &methodNotAllowedHandler{} + } - // Namespace handlers - sub = nil - prefix += "/namespaces/{namespace}" - for _, route := range routes.Namespace { - if sub == nil { - sub = router.PathPrefix(prefix).Subrouter() - sub.MethodNotAllowedHandler = &methodNotAllowedHandler{} + useful = true + methods, err := methodsFromSpec(route.Path, route.Spec) + if err != nil { + return nil, err + } + sub.HandleFunc("/"+route.Path, route.Handler). + Methods(methods...) } - - useful = true - methods, err := methodsFromSpec(route.Path, route.Spec) - if err != nil { - return nil, err - } - sub.HandleFunc("/"+route.Path, route.Handler). - Methods(methods...) } } diff --git a/pkg/services/apiserver/service.go b/pkg/services/apiserver/service.go index 9246d08439a..65490a742d4 100644 --- a/pkg/services/apiserver/service.go +++ b/pkg/services/apiserver/service.go @@ -265,24 +265,29 @@ func (s *service) RegisterAPI(b builder.APIGroupBuilder) { func (s *service) start(ctx context.Context) error { // Get the list of groups the server will support builders := s.builders - groupVersions := make([]schema.GroupVersion, 0, len(builders)) + // Install schemas initialSize := len(kubeaggregator.APIVersionPriorities) for i, b := range builders { - groupVersions = append(groupVersions, b.GetGroupVersion()) + gvs := builder.GetGroupVersions(b) + groupVersions = append(groupVersions, gvs...) if err := b.InstallSchema(Scheme); err != nil { return err } - if s.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) { - // set the priority for the group+version - kubeaggregator.APIVersionPriorities[b.GetGroupVersion()] = kubeaggregator.Priority{Group: 15000, Version: int32(i + initialSize)} - } + for _, gv := range gvs { + if s.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) { + // set the priority for the group+version + kubeaggregator.APIVersionPriorities[gv] = kubeaggregator.Priority{Group: 15000, Version: int32(i + initialSize)} + } - auth := b.GetAuthorizer() - if auth != nil { - s.authorizer.Register(b.GetGroupVersion(), auth) + if a, ok := b.(builder.APIGroupAuthorizer); ok { + auth := a.GetAuthorizer() + if auth != nil { + s.authorizer.Register(gv, auth) + } + } } }