From d1cbef9157ef70282e56b83b1a2f91ac418c6caa Mon Sep 17 00:00:00 2001 From: Charandas <542168+charandas@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:53:31 -0800 Subject: [PATCH] K8s: use runtime config for API Builders (#114601) * Reapply "K8s: read resource configs from API Enablement for API Builders" (#114475) This reverts commit 4130bd9cd300ec7ce0fb492b3a7229968c9167b4. * revert part that broke things * FF service changes are gonna come later --- pkg/services/apiserver/builder/helper.go | 85 +++++++++++++------ pkg/services/apiserver/builder/openapi.go | 19 ++++- .../apiserver/builder/request_handler.go | 10 ++- pkg/services/apiserver/service.go | 8 +- 4 files changed, 90 insertions(+), 32 deletions(-) diff --git a/pkg/services/apiserver/builder/helper.go b/pkg/services/apiserver/builder/helper.go index 603c49f3cb2..a76a01dffba 100644 --- a/pkg/services/apiserver/builder/helper.go +++ b/pkg/services/apiserver/builder/helper.go @@ -22,6 +22,7 @@ import ( k8srequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" + serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/util/openapi" k8sscheme "k8s.io/client-go/kubernetes/scheme" k8stracing "k8s.io/component-base/tracing" @@ -78,7 +79,9 @@ func GetDefaultBuildHandlerChainFunc(builders []APIGroupBuilder, reg prometheus. delegateHandler, c.LoopbackClientConfig, builders, - reg) + reg, + c.MergedResourceConfig, + ) if err != nil { panic(fmt.Sprintf("could not build the request handler for specified API builders: %s", err.Error())) } @@ -105,6 +108,8 @@ func GetDefaultBuildHandlerChainFunc(builders []APIGroupBuilder, reg prometheus. } } +// SetupConfig sets up the server config for the API server +// specify isAggregator=true, if the chain is being constructed for kube-aggregator func SetupConfig( scheme *runtime.Scheme, serverConfig *genericapiserver.RecommendedConfig, @@ -114,6 +119,7 @@ func SetupConfig( gvs []schema.GroupVersion, additionalOpenAPIDefGetters []common.GetOpenAPIDefinitions, reg prometheus.Registerer, + apiResourceConfig *serverstorage.ResourceConfig, ) error { serverConfig.AdmissionControl = NewAdmissionFromBuilders(builders) defsGetter := GetOpenAPIDefinitions(builders, additionalOpenAPIDefGetters...) @@ -126,7 +132,7 @@ func SetupConfig( openapinamer.NewDefinitionNamer(scheme, k8sscheme.Scheme)) // Add the custom routes to service discovery - serverConfig.OpenAPIV3Config.PostProcessSpec = getOpenAPIPostProcessor(buildVersion, builders, gvs) + serverConfig.OpenAPIV3Config.PostProcessSpec = getOpenAPIPostProcessor(buildVersion, builders, gvs, apiResourceConfig) serverConfig.OpenAPIV3Config.GetOperationIDAndTagsFromRoute = func(r common.Route) (string, []string, error) { meta := r.Metadata() kind := "" @@ -287,6 +293,7 @@ func InstallAPIs( features featuremgmt.FeatureToggles, dualWriterMetrics *grafanarest.DualWriterMetrics, builderMetrics *BuilderMetrics, + apiResourceConfig *serverstorage.ResourceConfig, ) error { // dual writing is only enabled when the storage type is not legacy. // this is needed to support setting a default RESTOptionsGetter for new APIs that don't @@ -401,34 +408,9 @@ func InstallAPIs( for group, buildersForGroup := range buildersGroupMap { g := genericapiserver.NewDefaultAPIGroupInfo(group, scheme, metav1.ParameterCodec, codecs) for _, b := range buildersForGroup { - if err := b.UpdateAPIGroupInfo(&g, APIGroupOptions{ - Scheme: scheme, - OptsGetter: optsGetter, - DualWriteBuilder: dualWrite, - MetricsRegister: reg, - StorageOptsRegister: optsregister, - StorageOpts: storageOpts, - }); err != nil { + if err := installAPIGroupsForBuilder(&g, group, b, apiResourceConfig, scheme, optsGetter, dualWrite, reg, optsregister, storageOpts, features); err != nil { return err } - if len(g.PrioritizedVersions) < 1 { - continue - } - - // if grafanaAPIServerWithExperimentalAPIs is not enabled, remove v0alpha1 resources unless explicitly allowed - //nolint:staticcheck // not yet migrated to OpenFeature - if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { - if resources, ok := g.VersionedResourcesStorageMap["v0alpha1"]; ok { - for name := range resources { - if !allowRegisteringResourceByInfo(b.AllowedV0Alpha1Resources(), name) { - delete(resources, name) - } - } - if len(resources) == 0 { - delete(g.VersionedResourcesStorageMap, "v0alpha1") - } - } - } } // skip installing the group if there are no resources left after filtering @@ -445,6 +427,53 @@ func InstallAPIs( return nil } +func installAPIGroupsForBuilder(g *genericapiserver.APIGroupInfo, group string, b APIGroupBuilder, apiResourceConfig *serverstorage.ResourceConfig, scheme *runtime.Scheme, + optsGetter generic.RESTOptionsGetter, dualWrite grafanarest.DualWriteBuilder, reg prometheus.Registerer, optsregister apistore.StorageOptionsRegister, + storageOpts *options.StorageOptions, features featuremgmt.FeatureToggles) error { + if err := b.UpdateAPIGroupInfo(g, APIGroupOptions{ + Scheme: scheme, + OptsGetter: optsGetter, + DualWriteBuilder: dualWrite, + MetricsRegister: reg, + StorageOptsRegister: optsregister, + StorageOpts: storageOpts, + }); err != nil { + return err + } + if len(g.PrioritizedVersions) < 1 { + return nil + } + + // filter out api groups that are disabled in APIEnablementOptions + for version := range g.VersionedResourcesStorageMap { + gvr := schema.GroupVersionResource{ + Group: group, + Version: version, + } + if apiResourceConfig != nil && !apiResourceConfig.ResourceEnabled(gvr) { + klog.InfoS("Skipping storage for disabled resource", "gvr", gvr.String()) + delete(g.VersionedResourcesStorageMap, version) + } + } + + // if grafanaAPIServerWithExperimentalAPIs is not enabled, remove v0alpha1 resources unless explicitly allowed + //nolint:staticcheck // not yet migrated to OpenFeature + if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { + if resources, ok := g.VersionedResourcesStorageMap["v0alpha1"]; ok { + for name := range resources { + if !allowRegisteringResourceByInfo(b.AllowedV0Alpha1Resources(), name) { + delete(resources, name) + } + } + if len(resources) == 0 { + delete(g.VersionedResourcesStorageMap, "v0alpha1") + } + } + } + + return nil +} + // AddPostStartHooks adds post start hooks to a generic API server config func AddPostStartHooks( config *genericapiserver.RecommendedConfig, diff --git a/pkg/services/apiserver/builder/openapi.go b/pkg/services/apiserver/builder/openapi.go index 9cb7cc17f7c..6d3b33f4baf 100644 --- a/pkg/services/apiserver/builder/openapi.go +++ b/pkg/services/apiserver/builder/openapi.go @@ -9,6 +9,8 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/runtime/schema" + serverstorage "k8s.io/apiserver/pkg/server/storage" + "k8s.io/klog/v2" openapi "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/spec3" spec "k8s.io/kube-openapi/pkg/validation/spec" @@ -76,6 +78,7 @@ func addBuilderRoutes( targetGroupVersion schema.GroupVersion, openAPISpec *spec3.OpenAPI, apiGroupBuilders []APIGroupBuilder, + apiResourceConfig *serverstorage.ResourceConfig, ) (*spec3.OpenAPI, error) { for _, apiGroupBuilder := range apiGroupBuilders { // Optionally include raw http handlers for all builders @@ -107,12 +110,24 @@ func addBuilderRoutes( } } } + + // filter out api groups that are disabled in APIEnablementOptions + for path := range openAPISpec.Paths.Paths { + if strings.HasPrefix(path, "/apis/"+targetGroupVersion.String()+"/") { + gv := targetGroupVersion.WithResource("") + if apiResourceConfig != nil && !apiResourceConfig.ResourceEnabled(gv) { + klog.InfoS("removing openapi routes for disabled resource", "gv", gv.String()) + delete(openAPISpec.Paths.Paths, path) + } + } + } + return openAPISpec, nil } // Modify the OpenAPI spec to include the additional routes. // nolint:gocyclo -func getOpenAPIPostProcessor(version string, builders []APIGroupBuilder, gvs []schema.GroupVersion) func(*spec3.OpenAPI) (*spec3.OpenAPI, error) { +func getOpenAPIPostProcessor(version string, builders []APIGroupBuilder, gvs []schema.GroupVersion, apiResourceConfig *serverstorage.ResourceConfig) func(*spec3.OpenAPI) (*spec3.OpenAPI, error) { return func(s *spec3.OpenAPI) (*spec3.OpenAPI, error) { if s.Paths == nil { return s, nil @@ -227,7 +242,7 @@ func getOpenAPIPostProcessor(version string, builders []APIGroupBuilder, gvs []s } } } - return addBuilderRoutes(gv, ©, builders) + return addBuilderRoutes(gv, ©, builders, apiResourceConfig) } } return s, nil diff --git a/pkg/services/apiserver/builder/request_handler.go b/pkg/services/apiserver/builder/request_handler.go index 80fbf6d0eaa..50761f1d42c 100644 --- a/pkg/services/apiserver/builder/request_handler.go +++ b/pkg/services/apiserver/builder/request_handler.go @@ -6,7 +6,9 @@ import ( "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus" + serverstorage "k8s.io/apiserver/pkg/server/storage" restclient "k8s.io/client-go/rest" + klog "k8s.io/klog/v2" "k8s.io/kube-openapi/pkg/spec3" ) @@ -14,7 +16,7 @@ type requestHandler struct { router *mux.Router } -func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient.Config, builders []APIGroupBuilder, metricsRegistry prometheus.Registerer) (http.Handler, error) { +func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient.Config, builders []APIGroupBuilder, metricsRegistry prometheus.Registerer, apiResourceConfig *serverstorage.ResourceConfig) (http.Handler, error) { useful := false // only true if any routes exist anywhere router := mux.NewRouter() @@ -27,6 +29,12 @@ func GetCustomRoutesHandler(delegateHandler http.Handler, restConfig *restclient } for _, gv := range GetGroupVersions(builder) { + // filter out api groups that are disabled in APIEnablementOptions + gvr := gv.WithResource("") + if apiResourceConfig != nil && !apiResourceConfig.ResourceEnabled(gvr) { + klog.InfoS("Skipping custom route handler for disabled group version", "gv", gv.String()) + continue + } routes := provider.GetAPIRoutes(gv) if routes == nil { continue diff --git a/pkg/services/apiserver/service.go b/pkg/services/apiserver/service.go index 28da416536c..605016c112d 100644 --- a/pkg/services/apiserver/service.go +++ b/pkg/services/apiserver/service.go @@ -316,7 +316,11 @@ func (s *service) start(ctx context.Context) error { s.cfg.BuildBranch, ) - if err := o.APIEnablementOptions.ApplyTo(&serverConfig.Config, appinstaller.NewAPIResourceConfig(s.appInstallers), s.scheme); err != nil { + apiResourceConfig := appinstaller.NewAPIResourceConfig(s.appInstallers) + // add the builder group versions to the api resource config + apiResourceConfig.EnableVersions(groupVersions...) + + if err := o.APIEnablementOptions.ApplyTo(&serverConfig.Config, apiResourceConfig, s.scheme); err != nil { return err } @@ -359,6 +363,7 @@ func (s *service) start(ctx context.Context) error { groupVersions, defGetters, s.metrics, + apiResourceConfig, ) if err != nil { return err @@ -400,6 +405,7 @@ func (s *service) start(ctx context.Context) error { s.features, s.dualWriterMetrics, s.builderMetrics, + apiResourceConfig, ) if err != nil { return err