diff --git a/go.mod b/go.mod index c74c5a61c0a..eb285ab9c67 100644 --- a/go.mod +++ b/go.mod @@ -275,11 +275,11 @@ require ( ) require ( - github.com/grafana/grafana-apiserver v0.0.0-20230713001719-88a9ed41992d // @grafana/grafana-app-platform-squad go.opentelemetry.io/otel v1.19.0 // @grafana/backend-platform k8s.io/apimachinery v0.27.1 // @grafana/grafana-app-platform-squad k8s.io/apiserver v0.27.1 // @grafana/grafana-app-platform-squad k8s.io/client-go v0.27.1 // @grafana/grafana-app-platform-squad + k8s.io/component-base v0.27.1 // @grafana/grafana-app-platform-squad k8s.io/klog/v2 v2.90.1 // @grafana/grafana-app-platform-squad k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // @grafana/grafana-app-platform-squad ) @@ -411,7 +411,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect k8s.io/api v0.27.1 // indirect - k8s.io/component-base v0.27.1 // indirect k8s.io/kms v0.27.1 // indirect lukechampine.com/uint128 v1.2.0 // indirect modernc.org/cc/v3 v3.40.0 // indirect diff --git a/go.sum b/go.sum index 750fb55ec7b..fd61c367c7e 100644 --- a/go.sum +++ b/go.sum @@ -1812,8 +1812,6 @@ github.com/grafana/dskit v0.0.0-20230706162620-5081d8ed53e6 h1:/19OPOCKP95g9hKLn github.com/grafana/dskit v0.0.0-20230706162620-5081d8ed53e6/go.mod h1:M03k2fzuQ2n9TVE1xfVKTESibxsXdw0wYfWT3+9Owp4= github.com/grafana/gofpdf v0.0.0-20231002120153-857cc45be447 h1:jxJJ5z0GxqhWFbQUsys3BHG8jnmniJ2Q74tXAG1NaDo= github.com/grafana/gofpdf v0.0.0-20231002120153-857cc45be447/go.mod h1:IxsY6mns6Q5sAnWcrptrgUrSglTZJXH/kXr9nbpb/9I= -github.com/grafana/grafana-apiserver v0.0.0-20230713001719-88a9ed41992d h1:fjc6vlNnKCDewr/5GiObiiGLeqr6YnXUbZ44YZtlu5Q= -github.com/grafana/grafana-apiserver v0.0.0-20230713001719-88a9ed41992d/go.mod h1:2g9qGdCeU6x/69QAs82WM52bwBUT5/CBaBD0I94+txU= github.com/grafana/grafana-aws-sdk v0.19.1 h1:5GBiOv2AgdyjwlgAX+dtgPtXU4FgMTD9rfQUPQseEpQ= github.com/grafana/grafana-aws-sdk v0.19.1/go.mod h1:ntq2NDH12Y2Fkbc6fozpF8kYsJM9k6KNr+Xfo5w3/iM= github.com/grafana/grafana-azure-sdk-go v1.9.0 h1:4JRwlqgUtPRAQSoiV4DFZDQ3lbNsauHqj9kC6SMR9Ak= diff --git a/pkg/services/grafana-apiserver/README.md b/pkg/services/grafana-apiserver/README.md index 2b882b0513f..8beebc853e9 100644 --- a/pkg/services/grafana-apiserver/README.md +++ b/pkg/services/grafana-apiserver/README.md @@ -23,7 +23,7 @@ Start `etcd`: make devenv sources=etcd ``` -Enable dual write to `etcd`: +Add etcd server to `custom.ini`: ```ini [grafana-apiserver] @@ -32,10 +32,9 @@ etcd_servers = 127.0.0.1:2379 ### `kubectl` access -From the root of the repository: - +From the root of the Grafanaa repository, run the following: ```bash -export KUBECONFIG=$PWD/data/k8s/grafana.kubeconfig +export KUBECONFIG=$PWD/data/grafana-apiserver/grafana.kubeconfig kubectl api-resources ``` diff --git a/pkg/services/grafana-apiserver/config.go b/pkg/services/grafana-apiserver/config.go index 3287718c971..75916af55e5 100644 --- a/pkg/services/grafana-apiserver/config.go +++ b/pkg/services/grafana-apiserver/config.go @@ -2,7 +2,9 @@ package grafanaapiserver import ( "fmt" + "net" "path/filepath" + "strconv" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" @@ -12,8 +14,11 @@ type config struct { enabled bool devMode bool - host string - appURL string + ip net.IP + port int + host string + apiURL string + etcdServers []string dataPath string @@ -22,17 +27,31 @@ type config struct { func newConfig(cfg *setting.Cfg) *config { defaultLogLevel := 0 + ip := net.ParseIP(cfg.HTTPAddr) + apiURL := cfg.AppURL + port, err := strconv.Atoi(cfg.HTTPPort) + if err != nil { + port = 3000 + } + if cfg.Env == setting.Dev { defaultLogLevel = 10 + port = 6443 + ip = net.ParseIP("127.0.0.1") + apiURL = fmt.Sprintf("https://%s:%d", ip, port) } + host := fmt.Sprintf("%s:%d", ip, port) + return &config{ enabled: cfg.IsFeatureToggleEnabled(featuremgmt.FlagGrafanaAPIServer), devMode: cfg.Env == setting.Dev, dataPath: filepath.Join(cfg.DataPath, "grafana-apiserver"), - host: fmt.Sprintf("%s:%s", cfg.HTTPAddr, cfg.HTTPPort), + ip: ip, + port: port, + host: host, logLevel: cfg.SectionWithEnvOverrides("grafana-apiserver").Key("log_level").MustInt(defaultLogLevel), etcdServers: cfg.SectionWithEnvOverrides("grafana-apiserver").Key("etcd_servers").Strings(","), - appURL: cfg.AppURL, + apiURL: apiURL, } } diff --git a/pkg/services/grafana-apiserver/config_test.go b/pkg/services/grafana-apiserver/config_test.go index dfd4e68ca1c..aa0eb211d25 100644 --- a/pkg/services/grafana-apiserver/config_test.go +++ b/pkg/services/grafana-apiserver/config_test.go @@ -1,6 +1,7 @@ package grafanaapiserver import ( + "net" "testing" "github.com/stretchr/testify/require" @@ -10,9 +11,9 @@ import ( func TestNewConfig(t *testing.T) { cfg := setting.NewCfg() - cfg.Env = setting.Dev + cfg.Env = setting.Prod cfg.DataPath = "/tmp/grafana" - cfg.HTTPAddr = "test" + cfg.HTTPAddr = "10.0.0.1" cfg.HTTPPort = "4000" cfg.IsFeatureToggleEnabled = func(_ string) bool { return true } cfg.AppURL = "http://test:4000" @@ -25,10 +26,12 @@ func TestNewConfig(t *testing.T) { expected := &config{ enabled: true, - devMode: true, + devMode: false, etcdServers: []string{"http://localhost:2379"}, - appURL: "http://test:4000", - host: "test:4000", + apiURL: "http://test:4000", + ip: net.ParseIP("10.0.0.1"), + port: 4000, + host: "10.0.0.1:4000", dataPath: "/tmp/grafana/grafana-apiserver", logLevel: 5, } diff --git a/pkg/services/grafana-apiserver/log.go b/pkg/services/grafana-apiserver/log.go index ff1e6705565..edf10c0d3d3 100644 --- a/pkg/services/grafana-apiserver/log.go +++ b/pkg/services/grafana-apiserver/log.go @@ -1,7 +1,8 @@ package grafanaapiserver import ( - "cuelang.org/go/pkg/strings" + "strings" + "github.com/go-logr/logr" "github.com/grafana/grafana/pkg/infra/log" diff --git a/pkg/services/grafana-apiserver/service.go b/pkg/services/grafana-apiserver/service.go index 9679cd05ef6..9a6bdac055b 100644 --- a/pkg/services/grafana-apiserver/service.go +++ b/pkg/services/grafana-apiserver/service.go @@ -2,23 +2,18 @@ package grafanaapiserver import ( "context" - "crypto/x509" "fmt" - "net" "net/http" "path" "strconv" "github.com/go-logr/logr" "github.com/grafana/dskit/services" - "github.com/grafana/grafana-apiserver/pkg/certgenerator" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apiserver/pkg/authentication/authenticator" - "k8s.io/apiserver/pkg/authentication/request/headerrequest" "k8s.io/apiserver/pkg/authorization/authorizer" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" "k8s.io/apiserver/pkg/endpoints/responsewriter" @@ -29,6 +24,7 @@ import ( clientrest "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/component-base/logs" "k8s.io/klog/v2" "github.com/grafana/grafana/pkg/api/routing" @@ -41,18 +37,12 @@ import ( "github.com/grafana/grafana/pkg/web" ) -const ( - DefaultAPIServerHost = "https://" + certgenerator.DefaultAPIServerIp + ":6443" -) - var ( _ Service = (*service)(nil) _ RestConfigProvider = (*service)(nil) _ registry.BackgroundService = (*service)(nil) _ registry.CanBeDisabled = (*service)(nil) -) -var ( Scheme = runtime.NewScheme() Codecs = serializer.NewCodecFactory(Scheme) @@ -169,9 +159,13 @@ func (s *service) RegisterAPI(builder APIGroupBuilder) { func (s *service) start(ctx context.Context) error { logger := logr.New(newLogAdapter(s.config.logLevel)) klog.SetLoggerWithOptions(logger, klog.ContextualLogger(true)) + if _, err := logs.GlogSetter(strconv.Itoa(s.config.logLevel)); err != nil { + logger.Error(err, "failed to set log level") + } o := options.NewRecommendedOptions("", unstructured.UnstructuredJSONScheme) - o.SecureServing.BindPort = 6443 + o.SecureServing.BindAddress = s.config.ip + o.SecureServing.BindPort = s.config.port o.Authentication.RemoteKubeConfigFileOptional = true o.Authorization.RemoteKubeConfigFileOptional = true o.Etcd.StorageConfig.Transport.ServerList = s.config.etcdServers @@ -182,55 +176,52 @@ func (s *service) start(ctx context.Context) error { o.Etcd = nil } - // Get the util to get the paths to pre-generated certs - certUtil := certgenerator.CertUtil{ - K8sDataPath: s.config.dataPath, - } - - if err := certUtil.InitializeCACertPKI(); err != nil { - return err - } - - if err := certUtil.EnsureApiServerPKI(certgenerator.DefaultAPIServerIp); err != nil { - return err - } - - o.SecureServing.BindAddress = net.ParseIP(certgenerator.DefaultAPIServerIp) - o.SecureServing.ServerCert.CertKey = options.CertKey{ - CertFile: certUtil.APIServerCertFile(), - KeyFile: certUtil.APIServerKeyFile(), - } - if err := o.Validate(); len(err) > 0 { return err[0] } serverConfig := genericapiserver.NewRecommendedConfig(Codecs) - err := o.ApplyTo(serverConfig) - if err != nil { - return err + serverConfig.ExternalAddress = s.config.host + + if s.config.devMode { + // SecureServingOptions is used when the apiserver needs it's own listener. + // this is not needed in production, but it's useful for development kubectl access. + if err := o.SecureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil { + return err + } + // AuthenticationOptions is needed to authenticate requests from kubectl in dev mode. + if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, serverConfig.OpenAPIConfig); err != nil { + return err + } } - rootCert, err := certUtil.GetK8sCACert() - if err != nil { - return err + // override ExternalAddress and LoopbackClientConfig in prod mode. + // in dev mode we want to use the loopback client config + // and address provided by SecureServingOptions. + if !s.config.devMode { + serverConfig.ExternalAddress = s.config.host + serverConfig.LoopbackClientConfig = &clientrest.Config{ + Host: s.config.apiURL, + TLSClientConfig: clientrest.TLSClientConfig{ + Insecure: true, + }, + } } - authenticator, err := newAuthenticator(rootCert) - if err != nil { - return err + if o.Etcd != nil { + if err := o.Etcd.ApplyTo(&serverConfig.Config); err != nil { + return err + } } serverConfig.Authorization.Authorizer = s.authorizer - serverConfig.Authentication.Authenticator = authenticator // Get the list of groups the server will support builders := s.builders // Install schemas for _, b := range builders { - err = b.InstallSchema(Scheme) // previously was in init - if err != nil { + if err := b.InstallSchema(Scheme); err != nil { return err } } @@ -282,12 +273,13 @@ func (s *service) start(ctx context.Context) error { } s.restConfig = server.LoopbackClientConfig - err = s.writeKubeConfiguration(s.restConfig) - if err != nil { - return err - } - prepared := server.PrepareRun() + // only write kubeconfig in dev mode + if s.config.devMode { + if err := s.ensureKubeConfig(); err != nil { + return err + } + } // TODO: this is a hack. see note in ProvideService s.handler = func(c *contextmodel.ReqContext) { @@ -308,17 +300,28 @@ func (s *service) start(ctx context.Context) error { req.Header.Set("X-Remote-Group", "grafana") resp := responsewriter.WrapForHTTP1Or2(c.Resp) - prepared.GenericAPIServer.Handler.ServeHTTP(resp, req) + server.Handler.ServeHTTP(resp, req) } + // skip starting the server in prod mode + if !s.config.devMode { + return nil + } + + prepared := server.PrepareRun() go func() { s.stoppedCh <- prepared.Run(s.stopCh) }() - return nil } func (s *service) running(ctx context.Context) error { + // skip waiting for the server in prod mode + if !s.config.devMode { + <-ctx.Done() + return nil + } + select { case err := <-s.stoppedCh: if err != nil { @@ -330,10 +333,10 @@ func (s *service) running(ctx context.Context) error { return nil } -func (s *service) writeKubeConfiguration(restConfig *clientrest.Config) error { +func (s *service) ensureKubeConfig() error { clusters := make(map[string]*clientcmdapi.Cluster) clusters["default-cluster"] = &clientcmdapi.Cluster{ - Server: restConfig.Host, + Server: s.restConfig.Host, InsecureSkipTLSVerify: true, } @@ -346,7 +349,7 @@ func (s *service) writeKubeConfiguration(restConfig *clientrest.Config) error { authinfos := make(map[string]*clientcmdapi.AuthInfo) authinfos["default"] = &clientcmdapi.AuthInfo{ - Token: restConfig.BearerToken, + Token: s.restConfig.BearerToken, } clientConfig := clientcmdapi.Config{ @@ -357,24 +360,6 @@ func (s *service) writeKubeConfiguration(restConfig *clientrest.Config) error { CurrentContext: "default-context", AuthInfos: authinfos, } + return clientcmd.WriteToFile(clientConfig, path.Join(s.config.dataPath, "grafana.kubeconfig")) } - -func newAuthenticator(cert *x509.Certificate) (authenticator.Request, error) { - reqHeaderOptions := options.RequestHeaderAuthenticationOptions{ - UsernameHeaders: []string{"X-Remote-User"}, - GroupHeaders: []string{"X-Remote-Group"}, - ExtraHeaderPrefixes: []string{"X-Remote-Extra-"}, - } - - requestHeaderAuthenticator, err := headerrequest.New( - reqHeaderOptions.UsernameHeaders, - reqHeaderOptions.GroupHeaders, - reqHeaderOptions.ExtraHeaderPrefixes, - ) - if err != nil { - return nil, err - } - - return requestHeaderAuthenticator, nil -}