API Server: Standalone observability (#84789)

Adds support for logs (specify level), metrics (enable metrics and Prometheus /metrics endpoint 
and traces (jaeger or otlp) for standalone API server. This will allow any grafana core service 
part of standalone apiserver to use logging, metrics and traces as normal.

pkg/cmd/grafana-cli/commands/conflict_user_command.go

@@ -79,7 +79,12 @@ func initializeConflictResolver(cmd *utils.ContextCommandLine, f Formatter, ctx
 }
 
 func getSqlStore(cfg *setting.Cfg, features featuremgmt.FeatureToggles) (*sqlstore.SQLStore, error) {
-	tracer, err := tracing.ProvideService(cfg)
+	tracingCfg, err := tracing.ProvideTracingConfig(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("%v: %w", "failed to initialize tracer config", err)
+	}
+
+	tracer, err := tracing.ProvideService(tracingCfg)
 	if err != nil {
 		return nil, fmt.Errorf("%v: %w", "failed to initialize tracer service", err)
 	}

pkg/cmd/grafana-server/commands/buildinfo.go

@@ -16,7 +16,7 @@ func getBuildstamp(opts ServerOptions) int64 {
 	return buildstampInt64
 }
 
-func setBuildInfo(opts ServerOptions) {
+func SetBuildInfo(opts ServerOptions) {
 	setting.BuildVersion = opts.Version
 	setting.BuildCommit = opts.Commit
 	setting.EnterpriseBuildCommit = opts.EnterpriseCommit

pkg/cmd/grafana-server/commands/cli.go

@@ -98,7 +98,7 @@ func RunServer(opts ServerOptions) error {
 		}
 	}()
 
-	setBuildInfo(opts)
+	SetBuildInfo(opts)
 	checkPrivileges()
 
 	configOptions := strings.Split(ConfigOverrides, " ")
@@ -112,7 +112,7 @@ func RunServer(opts ServerOptions) error {
 		return err
 	}
 
-	metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
+	metrics.SetBuildInformation(metrics.ProvideRegisterer(), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
 
 	s, err := server.Initialize(
 		cfg,

pkg/cmd/grafana-server/commands/target.go

@@ -75,7 +75,7 @@ func RunTargetServer(opts ServerOptions) error {
 		}
 	}()
 
-	setBuildInfo(opts)
+	SetBuildInfo(opts)
 	checkPrivileges()
 
 	configOptions := strings.Split(ConfigOverrides, " ")
@@ -89,7 +89,7 @@ func RunTargetServer(opts ServerOptions) error {
 		return err
 	}
 
-	metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
+	metrics.SetBuildInformation(metrics.ProvideRegisterer(), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
 
 	s, err := server.InitializeModuleServer(
 		cfg,

pkg/cmd/grafana/apiserver/apiserver.md

@@ -1,6 +1,6 @@
 # grafana apiserver (standalone)
 
-The example-apiserver closely resembles the 
+The example-apiserver closely resembles the
 [sample-apiserver](https://github.com/kubernetes/sample-apiserver/tree/master) project in code and thus
 allows the same
 [CLI flags](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) as kube-apiserver.
@@ -32,3 +32,25 @@ dummy                  example.grafana.app/v0alpha1   true         DummyResource
 runtime                example.grafana.app/v0alpha1   false        RuntimeInfo
 ```
 
+### Observability
+
+Logs, metrics and traces are supported. See `--grafana.log.*`, `--grafana.metrics.*` and `--grafana.tracing.*` flags for details.
+
+```shell
+go run ./pkg/cmd/grafana apiserver \
+  --runtime-config=example.grafana.app/v0alpha1=true \
+  --help
+```
+
+For example, to enable debug logs, metrics and traces (using [self-instrumentation](../../../../devenv/docker/blocks/self-instrumentation/readme.md)) use the following:
+
+```shell
+go run ./pkg/cmd/grafana apiserver \
+  --runtime-config=example.grafana.app/v0alpha1=true \
+  --secure-port=7443 \
+  --grafana.log.level=debug \
+  --verbosity=10 \
+  --grafana.metrics.enable \
+  --grafana.tracing.jaeger.address=http://localhost:14268/api/traces \
+  --grafana.tracing.sampler-param=1
+```

pkg/cmd/grafana/apiserver/cmd.go

@@ -7,6 +7,7 @@ import (
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/component-base/cli"
 
+	"github.com/grafana/grafana/pkg/cmd/grafana-server/commands"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/server"
 	"github.com/grafana/grafana/pkg/services/apiserver/standalone"
@@ -64,27 +65,23 @@ func newCommandStartExampleAPIServer(o *APIServerOptions, stopCh <-chan struct{}
 			if err := o.RunAPIServer(config, stopCh); err != nil {
 				return err
 			}
+
 			return nil
 		},
 	}
 
 	cmd.Flags().StringVar(&runtimeConfig, "runtime-config", "", "A set of key=value pairs that enable or disable built-in APIs.")
 
-	if factoryOptions := o.factory.GetOptions(); factoryOptions != nil {
-		factoryOptions.AddFlags(cmd.Flags())
-	}
-
-	o.ExtraOptions.AddFlags(cmd.Flags())
-
-	// Register standard k8s flags with the command line
-	o.RecommendedOptions.AddFlags(cmd.Flags())
+	o.AddFlags(cmd.Flags())
 
 	return cmd
 }
 
-func RunCLI() int {
+func RunCLI(opts commands.ServerOptions) int {
 	stopCh := genericapiserver.SetupSignalHandler()
 
+	commands.SetBuildInfo(opts)
+
 	options := newAPIServerOptions(os.Stdout, os.Stderr)
 	cmd := newCommandStartExampleAPIServer(options, stopCh)
 

pkg/cmd/grafana/apiserver/server.go

@@ -9,16 +9,17 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	genericapiserver "k8s.io/apiserver/pkg/server"
-	"k8s.io/apiserver/pkg/server/options"
 	"k8s.io/client-go/tools/clientcmd"
 	netutils "k8s.io/utils/net"
 
 	"github.com/grafana/grafana/pkg/apiserver/builder"
+	"github.com/grafana/grafana/pkg/infra/log"
 	grafanaAPIServer "github.com/grafana/grafana/pkg/services/apiserver"
-	grafanaAPIServerOptions "github.com/grafana/grafana/pkg/services/apiserver/options"
 	"github.com/grafana/grafana/pkg/services/apiserver/standalone"
+	standaloneoptions "github.com/grafana/grafana/pkg/services/apiserver/standalone/options"
 	"github.com/grafana/grafana/pkg/services/apiserver/utils"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/spf13/pflag"
 )
 
 const (
@@ -28,25 +29,24 @@ const (
 
 // APIServerOptions contains the state for the apiserver
 type APIServerOptions struct {
-	factory            standalone.APIServerFactory
-	builders           []builder.APIGroupBuilder
-	ExtraOptions       *grafanaAPIServerOptions.ExtraOptions
-	RecommendedOptions *options.RecommendedOptions
-	AlternateDNS       []string
+	factory      standalone.APIServerFactory
+	builders     []builder.APIGroupBuilder
+	Options      *standaloneoptions.Options
+	AlternateDNS []string
+	logger       log.Logger
 
 	StdOut io.Writer
 	StdErr io.Writer
 }
 
 func newAPIServerOptions(out, errOut io.Writer) *APIServerOptions {
+	logger := log.New("grafana-apiserver")
+
 	return &APIServerOptions{
-		StdOut: out,
-		StdErr: errOut,
-		RecommendedOptions: options.NewRecommendedOptions(
-			defaultEtcdPathPrefix,
-			grafanaAPIServer.Codecs.LegacyCodec(), // the codec is passed to etcd and not used
-		),
-		ExtraOptions: grafanaAPIServerOptions.NewExtraOptions(),
+		logger:  logger,
+		StdOut:  out,
+		StdErr:  errOut,
+		Options: standaloneoptions.New(logger, grafanaAPIServer.Codecs.LegacyCodec()),
 	}
 }
 
@@ -73,92 +73,31 @@ func (o *APIServerOptions) loadAPIGroupBuilders(apis []schema.GroupVersion) erro
 	return nil
 }
 
-// A copy of ApplyTo in recommended.go, but for >= 0.28, server pkg in apiserver does a bit extra causing
-// a panic when CoreAPI is set to nil
-func (o *APIServerOptions) ModifiedApplyTo(config *genericapiserver.RecommendedConfig) error {
-	if err := o.RecommendedOptions.Etcd.ApplyTo(&config.Config); err != nil {
-		return err
-	}
-	if err := o.RecommendedOptions.EgressSelector.ApplyTo(&config.Config); err != nil {
-		return err
-	}
-	if err := o.RecommendedOptions.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil {
-		return err
-	}
-	if err := o.RecommendedOptions.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil {
-		return err
-	}
-	if err := o.RecommendedOptions.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil {
-		return err
-	}
-	if err := o.RecommendedOptions.Authorization.ApplyTo(&config.Config.Authorization); err != nil {
-		return err
-	}
-	if err := o.RecommendedOptions.Audit.ApplyTo(&config.Config); err != nil {
-		return err
-	}
-
-	// TODO: determine whether we need flow control (API priority and fairness)
-	// We can't assume that a shared informers config was provided in standalone mode and will need a guard
-	// when enabling below
-	/* kubeClient, err := kubernetes.NewForConfig(config.ClientConfig)
-	if err != nil {
-		return err
-	}
-
-	if err := o.RecommendedOptions.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil {
-		return err
-	} */
-
-	if err := o.RecommendedOptions.CoreAPI.ApplyTo(config); err != nil {
-		return err
-	}
-
-	_, err := o.RecommendedOptions.ExtraAdmissionInitializers(config)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (o *APIServerOptions) Config() (*genericapiserver.RecommendedConfig, error) {
-	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts(
+	if err := o.Options.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts(
 		"localhost", o.AlternateDNS, []net.IP{netutils.ParseIPSloppy("127.0.0.1")},
 	); err != nil {
 		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
 	}
 
-	o.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true
+	o.Options.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true
 
 	// TODO: determine authorization, currently insecure because Authorization provided by recommended options doesn't work
 	// reason: an aggregated server won't be able to post subjectaccessreviews (Grafana doesn't have this kind)
 	// exact error: the server could not find the requested resource (post subjectaccessreviews.authorization.k8s.io)
-	o.RecommendedOptions.Authorization = nil
+	o.Options.RecommendedOptions.Authorization = nil
 
-	o.RecommendedOptions.Admission = nil
-	o.RecommendedOptions.Etcd = nil
+	o.Options.RecommendedOptions.Admission = nil
+	o.Options.RecommendedOptions.Etcd = nil
 
-	if o.RecommendedOptions.CoreAPI.CoreAPIKubeconfigPath == "" {
-		o.RecommendedOptions.CoreAPI = nil
+	if o.Options.RecommendedOptions.CoreAPI.CoreAPIKubeconfigPath == "" {
+		o.Options.RecommendedOptions.CoreAPI = nil
 	}
 
 	serverConfig := genericapiserver.NewRecommendedConfig(grafanaAPIServer.Codecs)
 
-	if o.RecommendedOptions.CoreAPI == nil {
-		if err := o.ModifiedApplyTo(serverConfig); err != nil {
-			return nil, err
-		}
-	} else {
-		if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
-			return nil, err
-		}
-	}
-
-	if o.ExtraOptions != nil {
-		if err := o.ExtraOptions.ApplyTo(serverConfig); err != nil {
-			return nil, err
-		}
+	if err := o.Options.ApplyTo(serverConfig); err != nil {
+		return nil, fmt.Errorf("failed to apply options to server config: %w", err)
 	}
 
 	serverConfig.DisabledPostStartHooks = serverConfig.DisabledPostStartHooks.Insert("generic-apiserver-start-informers")
@@ -177,16 +116,26 @@ func (o *APIServerOptions) Config() (*genericapiserver.RecommendedConfig, error)
 	return serverConfig, err
 }
 
+func (o *APIServerOptions) AddFlags(fs *pflag.FlagSet) {
+	o.Options.AddFlags(fs)
+
+	if factoryOptions := o.factory.GetOptions(); factoryOptions != nil {
+		factoryOptions.AddFlags(fs)
+	}
+}
+
 // Validate validates APIServerOptions
 func (o *APIServerOptions) Validate() error {
 	errors := make([]error, 0)
-	// NOTE: we don't call validate on the top level recommended options as it doesn't like skipping etcd-servers
-	// the function is left here for troubleshooting any other config issues
-	// errors = append(errors, o.RecommendedOptions.Validate()...)
+
 	if factoryOptions := o.factory.GetOptions(); factoryOptions != nil {
 		errors = append(errors, factoryOptions.ValidateOptions()...)
 	}
 
+	if errs := o.Options.Validate(); len(errs) > 0 {
+		errors = append(errors, errors...)
+	}
+
 	return utilerrors.NewAggregate(errors)
 }
 
@@ -211,7 +160,7 @@ func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConf
 	}
 
 	// write the local config to disk
-	if o.ExtraOptions.DevMode {
+	if o.Options.ExtraOptions.DevMode {
 		if err = clientcmd.WriteToFile(
 			utils.FormatKubeConfig(server.LoopbackClientConfig),
 			path.Join(dataPath, "apiserver.kubeconfig"),

pkg/cmd/grafana/main.go

@@ -41,7 +41,14 @@ func main() {
 				SkipFlagParsing: true,
 				Action: func(context *cli.Context) error {
 					// exit here because apiserver handles its own error output
-					os.Exit(apiserver.RunCLI())
+					os.Exit(apiserver.RunCLI(gsrv.ServerOptions{
+						Version:          version,
+						Commit:           commit,
+						EnterpriseCommit: enterpriseCommit,
+						BuildBranch:      buildBranch,
+						BuildStamp:       buildstamp,
+						Context:          context,
+					}))
 					return nil
 				},
 			},