Storage: Unified Storage based on Entity API (#71977)

* first round of entityapi updates

- quote column names and clean up insert/update queries
- replace grn with guid
- streamline table structure

fixes

streamline entity history

move EntitySummary into proto

remove EntitySummary

add guid to json

fix tests

change DB_Uuid to DB_NVarchar

fix folder test

convert interface to any

more cleanup

start entity store under grafana-apiserver dskit target

CRUD working, kind of

rough cut of wiring entity api to kube-apiserver

fake grafana user in context

add key to entity

list working

revert unnecessary changes

move entity storage files to their own package, clean up

use accessor to read/write grafana annotations

implement separate Create and Update functions

* go mod tidy

* switch from Kind to resource

* basic grpc storage server

* basic support for grpc entity store

* don't connect to database unless it's needed, pass user identity over grpc

* support getting user from k8s context, fix some mysql issues

* assign owner to snowflake dependency

* switch from ulid to uuid for guids

* cleanup, rename Search to List

* remove entityListResult

* EntityAPI: remove extra user abstraction (#79033)

* remove extra user abstraction

* add test stub (but

* move grpc context setup into client wrapper, fix lint issue

* remove unused constants

* remove custom json stuff

* basic list filtering, add todo

* change target to storage-server, allow entityStore flag in prod mode

* fix issue with Update

* EntityAPI: make test work, need to resolve expected differences (#79123)

* make test work, need to resolve expected differences

* remove the fields not supported by legacy

* sanitize out the bits legacy does not support

* sanitize out the bits legacy does not support

---------

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

* update feature toggle generated files

* remove unused http headers

* update feature flag strategy

* devmode

* update readme

* spelling

* readme

---------

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

pkg/services/featuremgmt/registry.go

@@ -222,10 +222,11 @@ var (
 			Created:           time.Date(2022, time.September, 27, 12, 0, 0, 0, time.UTC),
 		},
 		{
-			Name:            "entityStore",
-			Description:     "SQL-based entity store (requires storage flag also)",
+			Name:            "unifiedStorage",
+			Description:     "SQL-based k8s storage",
 			Stage:           FeatureStageExperimental,
 			RequiresDevMode: true,
+			RequiresRestart: true, // new SQL tables created
 			Owner:           grafanaAppPlatformSquad,
 			Created:         time.Date(2022, time.December, 1, 12, 0, 0, 0, time.UTC),
 		},
@@ -738,6 +739,15 @@ var (
 			Owner:        grafanaAppPlatformSquad,
 			Created:      time.Date(2023, time.October, 6, 12, 0, 0, 0, time.UTC),
 		},
+		{
+			Name:            "grafanaAPIServerEnsureKubectlAccess",
+			Description:     "Start an additional https handler and write kubectl options",
+			Stage:           FeatureStageExperimental,
+			RequiresDevMode: true,
+			RequiresRestart: true,
+			Owner:           grafanaAppPlatformSquad,
+			Created:         time.Date(2023, time.December, 6, 12, 0, 0, 0, time.UTC),
+		},
 		{
 			Name:            "featureToggleAdminPage",
 			Description:     "Enable admin page for managing feature toggles from the Grafana front-end",

pkg/services/featuremgmt/toggles_gen.csv

@@ -24,7 +24,7 @@ dataConnectionsConsole,GA,@grafana/plugins-platform-backend,2022-06-01,false,fal
 topnav,deprecated,@grafana/grafana-frontend-platform,2022-06-20,false,false,false,false
 dockedMegaMenu,experimental,@grafana/grafana-frontend-platform,2023-09-18,false,false,false,true
 grpcServer,preview,@grafana/grafana-app-platform-squad,2022-09-27,false,false,false,false
-entityStore,experimental,@grafana/grafana-app-platform-squad,2022-12-01,true,false,false,false
+unifiedStorage,experimental,@grafana/grafana-app-platform-squad,2022-12-01,true,false,true,false
 cloudWatchCrossAccountQuerying,GA,@grafana/aws-datasources,2022-11-28,false,false,false,false
 redshiftAsyncQueryDataSupport,GA,@grafana/aws-datasources,2022-08-27,false,false,false,false
 athenaAsyncQueryDataSupport,GA,@grafana/aws-datasources,2022-08-27,false,false,false,true
@@ -86,6 +86,7 @@ traceQLStreaming,experimental,@grafana/observability-traces-and-profiling,2023-0
 metricsSummary,experimental,@grafana/observability-traces-and-profiling,2023-08-28,false,false,false,true
 grafanaAPIServer,experimental,@grafana/grafana-app-platform-squad,2023-07-14,false,false,false,false
 grafanaAPIServerWithExperimentalAPIs,experimental,@grafana/grafana-app-platform-squad,2023-10-06,false,false,false,false
+grafanaAPIServerEnsureKubectlAccess,experimental,@grafana/grafana-app-platform-squad,2023-12-06,true,false,true,false
 featureToggleAdminPage,experimental,@grafana/grafana-operator-experience-squad,2023-07-18,false,false,true,false
 awsAsyncQueryCaching,preview,@grafana/aws-datasources,2023-07-21,false,false,false,false
 splitScopes,preview,@grafana/identity-access-team,2023-07-21,false,false,true,false

pkg/services/featuremgmt/toggles_gen.go

@@ -107,9 +107,9 @@ const (
 	// Run the GRPC server
 	FlagGrpcServer = "grpcServer"
 
-	// FlagEntityStore
-	// SQL-based entity store (requires storage flag also)
-	FlagEntityStore = "entityStore"
+	// FlagUnifiedStorage
+	// SQL-based k8s storage
+	FlagUnifiedStorage = "unifiedStorage"
 
 	// FlagCloudWatchCrossAccountQuerying
 	// Enables cross-account querying in CloudWatch datasources
@@ -355,6 +355,10 @@ const (
 	// Register experimental APIs with the k8s API server
 	FlagGrafanaAPIServerWithExperimentalAPIs = "grafanaAPIServerWithExperimentalAPIs"
 
+	// FlagGrafanaAPIServerEnsureKubectlAccess
+	// Start an additional https handler and write kubectl options
+	FlagGrafanaAPIServerEnsureKubectlAccess = "grafanaAPIServerEnsureKubectlAccess"
+
 	// FlagFeatureToggleAdminPage
 	// Enable admin page for managing feature toggles from the Grafana front-end
 	FlagFeatureToggleAdminPage = "featureToggleAdminPage"

pkg/services/featuremgmt/toggles_gen_test.go

@@ -22,10 +22,7 @@ import (
 
 func TestFeatureToggleFiles(t *testing.T) {
 	legacyNames := map[string]bool{
-		"httpclientprovider_azure_auth": true,
-		"service-accounts":              true,
-		"database_metrics":              true,
-		"live-service-web-worker":       true,
+		"live-service-web-worker": true,
 	}
 
 	t.Run("check registry constraints", func(t *testing.T) {