Storage: Unified Storage based on Entity API (#71977)

* first round of entityapi updates - quote column names and clean up insert/update queries - replace grn with guid - streamline table structure fixes streamline entity history move EntitySummary into proto remove EntitySummary add guid to json fix tests change DB_Uuid to DB_NVarchar fix folder test convert interface to any more cleanup start entity store under grafana-apiserver dskit target CRUD working, kind of rough cut of wiring entity api to kube-apiserver fake grafana user in context add key to entity list working revert unnecessary changes move entity storage files to their own package, clean up use accessor to read/write grafana annotations implement separate Create and Update functions * go mod tidy * switch from Kind to resource * basic grpc storage server * basic support for grpc entity store * don't connect to database unless it's needed, pass user identity over grpc * support getting user from k8s context, fix some mysql issues * assign owner to snowflake dependency * switch from ulid to uuid for guids * cleanup, rename Search to List * remove entityListResult * EntityAPI: remove extra user abstraction (#79033) * remove extra user abstraction * add test stub (but * move grpc context setup into client wrapper, fix lint issue * remove unused constants * remove custom json stuff * basic list filtering, add todo * change target to storage-server, allow entityStore flag in prod mode * fix issue with Update * EntityAPI: make test work, need to resolve expected differences (#79123) * make test work, need to resolve expected differences * remove the fields not supported by legacy * sanitize out the bits legacy does not support * sanitize out the bits legacy does not support --------- Co-authored-by: Ryan McKinley <ryantxu@gmail.com> * update feature toggle generated files * remove unused http headers * update feature flag strategy * devmode * update readme * spelling * readme --------- Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
2023-12-06 15:21:21 -05:00
parent 07915703fe
commit c4c9bfaf2e
42 changed files with 4357 additions and 2388 deletions
@@ -0,0 +1,153 @@
+package db
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"xorm.io/xorm"
+
+	"github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+
+	// "github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/services/sqlstore/session"
+	"github.com/grafana/grafana/pkg/services/store/entity/migrations"
+	"github.com/grafana/grafana/pkg/services/store/entity/sqlstash"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/util"
+)
+
+var _ sqlstash.EntityDB = (*EntityDB)(nil)
+
+func ProvideEntityDB(db db.DB, cfg *setting.Cfg, features featuremgmt.FeatureToggles) (*EntityDB, error) {
+	return &EntityDB{
+		db:       db,
+		cfg:      cfg,
+		features: features,
+	}, nil
+}
+
+type EntityDB struct {
+	db       db.DB
+	features featuremgmt.FeatureToggles
+	engine   *xorm.Engine
+	cfg      *setting.Cfg
+}
+
+func (db *EntityDB) Init() error {
+	_, err := db.GetEngine()
+	return err
+}
+
+func (db *EntityDB) GetEngine() (*xorm.Engine, error) {
+	if db.engine != nil {
+		return db.engine, nil
+	}
+
+	var engine *xorm.Engine
+	var err error
+
+	cfgSection := db.cfg.SectionWithEnvOverrides("entity_api")
+	dbType := cfgSection.Key("db_type").MustString("")
+
+	// if explicit connection settings are provided, use them
+	if dbType != "" {
+		dbHost := cfgSection.Key("db_host").MustString("")
+		dbName := cfgSection.Key("db_name").MustString("")
+		dbUser := cfgSection.Key("db_user").MustString("")
+		dbPass := cfgSection.Key("db_pass").MustString("")
+
+		if dbType == "postgres" {
+			// TODO: support all postgres connection options
+			dbSslMode := cfgSection.Key("db_sslmode").MustString("disable")
+
+			addr, err := util.SplitHostPortDefault(dbHost, "127.0.0.1", "5432")
+			if err != nil {
+				return nil, fmt.Errorf("invalid host specifier '%s': %w", dbHost, err)
+			}
+
+			connectionString := fmt.Sprintf(
+				"user=%s password=%s host=%s port=%s dbname=%s sslmode=%s", // sslcert=%s sslkey=%s sslrootcert=%s",
+				dbUser, dbPass, addr.Host, addr.Port, dbName, dbSslMode, // ss.dbCfg.ClientCertPath, ss.dbCfg.ClientKeyPath, ss.dbCfg.CaCertPath
+			)
+
+			engine, err = xorm.NewEngine("postgres", connectionString)
+			if err != nil {
+				return nil, err
+			}
+
+			_, err = engine.Exec("SET SESSION enable_experimental_alter_column_type_general=true")
+			if err != nil {
+				return nil, err
+			}
+		} else if dbType == "mysql" {
+			// TODO: support all mysql connection options
+			protocol := "tcp"
+			if strings.HasPrefix(dbHost, "/") {
+				protocol = "unix"
+			}
+
+			connectionString := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&allowNativePasswords=true&clientFoundRows=true",
+				dbUser, dbPass, protocol, dbHost, dbName)
+
+			engine, err = xorm.NewEngine("mysql", connectionString)
+			if err != nil {
+				return nil, err
+			}
+
+			engine.SetMaxOpenConns(0)
+			engine.SetMaxIdleConns(2)
+			engine.SetConnMaxLifetime(time.Second * time.Duration(14400))
+
+			_, err = engine.Exec("SELECT 1")
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			// TODO: sqlite support
+			return nil, fmt.Errorf("invalid db type specified: %s", dbType)
+		}
+
+		// configure sql logging
+		debugSQL := cfgSection.Key("log_queries").MustBool(false)
+		if !debugSQL {
+			engine.SetLogger(&xorm.DiscardLogger{})
+		} else {
+			// add stack to database calls to be able to see what repository initiated queries. Top 7 items from the stack as they are likely in the xorm library.
+			// engine.SetLogger(sqlstore.NewXormLogger(log.LvlInfo, log.WithSuffix(log.New("sqlstore.xorm"), log.CallerContextKey, log.StackCaller(log.DefaultCallerDepth))))
+			engine.ShowSQL(true)
+			engine.ShowExecTime(true)
+		}
+		// otherwise, try to use the grafana db connection
+	} else {
+		if db.db == nil {
+			return nil, fmt.Errorf("no db connection provided")
+		}
+
+		engine = db.db.GetEngine()
+	}
+
+	db.engine = engine
+
+	if err := migrations.MigrateEntityStore(db, db.features); err != nil {
+		db.engine = nil
+		return nil, err
+	}
+
+	return db.engine, nil
+}
+
+func (db *EntityDB) GetSession() (*session.SessionDB, error) {
+	engine, err := db.GetEngine()
+	if err != nil {
+		return nil, err
+	}
+
+	return session.GetSession(sqlx.NewDb(engine.DB().DB, engine.DriverName())), nil
+}
+
+func (db *EntityDB) GetCfg() *setting.Cfg {
+	return db.cfg
+}