Unified Storage: Fix Create, Update and Delete wrt Resource Versions (#88183)

* add sqltemplate utilities, improve tests and documentation * bunch of things * remove unnecessary message * add queries * add queries * add queries * add folders support * fix diff * fix linters * fix diff * fix linters * fix linters * fix typo * fix linters * fix linters * fix linters * several fixes * several fixes * temporarily disable k8s integration tests for Entity Server * postpone some tests * postpone documentation changes * Fix bug in create * improve error reporting * fix PostgeSQL parameters * fix MySQL sqlmode * fix MySQL-5.7 * reduce but document the number of database connection options * remove unused code and improve docs
2024-06-05 14:23:32 -03:00
parent cbe7521a56
commit 6fcd7d9e03
43 changed files with 1867 additions and 1298 deletions
@@ -3,13 +3,16 @@ package sqlstash
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"strings"

 	folder "github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
-	"github.com/grafana/grafana/pkg/services/sqlstore/session"
+	"github.com/grafana/grafana/pkg/services/store/entity/db"
+	"github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
 )

 type folderInfo struct {
-	Guid string `json:"guid"`
+	GUID string `json:"guid"`

 	UID      string `json:"uid"`
 	Name     string `json:"name"` // original display name
@@ -23,7 +26,7 @@ type folderInfo struct {
 	right int32

 	// Build the tree
-	parentUID string
+	ParentUID string

 	// Calculated after query
 	parent   *folderInfo
@@ -31,56 +34,101 @@ type folderInfo struct {
 	stack    []*folderInfo
 }

-// This will replace all entries in `entity_folder`
-// This is pretty heavy weight, but it does give us a sorted folder list
-// NOTE: this could be done async with a mutex/lock?  reconciler pattern
-func (s *sqlEntityServer) updateFolderTree(ctx context.Context, tx *session.SessionTx, namespace string) error {
-	_, err := tx.Exec(ctx, "DELETE FROM entity_folder WHERE namespace=?", namespace)
-	if err != nil {
-		return err
+func (fi *folderInfo) buildInsertItems(items *[]*sqlEntityFolderInsertRequestItem, namespace string, isLost bool) error {
+	var js strings.Builder
+	if err := json.NewEncoder(&js).Encode(fi.stack); err != nil {
+		return fmt.Errorf("marshal stack of folder %q to JSON: %w", fi.SlugPath, err)
 	}

-	query := "SELECT guid,name,folder,name,slug" +
-		" FROM entity" +
-		" WHERE " + s.dialect.Quote("group") + "=? AND resource=? AND namespace=?" +
-		" ORDER BY slug asc"
-	args := []interface{}{folder.GROUP, folder.RESOURCE, namespace}
+	*items = append(*items, &sqlEntityFolderInsertRequestItem{
+		GUID:      fi.GUID,
+		Namespace: namespace,
+		UID:       fi.UID,
+		SlugPath:  fi.SlugPath,
+		JS:        js.String(),
+		Depth:     fi.depth,
+		Left:      fi.left,
+		Right:     fi.right,
+		Detached:  isLost,
+	})

-	all := []*folderInfo{}
-	rows, err := tx.Query(ctx, query, args...)
-	if err != nil {
-		return err
-	}
-	defer func() { _ = rows.Close() }()
-
-	for rows.Next() {
-		folder := folderInfo{
-			children: []*folderInfo{},
-		}
-		err = rows.Scan(&folder.Guid, &folder.UID, &folder.parentUID, &folder.Name, &folder.Slug)
-		if err != nil {
-			return err
-		}
-		all = append(all, &folder)
-	}
-
-	root, lost, err := buildFolderTree(all)
-	if err != nil {
-		return err
-	}
-
-	err = insertFolderInfo(ctx, tx, namespace, root, false)
-	if err != nil {
-		return err
-	}
-
-	for _, folder := range lost {
-		err = insertFolderInfo(ctx, tx, namespace, folder, true)
-		if err != nil {
-			return err
+	for _, sub := range fi.children {
+		if err := sub.buildInsertItems(items, namespace, isLost); err != nil {
+			return nil
 		}
 	}
-	return err
+
+	return nil
+}
+
+// This rebuilds the whole folders structure for a given namespace. This has to
+// be done each time an entity is created or deleted.
+// FIXME: This is very inefficient and time consuming. This could be implemented
+// with a different approach instead of MPTT, or at least mitigated by an async
+// job?
+// FIXME: This algorithm apparently allows lost trees which are called
+// "detached"? We should probably migrate to something safer.
+func (s *sqlEntityServer) updateFolderTree(ctx context.Context, x db.ContextExecer, namespace string) error {
+	_, err := x.ExecContext(ctx, "DELETE FROM entity_folder WHERE namespace=?", namespace)
+	if err != nil {
+		return fmt.Errorf("clear entity_folder for namespace %q: %w", namespace, err)
+	}
+
+	listReq := sqlEntityListFolderElementsRequest{
+		SQLTemplate: sqltemplate.New(s.sqlDialect),
+		Group:       folder.GROUP,
+		Resource:    folder.RESOURCE,
+		Namespace:   namespace,
+		FolderInfo:  new(folderInfo),
+	}
+	query, err := sqltemplate.Execute(sqlEntityListFolderElements, listReq)
+	if err != nil {
+		return fmt.Errorf("execute SQL template to list folder items in namespace %q: %w", namespace, err)
+	}
+
+	rows, err := x.QueryContext(ctx, query, listReq.GetArgs()...)
+	if err != nil {
+		return fmt.Errorf("list folder items in namespace %q: %w", namespace, err)
+	}
+
+	var itemList []*folderInfo
+	for i := 1; rows.Next(); i++ {
+		if err := rows.Scan(listReq.GetScanDest()...); err != nil {
+			return fmt.Errorf("scan row #%d listing folder items in namespace %q: %w", i, namespace, err)
+		}
+		fi := *listReq.FolderInfo
+		itemList = append(itemList, &fi)
+	}
+
+	if err := rows.Close(); err != nil {
+		return fmt.Errorf("close rows after listing folder items in namespace %q: %w", namespace, err)
+	}
+
+	root, lost, err := buildFolderTree(itemList)
+	if err != nil {
+		return fmt.Errorf("build folder tree for namespace %q: %w", namespace, err)
+	}
+
+	var insertItems []*sqlEntityFolderInsertRequestItem
+	if err = root.buildInsertItems(&insertItems, namespace, false); err != nil {
+		return fmt.Errorf("build insert items for root tree in namespace %q: %w", namespace, err)
+	}
+
+	for i, lostItem := range lost {
+		if err = lostItem.buildInsertItems(&insertItems, namespace, false); err != nil {
+			return fmt.Errorf("build insert items for lost folder #%d tree in namespace %q: %w", i, namespace, err)
+		}
+	}
+
+	insReq := sqlEntityFolderInsertRequest{
+		SQLTemplate: sqltemplate.New(s.sqlDialect),
+		Items:       insertItems,
+	}
+	if _, err = exec(ctx, x, sqlEntityFolderInsert, insReq); err != nil {
+		return fmt.Errorf("insert rebuilt tree for namespace %q: %w", namespace, err)
+	}
+
+	return nil
 }

 func buildFolderTree(all []*folderInfo) (*folderInfo, []*folderInfo, error) {
@@ -100,7 +148,7 @@ func buildFolderTree(all []*folderInfo) (*folderInfo, []*folderInfo, error) {

 	// already sorted by slug
 	for _, folder := range all {
-		parent, ok := lookup[folder.parentUID]
+		parent, ok := lookup[folder.ParentUID]
 		if ok {
 			folder.parent = parent
 			parent.children = append(parent.children, folder)
@@ -136,32 +184,3 @@ func setMPTTOrder(folder *folderInfo, stack []*folderInfo, idx int32) (int32, er
 	folder.right = idx + 1
 	return folder.right, nil
 }
-
-func insertFolderInfo(ctx context.Context, tx *session.SessionTx, namespace string, folder *folderInfo, isDetached bool) error {
-	js, _ := json.Marshal(folder.stack)
-	_, err := tx.Exec(ctx,
-		`INSERT INTO entity_folder `+
-			"(guid, namespace, name, slug_path, tree, depth, lft, rgt, detached) "+
-			`VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-		folder.Guid,
-		namespace,
-		folder.UID,
-		folder.SlugPath,
-		string(js),
-		folder.depth,
-		folder.left,
-		folder.right,
-		isDetached,
-	)
-	if err != nil {
-		return err
-	}
-
-	for _, sub := range folder.children {
-		err := insertFolderInfo(ctx, tx, namespace, sub, isDetached)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}