EntityAPI: Save nested summary info in the SQL database (#61732)

pkg/services/store/entity/sqlstash/folder_support.go

@@ -11,15 +11,23 @@ import (
 
 type folderInfo struct {
 	UID  string `json:"uid"`
-	Name string `json:"name"`
-	Slug string `json:"slug"`
+	Name string `json:"name"` // original display name
+	Slug string `json:"slug"` // full slug
+
+	// original slug
+	originalSlug string
+
+	depth int32
+	left  int32
+	right int32
 
 	// Build the tree
-	ParentUID string `json:"-"`
+	parentUID string
 
-	// Added after query
-	children []*folderInfo
+	// Calculated after query
 	parent   *folderInfo
+	children []*folderInfo
+	stack    []*folderInfo
 }
 
 // This will replace all entries in `entity_folder`
@@ -32,7 +40,6 @@ func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenant int64)
 	}
 
 	all := []*folderInfo{}
-	lookup := make(map[string]*folderInfo)
 	rows, err := tx.Query(ctx, "SELECT uid,folder,name,slug FROM entity WHERE kind=? AND tenant_id=? ORDER BY slug asc;",
 		models.StandardKindFolder, tenant)
 	if err != nil {
@@ -42,11 +49,10 @@ func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenant int64)
 		folder := folderInfo{
 			children: []*folderInfo{},
 		}
-		err = rows.Scan(&folder.UID, &folder.ParentUID, &folder.Name, &folder.Slug)
+		err = rows.Scan(&folder.UID, &folder.parentUID, &folder.Name, &folder.originalSlug)
 		if err != nil {
 			return err
 		}
-		lookup[folder.UID] = &folder
 		all = append(all, &folder)
 	}
 	err = rows.Close()
@@ -54,16 +60,43 @@ func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenant int64)
 		return err
 	}
 
+	root, lost, err := buildFolderTree(all)
+	if err != nil {
+		return err
+	}
+
+	err = insertFolderInfo(ctx, tx, tenant, root, false)
+	if err != nil {
+		return err
+	}
+
+	for _, folder := range lost {
+		err = insertFolderInfo(ctx, tx, tenant, folder, true)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
+func buildFolderTree(all []*folderInfo) (*folderInfo, []*folderInfo, error) {
+	lost := []*folderInfo{}
+	lookup := make(map[string]*folderInfo)
+	for _, folder := range all {
+		lookup[folder.UID] = folder
+	}
+
 	root := &folderInfo{
 		Name:     "Root",
+		UID:      "",
 		children: []*folderInfo{},
+		left:     1,
 	}
 	lookup[""] = root
-	lost := []*folderInfo{}
 
 	// already sorted by slug
 	for _, folder := range all {
-		parent, ok := lookup[folder.ParentUID]
+		parent, ok := lookup[folder.parentUID]
 		if ok {
 			folder.parent = parent
 			parent.children = append(parent.children, folder)
@@ -72,40 +105,49 @@ func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenant int64)
 		}
 	}
 
-	for _, folder := range root.children {
-		err = addFolderInfo(ctx, tx, tenant, []*folderInfo{folder}, false)
-		if err != nil {
-			return err
-		}
-	}
-	for _, folder := range lost {
-		err = addFolderInfo(ctx, tx, tenant, []*folderInfo{folder}, true)
-		if err != nil {
-			return err
-		}
-	}
-	return err
+	_, err := setMPTTOrder(root, []*folderInfo{}, int32(1))
+	return root, lost, err
 }
 
-func addFolderInfo(ctx context.Context, tx *session.SessionTx, tenant int64, tree []*folderInfo, isDetached bool) error {
-	folder := tree[len(tree)-1] // last item in the tree
+// https://imrannazar.com/Modified-Preorder-Tree-Traversal
+func setMPTTOrder(folder *folderInfo, stack []*folderInfo, idx int32) (int32, error) {
+	var err error
+	folder.depth = int32(len(stack))
+	folder.left = idx
+	folder.stack = stack
 
-	js, _ := json.Marshal(tree)
-	slugPath := "/"
-	for _, f := range tree {
-		slugPath += f.Slug + "/"
+	if folder.depth > 0 {
+		folder.Slug = "/"
+		for _, f := range stack {
+			folder.Slug += f.originalSlug + "/"
+		}
 	}
+
+	for _, child := range folder.children {
+		idx, err = setMPTTOrder(child, append(stack, child), idx+1)
+		if err != nil {
+			return idx, err
+		}
+	}
+	folder.right = idx + 1
+	return folder.right, nil
+}
+
+func insertFolderInfo(ctx context.Context, tx *session.SessionTx, tenant int64, folder *folderInfo, isDetached bool) error {
+	js, _ := json.Marshal(folder.stack)
 	grn := entity.GRN{TenantId: tenant, Kind: models.StandardKindFolder, UID: folder.UID}
 	_, err := tx.Exec(ctx,
 		`INSERT INTO entity_folder `+
-			"(grn, tenant_id, uid, slug_path, tree, depth, detached) "+
-			`VALUES (?, ?, ?, ?, ?, ?, ?)`,
+			"(grn, tenant_id, uid, slug_path, tree, depth, left, right, detached) "+
+			`VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 		grn.ToGRNString(),
 		tenant,
 		folder.UID,
-		slugPath,
+		folder.Slug,
 		string(js),
-		len(tree),
+		folder.depth,
+		folder.left,
+		folder.right,
 		isDetached,
 	)
 	if err != nil {
@@ -113,7 +155,7 @@ func addFolderInfo(ctx context.Context, tx *session.SessionTx, tenant int64, tre
 	}
 
 	for _, sub := range folder.children {
-		err := addFolderInfo(ctx, tx, tenant, append(tree, sub), isDetached)
+		err := insertFolderInfo(ctx, tx, tenant, sub, isDetached)
 		if err != nil {
 			return err
 		}

pkg/services/store/entity/sqlstash/folder_support_test.go

@@ -0,0 +1,63 @@
+package sqlstash
+
+import (
+	_ "embed"
+	"encoding/json"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana-plugin-sdk-go/experimental"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFolderSupport(t *testing.T) {
+	root, lost, err := buildFolderTree([]*folderInfo{
+		{UID: "A", parentUID: "", Name: "A", originalSlug: "a"},
+		{UID: "AA", parentUID: "A", Name: "AA", originalSlug: "aa"},
+		{UID: "B", parentUID: "", Name: "B", originalSlug: "b"},
+	})
+	require.NoError(t, err)
+	require.NotNil(t, root)
+	require.NotNil(t, lost)
+	require.Empty(t, lost)
+
+	frame := treeToFrame(root)
+	experimental.CheckGoldenJSONFrame(t, "testdata", "simple", frame, true)
+}
+
+func treeToFrame(root *folderInfo) *data.Frame {
+	frame := data.NewFrame("",
+		data.NewFieldFromFieldType(data.FieldTypeString, 0), // UID
+		data.NewFieldFromFieldType(data.FieldTypeString, 0), // Name
+		data.NewFieldFromFieldType(data.FieldTypeString, 0), // Slug
+		data.NewFieldFromFieldType(data.FieldTypeInt32, 0),  // Depth
+		data.NewFieldFromFieldType(data.FieldTypeInt32, 0),  // Left
+		data.NewFieldFromFieldType(data.FieldTypeInt32, 0),  // Right
+		data.NewFieldFromFieldType(data.FieldTypeJSON, 0),   // Tree
+	)
+	frame.Fields[0].Name = "UID"
+	frame.Fields[1].Name = "name"
+	frame.Fields[2].Name = "slug"
+	frame.Fields[3].Name = "depth"
+	frame.Fields[4].Name = "left"
+	frame.Fields[5].Name = "right"
+	frame.Fields[6].Name = "tree"
+	appendFolder(root, frame)
+	return frame
+}
+
+func appendFolder(folder *folderInfo, frame *data.Frame) {
+	b, _ := json.Marshal(folder.stack)
+	frame.AppendRow(
+		folder.UID,
+		folder.Name,
+		folder.Slug,
+		folder.depth,
+		folder.left,
+		folder.right,
+		json.RawMessage(b),
+	)
+	for _, sub := range folder.children {
+		appendFolder(sub, frame)
+	}
+}

pkg/services/store/entity/sqlstash/sql_storage_server.go

@@ -304,7 +304,6 @@ func (s *sqlEntityServer) AdminWrite(ctx context.Context, r *entity.AdminWriteEn
 		return nil, err
 	}
 
-	isFolder := models.StandardKindFolder == r.GRN.Kind
 	etag := createContentsHash(body)
 	rsp := &entity.WriteEntityResponse{
 		GRN:    grn,
@@ -372,10 +371,13 @@ func (s *sqlEntityServer) AdminWrite(ctx context.Context, r *entity.AdminWriteEn
 
 		if isUpdate {
 			// Clear the labels+refs
-			if _, err := tx.Exec(ctx, "DELETE FROM entity_labels WHERE grn=?", oid); err != nil {
+			if _, err := tx.Exec(ctx, "DELETE FROM entity_labels WHERE grn=? OR parent_grn=?", oid, oid); err != nil {
 				return err
 			}
-			if _, err := tx.Exec(ctx, "DELETE FROM entity_ref WHERE grn=?", oid); err != nil {
+			if _, err := tx.Exec(ctx, "DELETE FROM entity_ref WHERE grn=? OR parent_grn=?", oid, oid); err != nil {
+				return err
+			}
+			if _, err := tx.Exec(ctx, "DELETE FROM entity_nested WHERE parent_grn=?", oid); err != nil {
 				return err
 			}
 		}
@@ -398,37 +400,6 @@ func (s *sqlEntityServer) AdminWrite(ctx context.Context, r *entity.AdminWriteEn
 			return err
 		}
 
-		// 2. Add the labels rows
-		for k, v := range summary.model.Labels {
-			_, err = tx.Exec(ctx,
-				`INSERT INTO entity_labels `+
-					"(grn, label, value) "+
-					`VALUES (?, ?, ?)`,
-				oid, k, v,
-			)
-			if err != nil {
-				return err
-			}
-		}
-
-		// 3. Add the references rows
-		for _, ref := range summary.model.References {
-			resolved, err := s.resolver.Resolve(ctx, ref)
-			if err != nil {
-				return err
-			}
-			_, err = tx.Exec(ctx, `INSERT INTO entity_ref (`+
-				"grn, kind, type, uid, "+
-				"resolved_ok, resolved_to, resolved_warning, resolved_time) "+
-				`VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
-				oid, ref.Kind, ref.Type, ref.UID,
-				resolved.OK, resolved.Key, resolved.Warning, resolved.Timestamp,
-			)
-			if err != nil {
-				return err
-			}
-		}
-
 		// 5. Add/update the main `entity` table
 		rsp.Entity = versionInfo
 		if isUpdate {
@@ -447,43 +418,43 @@ func (s *sqlEntityServer) AdminWrite(ctx context.Context, r *entity.AdminWriteEn
 				origin.Source, origin.Key, timestamp,
 				oid,
 			)
-
-			if isFolder && err == nil {
-				err = updateFolderTree(ctx, tx, grn.TenantId)
+		} else {
+			if createdAt < 1000 {
+				createdAt = updatedAt
+			}
+			if createdBy == "" {
+				createdBy = updatedBy
 			}
-			return err
-		}
 
-		if createdAt < 1000 {
-			createdAt = updatedAt
+			_, err = tx.Exec(ctx, "INSERT INTO entity ("+
+				"grn, tenant_id, kind, uid, folder, "+
+				"size, body, etag, version, "+
+				"updated_at, updated_by, created_at, created_by, "+
+				"name, description, slug, "+
+				"labels, fields, errors, "+
+				"origin, origin_key, origin_ts) "+
+				"VALUES (?, ?, ?, ?, ?, "+
+				" ?, ?, ?, ?, "+
+				" ?, ?, ?, ?, "+
+				" ?, ?, ?, "+
+				" ?, ?, ?, "+
+				" ?, ?, ?)",
+				oid, grn.TenantId, grn.Kind, grn.UID, r.Folder,
+				versionInfo.Size, body, etag, versionInfo.Version,
+				updatedAt, createdBy, createdAt, createdBy,
+				summary.model.Name, summary.model.Description, summary.model.Slug,
+				summary.labels, summary.fields, summary.errors,
+				origin.Source, origin.Key, origin.Time,
+			)
 		}
-		if createdBy == "" {
-			createdBy = updatedBy
-		}
-
-		_, err = tx.Exec(ctx, "INSERT INTO entity ("+
-			"grn, tenant_id, kind, uid, folder, "+
-			"size, body, etag, version, "+
-			"updated_at, updated_by, created_at, created_by, "+
-			"name, description, slug, "+
-			"labels, fields, errors, "+
-			"origin, origin_key, origin_ts) "+
-			"VALUES (?, ?, ?, ?, ?, "+
-			" ?, ?, ?, ?, "+
-			" ?, ?, ?, ?, "+
-			" ?, ?, ?, "+
-			" ?, ?, ?, "+
-			" ?, ?, ?)",
-			oid, grn.TenantId, grn.Kind, grn.UID, r.Folder,
-			versionInfo.Size, body, etag, versionInfo.Version,
-			updatedAt, createdBy, createdAt, createdBy,
-			summary.model.Name, summary.model.Description, summary.model.Slug,
-			summary.labels, summary.fields, summary.errors,
-			origin.Source, origin.Key, origin.Time,
-		)
-		if isFolder && err == nil {
+		if err == nil && models.StandardKindFolder == r.GRN.Kind {
 			err = updateFolderTree(ctx, tx, grn.TenantId)
 		}
+		if err == nil {
+			summary.folder = r.Folder
+			summary.parent_grn = grn
+			return s.writeSearchInfo(ctx, tx, oid, summary)
+		}
 		return err
 	})
 	rsp.SummaryJson = summary.marshaled
@@ -534,6 +505,92 @@ func (s *sqlEntityServer) selectForUpdate(ctx context.Context, tx *session.Sessi
 	return current, err
 }
 
+func (s *sqlEntityServer) writeSearchInfo(
+	ctx context.Context,
+	tx *session.SessionTx,
+	grn string,
+	summary *summarySupport,
+) error {
+	parent_grn := summary.getParentGRN()
+
+	// Add the labels rows
+	for k, v := range summary.model.Labels {
+		_, err := tx.Exec(ctx,
+			`INSERT INTO entity_labels `+
+				"(grn, label, value, parent_grn) "+
+				`VALUES (?, ?, ?, ?)`,
+			grn, k, v, parent_grn,
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Resolve references
+	for _, ref := range summary.model.References {
+		resolved, err := s.resolver.Resolve(ctx, ref)
+		if err != nil {
+			return err
+		}
+		_, err = tx.Exec(ctx, `INSERT INTO entity_ref (`+
+			"grn, parent_grn, kind, type, uid, "+
+			"resolved_ok, resolved_to, resolved_warning, resolved_time) "+
+			`VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+			grn, parent_grn, ref.Kind, ref.Type, ref.UID,
+			resolved.OK, resolved.Key, resolved.Warning, resolved.Timestamp,
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Traverse entities and insert refs
+	if summary.model.Nested != nil {
+		for _, childModel := range summary.model.Nested {
+			grn = (&entity.GRN{
+				TenantId: summary.parent_grn.TenantId,
+				Kind:     childModel.Kind,
+				UID:      childModel.UID, // append???
+			}).ToGRNString()
+
+			child, err := newSummarySupport(childModel)
+			if err != nil {
+				return err
+			}
+			child.isNested = true
+			child.folder = summary.folder
+			child.parent_grn = summary.parent_grn
+			parent_grn := child.getParentGRN()
+
+			_, err = tx.Exec(ctx, "INSERT INTO entity_nested ("+
+				"parent_grn, grn, "+
+				"tenant_id, kind, uid, folder, "+
+				"name, description, "+
+				"labels, fields, errors) "+
+				"VALUES (?, ?,"+
+				" ?, ?, ?, ?,"+
+				" ?, ?,"+
+				" ?, ?, ?)",
+				*parent_grn, grn,
+				summary.parent_grn.TenantId, childModel.Kind, childModel.UID, summary.folder,
+				child.name, child.description,
+				child.labels, child.fields, child.errors,
+			)
+
+			if err != nil {
+				return err
+			}
+
+			err = s.writeSearchInfo(ctx, tx, grn, child)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 func (s *sqlEntityServer) prepare(ctx context.Context, r *entity.AdminWriteEntityRequest) (*summarySupport, []byte, error) {
 	grn := r.GRN
 	builder := s.kinds.GetSummaryBuilder(grn.Kind)
@@ -589,14 +646,26 @@ func doDelete(ctx context.Context, tx *session.SessionTx, grn *entity.GRN) (bool
 	}
 
 	// TODO: keep history? would need current version bump, and the "write" would have to get from history
-	_, _ = tx.Exec(ctx, "DELETE FROM entity_history WHERE grn=?", str)
-	_, _ = tx.Exec(ctx, "DELETE FROM entity_labels WHERE grn=?", str)
-	_, _ = tx.Exec(ctx, "DELETE FROM entity_ref WHERE grn=?", str)
+	_, err = tx.Exec(ctx, "DELETE FROM entity_history WHERE grn=?", str)
+	if err != nil {
+		return false, err
+	}
+	_, err = tx.Exec(ctx, "DELETE FROM entity_labels WHERE grn=? OR parent_grn=?", str, str)
+	if err != nil {
+		return false, err
+	}
+	_, err = tx.Exec(ctx, "DELETE FROM entity_ref WHERE grn=? OR parent_grn=?", str, str)
+	if err != nil {
+		return false, err
+	}
+	_, err = tx.Exec(ctx, "DELETE FROM entity_nested WHERE parent_grn=?", str)
+	if err != nil {
+		return false, err
+	}
 
 	if grn.Kind == models.StandardKindFolder {
 		err = updateFolderTree(ctx, tx, grn.TenantId)
 	}
-
 	return rows > 0, err
 }
 

pkg/services/store/entity/sqlstash/summary_handler.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/store/entity"
 )
 
 type summarySupport struct {
@@ -15,6 +16,11 @@ type summarySupport struct {
 	fields      *string
 	errors      *string // should not allow saving with this!
 	marshaled   []byte
+
+	// metadata for nested objects
+	parent_grn *entity.GRN
+	folder     string
+	isNested   bool // set when this is for a nested item
 }
 
 func newSummarySupport(summary *models.EntitySummary) (*summarySupport, error) {
@@ -100,3 +106,11 @@ func (s summarySupport) toEntitySummary() (*models.EntitySummary, error) {
 	}
 	return summary, err
 }
+
+func (s *summarySupport) getParentGRN() *string {
+	if s.isNested {
+		t := s.parent_grn.ToGRNString()
+		return &t
+	}
+	return nil
+}

pkg/services/store/entity/sqlstash/testdata/simple.jsonc

@@ -0,0 +1,147 @@
+//  🌟 This was machine generated.  Do not edit. 🌟
+//  
+//  Frame[0] 
+//  Name: 
+//  Dimensions: 7 Fields by 4 Rows
+//  +----------------+----------------+----------------+---------------+---------------+---------------+--------------------------------------------------------------------------------+
+//  | Name: UID      | Name: name     | Name: slug     | Name: depth   | Name: left    | Name: right   | Name: tree                                                                     |
+//  | Labels:        | Labels:        | Labels:        | Labels:       | Labels:       | Labels:       | Labels:                                                                        |
+//  | Type: []string | Type: []string | Type: []string | Type: []int32 | Type: []int32 | Type: []int32 | Type: []json.RawMessage                                                        |
+//  +----------------+----------------+----------------+---------------+---------------+---------------+--------------------------------------------------------------------------------+
+//  |                | Root           |                | 0             | 1             | 8             | []                                                                             |
+//  | A              | A              | /a/            | 1             | 2             | 5             | [{"uid":"A","name":"A","slug":"/a/"}]                                          |
+//  | AA             | AA             | /a/aa/         | 2             | 3             | 4             | [{"uid":"A","name":"A","slug":"/a/"},{"uid":"AA","name":"AA","slug":"/a/aa/"}] |
+//  | B              | B              | /b/            | 1             | 6             | 7             | [{"uid":"B","name":"B","slug":"/b/"}]                                          |
+//  +----------------+----------------+----------------+---------------+---------------+---------------+--------------------------------------------------------------------------------+
+//  
+//  
+//  🌟 This was machine generated.  Do not edit. 🌟
+{
+  "status": 200,
+  "frames": [
+    {
+      "schema": {
+        "fields": [
+          {
+            "name": "UID",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          },
+          {
+            "name": "name",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          },
+          {
+            "name": "slug",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          },
+          {
+            "name": "depth",
+            "type": "number",
+            "typeInfo": {
+              "frame": "int32"
+            }
+          },
+          {
+            "name": "left",
+            "type": "number",
+            "typeInfo": {
+              "frame": "int32"
+            }
+          },
+          {
+            "name": "right",
+            "type": "number",
+            "typeInfo": {
+              "frame": "int32"
+            }
+          },
+          {
+            "name": "tree",
+            "type": "other",
+            "typeInfo": {
+              "frame": "json.RawMessage"
+            }
+          }
+        ]
+      },
+      "data": {
+        "values": [
+          [
+            "",
+            "A",
+            "AA",
+            "B"
+          ],
+          [
+            "Root",
+            "A",
+            "AA",
+            "B"
+          ],
+          [
+            "",
+            "/a/",
+            "/a/aa/",
+            "/b/"
+          ],
+          [
+            0,
+            1,
+            2,
+            1
+          ],
+          [
+            1,
+            2,
+            3,
+            6
+          ],
+          [
+            8,
+            5,
+            4,
+            7
+          ],
+          [
+            [],
+            [
+              {
+                "uid": "A",
+                "name": "A",
+                "slug": "/a/"
+              }
+            ],
+            [
+              {
+                "uid": "A",
+                "name": "A",
+                "slug": "/a/"
+              },
+              {
+                "uid": "AA",
+                "name": "AA",
+                "slug": "/a/aa/"
+              }
+            ],
+            [
+              {
+                "uid": "B",
+                "name": "B",
+                "slug": "/b/"
+              }
+            ]
+          ]
+        ]
+      }
+    }
+  ]
+}