Storage: Unified Storage based on Entity API (#71977)
* first round of entityapi updates - quote column names and clean up insert/update queries - replace grn with guid - streamline table structure fixes streamline entity history move EntitySummary into proto remove EntitySummary add guid to json fix tests change DB_Uuid to DB_NVarchar fix folder test convert interface to any more cleanup start entity store under grafana-apiserver dskit target CRUD working, kind of rough cut of wiring entity api to kube-apiserver fake grafana user in context add key to entity list working revert unnecessary changes move entity storage files to their own package, clean up use accessor to read/write grafana annotations implement separate Create and Update functions * go mod tidy * switch from Kind to resource * basic grpc storage server * basic support for grpc entity store * don't connect to database unless it's needed, pass user identity over grpc * support getting user from k8s context, fix some mysql issues * assign owner to snowflake dependency * switch from ulid to uuid for guids * cleanup, rename Search to List * remove entityListResult * EntityAPI: remove extra user abstraction (#79033) * remove extra user abstraction * add test stub (but * move grpc context setup into client wrapper, fix lint issue * remove unused constants * remove custom json stuff * basic list filtering, add todo * change target to storage-server, allow entityStore flag in prod mode * fix issue with Update * EntityAPI: make test work, need to resolve expected differences (#79123) * make test work, need to resolve expected differences * remove the fields not supported by legacy * sanitize out the bits legacy does not support * sanitize out the bits legacy does not support --------- Co-authored-by: Ryan McKinley <ryantxu@gmail.com> * update feature toggle generated files * remove unused http headers * update feature flag strategy * devmode * update readme * spelling * readme --------- Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
This commit is contained in:
Dan Cech
co-authored by
Ryan McKinley
Ryan McKinley
parent
07915703fe
commit
c4c9bfaf2e
@@ -4,18 +4,19 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/grn"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/session"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity"
|
||||
)
|
||||
|
||||
type folderInfo struct {
|
||||
UID string `json:"uid"`
|
||||
Name string `json:"name"` // original display name
|
||||
Slug string `json:"slug"` // full slug
|
||||
Guid string `json:"guid"`
|
||||
|
||||
UID string `json:"uid"`
|
||||
Name string `json:"name"` // original display name
|
||||
SlugPath string `json:"slug"` // full slug path
|
||||
|
||||
// original slug
|
||||
originalSlug string
|
||||
Slug string `json:"-"`
|
||||
|
||||
depth int32
|
||||
left int32
|
||||
@@ -33,51 +34,48 @@ type folderInfo struct {
|
||||
// This will replace all entries in `entity_folder`
|
||||
// This is pretty heavy weight, but it does give us a sorted folder list
|
||||
// NOTE: this could be done async with a mutex/lock? reconciler pattern
|
||||
func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenant int64) error {
|
||||
_, err := tx.Exec(ctx, "DELETE FROM entity_folder WHERE tenant_id=?", tenant)
|
||||
func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenantId int64) error {
|
||||
_, err := tx.Exec(ctx, "DELETE FROM entity_folder WHERE tenant_id=?", tenantId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := "SELECT guid,uid,folder,name,slug" +
|
||||
" FROM entity" +
|
||||
" WHERE kind=? AND tenant_id=?" +
|
||||
" ORDER BY slug asc"
|
||||
args := []interface{}{entity.StandardKindFolder, tenantId}
|
||||
|
||||
all := []*folderInfo{}
|
||||
rows, err := tx.Query(ctx, "SELECT uid,folder,name,slug FROM entity WHERE kind=? AND tenant_id=? ORDER BY slug asc;",
|
||||
entity.StandardKindFolder, tenant)
|
||||
rows, err := tx.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
for rows.Next() {
|
||||
folder := folderInfo{
|
||||
children: []*folderInfo{},
|
||||
}
|
||||
err = rows.Scan(&folder.UID, &folder.parentUID, &folder.Name, &folder.originalSlug)
|
||||
err = rows.Scan(&folder.Guid, &folder.UID, &folder.parentUID, &folder.Name, &folder.Slug)
|
||||
if err != nil {
|
||||
break
|
||||
return err
|
||||
}
|
||||
all = append(all, &folder)
|
||||
}
|
||||
errClose := rows.Close()
|
||||
// TODO: Use some kind of multi-error.
|
||||
// Until then, we want to prioritize errors coming from the .Scan
|
||||
// over those coming from .Close.
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if errClose != nil {
|
||||
return errClose
|
||||
}
|
||||
|
||||
root, lost, err := buildFolderTree(all)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = insertFolderInfo(ctx, tx, tenant, root, false)
|
||||
err = insertFolderInfo(ctx, tx, tenantId, root, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, folder := range lost {
|
||||
err = insertFolderInfo(ctx, tx, tenant, folder, true)
|
||||
err = insertFolderInfo(ctx, tx, tenantId, folder, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -123,9 +121,9 @@ func setMPTTOrder(folder *folderInfo, stack []*folderInfo, idx int32) (int32, er
|
||||
folder.stack = stack
|
||||
|
||||
if folder.depth > 0 {
|
||||
folder.Slug = "/"
|
||||
folder.SlugPath = "/"
|
||||
for _, f := range stack {
|
||||
folder.Slug += f.originalSlug + "/"
|
||||
folder.SlugPath += f.Slug + "/"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,17 +137,16 @@ func setMPTTOrder(folder *folderInfo, stack []*folderInfo, idx int32) (int32, er
|
||||
return folder.right, nil
|
||||
}
|
||||
|
||||
func insertFolderInfo(ctx context.Context, tx *session.SessionTx, tenant int64, folder *folderInfo, isDetached bool) error {
|
||||
func insertFolderInfo(ctx context.Context, tx *session.SessionTx, tenantId int64, folder *folderInfo, isDetached bool) error {
|
||||
js, _ := json.Marshal(folder.stack)
|
||||
grn2 := grn.GRN{TenantID: tenant, ResourceKind: entity.StandardKindFolder, ResourceIdentifier: folder.UID}
|
||||
_, err := tx.Exec(ctx,
|
||||
`INSERT INTO entity_folder `+
|
||||
"(grn, tenant_id, uid, slug_path, tree, depth, left, right, detached) "+
|
||||
"(guid, tenant_id, uid, slug_path, tree, depth, lft, rgt, detached) "+
|
||||
`VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
grn2.ToGRNString(),
|
||||
tenant,
|
||||
folder.Guid,
|
||||
tenantId,
|
||||
folder.UID,
|
||||
folder.Slug,
|
||||
folder.SlugPath,
|
||||
string(js),
|
||||
folder.depth,
|
||||
folder.left,
|
||||
@@ -161,7 +158,7 @@ func insertFolderInfo(ctx context.Context, tx *session.SessionTx, tenant int64,
|
||||
}
|
||||
|
||||
for _, sub := range folder.children {
|
||||
err := insertFolderInfo(ctx, tx, tenant, sub, isDetached)
|
||||
err := insertFolderInfo(ctx, tx, tenantId, sub, isDetached)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -12,9 +12,9 @@ import (
|
||||
|
||||
func TestFolderSupport(t *testing.T) {
|
||||
root, lost, err := buildFolderTree([]*folderInfo{
|
||||
{UID: "A", parentUID: "", Name: "A", originalSlug: "a"},
|
||||
{UID: "AA", parentUID: "A", Name: "AA", originalSlug: "aa"},
|
||||
{UID: "B", parentUID: "", Name: "B", originalSlug: "b"},
|
||||
{Guid: "GA", UID: "A", parentUID: "", Name: "A", Slug: "a"},
|
||||
{Guid: "GAA", UID: "AA", parentUID: "A", Name: "AA", Slug: "aa"},
|
||||
{Guid: "GB", UID: "B", parentUID: "", Name: "B", Slug: "b"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, root)
|
||||
@@ -51,7 +51,7 @@ func appendFolder(folder *folderInfo, frame *data.Frame) {
|
||||
frame.AppendRow(
|
||||
folder.UID,
|
||||
folder.Name,
|
||||
folder.Slug,
|
||||
folder.SlugPath,
|
||||
folder.depth,
|
||||
folder.left,
|
||||
folder.right,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,116 +0,0 @@
|
||||
package sqlstash
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/grn"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity"
|
||||
)
|
||||
|
||||
type summarySupport struct {
|
||||
model *entity.EntitySummary
|
||||
name string
|
||||
description *string // null or empty
|
||||
slug *string // null or empty
|
||||
labels *string
|
||||
fields *string
|
||||
errors *string // should not allow saving with this!
|
||||
marshaled []byte
|
||||
|
||||
// metadata for nested objects
|
||||
parent_grn *grn.GRN
|
||||
folder string
|
||||
isNested bool // set when this is for a nested item
|
||||
}
|
||||
|
||||
func newSummarySupport(summary *entity.EntitySummary) (*summarySupport, error) {
|
||||
var err error
|
||||
var js []byte
|
||||
s := &summarySupport{
|
||||
model: summary,
|
||||
}
|
||||
if summary != nil {
|
||||
s.marshaled, err = json.Marshal(summary)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
|
||||
s.name = summary.Name
|
||||
if summary.Description != "" {
|
||||
s.description = &summary.Description
|
||||
}
|
||||
if summary.Slug != "" {
|
||||
s.slug = &summary.Slug
|
||||
}
|
||||
if len(summary.Labels) > 0 {
|
||||
js, err = json.Marshal(summary.Labels)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
str := string(js)
|
||||
s.labels = &str
|
||||
}
|
||||
|
||||
if len(summary.Fields) > 0 {
|
||||
js, err = json.Marshal(summary.Fields)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
str := string(js)
|
||||
s.fields = &str
|
||||
}
|
||||
|
||||
if summary.Error != nil {
|
||||
js, err = json.Marshal(summary.Error)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
str := string(js)
|
||||
s.errors = &str
|
||||
}
|
||||
}
|
||||
return s, err
|
||||
}
|
||||
|
||||
func (s summarySupport) toEntitySummary() (*entity.EntitySummary, error) {
|
||||
var err error
|
||||
summary := &entity.EntitySummary{
|
||||
Name: s.name,
|
||||
}
|
||||
if s.description != nil {
|
||||
summary.Description = *s.description
|
||||
}
|
||||
if s.slug != nil {
|
||||
summary.Slug = *s.slug
|
||||
}
|
||||
if s.labels != nil {
|
||||
b := []byte(*s.labels)
|
||||
err = json.Unmarshal(b, &summary.Labels)
|
||||
if err != nil {
|
||||
return summary, err
|
||||
}
|
||||
}
|
||||
if s.fields != nil {
|
||||
b := []byte(*s.fields)
|
||||
err = json.Unmarshal(b, &summary.Fields)
|
||||
if err != nil {
|
||||
return summary, err
|
||||
}
|
||||
}
|
||||
if s.errors != nil {
|
||||
b := []byte(*s.errors)
|
||||
err = json.Unmarshal(b, &summary.Error)
|
||||
if err != nil {
|
||||
return summary, err
|
||||
}
|
||||
}
|
||||
return summary, err
|
||||
}
|
||||
|
||||
func (s *summarySupport) getParentGRN() *string {
|
||||
if s.isNested {
|
||||
t := s.parent_grn.ToGRNString()
|
||||
return &t
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+14
-10
@@ -3,16 +3,16 @@
|
||||
// Frame[0]
|
||||
// Name:
|
||||
// Dimensions: 7 Fields by 4 Rows
|
||||
// +----------------+----------------+----------------+---------------+---------------+---------------+--------------------------------------------------------------------------------+
|
||||
// | Name: UID | Name: name | Name: slug | Name: depth | Name: left | Name: right | Name: tree |
|
||||
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
|
||||
// | Type: []string | Type: []string | Type: []string | Type: []int32 | Type: []int32 | Type: []int32 | Type: []json.RawMessage |
|
||||
// +----------------+----------------+----------------+---------------+---------------+---------------+--------------------------------------------------------------------------------+
|
||||
// | | Root | | 0 | 1 | 8 | [] |
|
||||
// | A | A | /a/ | 1 | 2 | 5 | [{"uid":"A","name":"A","slug":"/a/"}] |
|
||||
// | AA | AA | /a/aa/ | 2 | 3 | 4 | [{"uid":"A","name":"A","slug":"/a/"},{"uid":"AA","name":"AA","slug":"/a/aa/"}] |
|
||||
// | B | B | /b/ | 1 | 6 | 7 | [{"uid":"B","name":"B","slug":"/b/"}] |
|
||||
// +----------------+----------------+----------------+---------------+---------------+---------------+--------------------------------------------------------------------------------+
|
||||
// +----------------+----------------+----------------+---------------+---------------+---------------+---------------------------------------------------------------------------------------------------------+
|
||||
// | Name: UID | Name: name | Name: slug | Name: depth | Name: left | Name: right | Name: tree |
|
||||
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
|
||||
// | Type: []string | Type: []string | Type: []string | Type: []int32 | Type: []int32 | Type: []int32 | Type: []json.RawMessage |
|
||||
// +----------------+----------------+----------------+---------------+---------------+---------------+---------------------------------------------------------------------------------------------------------+
|
||||
// | | Root | | 0 | 1 | 8 | [] |
|
||||
// | A | A | /a/ | 1 | 2 | 5 | [{"guid":"GA","uid":"A","name":"A","slug":"/a/"}] |
|
||||
// | AA | AA | /a/aa/ | 2 | 3 | 4 | [{"guid":"GA","uid":"A","name":"A","slug":"/a/"},{"guid":"GAA","uid":"AA","name":"AA","slug":"/a/aa/"}] |
|
||||
// | B | B | /b/ | 1 | 6 | 7 | [{"guid":"GB","uid":"B","name":"B","slug":"/b/"}] |
|
||||
// +----------------+----------------+----------------+---------------+---------------+---------------+---------------------------------------------------------------------------------------------------------+
|
||||
//
|
||||
//
|
||||
// 🌟 This was machine generated. Do not edit. 🌟
|
||||
@@ -115,6 +115,7 @@
|
||||
[],
|
||||
[
|
||||
{
|
||||
"guid": "GA",
|
||||
"uid": "A",
|
||||
"name": "A",
|
||||
"slug": "/a/"
|
||||
@@ -122,11 +123,13 @@
|
||||
],
|
||||
[
|
||||
{
|
||||
"guid": "GA",
|
||||
"uid": "A",
|
||||
"name": "A",
|
||||
"slug": "/a/"
|
||||
},
|
||||
{
|
||||
"guid": "GAA",
|
||||
"uid": "AA",
|
||||
"name": "AA",
|
||||
"slug": "/a/aa/"
|
||||
@@ -134,6 +137,7 @@
|
||||
],
|
||||
[
|
||||
{
|
||||
"guid": "GB",
|
||||
"uid": "B",
|
||||
"name": "B",
|
||||
"slug": "/b/"
|
||||
|
||||
@@ -5,7 +5,11 @@ import (
|
||||
"encoding/hex"
|
||||
)
|
||||
|
||||
func createContentsHash(contents []byte) string {
|
||||
hash := md5.Sum(contents)
|
||||
func createContentsHash(body []byte, meta []byte, status []byte) string {
|
||||
h := md5.New()
|
||||
_, _ = h.Write(meta)
|
||||
_, _ = h.Write(body)
|
||||
_, _ = h.Write(status)
|
||||
hash := h.Sum(nil)
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user