public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unified-storage: Reduce calls to the user service (#102934)

Browse Source 
* Create ListByIdOrUID in user service

* create UnstructuredToLegacyFolderList 

* update GetFolders to use list parser

* update GetDescendants to use list parser

* update UnstructuredToLegacyFolder to also make a single call to the user service

---------

Co-authored-by: Stephanie Hingtgen <stephanie.hingtgen@grafana.com>
This commit is contained in:
Will Assis
2025-03-27 22:01:07 +02:00
committed by GitHub
co-authored by Stephanie Hingtgen
parent 7e3efb3df2
commit 51825cfffe
13 changed files with 643 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/folder/folderimpl/conversions.go
+120 -30
View File
@@ -2,7 +2,7 @@ package folderimpl
import (
"context"
"errors"
"fmt"
"strconv"
"strings"
@@ -13,13 +13,12 @@ import (
"github.com/grafana/grafana/pkg/infra/slugify"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/user"
)
func (ss *FolderUnifiedStoreImpl) UnstructuredToLegacyFolder(ctx context.Context, item *unstructured.Unstructured) (*folder.Folder, error) {
func parseUnstructuredToLegacyFolder(item *unstructured.Unstructured) (*folder.Folder, string, string, error) {
meta, err := utils.MetaAccessor(item)
if err != nil {
return nil, err
return nil, "", "", err
}
info, _ := authlib.ParseNamespace(meta.GetNamespace())
@@ -46,16 +45,9 @@ func (ss *FolderUnifiedStoreImpl) UnstructuredToLegacyFolder(ctx context.Context
updated = &tmp
}
creator, err := ss.getUserFromMeta(ctx, meta.GetCreatedBy())
if err != nil {
return nil, err
}
updater, err := ss.getUserFromMeta(ctx, meta.GetUpdatedBy())
if err != nil {
return nil, err
}
if updater.UID == "" {
creator := meta.GetCreatedBy()
updater := meta.GetUpdatedBy()
if updater == "" {
updater = creator
}
@@ -75,34 +67,132 @@ func (ss *FolderUnifiedStoreImpl) UnstructuredToLegacyFolder(ctx context.Context
Created: created,
Updated: *updated,
OrgID: info.OrgID,
CreatedBy: creator.ID,
UpdatedBy: updater.ID,
}, nil
}, creator, updater, nil
}
func (ss *FolderUnifiedStoreImpl) getUserFromMeta(ctx context.Context, userMeta string) (*user.User, error) {
if userMeta == "" || toUID(userMeta) == "" {
return &user.User{}, nil
func (ss *FolderUnifiedStoreImpl) UnstructuredToLegacyFolder(ctx context.Context, item *unstructured.Unstructured) (*folder.Folder, error) {
folder, creatorRaw, updaterRaw, err := parseUnstructuredToLegacyFolder(item)
if err != nil {
return nil, err
}
usr, err := ss.getUser(ctx, toUID(userMeta))
if err != nil && errors.Is(err, user.ErrUserNotFound) {
return &user.User{}, nil
userUIDtoIDmapping, err := ss.getUserUIDtoIDmappingFromIdentifiers(ctx, []string{creatorRaw, updaterRaw})
if err != nil {
return nil, err
}
return usr, err
creatorId := getIdFromMapping(creatorRaw, userUIDtoIDmapping)
updaterId := getIdFromMapping(updaterRaw, userUIDtoIDmapping)
if updaterId == 0 {
updaterId = creatorId
}
folder.CreatedBy = creatorId
folder.UpdatedBy = updaterId
return folder, nil
}
func (ss *FolderUnifiedStoreImpl) getUser(ctx context.Context, uid string) (*user.User, error) {
userID, err := strconv.ParseInt(uid, 10, 64)
if err == nil {
return ss.userService.GetByID(ctx, &user.GetUserByIDQuery{ID: userID})
func (ss *FolderUnifiedStoreImpl) UnstructuredToLegacyFolderList(ctx context.Context, unstructuredList *unstructured.UnstructuredList) ([]*folder.Folder, error) {
folders := make([]*folder.Folder, 0)
identifiers := make([]string, 0)
for _, item := range unstructuredList.Items {
meta, err := utils.MetaAccessor(&item)
if err != nil {
return nil, fmt.Errorf("unable to convert unstructured item to legacy folder %w", err)
}
identifiers = append(identifiers, meta.GetCreatedBy(), meta.GetUpdatedBy())
}
return ss.userService.GetByUID(ctx, &user.GetUserByUIDQuery{UID: uid})
userUIDtoIDmapping, err := ss.getUserUIDtoIDmappingFromIdentifiers(ctx, identifiers)
if err != nil {
return nil, err
}
for _, item := range unstructuredList.Items {
folder, creatorRaw, updaterRaw, err := parseUnstructuredToLegacyFolder(&item)
if err != nil {
return nil, err
}
creatorId := getIdFromMapping(creatorRaw, userUIDtoIDmapping)
updaterId := getIdFromMapping(updaterRaw, userUIDtoIDmapping)
if updaterId == 0 {
updaterId = creatorId
}
folder.CreatedBy = creatorId
folder.UpdatedBy = updaterId
folders = append(folders, folder)
}
return folders, nil
}
func toUID(rawIdentifier string) string {
func (ss *FolderUnifiedStoreImpl) getUserUIDtoIDmappingFromIdentifiers(ctx context.Context, rawIdentifiers []string) (map[string]int64, error) {
userUIDs, userIds := parseIdentifiers(rawIdentifiers)
allUsers, err := ss.userService.ListByIdOrUID(ctx, userUIDs, userIds)
if err != nil {
return nil, err
}
mapping := make(map[string]int64)
for _, user := range allUsers {
mapping[user.UID] = user.ID
}
return mapping, nil
}
func getIdentifier(rawIdentifier string) string {
parts := strings.Split(rawIdentifier, ":")
if len(parts) < 2 {
return ""
}
if parts[0] != "user" {
return ""
}
return parts[1]
}
func parseIdentifiers(rawIdentifiers []string) ([]string, []int64) {
uids := make([]string, 0)
ids := make([]int64, 0)
for _, rawIdentifier := range rawIdentifiers {
identifier := getIdentifier(rawIdentifier)
if identifier == "" {
continue
}
id, err := strconv.ParseInt(identifier, 10, 64)
if err == nil {
ids = append(ids, id)
} else if identifier != "" {
uids = append(uids, identifier)
}
}
return uids, ids
}
func getIdFromMapping(rawIdentifier string, mapping map[string]int64) int64 {
identifier := getIdentifier(rawIdentifier)
if identifier == "" {
return 0
}
id, err := strconv.ParseInt(identifier, 10, 64)
if err == nil {
return id
}
uid, ok := mapping[identifier]
if ok {
return uid
}
return 0
}
pkg/services/folder/folderimpl/conversions_test.go
+296 -4
View File
@@ -34,7 +34,7 @@ func TestFolderConversions(t *testing.T) {
"grafana.app/updatedTimestamp": "2022-12-02T07:02:02Z",
"grafana.app/repoName": "example-repo",
"grafana.app/createdBy": "user:useruid",
"grafana.app/updatedBy": "user:useruid"
"grafana.app/updatedBy": "user:2"
}
},
"spec": {
@@ -49,12 +49,23 @@ func TestFolderConversions(t *testing.T) {
require.NoError(t, err)
fake := usertest.NewUserServiceFake()
fake.ExpectedUser = &user.User{ID: 10, UID: "useruid"}
fake.ExpectedListUsersByIdOrUid = []*user.User{
{
ID: 1,
UID: "useruid",
},
{
ID: 2,
UID: "useruid2",
},
}
fs := ProvideUnifiedStore(nil, fake)
converted, err := fs.UnstructuredToLegacyFolder(context.Background(), input)
require.NoError(t, err)
require.Equal(t, 1, len(fake.ListUsersByIdOrUidCalls)) // only one call to the user service
require.Equal(t, usertest.ListUsersByIdOrUidCall{Uids: []string{"useruid"}, Ids: []int64{2}}, fake.ListUsersByIdOrUidCalls[0])
require.Equal(t, folder.Folder{
ID: 234,
OrgID: 1,
@@ -67,7 +78,288 @@ func TestFolderConversions(t *testing.T) {
ManagedBy: utils.ManagerKindRepo,
Created: created,
Updated: created.Add(time.Hour * 5),
CreatedBy: 10,
UpdatedBy: 10,
CreatedBy: 1,
UpdatedBy: 2,
}, *converted)
}
func TestFolderListConversions(t *testing.T) {
input := &unstructured.UnstructuredList{}
err := input.UnmarshalJSON([]byte(`{
"apiVersion": "folder.grafana.app/v0alpha1",
"items": [
{
"apiVersion": "folder.grafana.app/v0alpha1",
"kind": "Folder",
"metadata": {
"annotations": {
"grafana.app/createdBy": "access-policy:service"
},
"creationTimestamp": "2022-12-02T02:02:02Z",
"generation": 1,
"labels": {
"grafana.app/deprecatedInternalID": "4",
"grafana.app/fullpath": "somefullpath",
"grafana.app/fullpathUIDs": "somefullpathuids"
},
"name": "foldername1",
"namespace": "default",
"resourceVersion": "1741881243564020",
"uid": "dd669a77-7872-4c96-8fc7-899a5d8fdb6c"
},
"spec": {
"title": "gdev dashboards",
"description": "description gdev"
}
},
{
"apiVersion": "folder.grafana.app/v0alpha1",
"kind": "Folder",
"metadata": {
"annotations": {
"grafana.app/createdBy": "user:uuuuuuuuuuuuuu"
},
"creationTimestamp": "2022-12-02T02:02:02Z",
"generation": 1,
"labels": {
"grafana.app/deprecatedInternalID": "149"
},
"name": "foldername2",
"namespace": "default",
"resourceVersion": "1742998826046994",
"uid": "941bcef6-1579-48b0-9b25-7cc227910aae"
},
"spec": {
"title": "yeye",
"description": "description yeye"
}
},
{
"apiVersion": "folder.grafana.app/v0alpha1",
"kind": "Folder",
"metadata": {
"annotations": {
"grafana.app/createdBy": "user:iiiiiiiiiiiiii",
"grafana.app/updatedBy": "user:jjjjjjjjjjjjjj"
},
"creationTimestamp": "2022-12-02T02:02:02Z",
"generation": 1,
"labels": {
"grafana.app/deprecatedInternalID": "145"
},
"name": "foldername3",
"namespace": "default",
"resourceVersion": "1743003591477996",
"uid": "96083f1d-8501-425a-bbeb-c1c4ef1e9985"
},
"spec": {
"title": "yoyo",
"description": "description yoyo"
}
},
{
"apiVersion": "folder.grafana.app/v0alpha1",
"kind": "Folder",
"metadata": {
"annotations": {
"grafana.app/createdBy": "user:1"
},
"creationTimestamp": "2022-12-02T02:02:02Z",
"generation": 1,
"labels": {
"grafana.app/deprecatedInternalID": "146"
},
"name": "foldername4",
"namespace": "default",
"resourceVersion": "1742998754624006",
"uid": "f42e5c38-8ad3-43ee-813e-575592ac88b2"
},
"spec": {
"title": "yaya",
"description": "description yaya"
}
},
{
"apiVersion": "folder.grafana.app/v0alpha1",
"kind": "Folder",
"metadata": {
"annotations": {
"grafana.app/createdBy": "user:2",
"grafana.app/updatedBy": "user:3"
},
"creationTimestamp": "2022-12-02T02:02:02Z",
"generation": 1,
"labels": {
"grafana.app/deprecatedInternalID": "147"
},
"name": "foldername5",
"namespace": "default",
"resourceVersion": "1742317244002993",
"uid": "b52ef257-2fed-4edc-a559-a1f68c46e75c"
},
"spec": {
"title": "yiyi",
"description": "description yiyi"
}
},
{
"apiVersion": "folder.grafana.app/v0alpha1",
"kind": "Folder",
"metadata": {
"annotations": {},
"creationTimestamp": "2022-12-02T02:02:02Z",
"generation": 1,
"labels": {
"grafana.app/deprecatedInternalID": "148"
},
"name": "foldername6",
"namespace": "default",
"resourceVersion": "1742998544679983",
"uid": "771747f4-d93c-4b4d-b5be-db287e559c64"
},
"spec": {
"title": "yuyu",
"description": "description yuyu"
}
}
],
"kind": "FolderList",
"metadata": {
"resourceVersion": "1743003591477997"
}
}`))
require.NoError(t, err)
created, err := time.Parse(time.RFC3339, "2022-12-02T02:02:02Z")
created = created.UTC()
require.NoError(t, err)
fake := usertest.NewUserServiceFake()
fake.ExpectedListUsersByIdOrUid = []*user.User{
{
ID: 1,
UID: "aaaaaaaaaaaaaa",
},
{
ID: 2,
UID: "oooooooooooooo",
},
{
ID: 3,
UID: "eeeeeeeeeeeeee",
},
{
ID: 4,
UID: "uuuuuuuuuuuuuu",
},
{
ID: 5,
UID: "iiiiiiiiiiiiii",
},
{
ID: 6,
UID: "jjjjjjjjjjjjjj",
},
}
fs := ProvideUnifiedStore(nil, fake)
converted, err := fs.UnstructuredToLegacyFolderList(context.Background(), input)
require.NoError(t, err)
require.Equal(t, 1, len(fake.ListUsersByIdOrUidCalls)) // only one call to the user service
require.Equal(t, usertest.ListUsersByIdOrUidCall{Uids: []string{"uuuuuuuuuuuuuu", "iiiiiiiiiiiiii", "jjjjjjjjjjjjjj"}, Ids: []int64{1, 2, 3}}, fake.ListUsersByIdOrUidCalls[0])
require.Equal(t, 6, len(converted))
require.Equal(t, []*folder.Folder{
{
ID: 4,
OrgID: 1,
Version: 1,
UID: "foldername1",
ParentUID: "",
Title: "gdev dashboards",
Description: "description gdev",
URL: "/dashboards/f/foldername1/gdev-dashboards",
ManagedBy: "",
Created: created,
Updated: created,
CreatedBy: 0, // service account
UpdatedBy: 0, // service account,
},
{
ID: 149,
OrgID: 1,
Version: 1,
UID: "foldername2",
ParentUID: "",
Title: "yeye",
Description: "description yeye",
URL: "/dashboards/f/foldername2/yeye",
ManagedBy: "",
Created: created,
Updated: created,
CreatedBy: 4,
UpdatedBy: 4,
},
{
ID: 145,
OrgID: 1,
Version: 1,
UID: "foldername3",
ParentUID: "",
Title: "yoyo",
Description: "description yoyo",
URL: "/dashboards/f/foldername3/yoyo",
ManagedBy: "",
Created: created,
Updated: created,
CreatedBy: 5,
UpdatedBy: 6,
},
{
ID: 146,
OrgID: 1,
Version: 1,
UID: "foldername4",
ParentUID: "",
Title: "yaya",
Description: "description yaya",
URL: "/dashboards/f/foldername4/yaya",
ManagedBy: "",
Created: created,
Updated: created,
CreatedBy: 1,
UpdatedBy: 1,
},
{
ID: 147,
OrgID: 1,
Version: 1,
UID: "foldername5",
ParentUID: "",
Title: "yiyi",
Description: "description yiyi",
URL: "/dashboards/f/foldername5/yiyi",
ManagedBy: "",
Created: created,
Updated: created,
CreatedBy: 2,
UpdatedBy: 3,
},
{
ID: 148,
OrgID: 1,
Version: 1,
UID: "foldername6",
ParentUID: "",
Title: "yuyu",
Description: "description yuyu",
URL: "/dashboards/f/foldername6/yuyu",
ManagedBy: "",
Created: created,
Updated: created,
CreatedBy: 0, // no createdby
UpdatedBy: 0,
},
}, converted)
}
pkg/services/folder/folderimpl/unifiedstore.go
+13 -14
View File
@@ -319,15 +319,14 @@ func (ss *FolderUnifiedStoreImpl) GetFolders(ctx context.Context, q folder.GetFo
if err != nil {
return nil, err
}
// convert item to legacy folder format
folders, err := ss.UnstructuredToLegacyFolderList(ctx, out)
if err != nil {
return nil, err
}
m := map[string]*folder.Folder{}
for _, item := range out.Items {
// convert item to legacy folder format
f, err := ss.UnstructuredToLegacyFolder(ctx, &item)
if f == nil {
return nil, fmt.Errorf("unable to convert unstructured item to legacy folder %w", err)
}
for _, f := range folders {
if (q.WithFullpath || q.WithFullpathUIDs) && f.Fullpath == "" {
parents, err := ss.GetParents(ctx, folder.GetParentsQuery{UID: f.UID, OrgID: q.OrgID})
if err != nil {
@@ -375,14 +374,14 @@ func (ss *FolderUnifiedStoreImpl) GetDescendants(ctx context.Context, orgID int6
return nil, err
}
nodes := map[string]*folder.Folder{}
for _, item := range out.Items {
// convert item to legacy folder format
f, err := ss.UnstructuredToLegacyFolder(ctx, &item)
if f == nil {
return nil, fmt.Errorf("unable to convert unstructured item to legacy folder %w", err)
}
// convert item to legacy folder format
folders, err := ss.UnstructuredToLegacyFolderList(ctx, out)
if err != nil {
return nil, err
}
nodes := map[string]*folder.Folder{}
for _, f := range folders {
nodes[f.UID] = f
}
pkg/services/folder/folderimpl/unifiedstore_test.go
+5 -2
View File
@@ -11,6 +11,7 @@ import (
"github.com/grafana/grafana/pkg/services/apiserver/client"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/user/usertest"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
@@ -93,7 +94,8 @@ func TestComputeFullPath(t *testing.T) {
func TestGetParents(t *testing.T) {
mockCli := new(client.MockK8sHandler)
store := FolderUnifiedStoreImpl{
k8sclient: mockCli,
k8sclient: mockCli,
userService: usertest.NewUserServiceFake(),
}
ctx := context.Background()
@@ -187,7 +189,8 @@ func TestGetParents(t *testing.T) {
func TestGetChildren(t *testing.T) {
mockCli := new(client.MockK8sHandler)
store := FolderUnifiedStoreImpl{
k8sclient: mockCli,
k8sclient: mockCli,
userService: usertest.NewUserServiceFake(),
}
ctx := context.Background()