public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(unified-storage): add pagination support for folder list (#105444)

Browse Source
This commit is contained in:
Mustafa Sencer Özcan
2025-05-15 18:43:44 +02:00
committed by GitHub
parent d4d1514ecb
commit 167b201525
3 changed files with 281 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/folder/folderimpl/folder_unifiedstorage.go
+1
View File
@@ -34,6 +34,7 @@ import (
)
const folderSearchLimit = 100000
const folderListLimit = 100
func (s *Service) getFoldersFromApiServer(ctx context.Context, q folder.GetFoldersQuery) ([]*folder.Folder, error) {
ctx, span := s.tracer.Start(ctx, "folder.getFoldersFromApiServer")
pkg/services/folder/folderimpl/unifiedstore.go
+44 -5
View File
@@ -317,16 +317,14 @@ func (ss *FolderUnifiedStoreImpl) GetHeight(ctx context.Context, foldrUID string
// The full path UIDs of B is "uid1/uid2".
// The full path UIDs of A is "uid1".
func (ss *FolderUnifiedStoreImpl) GetFolders(ctx context.Context, q folder.GetFoldersFromStoreQuery) ([]*folder.Folder, error) {
opts := v1.ListOptions{
Limit: folderSearchLimit,
}
opts := v1.ListOptions{}
if q.WithFullpath || q.WithFullpathUIDs {
// only supported in modes 0-2, to keep the alerting queries from causing tons of get folder requests
// to retrieve the parent for all folders in grafana
opts.LabelSelector = utils.LabelGetFullpath + "=true"
}
out, err := ss.k8sclient.List(ctx, q.OrgID, opts)
out, err := ss.list(ctx, q.OrgID, opts)
if err != nil {
return nil, err
}
@@ -371,7 +369,7 @@ func (ss *FolderUnifiedStoreImpl) GetFolders(ctx context.Context, q folder.GetFo
}
func (ss *FolderUnifiedStoreImpl) GetDescendants(ctx context.Context, orgID int64, ancestor_uid string) ([]*folder.Folder, error) {
out, err := ss.k8sclient.List(ctx, orgID, v1.ListOptions{})
out, err := ss.list(ctx, orgID, v1.ListOptions{})
if err != nil {
return nil, err
}
@@ -447,6 +445,47 @@ func (ss *FolderUnifiedStoreImpl) CountInOrg(ctx context.Context, orgID int64) (
return resp.Stats[0].Count, nil
}
func (ss *FolderUnifiedStoreImpl) list(ctx context.Context, orgID int64, opts v1.ListOptions) (*unstructured.UnstructuredList, error) {
var allItems []unstructured.Unstructured
listOpts := opts.DeepCopy()
if listOpts.Limit == 0 {
listOpts.Limit = folderListLimit
}
for {
out, err := ss.k8sclient.List(ctx, orgID, *listOpts)
if err != nil {
return nil, err
}
if out == nil {
return nil, fmt.Errorf("k8s folder list returned nil")
}
if len(out.Items) > 0 {
allItems = append(allItems, out.Items...)
}
if out.GetContinue() == "" || (opts.Limit > 0 && int64(len(allItems)) >= opts.Limit) {
break
}
listOpts.Continue = out.GetContinue()
}
result := &unstructured.UnstructuredList{
Items: allItems,
}
if opts.Limit > 0 && int64(len(allItems)) > opts.Limit {
result.Items = allItems[:opts.Limit]
}
return result, nil
}
func toFolderLegacyCounts(u *unstructured.Unstructured) (*folder.DescendantCounts, error) {
ds, err := folderv1.UnstructuredToDescendantCounts(u)
if err != nil {
pkg/services/folder/folderimpl/unifiedstore_test.go
+236 -5
View File
@@ -465,7 +465,7 @@ func TestGetFolders(t *testing.T) {
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, orgID, metav1.ListOptions{
Limit: 100000,
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
}).Return(&unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
@@ -521,7 +521,7 @@ func TestGetFolders(t *testing.T) {
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, orgID, metav1.ListOptions{
Limit: 100000,
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
}).Return(&unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
@@ -588,7 +588,7 @@ func TestGetFolders(t *testing.T) {
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, orgID, metav1.ListOptions{
Limit: 100000,
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
LabelSelector: "grafana.app/fullpath=true",
}).Return(&unstructured.UnstructuredList{
@@ -650,7 +650,7 @@ func TestGetFolders(t *testing.T) {
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, orgID, metav1.ListOptions{
Limit: 100000,
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
LabelSelector: "grafana.app/fullpath=true",
}).Return(&unstructured.UnstructuredList{
@@ -724,7 +724,7 @@ func TestGetFolders(t *testing.T) {
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, orgID, metav1.ListOptions{
Limit: 100000,
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
}).Return(nil, apierrors.NewNotFound(schema.GroupResource{Group: "folders.folder.grafana.app", Resource: "folder"}, "folder1")).Once()
},
@@ -880,3 +880,234 @@ func TestBuildFolderFullPaths(t *testing.T) {
})
}
}
func TestList(t *testing.T) {
type args struct {
ctx context.Context
orgID int64
opts metav1.ListOptions
}
tests := []struct {
name string
args args
mock func(mockCli *client.MockK8sHandler)
want *unstructured.UnstructuredList
wantErr bool
}{
{
name: "should return all folders",
args: args{
ctx: context.Background(),
orgID: orgID,
opts: metav1.ListOptions{
Limit: 0,
TypeMeta: metav1.TypeMeta{},
},
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, int64(1), metav1.ListOptions{
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
}).Return(&unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder2",
"uid": "folder2",
},
"spec": map[string]interface{}{
"title": "folder2",
},
},
},
},
}, nil).Once()
},
want: &unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder2",
"uid": "folder2",
},
"spec": map[string]interface{}{
"title": "folder2",
},
},
},
},
},
},
{
name: "should return folders with limit",
args: args{
ctx: context.Background(),
orgID: orgID,
opts: metav1.ListOptions{
Limit: 1,
TypeMeta: metav1.TypeMeta{},
},
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, int64(1), metav1.ListOptions{
Limit: 1,
TypeMeta: metav1.TypeMeta{},
}).Return(&unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder2",
"uid": "folder2",
},
"spec": map[string]interface{}{
"title": "folder2",
},
},
},
},
}, nil).Once()
},
want: &unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
},
},
wantErr: false,
},
{
name: "should return folders with continue token",
args: args{
ctx: context.Background(),
orgID: orgID,
opts: metav1.ListOptions{
Limit: 1,
TypeMeta: metav1.TypeMeta{},
},
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, int64(1), metav1.ListOptions{
Limit: 1,
TypeMeta: metav1.TypeMeta{},
}).Return(&unstructured.UnstructuredList{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"continue": "continue-token",
},
},
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
},
}, nil).Once()
},
want: &unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
},
},
wantErr: false,
},
{
name: "should return error if k8s returns error",
args: args{
ctx: context.Background(),
orgID: orgID,
opts: metav1.ListOptions{
Limit: 0,
TypeMeta: metav1.TypeMeta{},
},
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, int64(1), metav1.ListOptions{
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
}).Return(nil, apierrors.NewNotFound(schema.GroupResource{Group: "folders.folder.grafana.app", Resource: "folder"}, "folder1")).Once()
},
want: nil,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockCLI := new(client.MockK8sHandler)
tt.mock(mockCLI)
ss := &FolderUnifiedStoreImpl{
k8sclient: mockCLI,
userService: usertest.NewUserServiceFake(),
}
got, err := ss.list(tt.args.ctx, tt.args.orgID, tt.args.opts)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
require.Equal(t, tt.want, got)
})
}
}