public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

K8s: Folders: Add pagination for children (#100978)

Browse Source
This commit is contained in:
Stephanie Hingtgen
2025-02-19 11:06:26 -07:00
committed by GitHub
parent ff1b22297c
commit 95278d7552
4 changed files with 233 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/folder/folderimpl/folder_unifiedstorage.go
+6 -4
View File
@@ -33,6 +33,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const folderSearchLimit = 100000
func (s *Service) getFoldersFromApiServer(ctx context.Context, q folder.GetFoldersQuery) ([]*folder.Folder, error) {
if q.SignedInUser == nil {
return nil, folder.ErrBadRequest.Errorf("missing signed in user")
@@ -171,7 +173,7 @@ func (s *Service) searchFoldersFromApiServer(ctx context.Context, query folder.S
Fields: []*resource.Requirement{},
Labels: []*resource.Requirement{},
},
Limit: 100000}
Limit: folderSearchLimit}
if len(query.UIDs) > 0 {
request.Options.Fields = []*resource.Requirement{{
@@ -269,7 +271,7 @@ func (s *Service) getFolderByIDFromApiServer(ctx context.Context, id int64, orgI
},
},
},
Limit: 100000}
Limit: folderSearchLimit}
res, err := s.k8sclient.Search(ctx, orgID, request)
if err != nil {
@@ -322,7 +324,7 @@ func (s *Service) getFolderByTitleFromApiServer(ctx context.Context, orgID int64
},
Labels: []*resource.Requirement{},
},
Limit: 100000}
Limit: folderSearchLimit}
if parentUID != nil {
req := []*resource.Requirement{{
@@ -671,7 +673,7 @@ func (s *Service) deleteFromApiServer(ctx context.Context, cmd *folder.DeleteFol
},
},
},
Limit: 100000}
Limit: folderSearchLimit}
res, err := s.dashboardK8sClient.Search(ctx, cmd.OrgID, request)
if err != nil {
pkg/services/folder/folderimpl/folder_unifiedstorage_test.go
+4 -4
View File
@@ -554,7 +554,7 @@ func TestSearchFoldersFromApiServer(t *testing.T) {
},
Labels: []*resource.Requirement{},
},
Limit: 100000}).Return(&resource.ResourceSearchResponse{
Limit: folderSearchLimit}).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{
{
@@ -649,7 +649,7 @@ func TestSearchFoldersFromApiServer(t *testing.T) {
},
},
},
Limit: 100000}).Return(&resource.ResourceSearchResponse{
Limit: folderSearchLimit}).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{
{
@@ -716,7 +716,7 @@ func TestSearchFoldersFromApiServer(t *testing.T) {
},
Query: "*test*",
Fields: dashboardsearch.IncludeFields,
Limit: 100000}).Return(&resource.ResourceSearchResponse{
Limit: folderSearchLimit}).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{
{
@@ -829,7 +829,7 @@ func TestDeleteFoldersFromApiServer(t *testing.T) {
},
},
},
Limit: 100000}).Return(&resource.ResourceSearchResponse{
Limit: folderSearchLimit}).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{
{
pkg/services/folder/folderimpl/unifiedstore.go
+62 -12
View File
@@ -8,14 +8,19 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/selection"
claims "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
"github.com/grafana/grafana/pkg/infra/log"
internalfolders "github.com/grafana/grafana/pkg/registry/apis/folders"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/apiserver/client"
"github.com/grafana/grafana/pkg/services/dashboards"
dashboardsearch "github.com/grafana/grafana/pkg/services/dashboards/service/search"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/util"
@@ -159,26 +164,71 @@ func (ss *FolderUnifiedStoreImpl) GetParents(ctx context.Context, q folder.GetPa
}
func (ss *FolderUnifiedStoreImpl) GetChildren(ctx context.Context, q folder.GetChildrenQuery) ([]*folder.Folder, error) {
out, err := ss.k8sclient.List(ctx, q.OrgID, v1.ListOptions{})
// the general folder is saved as an empty string in the database
if q.UID == folder.GeneralFolderUID {
q.UID = ""
}
if q.Limit == 0 {
q.Limit = folderSearchLimit
}
if q.Page == 0 {
q.Page = 1
}
req := &resource.ResourceSearchRequest{
Options: &resource.ListOptions{
Fields: []*resource.Requirement{
{
Key: resource.SEARCH_FIELD_FOLDER,
Operator: string(selection.In),
Values: []string{q.UID},
},
},
},
Limit: q.Limit,
// legacy fallback search requires page, unistore requires offset,
// so set both
Offset: q.Limit * (q.Page - 1),
Page: q.Page,
}
// only filter the folder UIDs if they are provided in the query
if len(q.FolderUIDs) > 0 {
req.Options.Fields = append(req.Options.Fields, &resource.Requirement{
Key: resource.SEARCH_FIELD_NAME,
Operator: string(selection.In),
Values: q.FolderUIDs,
})
}
// now, get children of the parent folder
out, err := ss.k8sclient.Search(ctx, q.OrgID, req)
if err != nil {
return nil, err
}
res, err := dashboardsearch.ParseResults(out, 0)
if err != nil {
return nil, err
}
allowK6Folder := (q.SignedInUser != nil && q.SignedInUser.IsIdentityType(claims.TypeServiceAccount))
hits := make([]*folder.Folder, 0)
for _, item := range out.Items {
// convert item to legacy folder format
f, err := ss.UnstructuredToLegacyFolder(ctx, &item)
if f == nil {
return nil, fmt.Errorf("unable to convert unstructured item to legacy folder %w", err)
for _, item := range res.Hits {
// filter out k6 folders if request is not from a service account
if item.Name == accesscontrol.K6FolderUID && !allowK6Folder {
continue
}
// it we are at root level, skip subfolder
if q.UID == "" && f.ParentUID != "" {
continue // query filter
// search only returns a subset of info, get all info of the folder
item, err := ss.k8sclient.Get(ctx, item.Name, q.OrgID, v1.GetOptions{})
if err != nil {
return nil, err
}
// if we are at a nested folder, then skip folders that don't belong to parentUid
if q.UID != "" && !strings.EqualFold(f.ParentUID, q.UID) {
continue
f, err := ss.UnstructuredToLegacyFolder(ctx, item)
if err != nil {
return nil, err
}
hits = append(hits, f)
pkg/services/folder/folderimpl/unifiedstore_test.go
+161
View File
@@ -1,10 +1,19 @@
package folderimpl
import (
"context"
"testing"
claims "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/apiserver/client"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/selection"
)
func TestComputeFullPath(t *testing.T) {
@@ -75,3 +84,155 @@ func TestComputeFullPath(t *testing.T) {
})
}
}
func TestGetChildren(t *testing.T) {
mockCli := new(client.MockK8sHandler)
store := FolderUnifiedStoreImpl{
k8sclient: mockCli,
}
ctx := context.Background()
orgID := int64(2)
t.Run("should be able to find children folders, and set defaults for pages", func(t *testing.T) {
mockCli.On("Search", mock.Anything, orgID, &resource.ResourceSearchRequest{
Options: &resource.ListOptions{
Fields: []*resource.Requirement{
{
Key: resource.SEARCH_FIELD_FOLDER,
Operator: string(selection.In),
Values: []string{"folder1"},
},
},
},
Limit: folderSearchLimit, // should default to folderSearchLimit
Offset: 0, // should be set as limit * (page - 1)
Page: 1, // should be set to 1 by default
}).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{
{Name: "folder", Type: resource.ResourceTableColumnDefinition_STRING},
},
Rows: []*resource.ResourceTableRow{
{
Key: &resource.ResourceKey{Name: "folder2", Resource: "folder"},
Cells: [][]byte{[]byte("folder1")},
},
{
Key: &resource.ResourceKey{Name: "folder3", Resource: "folder"},
Cells: [][]byte{[]byte("folder1")},
},
},
},
TotalHits: 1,
}, nil).Once()
mockCli.On("Get", mock.Anything, "folder2", orgID, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{"name": "folder2"},
},
}, nil).Once()
mockCli.On("Get", mock.Anything, "folder3", orgID, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{"name": "folder3"},
},
}, nil).Once()
// don't set page or limit - should be automatically added
result, err := store.GetChildren(ctx, folder.GetChildrenQuery{
UID: "folder1",
OrgID: orgID,
})
require.NoError(t, err)
require.Len(t, result, 2)
require.Equal(t, "folder2", result[0].UID)
require.Equal(t, "folder3", result[1].UID)
})
t.Run("pages should be able to be set, general folder should be turned to empty string, and folder uids should be passed in", func(t *testing.T) {
mockCli.On("Search", mock.Anything, orgID, &resource.ResourceSearchRequest{
Options: &resource.ListOptions{
Fields: []*resource.Requirement{
{
Key: resource.SEARCH_FIELD_FOLDER,
Operator: string(selection.In),
Values: []string{""}, // should be an empty string if general is passed in
},
{
Key: resource.SEARCH_FIELD_NAME,
Operator: string(selection.In),
Values: []string{"folder2"},
},
},
},
Limit: 10,
Offset: 20, // should be set as limit * (page - 1)
Page: 3,
}).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{
{Name: "folder", Type: resource.ResourceTableColumnDefinition_STRING},
},
Rows: []*resource.ResourceTableRow{
{
Key: &resource.ResourceKey{Name: "folder2", Resource: "folder"},
Cells: [][]byte{[]byte("folder1")},
},
},
},
TotalHits: 1,
}, nil).Once()
mockCli.On("Get", mock.Anything, "folder2", orgID, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{"name": "folder2"},
},
}, nil).Once()
result, err := store.GetChildren(ctx, folder.GetChildrenQuery{
UID: "general",
OrgID: orgID,
Limit: 10,
Page: 3,
FolderUIDs: []string{"folder2"},
})
require.NoError(t, err)
require.Len(t, result, 1)
require.Equal(t, "folder2", result[0].UID)
})
t.Run("k6 folder should only be returned to service accounts", func(t *testing.T) {
mockCli.On("Search", mock.Anything, orgID, mock.Anything).Return(&resource.ResourceSearchResponse{
Results: &resource.ResourceTable{
Columns: []*resource.ResourceTableColumnDefinition{
{Name: "folder", Type: resource.ResourceTableColumnDefinition_STRING},
},
Rows: []*resource.ResourceTableRow{
{
Key: &resource.ResourceKey{Name: accesscontrol.K6FolderUID, Resource: "folder"},
Cells: [][]byte{[]byte("folder1")},
},
},
},
TotalHits: 1,
}, nil)
mockCli.On("Get", mock.Anything, accesscontrol.K6FolderUID, orgID, mock.Anything, mock.Anything).Return(&unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{"name": accesscontrol.K6FolderUID},
},
}, nil)
result, err := store.GetChildren(ctx, folder.GetChildrenQuery{
UID: "folder",
OrgID: orgID,
})
require.NoError(t, err)
require.Len(t, result, 0)
result, err = store.GetChildren(ctx, folder.GetChildrenQuery{
UID: "folder",
OrgID: orgID,
SignedInUser: &identity.StaticRequester{Type: claims.TypeServiceAccount},
})
require.NoError(t, err)
require.Len(t, result, 1)
})
}