From 76372a240ce7c009ba5a19351a7b66b1b2a73a23 Mon Sep 17 00:00:00 2001 From: Tania Date: Thu, 24 Nov 2022 15:15:32 +0100 Subject: [PATCH 001/131] Storage: Add system users service (#57767) * Storage: Add access for reporting * reporting upload user per org * add some basic comments * Move reporting storage to enterprise * add comments Co-authored-by: Artur Wierzbicki --- pkg/server/wire.go | 1 + pkg/services/quota/quotaimpl/quota_test.go | 2 +- pkg/services/store/service.go | 37 ++----- pkg/services/store/service_test.go | 12 +-- pkg/services/store/system_users.go | 109 +++++++++++++++++++++ pkg/services/store/system_users_test.go | 63 ++++++++++++ 6 files changed, 190 insertions(+), 34 deletions(-) create mode 100644 pkg/services/store/system_users.go create mode 100644 pkg/services/store/system_users_test.go diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 7c4dfeadb54..3bc71a93c21 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -246,6 +246,7 @@ var wireBasicSet = wire.NewSet( searchV2.ProvideService, searchV2.ProvideSearchHTTPService, store.ProvideService, + store.ProvideSystemUsersService, export.ProvideService, live.ProvideService, pushhttp.ProvideService, diff --git a/pkg/services/quota/quotaimpl/quota_test.go b/pkg/services/quota/quotaimpl/quota_test.go index 6614bcbdaf9..54d5c127cd1 100644 --- a/pkg/services/quota/quotaimpl/quota_test.go +++ b/pkg/services/quota/quotaimpl/quota_test.go @@ -479,6 +479,6 @@ func setupEnv(t *testing.T, sqlStore *sqlstore.SQLStore, b bus.Bus, quotaService secretsService, nil, m, &foldertest.FakeService{}, &acmock.Mock{}, &dashboards.FakeDashboardService{}, nil, b, &acmock.Mock{}, annotationstest.NewFakeAnnotationsRepo(), ) require.NoError(t, err) - _, err = storesrv.ProvideService(sqlStore, featuremgmt.WithFeatures(), sqlStore.Cfg, quotaService) + _, err = storesrv.ProvideService(sqlStore, featuremgmt.WithFeatures(), sqlStore.Cfg, quotaService, storesrv.ProvideSystemUsersService()) require.NoError(t, err) } diff --git a/pkg/services/store/service.go b/pkg/services/store/service.go index cc3beab7d10..fc17c6b85b1 100644 --- a/pkg/services/store/service.go +++ b/pkg/services/store/service.go @@ -40,14 +40,6 @@ const RootContent = "content" const RootDevenv = "devenv" const RootSystem = "system" -const brandingStorage = "branding" -const SystemBrandingStorage = "system/" + brandingStorage - -var ( - SystemBrandingReader = &user.SignedInUser{OrgID: ac.GlobalOrgID} - SystemBrandingAdmin = &user.SignedInUser{OrgID: ac.GlobalOrgID} -) - const MAX_UPLOAD_SIZE = 1 * 1024 * 1024 // 3MB type DeleteFolderCmd struct { @@ -96,6 +88,7 @@ type standardStorageService struct { cfg *GlobalStorageConfig authService storageAuthService quotaService quota.Service + systemUsers SystemUsersFilterProvider } func ProvideService( @@ -103,6 +96,7 @@ func ProvideService( features featuremgmt.FeatureToggles, cfg *setting.Cfg, quotaService quota.Service, + systemUsersService SystemUsers, ) (StorageService, error) { settings, err := LoadStorageConfig(cfg, features) if err != nil { @@ -208,22 +202,17 @@ func ProvideService( } if storageName == RootSystem { - if user == SystemBrandingReader { + filter, err := systemUsersService.GetFilter(user) + if err != nil { + grafanaStorageLogger.Error("failed to create path filter for system user", "userID", user.UserID, "userLogin", user.Login, "err", err) return map[string]filestorage.PathFilter{ - ActionFilesRead: createSystemBrandingPathFilter(), + ActionFilesRead: denyAllPathFilter, ActionFilesWrite: denyAllPathFilter, ActionFilesDelete: denyAllPathFilter, } } - if user == SystemBrandingAdmin { - systemBrandingFilter := createSystemBrandingPathFilter() - return map[string]filestorage.PathFilter{ - ActionFilesRead: systemBrandingFilter, - ActionFilesWrite: systemBrandingFilter, - ActionFilesDelete: systemBrandingFilter, - } - } + return filter } if storageName == RootContent { @@ -262,7 +251,7 @@ func ProvideService( } }) - s := newStandardStorageService(sql, globalRoots, initializeOrgStorages, authService, cfg) + s := newStandardStorageService(sql, globalRoots, initializeOrgStorages, authService, cfg, systemUsersService) s.quotaService = quotaService s.cfg = settings @@ -298,20 +287,13 @@ func readQuotaConfig(cfg *setting.Cfg) (*quota.Map, error) { return limits, nil } -func createSystemBrandingPathFilter() filestorage.PathFilter { - return filestorage.NewPathFilter( - []string{filestorage.Delimiter + brandingStorage + filestorage.Delimiter}, // access to all folders and files inside `/branding/` - []string{filestorage.Delimiter + brandingStorage}, // access to the `/branding` folder itself, but not to any other sibling folder - nil, - nil) -} - func newStandardStorageService( sql db.DB, globalRoots []storageRuntime, initializeOrgStorages func(orgId int64) []storageRuntime, authService storageAuthService, cfg *setting.Cfg, + systemUsers SystemUsersFilterProvider, ) *standardStorageService { prefixes := make(map[string]bool) @@ -336,6 +318,7 @@ func newStandardStorageService( sql: sql, tree: res, authService: authService, + systemUsers: systemUsers, } } diff --git a/pkg/services/store/service_test.go b/pkg/services/store/service_test.go index c74b744af16..42683061453 100644 --- a/pkg/services/store/service_test.go +++ b/pkg/services/store/service_test.go @@ -74,7 +74,7 @@ func TestListFiles(t *testing.T) { store := newStandardStorageService(db.InitTestDB(t), roots, func(orgId int64) []storageRuntime { return make([]storageRuntime, 0) - }, allowAllAuthService, cfg) + }, allowAllAuthService, cfg, nil) frame, err := store.List(context.Background(), dummyUser, "public/testdata") require.NoError(t, err) @@ -94,7 +94,7 @@ func TestListFilesWithoutPermissions(t *testing.T) { store := newStandardStorageService(db.InitTestDB(t), roots, func(orgId int64) []storageRuntime { return make([]storageRuntime, 0) - }, denyAllAuthService, cfg) + }, denyAllAuthService, cfg, nil) frame, err := store.List(context.Background(), dummyUser, "public/testdata") require.NoError(t, err) rowLen, err := frame.RowLen() @@ -114,7 +114,7 @@ func setupUploadStore(t *testing.T, authService storageAuthService) (StorageServ } store := newStandardStorageService(db.InitTestDB(t), []storageRuntime{sqlStorage}, func(orgId int64) []storageRuntime { return make([]storageRuntime, 0) - }, authService, cfg) + }, authService, cfg, nil) store.cfg = &GlobalStorageConfig{ AllowUnsanitizedSvgUpload: true, } @@ -268,7 +268,7 @@ func TestSetupWithNonUniqueStoragePrefixes(t *testing.T) { newStandardStorageService(db.InitTestDB(t), []storageRuntime{sqlStorage, sqlStorage2}, func(orgId int64) []storageRuntime { return make([]storageRuntime, 0) - }, allowAllAuthService, cfg) + }, allowAllAuthService, cfg, nil) } func TestContentRootWithNestedStorage(t *testing.T) { @@ -293,7 +293,7 @@ func TestContentRootWithNestedStorage(t *testing.T) { store := newStandardStorageService(db.InitTestDB(t), []storageRuntime{contentStorage, nestedStorage}, func(orgId int64) []storageRuntime { return []storageRuntime{nestedOrgedStorage, contentStorage} - }, allowAllAuthService, cfg) + }, allowAllAuthService, cfg, nil) store.cfg = &GlobalStorageConfig{ AllowUnsanitizedSvgUpload: true, } @@ -531,7 +531,7 @@ func TestShadowingExistingFolderByNestedContentRoot(t *testing.T) { }) require.NoError(t, err) - store := newStandardStorageService(db, []storageRuntime{nestedStorage, contentStorage}, func(orgId int64) []storageRuntime { return make([]storageRuntime, 0) }, allowAllAuthService, cfg) + store := newStandardStorageService(db, []storageRuntime{nestedStorage, contentStorage}, func(orgId int64) []storageRuntime { return make([]storageRuntime, 0) }, allowAllAuthService, cfg, nil) store.cfg = &GlobalStorageConfig{ AllowUnsanitizedSvgUpload: true, } diff --git a/pkg/services/store/system_users.go b/pkg/services/store/system_users.go new file mode 100644 index 00000000000..048710a949d --- /dev/null +++ b/pkg/services/store/system_users.go @@ -0,0 +1,109 @@ +package store + +import ( + "fmt" + "sync" + + "github.com/grafana/grafana/pkg/infra/filestorage" + ac "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/user" +) + +type SystemUserType string + +// SystemUsersFilterProvider interface internal to `pkg/store` service. +// Used by the Storage service to retrieve path filter for system users +type SystemUsersFilterProvider interface { + GetFilter(user *user.SignedInUser) (map[string]filestorage.PathFilter, error) +} + +// SystemUsersProvider interface used by `pkg/store` clients +// Used by Grafana services to retrieve users having access only to their own slice of storage +// For example, service 'Dashboard' could have exclusive access to paths matching `system/dashboard/*` +// by creating a system user with appropriate permissions. +type SystemUsersProvider interface { + GetUser(userType SystemUserType, orgID int64) (*user.SignedInUser, error) +} + +type SystemUsers interface { + SystemUsersFilterProvider + SystemUsersProvider + + // RegisterUser extension point - allows other Grafana services to register their own user type and assign them path-based permissions + RegisterUser(userType SystemUserType, filterFn func() map[string]filestorage.PathFilter) +} + +func ProvideSystemUsersService() SystemUsers { + return &hardcodedSystemUsers{ + mutex: sync.RWMutex{}, + users: make(map[SystemUserType]map[int64]*user.SignedInUser), + createFilterByUser: make(map[*user.SignedInUser]func() map[string]filestorage.PathFilter), + } +} + +type hardcodedSystemUsers struct { + mutex sync.RWMutex + + // map of user type -> map of user per orgID + users map[SystemUserType]map[int64]*user.SignedInUser + + // map of user -> create filter function. all users of the same type will point to the same function + createFilterByUser map[*user.SignedInUser]func() map[string]filestorage.PathFilter +} + +func (h *hardcodedSystemUsers) GetFilter(user *user.SignedInUser) (map[string]filestorage.PathFilter, error) { + h.mutex.Lock() + defer h.mutex.Unlock() + + createFn, ok := h.createFilterByUser[user] + if !ok { + return nil, fmt.Errorf("user %s with id %d has not been initialized", user.Login, user.UserID) + } + + return createFn(), nil +} + +func (h *hardcodedSystemUsers) GetUser(userType SystemUserType, orgID int64) (*user.SignedInUser, error) { + h.mutex.Lock() + defer h.mutex.Unlock() + + userPerOrgIdMap, ok := h.users[userType] + if !ok { + return nil, fmt.Errorf("user type %s is unknown", userType) + } + + orgSignedInUser, ok := userPerOrgIdMap[orgID] + if ok { + return orgSignedInUser, nil + } + + // user for the given org does not yet exist - initialize it + + globalUser, globalUserExists := userPerOrgIdMap[ac.GlobalOrgID] + if !globalUserExists { + return nil, fmt.Errorf("initialization error: user type %s should exist for global org id: %d", userType, ac.GlobalOrgID) + } + + globalUserFn, globalUserFnExists := h.createFilterByUser[globalUser] + if !globalUserFnExists { + return nil, fmt.Errorf("initialization error: user type %s should be associated with a create filter function", userType) + } + + newUser := &user.SignedInUser{ + Login: string(userType), + OrgID: orgID, + } + userPerOrgIdMap[orgID] = newUser + h.createFilterByUser[newUser] = globalUserFn + return newUser, nil +} + +func (h *hardcodedSystemUsers) RegisterUser(userType SystemUserType, filterFn func() map[string]filestorage.PathFilter) { + h.mutex.Lock() + defer h.mutex.Unlock() + + globalUser := &user.SignedInUser{OrgID: ac.GlobalOrgID, Login: string(userType)} + h.users[userType] = map[int64]*user.SignedInUser{ac.GlobalOrgID: globalUser} + + h.createFilterByUser[globalUser] = filterFn +} diff --git a/pkg/services/store/system_users_test.go b/pkg/services/store/system_users_test.go new file mode 100644 index 00000000000..422d975974f --- /dev/null +++ b/pkg/services/store/system_users_test.go @@ -0,0 +1,63 @@ +package store + +import ( + "testing" + + "github.com/grafana/grafana/pkg/infra/filestorage" + "github.com/grafana/grafana/pkg/services/user" + "github.com/stretchr/testify/require" +) + +const admin SystemUserType = "storageAdmin" + +func TestRetrievalOfNotInitializedOrg(t *testing.T) { + service := setupSystemUsers() + + orgID := int64(1) + user, err := service.GetUser(admin, orgID) + require.NoError(t, err) + + require.Equal(t, string(admin), user.Login) + require.Equal(t, orgID, user.OrgID) + + userFromSubsequentCall, err := service.GetUser(admin, orgID) + require.NoError(t, err) + require.Same(t, user, userFromSubsequentCall) +} + +func TestRetrievalOfFilterForInitializedUser(t *testing.T) { + service := setupSystemUsers() + + orgID := int64(1) + reportsAdminUser, err := service.GetUser(admin, orgID) + require.NoError(t, err) + + filter, err := service.GetFilter(reportsAdminUser) + require.NoError(t, err) + require.NotNil(t, filter) +} + +func TestRetrievalOfFilterForNotInitializedUser(t *testing.T) { + service := setupSystemUsers() + + orgID := int64(1) + + filter, err := service.GetFilter(&user.SignedInUser{ + OrgID: orgID, + Login: string(admin), + }) + require.Error(t, err) + require.Nil(t, filter) +} + +func setupSystemUsers() SystemUsers { + service := ProvideSystemUsersService() + + service.RegisterUser(admin, func() map[string]filestorage.PathFilter { + return map[string]filestorage.PathFilter{ + ActionFilesRead: denyAllPathFilter, + } + }) + + return service +} From 45c759eb59fe75dc83f069b8eadbb8c1ad4d1fd9 Mon Sep 17 00:00:00 2001 From: Kevin Minehart Date: Thu, 24 Nov 2022 08:17:12 -0600 Subject: [PATCH 002/131] CI: Make the downstream enterprise test a check instead of comments (#59071) --- go.mod | 1 + go.sum | 3 + pkg/build/cmd/argcount_wrapper.go | 17 ++- pkg/build/cmd/enterprisecheck.go | 121 ++++++++++++++++++++++ pkg/build/cmd/enterprisecheck_test.go | 69 +++++++++++++ pkg/build/cmd/flags.go | 6 ++ pkg/build/cmd/main.go | 44 ++++++-- pkg/build/env/lookup.go | 18 ++++ pkg/build/env/lookup_test.go | 43 ++++++++ pkg/build/git/git.go | 143 ++++++++++++++++++++++++++ pkg/build/git/git_checks_test.go | 55 ++++++++++ pkg/build/git/git_issues_test.go | 134 ++++++++++++++++++++++++ pkg/build/git/git_test.go | 25 +++++ pkg/build/stringutil/contains.go | 10 ++ 14 files changed, 681 insertions(+), 8 deletions(-) create mode 100644 pkg/build/cmd/enterprisecheck.go create mode 100644 pkg/build/cmd/enterprisecheck_test.go create mode 100644 pkg/build/env/lookup.go create mode 100644 pkg/build/env/lookup_test.go create mode 100644 pkg/build/git/git.go create mode 100644 pkg/build/git/git_checks_test.go create mode 100644 pkg/build/git/git_issues_test.go create mode 100644 pkg/build/git/git_test.go create mode 100644 pkg/build/stringutil/contains.go diff --git a/go.mod b/go.mod index 8f29add2ffd..c04c7037488 100644 --- a/go.mod +++ b/go.mod @@ -284,6 +284,7 @@ require ( github.com/drone/envsubst v1.0.3 // indirect github.com/drone/runner-go v1.12.0 // indirect github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect + github.com/google/go-github/v31 v31.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa // indirect github.com/googleapis/go-type-adapters v1.0.0 // indirect diff --git a/go.sum b/go.sum index d41b9baea0d..c55c5495966 100644 --- a/go.sum +++ b/go.sum @@ -1240,6 +1240,8 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-github/v31 v31.0.0 h1:JJUxlP9lFK+ziXKimTCprajMApV1ecWD4NB6CCb0plo= +github.com/google/go-github/v31 v31.0.0/go.mod h1:NQPZol8/1sMoWYGN2yaALIBytu17gAWfhbweiEed3pM= github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI= github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -3303,6 +3305,7 @@ google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6r google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= google.golang.org/api v0.84.0 h1:NMB9J4cCxs9xEm+1Z9QiO3eFvn7EnQj3Eo3hN6ugVlg= google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/pkg/build/cmd/argcount_wrapper.go b/pkg/build/cmd/argcount_wrapper.go index d3a3cfd67f6..690695cd350 100644 --- a/pkg/build/cmd/argcount_wrapper.go +++ b/pkg/build/cmd/argcount_wrapper.go @@ -2,7 +2,22 @@ package main import "github.com/urfave/cli/v2" -func ArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc { +// ArgCountWrapper will cause the action to fail if there were not exactly `num` args provided. +func ArgCountWrapper(num int, action cli.ActionFunc) cli.ActionFunc { + return func(ctx *cli.Context) error { + if ctx.NArg() != num { + if err := cli.ShowSubcommandHelp(ctx); err != nil { + return cli.Exit(err.Error(), 1) + } + return cli.Exit("", 1) + } + + return action(ctx) + } +} + +// ArgCountWrapper will cause the action to fail if there were more than `num` args provided. +func MaxArgCountWrapper(max int, action cli.ActionFunc) cli.ActionFunc { return func(ctx *cli.Context) error { if ctx.NArg() > max { if err := cli.ShowSubcommandHelp(ctx); err != nil { diff --git a/pkg/build/cmd/enterprisecheck.go b/pkg/build/cmd/enterprisecheck.go new file mode 100644 index 00000000000..732ae728442 --- /dev/null +++ b/pkg/build/cmd/enterprisecheck.go @@ -0,0 +1,121 @@ +package main + +import ( + "os" + "strconv" + + "github.com/grafana/grafana/pkg/build/env" + "github.com/grafana/grafana/pkg/build/git" + "github.com/urfave/cli/v2" +) + +// checkOpts are options used to create a new GitHub check for the enterprise downstream test. +type checkOpts struct { + SHA string + URL string + Branch string + PR int +} + +func getCheckOpts(args []string) (*checkOpts, error) { + sha, ok := env.Lookup("SOURCE_COMMIT", args) + if !ok { + return nil, cli.Exit(`missing environment variable "SOURCE_COMMIT"`, 1) + } + + url, ok := env.Lookup("DRONE_BUILD_LINK", args) + if !ok { + return nil, cli.Exit(`missing environment variable "DRONE_BUILD_LINK"`, 1) + } + + branch, ok := env.Lookup("DRONE_SOURCE_BRANCH", args) + if !ok { + return nil, cli.Exit("Unable to retrieve build source branch", 1) + } + + prStr, ok := env.Lookup("OSS_PULL_REQUEST", args) + if !ok { + matches := git.PRCheckRegexp().FindStringSubmatch(branch) + if matches == nil || len(matches) <= 1 { + return nil, cli.Exit("Unable to retrieve PR number", 1) + } + + prStr = matches[1] + } + + pr, err := strconv.Atoi(prStr) + if err != nil { + return nil, err + } + + return &checkOpts{ + Branch: branch, + PR: pr, + SHA: sha, + URL: url, + }, nil +} + +// EnterpriseCheckBegin creates the GitHub check and signals the beginning of the downstream build / test process +func EnterpriseCheckBegin(c *cli.Context) error { + var ( + ctx = c.Context + client = git.NewGitHubClient(ctx, c.String("github-token")) + ) + + opts, err := getCheckOpts(os.Environ()) + if err != nil { + return err + } + + if _, err = git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, "pending"); err != nil { + return err + } + + return nil +} + +func EnterpriseCheckSuccess(c *cli.Context) error { + return completeEnterpriseCheck(c, true) +} + +func EnterpriseCheckFail(c *cli.Context) error { + return completeEnterpriseCheck(c, false) +} + +func completeEnterpriseCheck(c *cli.Context, success bool) error { + var ( + ctx = c.Context + client = git.NewGitHubClient(ctx, c.String("github-token")) + ) + + // Update the pull request labels + opts, err := getCheckOpts(os.Environ()) + if err != nil { + return err + } + + status := "failure" + if success { + status = "success" + } + + // Update the GitHub check... + if _, err := git.CreateEnterpriseStatus(ctx, client.Repositories, opts.SHA, opts.URL, status); err != nil { + return err + } + + // Delete branch if needed + if git.PRCheckRegexp().MatchString(opts.Branch) { + if err := git.DeleteEnterpriseBranch(ctx, client.Git, opts.Branch); err != nil { + return nil + } + } + + label := "enterprise-failed" + if success { + label = "enterprise-ok" + } + + return git.AddLabelToPR(ctx, client.Issues, opts.PR, label) +} diff --git a/pkg/build/cmd/enterprisecheck_test.go b/pkg/build/cmd/enterprisecheck_test.go new file mode 100644 index 00000000000..0eeb5bd5741 --- /dev/null +++ b/pkg/build/cmd/enterprisecheck_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetCheckOpts(t *testing.T) { + t.Run("it should return the checkOpts if the correct environment variables are set", func(t *testing.T) { + args := []string{ + "SOURCE_COMMIT=1234", + "DRONE_SOURCE_BRANCH=test", + "DRONE_BUILD_LINK=http://example.com", + "OSS_PULL_REQUEST=1", + } + + opts, err := getCheckOpts(args) + require.NoError(t, err) + require.Equal(t, opts.SHA, "1234") + require.Equal(t, opts.URL, "http://example.com") + }) + t.Run("it should return an error if SOURCE_COMMIT is not set", func(t *testing.T) { + args := []string{ + "DRONE_BUILD_LINK=http://example.com", + "DRONE_SOURCE_BRANCH=test", + "DRONE_BUILD_LINK=http://example.com", + "OSS_PULL_REQUEST=1", + } + + opts, err := getCheckOpts(args) + require.Nil(t, opts) + require.Error(t, err) + }) + t.Run("it should return an error if DRONE_BUILD_LINK is not set", func(t *testing.T) { + args := []string{ + "SOURCE_COMMIT=1234", + "DRONE_SOURCE_BRANCH=test", + "OSS_PULL_REQUEST=1", + } + + opts, err := getCheckOpts(args) + require.Nil(t, opts) + require.Error(t, err) + }) + t.Run("it should return an error if OSS_PULL_REQUEST is not set", func(t *testing.T) { + args := []string{ + "SOURCE_COMMIT=1234", + "DRONE_SOURCE_BRANCH=test", + "DRONE_BUILD_LINK=http://example.com", + } + + opts, err := getCheckOpts(args) + require.Nil(t, opts) + require.Error(t, err) + }) + t.Run("it should return an error if OSS_PULL_REQUEST is not an integer", func(t *testing.T) { + args := []string{ + "SOURCE_COMMIT=1234", + "DRONE_SOURCE_BRANCH=test", + "DRONE_BUILD_LINK=http://example.com", + "OSS_PULL_REQUEST=http://example.com", + } + + opts, err := getCheckOpts(args) + require.Nil(t, opts) + require.Error(t, err) + }) +} diff --git a/pkg/build/cmd/flags.go b/pkg/build/cmd/flags.go index 7aa6d98d026..e92ce0022d2 100644 --- a/pkg/build/cmd/flags.go +++ b/pkg/build/cmd/flags.go @@ -46,4 +46,10 @@ var ( Usage: "Google Cloud Platform key file", Required: true, } + gitHubTokenFlag = cli.StringFlag{ + Name: "github-token", + Value: "", + EnvVars: []string{"GITHUB_TOKEN"}, + Usage: "GitHub token", + } ) diff --git a/pkg/build/cmd/main.go b/pkg/build/cmd/main.go index 658cceb9011..d8863710085 100644 --- a/pkg/build/cmd/main.go +++ b/pkg/build/cmd/main.go @@ -16,7 +16,7 @@ func main() { Name: "build-backend", Usage: "Build one or more variants of back-end binaries", ArgsUsage: "[version]", - Action: ArgCountWrapper(1, BuildBackend), + Action: MaxArgCountWrapper(1, BuildBackend), Flags: []cli.Flag{ &jobsFlag, &variantsFlag, @@ -67,7 +67,7 @@ func main() { Name: "build-frontend", Usage: "Build front-end artifacts", ArgsUsage: "[version]", - Action: ArgCountWrapper(1, BuildFrontend), + Action: MaxArgCountWrapper(1, BuildFrontend), Flags: []cli.Flag{ &jobsFlag, &editionFlag, @@ -77,7 +77,7 @@ func main() { { Name: "build-docker", Usage: "Build Grafana Docker images", - Action: ArgCountWrapper(1, BuildDocker), + Action: MaxArgCountWrapper(1, BuildDocker), Flags: []cli.Flag{ &jobsFlag, &editionFlag, @@ -112,7 +112,7 @@ func main() { { Name: "build-plugins", Usage: "Build internal plug-ins", - Action: ArgCountWrapper(1, BuildInternalPlugins), + Action: MaxArgCountWrapper(1, BuildInternalPlugins), Flags: []cli.Flag{ &jobsFlag, &editionFlag, @@ -125,7 +125,7 @@ func main() { Name: "publish-metrics", Usage: "Publish a set of metrics from stdin", ArgsUsage: "", - Action: ArgCountWrapper(1, PublishMetrics), + Action: MaxArgCountWrapper(1, PublishMetrics), }, { Name: "verify-drone", @@ -141,7 +141,7 @@ func main() { Name: "package", Usage: "Package one or more Grafana variants", ArgsUsage: "[version]", - Action: ArgCountWrapper(1, Package), + Action: MaxArgCountWrapper(1, Package), Flags: []cli.Flag{ &jobsFlag, &variantsFlag, @@ -182,7 +182,7 @@ func main() { Name: "fetch", Usage: "Fetch Grafana Docker images", ArgsUsage: "[version]", - Action: ArgCountWrapper(1, FetchImages), + Action: MaxArgCountWrapper(1, FetchImages), Flags: []cli.Flag{ &editionFlag, }, @@ -277,6 +277,36 @@ func main() { }, }, }, + { + Name: "enterprise-check", + Usage: "Commands for testing against Grafana Enterprise", + Subcommands: cli.Commands{ + { + Name: "begin", + Usage: "Creates the GitHub check in a pull request and begins the tests", + Action: EnterpriseCheckBegin, + Flags: []cli.Flag{ + &gitHubTokenFlag, + }, + }, + { + Name: "success", + Usage: "Updates the GitHub check in a pull request to show a successful build and updates the pull request labels", + Action: EnterpriseCheckSuccess, + Flags: []cli.Flag{ + &gitHubTokenFlag, + }, + }, + { + Name: "fail", + Usage: "Updates the GitHub check in a pull request to show a failed build and updates the pull request labels", + Action: EnterpriseCheckFail, + Flags: []cli.Flag{ + &gitHubTokenFlag, + }, + }, + }, + }, } if err := app.Run(os.Args); err != nil { diff --git a/pkg/build/env/lookup.go b/pkg/build/env/lookup.go new file mode 100644 index 00000000000..993b7259e14 --- /dev/null +++ b/pkg/build/env/lookup.go @@ -0,0 +1,18 @@ +package env + +import ( + "strings" +) + +// Lookup is the equivalent of os.LookupEnv, only you are able to provide the list of environment variables. +// To use this as os.LookupEnv would be used, simply call +// `env.Lookup("ENVIRONMENT_VARIABLE", os.Environ())` +func Lookup(name string, vars []string) (string, bool) { + for _, v := range vars { + if strings.HasPrefix(v, name) { + return strings.TrimPrefix(v, name+"="), true + } + } + + return "", false +} diff --git a/pkg/build/env/lookup_test.go b/pkg/build/env/lookup_test.go new file mode 100644 index 00000000000..cebfb4fac29 --- /dev/null +++ b/pkg/build/env/lookup_test.go @@ -0,0 +1,43 @@ +package env_test + +import ( + "testing" + + "github.com/grafana/grafana/pkg/build/env" + + "github.com/stretchr/testify/require" +) + +func TestLookup(t *testing.T) { + values := []string{"ENV_1=a", "ENV_2=b", "ENV_3=c", "ENV_4_TEST="} + + { + v, ok := env.Lookup("ENV_1", values) + require.Equal(t, v, "a") + require.True(t, ok) + } + + { + v, ok := env.Lookup("ENV_2", values) + require.Equal(t, v, "b") + require.True(t, ok) + } + + { + v, ok := env.Lookup("ENV_3", values) + require.Equal(t, v, "c") + require.True(t, ok) + } + + { + v, ok := env.Lookup("ENV_4_TEST", values) + require.Equal(t, v, "") + require.True(t, ok) + } + + { + v, ok := env.Lookup("NOT_THERE", values) + require.Equal(t, v, "") + require.False(t, ok) + } +} diff --git a/pkg/build/git/git.go b/pkg/build/git/git.go new file mode 100644 index 00000000000..63fa5147f7c --- /dev/null +++ b/pkg/build/git/git.go @@ -0,0 +1,143 @@ +package git + +import ( + "context" + "errors" + "fmt" + "net/http" + "regexp" + + "github.com/google/go-github/v45/github" + "github.com/grafana/grafana/pkg/build/stringutil" + "golang.org/x/oauth2" +) + +const ( + MainBranch = "main" + HomeDir = "." + RepoOwner = "grafana" + OSSRepo = "grafana" + EnterpriseRepo = "grafana-enterprise" + EnterpriseCheckName = "Grafana Enterprise" + EnterpriseCheckDescription = "Downstream tests to ensure that your changes are compatible with Grafana Enterprise" +) + +var EnterpriseCheckLabels = []string{"enterprise-ok", "enterprise-failed", "enterprise-override"} + +var ( + ErrorNoDroneBuildLink = errors.New("no drone build link") +) + +type GitService interface { + DeleteRef(ctx context.Context, owner string, repo string, ref string) (*github.Response, error) +} + +type LabelsService interface { + ListLabelsByIssue(ctx context.Context, owner string, repo string, number int, opts *github.ListOptions) ([]*github.Label, *github.Response, error) + RemoveLabelForIssue(ctx context.Context, owner string, repo string, number int, label string) (*github.Response, error) + AddLabelsToIssue(ctx context.Context, owner string, repo string, number int, labels []string) ([]*github.Label, *github.Response, error) +} + +type CommentService interface { + CreateComment(ctx context.Context, owner string, repo string, number int, comment *github.IssueComment) (*github.IssueComment, *github.Response, error) +} + +type StatusesService interface { + CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, *github.Response, error) +} + +// NewGitHubClient creates a new Client using the provided GitHub token if not empty. +func NewGitHubClient(ctx context.Context, token string) *github.Client { + var tc *http.Client + if token != "" { + ts := oauth2.StaticTokenSource(&oauth2.Token{ + AccessToken: token, + }) + tc = oauth2.NewClient(ctx, ts) + } + + return github.NewClient(tc) +} + +func PRCheckRegexp() *regexp.Regexp { + reBranch, err := regexp.Compile(`^pr-check-([0-9]+)\/(.+)$`) + if err != nil { + panic(fmt.Sprintf("Failed to compile regexp: %s", err)) + } + + return reBranch +} + +func AddLabelToPR(ctx context.Context, client LabelsService, prID int, newLabel string) error { + // Check existing labels + labels, _, err := client.ListLabelsByIssue(ctx, RepoOwner, OSSRepo, prID, nil) + if err != nil { + return err + } + + duplicate := false + for _, label := range labels { + if *label.Name == newLabel { + duplicate = true + continue + } + + // Delete existing "enterprise-xx" labels + if stringutil.Contains(EnterpriseCheckLabels, *label.Name) { + _, err := client.RemoveLabelForIssue(ctx, RepoOwner, OSSRepo, prID, *label.Name) + if err != nil { + return err + } + } + } + + if duplicate { + return nil + } + + _, _, err = client.AddLabelsToIssue(ctx, RepoOwner, OSSRepo, prID, []string{newLabel}) + if err != nil { + return err + } + + return nil +} + +func DeleteEnterpriseBranch(ctx context.Context, client GitService, branchName string) error { + ref := "heads/" + branchName + _, err := client.DeleteRef(ctx, RepoOwner, EnterpriseRepo, ref) + if err != nil { + return err + } + + return nil +} + +// CreateEnterpriseStatus sets the status on a commit for the enterprise build check. +func CreateEnterpriseStatus(ctx context.Context, client StatusesService, sha, link, status string) (*github.RepoStatus, error) { + check, _, err := client.CreateStatus(ctx, RepoOwner, OSSRepo, sha, &github.RepoStatus{ + Context: github.String(EnterpriseCheckName), + Description: github.String(EnterpriseCheckDescription), + TargetURL: github.String(link), + State: github.String(status), + }) + + if err != nil { + return nil, err + } + + return check, nil +} + +func CreateEnterpriseBuildFailedComment(ctx context.Context, client CommentService, link string, prID int) error { + body := fmt.Sprintf("Drone build failed: %s", link) + + _, _, err := client.CreateComment(ctx, RepoOwner, OSSRepo, prID, &github.IssueComment{ + Body: &body, + }) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/build/git/git_checks_test.go b/pkg/build/git/git_checks_test.go new file mode 100644 index 00000000000..ed3d34a8d28 --- /dev/null +++ b/pkg/build/git/git_checks_test.go @@ -0,0 +1,55 @@ +package git_test + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-github/v45/github" + "github.com/grafana/grafana/pkg/build/git" + "github.com/stretchr/testify/require" +) + +type TestChecksService struct { + CreateCheckRunError error +} + +func (s *TestChecksService) CreateStatus(ctx context.Context, owner, repo, ref string, status *github.RepoStatus) (*github.RepoStatus, *github.Response, error) { + if s.CreateCheckRunError != nil { + return nil, nil, s.CreateCheckRunError + } + + return &github.RepoStatus{ + ID: github.Int64(1), + URL: status.URL, + }, nil, nil +} + +func TestCreateEnterpriseRepoStatus(t *testing.T) { + t.Run("It should create a repo status", func(t *testing.T) { + var ( + ctx = context.Background() + client = &TestChecksService{} + link = "http://example.com" + sha = "1234" + ) + + _, err := git.CreateEnterpriseStatus(ctx, client, link, sha, "success") + + require.NoError(t, err) + }) + t.Run("It should return an error if GitHub fails to create the status", func(t *testing.T) { + var ( + ctx = context.Background() + createCheckError = errors.New("create check run error") + client = &TestChecksService{ + CreateCheckRunError: createCheckError, + } + link = "http://example.com" + sha = "1234" + ) + + _, err := git.CreateEnterpriseStatus(ctx, client, link, sha, "success") + require.ErrorIs(t, err, createCheckError) + }) +} diff --git a/pkg/build/git/git_issues_test.go b/pkg/build/git/git_issues_test.go new file mode 100644 index 00000000000..4eaa3bb6169 --- /dev/null +++ b/pkg/build/git/git_issues_test.go @@ -0,0 +1,134 @@ +package git_test + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-github/v45/github" + "github.com/grafana/grafana/pkg/build/git" + "github.com/stretchr/testify/require" +) + +type TestLabelsService struct { + Labels []*github.Label + ListLabelsError error + RemoveLabelError error + AddLabelsError error +} + +func (s *TestLabelsService) ListLabelsByIssue(ctx context.Context, owner string, repo string, number int, opts *github.ListOptions) ([]*github.Label, *github.Response, error) { + if s.ListLabelsError != nil { + return nil, nil, s.ListLabelsError + } + + labels := s.Labels + if labels == nil { + labels = []*github.Label{} + } + + return labels, nil, nil +} + +func (s *TestLabelsService) RemoveLabelForIssue(ctx context.Context, owner string, repo string, number int, label string) (*github.Response, error) { + if s.RemoveLabelError != nil { + return nil, s.RemoveLabelError + } + + return &github.Response{}, nil +} + +func (s *TestLabelsService) AddLabelsToIssue(ctx context.Context, owner string, repo string, number int, labels []string) ([]*github.Label, *github.Response, error) { + if s.AddLabelsError != nil { + return nil, nil, s.AddLabelsError + } + + l := make([]*github.Label, len(labels)) + for i, v := range labels { + l[i] = &github.Label{ + Name: github.String(v), + } + } + + return l, nil, nil +} + +func TestAddLabelToPR(t *testing.T) { + t.Run("It should add a label to a pull request", func(t *testing.T) { + var ( + ctx = context.Background() + client = &TestLabelsService{} + pr = 20 + label = "test-label" + ) + + require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) + }) + t.Run("It should not return an error if the label already exists", func(t *testing.T) { + var ( + ctx = context.Background() + client = &TestLabelsService{ + Labels: []*github.Label{ + { + Name: github.String("test-label"), + }, + }, + } + pr = 20 + label = "test-label" + ) + + require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) + }) + + t.Run("It should return an error if GitHub returns an error when listing labels", func(t *testing.T) { + var ( + ctx = context.Background() + listLabelsError = errors.New("list labels error") + client = &TestLabelsService{ + ListLabelsError: listLabelsError, + Labels: []*github.Label{}, + } + pr = 20 + label = "test-label" + ) + + require.ErrorIs(t, git.AddLabelToPR(ctx, client, pr, label), listLabelsError) + }) + + t.Run("It should not return an error if there are existing enterprise-check labels.", func(t *testing.T) { + var ( + ctx = context.Background() + client = &TestLabelsService{ + Labels: []*github.Label{ + { + Name: github.String("enterprise-failed"), + }, + }, + } + pr = 20 + label = "test-label" + ) + + require.NoError(t, git.AddLabelToPR(ctx, client, pr, label)) + }) + + t.Run("It should return an error if GitHub returns an error when removing existing enterprise-check labels", func(t *testing.T) { + var ( + ctx = context.Background() + removeLabelError = errors.New("remove label error") + client = &TestLabelsService{ + RemoveLabelError: removeLabelError, + Labels: []*github.Label{ + { + Name: github.String("enterprise-failed"), + }, + }, + } + pr = 20 + label = "test-label" + ) + + require.ErrorIs(t, git.AddLabelToPR(ctx, client, pr, label), removeLabelError) + }) +} diff --git a/pkg/build/git/git_test.go b/pkg/build/git/git_test.go new file mode 100644 index 00000000000..ba629ca669f --- /dev/null +++ b/pkg/build/git/git_test.go @@ -0,0 +1,25 @@ +package git_test + +import ( + "testing" + + "github.com/grafana/grafana/pkg/build/git" + "github.com/stretchr/testify/assert" +) + +func TestPRCheckRegexp(t *testing.T) { + var ( + shouldMatch = []string{"pr-check-1/branch-name", "pr-check-111/branch/name", "pr-check-102930122/branch-name"} + shouldNotMatch = []string{"pr-check-a/branch", "km/test", "test", "pr-check", "pr-check/test", "price"} + ) + + regex := git.PRCheckRegexp() + + for _, v := range shouldMatch { + assert.Truef(t, regex.MatchString(v), "regex should match %s", v) + } + + for _, v := range shouldNotMatch { + assert.False(t, regex.MatchString(v), "regex should not match %s", v) + } +} diff --git a/pkg/build/stringutil/contains.go b/pkg/build/stringutil/contains.go new file mode 100644 index 00000000000..b53efe70759 --- /dev/null +++ b/pkg/build/stringutil/contains.go @@ -0,0 +1,10 @@ +package stringutil + +func Contains(arr []string, s string) bool { + for _, e := range arr { + if e == s { + return true + } + } + return false +} From a53f57cc430ec458a63144aa040516eaef14f075 Mon Sep 17 00:00:00 2001 From: Karl Persson Date: Thu, 24 Nov 2022 15:25:51 +0100 Subject: [PATCH 003/131] Navigation: Support rbac for new dashboard, import dashboard and new folder (#59303) RBAC: Support rbac for new dashboard, import dashboard and new folder page --- public/app/routes/routes.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/app/routes/routes.tsx b/public/app/routes/routes.tsx index 4e10c837f3e..bb3db15c368 100644 --- a/public/app/routes/routes.tsx +++ b/public/app/routes/routes.tsx @@ -83,7 +83,7 @@ export function getAppRoutes(): RouteDescriptor[] { }, { path: '/dashboard/new', - roles: () => ['Editor', 'Admin'], + roles: () => contextSrv.evaluatePermission(() => ['Editor', 'Admin'], [AccessControlAction.DashboardsCreate]), pageClass: 'page-dashboard', routeName: DashboardRoutes.New, component: SafeDynamicImport( @@ -120,7 +120,7 @@ export function getAppRoutes(): RouteDescriptor[] { }, { path: '/dashboard/import', - roles: () => ['Editor', 'Admin'], + roles: () => contextSrv.evaluatePermission(() => ['Editor', 'Admin'], [AccessControlAction.DashboardsCreate]), component: SafeDynamicImport( () => import(/* webpackChunkName: "DashboardImport"*/ 'app/features/manage-dashboards/DashboardImportPage') ), @@ -170,7 +170,7 @@ export function getAppRoutes(): RouteDescriptor[] { }, { path: '/dashboards/folder/new', - roles: () => ['Editor', 'Admin'], + roles: () => contextSrv.evaluatePermission(() => ['Editor', 'Admin'], [AccessControlAction.FoldersCreate]), component: SafeDynamicImport( () => import(/* webpackChunkName: "NewDashboardsFolder"*/ 'app/features/folders/components/NewDashboardsFolder') ), From a8bae3f0b0cfc376a1d9698c364aa2e8abe7b71e Mon Sep 17 00:00:00 2001 From: Ieva Date: Thu, 24 Nov 2022 14:38:55 +0000 Subject: [PATCH 004/131] Access Control: Clear user's permission cache after resource creation (#59101) * refresh user's permission cache after resource creation * clear the cache instead of reloading the permissions * don't error if can't clear cache * fix tests * fix tests again --- pkg/api/common_test.go | 20 ++++++----- pkg/api/dashboard.go | 8 ++++- pkg/api/dashboard_test.go | 3 ++ pkg/api/datasources.go | 6 ++++ pkg/api/datasources_test.go | 10 ++++-- pkg/api/folder.go | 6 ++++ pkg/api/folder_test.go | 10 +++--- pkg/api/team.go | 6 ++++ pkg/api/team_test.go | 4 +++ pkg/services/accesscontrol/accesscontrol.go | 2 ++ pkg/services/accesscontrol/acimpl/service.go | 8 +++++ pkg/services/accesscontrol/actest/fake.go | 2 ++ pkg/services/accesscontrol/mock/mock.go | 10 ++++++ pkg/services/serviceaccounts/api/api.go | 35 +++++++++++-------- pkg/services/serviceaccounts/api/api_test.go | 4 ++- .../serviceaccounts/manager/service.go | 2 +- 16 files changed, 104 insertions(+), 32 deletions(-) diff --git a/pkg/api/common_test.go b/pkg/api/common_test.go index 20650b3ff93..2f4e57414b3 100644 --- a/pkg/api/common_test.go +++ b/pkg/api/common_test.go @@ -25,6 +25,7 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/accesscontrol/acimpl" + "github.com/grafana/grafana/pkg/services/accesscontrol/actest" accesscontrolmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/accesscontrol/ossaccesscontrol" "github.com/grafana/grafana/pkg/services/annotations/annotationstest" @@ -250,15 +251,16 @@ func (s *fakeRenderService) Init() error { func setupAccessControlScenarioContext(t *testing.T, cfg *setting.Cfg, url string, permissions []accesscontrol.Permission) (*scenarioContext, *HTTPServer) { store := sqlstore.InitTestDB(t) hs := &HTTPServer{ - Cfg: cfg, - Live: newTestLive(t, store), - License: &licensing.OSSLicensingService{}, - Features: featuremgmt.WithFeatures(), - QuotaService: quotatest.New(false, nil), - RouteRegister: routing.NewRouteRegister(), - AccessControl: accesscontrolmock.New().WithPermissions(permissions), - searchUsersService: searchusers.ProvideUsersService(filters.ProvideOSSSearchUserFilter(), usertest.NewUserServiceFake()), - ldapGroups: ldap.ProvideGroupsService(), + Cfg: cfg, + Live: newTestLive(t, store), + License: &licensing.OSSLicensingService{}, + Features: featuremgmt.WithFeatures(), + QuotaService: quotatest.New(false, nil), + RouteRegister: routing.NewRouteRegister(), + AccessControl: accesscontrolmock.New().WithPermissions(permissions), + searchUsersService: searchusers.ProvideUsersService(filters.ProvideOSSSearchUserFilter(), usertest.NewUserServiceFake()), + ldapGroups: ldap.ProvideGroupsService(), + accesscontrolService: actest.FakeService{}, } sc := setupScenarioContext(t, url) diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index 84fc103a2d2..42b2d976114 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -471,7 +471,7 @@ func (hs *HTTPServer) postDashboard(c *models.ReqContext, cmd models.SaveDashboa } if liveerr != nil { - hs.log.Warn("unable to broadcast save event", "uid", dashboard.Uid, "error", err) + hs.log.Warn("unable to broadcast save event", "uid", dashboard.Uid, "error", liveerr) } } @@ -479,6 +479,12 @@ func (hs *HTTPServer) postDashboard(c *models.ReqContext, cmd models.SaveDashboa return apierrors.ToDashboardErrorResponse(ctx, hs.pluginStore, err) } + // Clear permission cache for the user who's created the dashboard, so that new permissions are fetched for their next call + // Required for cases when caller wants to immediately interact with the newly created object + if newDashboard && !hs.accesscontrolService.IsDisabled() { + hs.accesscontrolService.ClearUserPermissionCache(c.SignedInUser) + } + // connect library panels for this dashboard after the dashboard is stored and has an ID err = hs.LibraryPanelService.ConnectLibraryPanelsForDashboard(ctx, c.SignedInUser, dashboard) if err != nil { diff --git a/pkg/api/dashboard_test.go b/pkg/api/dashboard_test.go index 42bdd60a957..980642c1dfd 100644 --- a/pkg/api/dashboard_test.go +++ b/pkg/api/dashboard_test.go @@ -22,6 +22,7 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/registry/corekind" + "github.com/grafana/grafana/pkg/services/accesscontrol/actest" accesscontrolmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/annotations/annotationstest" @@ -1093,6 +1094,7 @@ func postDashboardScenario(t *testing.T, desc string, url string, routePattern s folderService: folderService, Features: featuremgmt.WithFeatures(), Kinds: corekind.NewBase(nil), + accesscontrolService: actest.FakeService{}, } sc := setupScenarioContext(t, url) @@ -1201,6 +1203,7 @@ func restoreDashboardVersionScenario(t *testing.T, desc string, url string, rout Features: featuremgmt.WithFeatures(), dashboardVersionService: fakeDashboardVersionService, Kinds: corekind.NewBase(nil), + accesscontrolService: actest.FakeService{}, } sc := setupScenarioContext(t, url) diff --git a/pkg/api/datasources.go b/pkg/api/datasources.go index 2ad595a1ba9..45458f36a26 100644 --- a/pkg/api/datasources.go +++ b/pkg/api/datasources.go @@ -396,6 +396,12 @@ func (hs *HTTPServer) AddDataSource(c *models.ReqContext) response.Response { return response.Error(500, "Failed to add datasource", err) } + // Clear permission cache for the user who's created the data source, so that new permissions are fetched for their next call + // Required for cases when caller wants to immediately interact with the newly created object + if !hs.AccessControl.IsDisabled() { + hs.accesscontrolService.ClearUserPermissionCache(c.SignedInUser) + } + ds := hs.convertModelToDtos(c.Req.Context(), cmd.Result) return response.JSON(http.StatusOK, util.DynMap{ "message": "Datasource added", diff --git a/pkg/api/datasources_test.go b/pkg/api/datasources_test.go index 7ef272919f9..ccfd15be3a1 100644 --- a/pkg/api/datasources_test.go +++ b/pkg/api/datasources_test.go @@ -19,6 +19,8 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" ac "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/acimpl" + "github.com/grafana/grafana/pkg/services/accesscontrol/actest" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/datasources/permissions" "github.com/grafana/grafana/pkg/services/org" @@ -112,7 +114,9 @@ func TestAddDataSource_URLWithoutProtocol(t *testing.T) { DataSourcesService: &dataSourcesServiceMock{ expectedDatasource: &datasources.DataSource{}, }, - Cfg: setting.NewCfg(), + Cfg: setting.NewCfg(), + AccessControl: acimpl.ProvideAccessControl(setting.NewCfg()), + accesscontrolService: actest.FakeService{}, } sc := setupScenarioContext(t, "/api/datasources") @@ -224,7 +228,9 @@ func TestUpdateDataSource_URLWithoutProtocol(t *testing.T) { DataSourcesService: &dataSourcesServiceMock{ expectedDatasource: &datasources.DataSource{}, }, - Cfg: setting.NewCfg(), + Cfg: setting.NewCfg(), + AccessControl: acimpl.ProvideAccessControl(setting.NewCfg()), + accesscontrolService: actest.FakeService{}, } sc := setupScenarioContext(t, "/api/datasources/1234") diff --git a/pkg/api/folder.go b/pkg/api/folder.go index 0c7432da976..688222c3474 100644 --- a/pkg/api/folder.go +++ b/pkg/api/folder.go @@ -129,6 +129,12 @@ func (hs *HTTPServer) CreateFolder(c *models.ReqContext) response.Response { return apierrors.ToFolderErrorResponse(err) } + // Clear permission cache for the user who's created the folder, so that new permissions are fetched for their next call + // Required for cases when caller wants to immediately interact with the newly created object + if !hs.AccessControl.IsDisabled() { + hs.accesscontrolService.ClearUserPermissionCache(c.SignedInUser) + } + g := guardian.New(c.Req.Context(), folder.ID, c.OrgID, c.SignedInUser) // TODO set ParentUID if nested folders are enabled return response.JSON(http.StatusOK, hs.newToFolderDto(c, g, folder)) diff --git a/pkg/api/folder_test.go b/pkg/api/folder_test.go index 5c6ebabf06c..777fe6a281e 100644 --- a/pkg/api/folder_test.go +++ b/pkg/api/folder_test.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/actest" acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/featuremgmt" @@ -242,10 +243,11 @@ func createFolderScenario(t *testing.T, desc string, url string, routePattern st store := mockstore.NewSQLStoreMock() guardian.InitLegacyGuardian(store, dashSvc, teamSvc) hs := HTTPServer{ - AccessControl: acmock.New(), - folderService: folderService, - Cfg: setting.NewCfg(), - Features: featuremgmt.WithFeatures(), + AccessControl: acmock.New(), + folderService: folderService, + Cfg: setting.NewCfg(), + Features: featuremgmt.WithFeatures(), + accesscontrolService: actest.FakeService{}, } sc := setupScenarioContext(t, url) diff --git a/pkg/api/team.go b/pkg/api/team.go index 01d33d69fa1..d85416e512b 100644 --- a/pkg/api/team.go +++ b/pkg/api/team.go @@ -41,6 +41,12 @@ func (hs *HTTPServer) CreateTeam(c *models.ReqContext) response.Response { return response.Error(500, "Failed to create Team", err) } + // Clear permission cache for the user who's created the team, so that new permissions are fetched for their next call + // Required for cases when caller wants to immediately interact with the newly created object + if !hs.AccessControl.IsDisabled() { + hs.accesscontrolService.ClearUserPermissionCache(c.SignedInUser) + } + if accessControlEnabled || (c.OrgRole == org.RoleEditor && hs.Cfg.EditorsCanAdmin) { // if the request is authenticated using API tokens // the SignedInUser is an empty struct therefore diff --git a/pkg/api/team_test.go b/pkg/api/team_test.go index af010b06ab0..3d5cb0e4d1f 100644 --- a/pkg/api/team_test.go +++ b/pkg/api/team_test.go @@ -15,6 +15,8 @@ import ( "github.com/grafana/grafana/pkg/infra/log/logtest" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/acimpl" + "github.com/grafana/grafana/pkg/services/accesscontrol/actest" "github.com/grafana/grafana/pkg/services/org" pref "github.com/grafana/grafana/pkg/services/preference" "github.com/grafana/grafana/pkg/services/preference/preftest" @@ -213,6 +215,8 @@ func TestTeamAPIEndpoint_CreateTeam_RBAC(t *testing.T) { server := SetupAPITestServer(t, func(hs *HTTPServer) { hs.Cfg = setting.NewCfg() hs.teamService = teamtest.NewFakeService() + hs.AccessControl = acimpl.ProvideAccessControl(setting.NewCfg()) + hs.accesscontrolService = actest.FakeService{} }) input := strings.NewReader(fmt.Sprintf(teamCmd, 1)) diff --git a/pkg/services/accesscontrol/accesscontrol.go b/pkg/services/accesscontrol/accesscontrol.go index 5bba73ea4fd..ecbdf55692d 100644 --- a/pkg/services/accesscontrol/accesscontrol.go +++ b/pkg/services/accesscontrol/accesscontrol.go @@ -26,6 +26,8 @@ type Service interface { registry.ProvidesUsageStats // GetUserPermissions returns user permissions with only action and scope fields set. GetUserPermissions(ctx context.Context, user *user.SignedInUser, options Options) ([]Permission, error) + // ClearUserPermissionCache removes the permission cache entry for the given user + ClearUserPermissionCache(user *user.SignedInUser) // DeleteUserPermissions removes all permissions user has in org and all permission to that user // If orgID is set to 0 remove permissions from all orgs DeleteUserPermissions(ctx context.Context, orgID, userID int64) error diff --git a/pkg/services/accesscontrol/acimpl/service.go b/pkg/services/accesscontrol/acimpl/service.go index 9bab820f8eb..b5ac20d5ad1 100644 --- a/pkg/services/accesscontrol/acimpl/service.go +++ b/pkg/services/accesscontrol/acimpl/service.go @@ -147,6 +147,14 @@ func (s *Service) getCachedUserPermissions(ctx context.Context, user *user.Signe return permissions, nil } +func (s *Service) ClearUserPermissionCache(user *user.SignedInUser) { + key, err := permissionCacheKey(user) + if err != nil { + return + } + s.cache.Delete(key) +} + func (s *Service) DeleteUserPermissions(ctx context.Context, orgID int64, userID int64) error { return s.store.DeleteUserPermissions(ctx, orgID, userID) } diff --git a/pkg/services/accesscontrol/actest/fake.go b/pkg/services/accesscontrol/actest/fake.go index 66fffd38079..d62f7a9424b 100644 --- a/pkg/services/accesscontrol/actest/fake.go +++ b/pkg/services/accesscontrol/actest/fake.go @@ -24,6 +24,8 @@ func (f FakeService) GetUserPermissions(ctx context.Context, user *user.SignedIn return f.ExpectedPermissions, f.ExpectedErr } +func (f FakeService) ClearUserPermissionCache(user *user.SignedInUser) {} + func (f FakeService) DeleteUserPermissions(ctx context.Context, orgID, userID int64) error { return f.ExpectedErr } diff --git a/pkg/services/accesscontrol/mock/mock.go b/pkg/services/accesscontrol/mock/mock.go index a06beb9df3b..663902e95ca 100644 --- a/pkg/services/accesscontrol/mock/mock.go +++ b/pkg/services/accesscontrol/mock/mock.go @@ -20,6 +20,7 @@ type fullAccessControl interface { type Calls struct { Evaluate []interface{} GetUserPermissions []interface{} + ClearUserPermissionCache []interface{} IsDisabled []interface{} DeclareFixedRoles []interface{} DeclarePluginRoles []interface{} @@ -43,6 +44,7 @@ type Mock struct { // Override functions EvaluateFunc func(context.Context, *user.SignedInUser, accesscontrol.Evaluator) (bool, error) GetUserPermissionsFunc func(context.Context, *user.SignedInUser, accesscontrol.Options) ([]accesscontrol.Permission, error) + ClearUserPermissionCacheFunc func(*user.SignedInUser) IsDisabledFunc func() bool DeclareFixedRolesFunc func(...accesscontrol.RoleRegistration) error DeclarePluginRolesFunc func(context.Context, string, string, []plugins.RoleRegistration) error @@ -138,6 +140,14 @@ func (m *Mock) GetUserPermissions(ctx context.Context, user *user.SignedInUser, return m.permissions, nil } +func (m *Mock) ClearUserPermissionCache(user *user.SignedInUser) { + m.Calls.ClearUserPermissionCache = append(m.Calls.ClearUserPermissionCache, []interface{}{user}) + // Use override if provided + if m.ClearUserPermissionCacheFunc != nil { + m.ClearUserPermissionCacheFunc(user) + } +} + // Middleware checks if service disabled or not to switch to fallback authorization. // This mock return m.disabled unless an override is provided. func (m *Mock) IsDisabled() bool { diff --git a/pkg/services/serviceaccounts/api/api.go b/pkg/services/serviceaccounts/api/api.go index 745209c09ed..549e5001d6c 100644 --- a/pkg/services/serviceaccounts/api/api.go +++ b/pkg/services/serviceaccounts/api/api.go @@ -21,31 +21,34 @@ import ( ) type ServiceAccountsAPI struct { - cfg *setting.Cfg - service serviceaccounts.Service - accesscontrol accesscontrol.AccessControl - RouterRegister routing.RouteRegister - store serviceaccounts.Store - log log.Logger - permissionService accesscontrol.ServiceAccountPermissionsService + cfg *setting.Cfg + service serviceaccounts.Service + accesscontrol accesscontrol.AccessControl + accesscontrolService accesscontrol.Service + RouterRegister routing.RouteRegister + store serviceaccounts.Store + log log.Logger + permissionService accesscontrol.ServiceAccountPermissionsService } func NewServiceAccountsAPI( cfg *setting.Cfg, service serviceaccounts.Service, accesscontrol accesscontrol.AccessControl, + accesscontrolService accesscontrol.Service, routerRegister routing.RouteRegister, store serviceaccounts.Store, permissionService accesscontrol.ServiceAccountPermissionsService, ) *ServiceAccountsAPI { return &ServiceAccountsAPI{ - cfg: cfg, - service: service, - accesscontrol: accesscontrol, - RouterRegister: routerRegister, - store: store, - log: log.New("serviceaccounts.api"), - permissionService: permissionService, + cfg: cfg, + service: service, + accesscontrol: accesscontrol, + accesscontrolService: accesscontrolService, + RouterRegister: routerRegister, + store: store, + log: log.New("serviceaccounts.api"), + permissionService: permissionService, } } @@ -127,6 +130,10 @@ func (api *ServiceAccountsAPI) CreateServiceAccount(c *models.ReqContext) respon return response.Error(http.StatusInternalServerError, "Failed to set permissions for service account creator", err) } } + + // Clear permission cache for the user who's created the service account, so that new permissions are fetched for their next call + // Required for cases when caller wants to immediately interact with the newly created object + api.accesscontrolService.ClearUserPermissionCache(c.SignedInUser) } return response.JSON(http.StatusCreated, serviceAccount) diff --git a/pkg/services/serviceaccounts/api/api_test.go b/pkg/services/serviceaccounts/api/api_test.go index 8abeeb43789..49cc8a63329 100644 --- a/pkg/services/serviceaccounts/api/api_test.go +++ b/pkg/services/serviceaccounts/api/api_test.go @@ -20,6 +20,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/actest" accesscontrolmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/accesscontrol/ossaccesscontrol" "github.com/grafana/grafana/pkg/services/apikey/apikeyimpl" @@ -296,8 +297,9 @@ func setupTestServer(t *testing.T, svc *tests.ServiceAccountMock, saPermissionService, err := ossaccesscontrol.ProvideServiceAccountPermissions( cfg, routing.NewRouteRegister(), sqlStore, acmock, &licensing.OSSLicensingService{}, saStore, acmock, teamSvc, userSvc) require.NoError(t, err) + acService := actest.FakeService{} - a := NewServiceAccountsAPI(cfg, svc, acmock, routerRegister, saStore, saPermissionService) + a := NewServiceAccountsAPI(cfg, svc, acmock, acService, routerRegister, saStore, saPermissionService) a.RegisterAPIEndpoints() a.cfg.ApiKeyMaxSecondsToLive = -1 // disable api key expiration diff --git a/pkg/services/serviceaccounts/manager/service.go b/pkg/services/serviceaccounts/manager/service.go index 1301bc90659..4365fcd52af 100644 --- a/pkg/services/serviceaccounts/manager/service.go +++ b/pkg/services/serviceaccounts/manager/service.go @@ -51,7 +51,7 @@ func ProvideServiceAccountsService( usageStats.RegisterMetricsFunc(s.getUsageMetrics) - serviceaccountsAPI := api.NewServiceAccountsAPI(cfg, s, ac, routeRegister, s.store, permissionService) + serviceaccountsAPI := api.NewServiceAccountsAPI(cfg, s, ac, accesscontrolService, routeRegister, s.store, permissionService) serviceaccountsAPI.RegisterAPIEndpoints() s.secretScanEnabled = cfg.SectionWithEnvOverrides("secretscan").Key("enabled").MustBool(false) From d76638338dde8e2d13cd69fc352543576d4e9145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 24 Nov 2022 16:03:26 +0100 Subject: [PATCH 005/131] PanelEdit: Fixes bug with not remembering panel options pane collapse/expand state (#59265) --- .../dashboard/components/PanelEditor/PanelEditor.tsx | 5 ----- .../dashboard/components/PanelEditor/VisualizationButton.tsx | 5 +++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/public/app/features/dashboard/components/PanelEditor/PanelEditor.tsx b/public/app/features/dashboard/components/PanelEditor/PanelEditor.tsx index d06cbfc7353..8dea8470fbe 100644 --- a/public/app/features/dashboard/components/PanelEditor/PanelEditor.tsx +++ b/public/app/features/dashboard/components/PanelEditor/PanelEditor.tsx @@ -193,11 +193,6 @@ export class PanelEditorUnconnected extends PureComponent { this.props.toggleTableView(); }; - onTogglePanelOptions = () => { - const { uiState, updatePanelEditorUIState } = this.props; - updatePanelEditorUIState({ isPanelOptionsVisible: !uiState.isPanelOptionsVisible }); - }; - renderPanel(styles: EditorStyles, isOnlyPanel: boolean) { const { dashboard, panel, uiState, tableViewEnabled, theme } = this.props; diff --git a/public/app/features/dashboard/components/PanelEditor/VisualizationButton.tsx b/public/app/features/dashboard/components/PanelEditor/VisualizationButton.tsx index 7fea2df9955..7143623a879 100644 --- a/public/app/features/dashboard/components/PanelEditor/VisualizationButton.tsx +++ b/public/app/features/dashboard/components/PanelEditor/VisualizationButton.tsx @@ -8,7 +8,8 @@ import { useDispatch, useSelector } from 'app/types'; import { PanelModel } from '../../state'; import { getPanelPluginWithFallback } from '../../state/selectors'; -import { setPanelEditorUIState, toggleVizPicker } from './state/reducers'; +import { updatePanelEditorUIState } from './state/actions'; +import { toggleVizPicker } from './state/reducers'; type Props = { panel: PanelModel; @@ -25,7 +26,7 @@ export const VisualizationButton = ({ panel }: Props) => { }; const onToggleOptionsPane = () => { - dispatch(setPanelEditorUIState({ isPanelOptionsVisible: !isPanelOptionsVisible })); + dispatch(updatePanelEditorUIState({ isPanelOptionsVisible: !isPanelOptionsVisible })); }; if (!plugin) { From 58fd0a8b44b6be53b66142a9d617e9d9cd649afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Bedi?= Date: Thu, 24 Nov 2022 16:23:13 +0100 Subject: [PATCH 006/131] Calcs: Fix difference percent in legend (#59243) --- packages/grafana-ui/src/components/uPlot/PlotLegend.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grafana-ui/src/components/uPlot/PlotLegend.tsx b/packages/grafana-ui/src/components/uPlot/PlotLegend.tsx index 5f465b4c597..12dbefd7d04 100644 --- a/packages/grafana-ui/src/components/uPlot/PlotLegend.tsx +++ b/packages/grafana-ui/src/components/uPlot/PlotLegend.tsx @@ -80,7 +80,7 @@ export const PlotLegend: React.FC = React.memo( ...field, config: { ...field.config, - unit: 'percent', + unit: 'percentunit', }, }, theme, From ff87a2641f866540a67b0ed04a505b6bdb48bdb2 Mon Sep 17 00:00:00 2001 From: Matias Chomicki Date: Thu, 24 Nov 2022 17:41:02 +0100 Subject: [PATCH 007/131] Loki Variable Query Editor: Improve stream selector tooltip text (#59311) * feat(loki-variable-query-editor): improve stream selector tooltip text * Chore: improve sample stream selector * Chore: update sample stream selector in tooltip * Chore: add example of interpolated stream selector --- .../datasource/loki/components/VariableQueryEditor.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/loki/components/VariableQueryEditor.tsx b/public/app/plugins/datasource/loki/components/VariableQueryEditor.tsx index 22898babca4..3da87fecfe8 100644 --- a/public/app/plugins/datasource/loki/components/VariableQueryEditor.tsx +++ b/public/app/plugins/datasource/loki/components/VariableQueryEditor.tsx @@ -103,7 +103,9 @@ export const LokiVariableQueryEditor: FC = ({ onChange, query, datasource labelWidth={20} tooltip={
- Optional. If defined, a list of values for the label in the specified log stream selector is returned. + { + 'Optional. If defined, a list of values for the specified log stream selector is returned. For example: {label="value"} or {label="$variable"}' + }
} > From 220295d26f5497b7e38b06897d0fcddd2bb819c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Bedi?= Date: Thu, 24 Nov 2022 18:01:05 +0100 Subject: [PATCH 008/131] SQL: Return no data response when no rows returned (#59121) --- pkg/tsdb/sqleng/sql_engine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tsdb/sqleng/sql_engine.go b/pkg/tsdb/sqleng/sql_engine.go index a837ec8bf63..e829d93ef14 100644 --- a/pkg/tsdb/sqleng/sql_engine.go +++ b/pkg/tsdb/sqleng/sql_engine.go @@ -297,7 +297,7 @@ func (e *DataSourceHandler) executeQuery(query backend.DataQuery, wg *sync.WaitG // If no rows were returned, no point checking anything else. if frame.Rows() == 0 { - queryResult.dataResponse.Frames = data.Frames{frame} + queryResult.dataResponse.Frames = data.Frames{} ch <- queryResult return } From 45d3125919f0bedb965316685af4723e0bfbb8d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laura=20Fern=C3=A1ndez?= Date: Thu, 24 Nov 2022 18:21:33 +0100 Subject: [PATCH 009/131] OptionsUI: SliderValueEditor does not get auto focused on slider change (#59209) --- .eslintrc | 6 +++++- public/app/core/components/OptionsUI/slider.tsx | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.eslintrc b/.eslintrc index 9535a971dd8..980edc7e524 100644 --- a/.eslintrc +++ b/.eslintrc @@ -87,7 +87,11 @@ "ignoreNonDOM": true } ], - "jsx-a11y/no-static-element-interactions": "off" + "jsx-a11y/no-static-element-interactions": "off", + "jsx-a11y/label-has-associated-control": [ "error", { + "controlComponents": ["NumberInput"], + "depth": 2 + }] } } ] diff --git a/public/app/core/components/OptionsUI/slider.tsx b/public/app/core/components/OptionsUI/slider.tsx index be4d46de1fd..a668a66868e 100644 --- a/public/app/core/components/OptionsUI/slider.tsx +++ b/public/app/core/components/OptionsUI/slider.tsx @@ -96,7 +96,7 @@ export const SliderValueEditor: React.FC {/** Slider tooltip's parent component is body and therefore we need Global component to do css overrides for it. */} -
+
+ ); }; From 5a3f0e869642d9273e50f1506329c1ae2aa49afa Mon Sep 17 00:00:00 2001 From: Timur Olzhabayev Date: Thu, 24 Nov 2022 19:00:33 +0100 Subject: [PATCH 010/131] Docs: Updating sign plugin docs (#59322) Updating sign plugin docs --- docs/sources/developers/plugins/sign-a-plugin.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/sources/developers/plugins/sign-a-plugin.md b/docs/sources/developers/plugins/sign-a-plugin.md index 4d5730a1d86..159136beca3 100644 --- a/docs/sources/developers/plugins/sign-a-plugin.md +++ b/docs/sources/developers/plugins/sign-a-plugin.md @@ -40,11 +40,9 @@ Public plugins need to be reviewed by the Grafana team before you can sign them. ```bash export GRAFANA_API_KEY= - npx @grafana/sign-plugin plugin:sign + npx @grafana/sign-plugin ``` -> **Note:** If running NPM 7+ the `npx` commands mentioned in this article may hang. The workaround is to use `npx --legacy-peer-deps `. - ## Sign a private plugin 1. In your plugin directory, sign the plugin with the API key you just created. Grafana Sign Plugin creates a [MANIFEST.txt](#plugin-manifest) file in the `dist` directory of your plugin. @@ -53,7 +51,7 @@ Public plugins need to be reviewed by the Grafana team before you can sign them. ```bash export GRAFANA_API_KEY= - npx @grafana/sign-plugin plugin:sign --rootUrls https://example.com/grafana + npx @grafana/sign-plugin --rootUrls https://example.com/grafana ``` ## Plugin signature levels From f5c41ea497886192db55de5878903b580a173160 Mon Sep 17 00:00:00 2001 From: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com> Date: Thu, 24 Nov 2022 20:28:53 +0200 Subject: [PATCH 011/131] Nested Folders: More API fixes (#59316) * Nested Folder: Fix create, use camel case for JSON properties * Fix get parents if the folder does not exist * Add store test for get parents --- pkg/services/folder/folderimpl/folder.go | 3 +- pkg/services/folder/folderimpl/sqlstore.go | 7 + .../folder/folderimpl/sqlstore_test.go | 5 + pkg/services/folder/model.go | 4 +- public/api-merged.json | 197 ++++++++-------- public/api-spec.json | 163 +++++++------ public/openapi3.json | 219 ++++++++++-------- 7 files changed, 310 insertions(+), 288 deletions(-) diff --git a/pkg/services/folder/folderimpl/folder.go b/pkg/services/folder/folderimpl/folder.go index f16aaa1abbc..bf25c6d107b 100644 --- a/pkg/services/folder/folderimpl/folder.go +++ b/pkg/services/folder/folderimpl/folder.go @@ -3,6 +3,7 @@ package folderimpl import ( "context" "errors" + "fmt" "strings" "github.com/grafana/grafana/pkg/bus" @@ -517,7 +518,7 @@ func (s *Service) nestedFolderCreate(ctx context.Context, cmd *folder.CreateFold func (s *Service) validateParent(ctx context.Context, orgID int64, parentUID string) error { ancestors, err := s.store.GetParents(ctx, folder.GetParentsQuery{UID: parentUID, OrgID: orgID}) if err != nil { - return err + return fmt.Errorf("failed to get parents: %w", err) } if len(ancestors) == folder.MaxNestedFolderDepth { diff --git a/pkg/services/folder/folderimpl/sqlstore.go b/pkg/services/folder/folderimpl/sqlstore.go index b8dbfbe9351..e870b736cd0 100644 --- a/pkg/services/folder/folderimpl/sqlstore.go +++ b/pkg/services/folder/folderimpl/sqlstore.go @@ -201,6 +201,13 @@ func (ss *sqlStore) GetParents(ctx context.Context, q folder.GetParentsQuery) ([ } return nil, err } + + if len(folders) < 1 { + // the query is expected to return at least the same folder + // if it's empty it means that the folder does not exist + return nil, folder.ErrFolderNotFound + } + return util.Reverse(folders[1:]), nil } diff --git a/pkg/services/folder/folderimpl/sqlstore_test.go b/pkg/services/folder/folderimpl/sqlstore_test.go index 4824d934d38..600f736a790 100644 --- a/pkg/services/folder/folderimpl/sqlstore_test.go +++ b/pkg/services/folder/folderimpl/sqlstore_test.go @@ -397,6 +397,11 @@ func TestIntegrationGetParents(t *testing.T) { require.NoError(t, err) }) + t.Run("get parents of unknown folder should return an error", func(t *testing.T) { + _, err := folderStore.GetParents(context.Background(), folder.GetParentsQuery{}) + require.ErrorIs(t, err, folder.ErrFolderNotFound) + }) + t.Run("get parents of 1-st level folder should be empty", func(t *testing.T) { parents, err := folderStore.GetParents(context.Background(), folder.GetParentsQuery{ UID: f.UID, diff --git a/pkg/services/folder/model.go b/pkg/services/folder/model.go index 87b592c5d97..d3de565aa81 100644 --- a/pkg/services/folder/model.go +++ b/pkg/services/folder/model.go @@ -66,7 +66,7 @@ type CreateFolderCommand struct { OrgID int64 `json:"-"` Title string `json:"title"` Description string `json:"description"` - ParentUID string `json:"parent_uid"` + ParentUID string `json:"parentUid"` SignedInUser *user.SignedInUser `json:"-"` } @@ -86,7 +86,7 @@ type UpdateFolderCommand struct { // to move a folder. type MoveFolderCommand struct { UID string `json:"uid"` - NewParentUID string `json:"new_parent_uid"` + NewParentUID string `json:"newParentUid"` OrgID int64 `json:"-"` SignedInUser *user.SignedInUser `json:"-"` diff --git a/public/api-merged.json b/public/api-merged.json index f0aebc474db..23aeab68549 100644 --- a/public/api-merged.json +++ b/public/api-merged.json @@ -1400,7 +1400,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/UpdateUserQuotaCmd" + "$ref": "#/definitions/UpdateQuotaCmd" } }, { @@ -2458,6 +2458,11 @@ "schema": { "$ref": "#/definitions/ProvisionedAlertRule" } + }, + { + "type": "string", + "name": "X-Disable-Provenance", + "in": "header" } ], "responses": { @@ -2527,6 +2532,11 @@ "schema": { "$ref": "#/definitions/ProvisionedAlertRule" } + }, + { + "type": "string", + "name": "X-Disable-Provenance", + "in": "header" } ], "responses": { @@ -6317,6 +6327,33 @@ } } }, + "/org/quotas": { + "get": { + "description": "If you are running Grafana Enterprise and have Fine-grained access control enabled, you need to have a permission with action `orgs.quotas:read` and scope `org:id:1` (orgIDScope).", + "tags": [ + "getCurrentOrg" + ], + "summary": "Fetch Organization quota.", + "operationId": "getCurrentOrgQuota", + "responses": { + "200": { + "$ref": "#/responses/getQuotaResponse" + }, + "401": { + "$ref": "#/responses/unauthorisedError" + }, + "403": { + "$ref": "#/responses/forbiddenError" + }, + "404": { + "$ref": "#/responses/notFoundError" + }, + "500": { + "$ref": "#/responses/internalServerError" + } + } + } + }, "/org/users": { "get": { "description": "Returns all org users within the current organization. Accessible to users with org admin role.\nIf you are running Grafana Enterprise and have Fine-grained access control enabled\nyou need to have a permission with action: `org.users:read` with scope `users:*`.", @@ -6840,7 +6877,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/UpdateOrgQuotaCmd" + "$ref": "#/definitions/UpdateQuotaCmd" } }, { @@ -11630,7 +11667,7 @@ "description": { "type": "string" }, - "parent_uid": { + "parentUid": { "type": "string" }, "title": { @@ -12989,7 +13026,7 @@ "type": "integer", "format": "int64" }, - "parent_uid": { + "parentUid": { "description": "only used if nested folders are enabled", "type": "string" }, @@ -13431,12 +13468,6 @@ "GettableNGalertConfig": { "type": "object", "properties": { - "alertmanagers": { - "type": "array", - "items": { - "type": "string" - } - }, "alertmanagersChoice": { "type": "string", "enum": [ @@ -14368,32 +14399,6 @@ } } }, - "Model": { - "description": "THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.\nEquivalent Go types at stable import paths are provided in https://github.com/grafana/grok.", - "type": "object", - "title": "Model is the Go representation of a playlist.", - "properties": { - "interval": { - "description": "Interval sets the time between switching views in a playlist.\nFIXME: Is this based on a standardized format or what options are available? Can datemath be used?", - "type": "string" - }, - "items": { - "description": "The ordered list of items that the playlist will iterate over.\nFIXME! This should not be optional, but changing it makes the godegen awkward", - "type": "array", - "items": { - "$ref": "#/definitions/PlaylistItem" - } - }, - "name": { - "description": "Name of the playlist.", - "type": "string" - }, - "uid": { - "description": "Unique playlist identifier. Generated on creation, either by the\ncreator of the playlist of by the application.", - "type": "string" - } - } - }, "MultiStatus": { "type": "object" }, @@ -14898,7 +14903,7 @@ "homeDashboardUID": { "type": "string" }, - "locale": { + "language": { "type": "string" }, "navbar": { @@ -14992,12 +14997,15 @@ "format": "int64" }, "interval": { + "description": "Interval sets the time between switching views in a playlist.\nFIXME: Is this based on a standardized format or what options are available? Can datemath be used?", "type": "string" }, "name": { + "description": "Name of the playlist.", "type": "string" }, "uid": { + "description": "Unique playlist identifier. Generated on creation, either by the\ncreator of the playlist of by the application.", "type": "string" } } @@ -15034,9 +15042,8 @@ } }, "PlaylistItem": { - "description": "THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.\nEquivalent Go types at stable import paths are provided in https://github.com/grafana/grok.", "type": "object", - "title": "PlaylistItem is the Go representation of a playlist.Item.", + "title": "PlaylistItem defines model for playlist.Item.", "properties": { "title": { "description": "Title is an unused property -- it will be removed in the future", @@ -15052,7 +15059,6 @@ } }, "PlaylistItemType": { - "description": "THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.\nEquivalent Go types at stable import paths are provided in https://github.com/grafana/grok.", "type": "string", "title": "Type of the item." }, @@ -15350,12 +15356,6 @@ "PostableNGalertConfig": { "type": "object", "properties": { - "alertmanagers": { - "type": "array", - "items": { - "type": "string" - } - }, "alertmanagersChoice": { "type": "string", "enum": [ @@ -15407,7 +15407,7 @@ "homeDashboardUID": { "type": "string" }, - "locale": { + "language": { "type": "string" }, "navbar": { @@ -15848,6 +15848,30 @@ } } }, + "QuotaDTO": { + "type": "object", + "properties": { + "limit": { + "type": "integer", + "format": "int64" + }, + "org_id": { + "type": "integer", + "format": "int64" + }, + "target": { + "type": "string" + }, + "used": { + "type": "integer", + "format": "int64" + }, + "user_id": { + "type": "integer", + "format": "int64" + } + } + }, "Receiver": { "type": "object", "title": "Receiver configuration provides configuration on how to contact a receiver.", @@ -17523,8 +17547,9 @@ "type": "string" }, "URL": { + "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.", "type": "object", - "title": "URL is a custom URL type that allows validation at configuration load time.", + "title": "A URL represents a parsed URL (technically, a URI reference).", "properties": { "ForceQuery": { "type": "boolean" @@ -17535,9 +17560,6 @@ "Host": { "type": "string" }, - "OmitHost": { - "type": "boolean" - }, "Opaque": { "type": "string" }, @@ -17758,6 +17780,9 @@ "UpdateFolderCommand": { "type": "object", "properties": { + "description": { + "type": "string" + }, "overwrite": { "type": "boolean" }, @@ -17804,18 +17829,6 @@ } } }, - "UpdateOrgQuotaCmd": { - "type": "object", - "properties": { - "limit": { - "type": "integer", - "format": "int64" - }, - "target": { - "type": "string" - } - } - }, "UpdateOrgUserCommand": { "type": "object", "properties": { @@ -17861,7 +17874,7 @@ "homeDashboardUID": { "type": "string" }, - "locale": { + "language": { "type": "string" }, "navbar": { @@ -17889,6 +17902,18 @@ } } }, + "UpdateQuotaCmd": { + "type": "object", + "properties": { + "limit": { + "type": "integer", + "format": "int64" + }, + "target": { + "type": "string" + } + } + }, "UpdateRoleCommand": { "type": "object", "properties": { @@ -17981,18 +18006,6 @@ } } }, - "UpdateUserQuotaCmd": { - "type": "object", - "properties": { - "limit": { - "type": "integer", - "format": "int64" - }, - "target": { - "type": "string" - } - } - }, "UserIdDTO": { "type": "object", "properties": { @@ -18097,26 +18110,6 @@ } } }, - "UserQuotaDTO": { - "type": "object", - "properties": { - "limit": { - "type": "integer", - "format": "int64" - }, - "target": { - "type": "string" - }, - "used": { - "type": "integer", - "format": "int64" - }, - "user_id": { - "type": "integer", - "format": "int64" - } - } - }, "UserSearchHitDTO": { "type": "object", "properties": { @@ -18362,6 +18355,7 @@ } }, "alertGroup": { + "description": "AlertGroup alert group", "type": "object", "required": [ "alerts", @@ -18490,7 +18484,6 @@ } }, "gettableAlert": { - "description": "GettableAlert gettable alert", "type": "object", "required": [ "labels", @@ -18553,6 +18546,7 @@ } }, "gettableSilence": { + "description": "GettableSilence gettable silence", "type": "object", "required": [ "comment", @@ -18601,6 +18595,7 @@ } }, "gettableSilences": { + "description": "GettableSilences gettable silences", "type": "array", "items": { "$ref": "#/definitions/gettableSilence" @@ -18751,6 +18746,7 @@ } }, "postableSilence": { + "description": "PostableSilence postable silence", "type": "object", "required": [ "comment", @@ -18788,7 +18784,6 @@ } }, "receiver": { - "description": "Receiver receiver", "type": "object", "required": [ "active", @@ -19579,7 +19574,7 @@ "getPlaylistResponse": { "description": "(empty)", "schema": { - "$ref": "#/definitions/Model" + "$ref": "#/definitions/Playlist" } }, "getPreferencesResponse": { @@ -19617,7 +19612,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/UserQuotaDTO" + "$ref": "#/definitions/QuotaDTO" } } }, @@ -20101,7 +20096,7 @@ "updatePlaylistResponse": { "description": "(empty)", "schema": { - "$ref": "#/definitions/Model" + "$ref": "#/definitions/Playlist" } }, "updateServiceAccountResponse": { diff --git a/public/api-spec.json b/public/api-spec.json index 6d479bd6e5b..1a16ebff937 100644 --- a/public/api-spec.json +++ b/public/api-spec.json @@ -1400,7 +1400,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/UpdateUserQuotaCmd" + "$ref": "#/definitions/UpdateQuotaCmd" } }, { @@ -5670,6 +5670,33 @@ } } }, + "/org/quotas": { + "get": { + "description": "If you are running Grafana Enterprise and have Fine-grained access control enabled, you need to have a permission with action `orgs.quotas:read` and scope `org:id:1` (orgIDScope).", + "tags": [ + "getCurrentOrg" + ], + "summary": "Fetch Organization quota.", + "operationId": "getCurrentOrgQuota", + "responses": { + "200": { + "$ref": "#/responses/getQuotaResponse" + }, + "401": { + "$ref": "#/responses/unauthorisedError" + }, + "403": { + "$ref": "#/responses/forbiddenError" + }, + "404": { + "$ref": "#/responses/notFoundError" + }, + "500": { + "$ref": "#/responses/internalServerError" + } + } + } + }, "/org/users": { "get": { "description": "Returns all org users within the current organization. Accessible to users with org admin role.\nIf you are running Grafana Enterprise and have Fine-grained access control enabled\nyou need to have a permission with action: `org.users:read` with scope `users:*`.", @@ -6193,7 +6220,7 @@ "in": "body", "required": true, "schema": { - "$ref": "#/definitions/UpdateOrgQuotaCmd" + "$ref": "#/definitions/UpdateQuotaCmd" } }, { @@ -10650,7 +10677,7 @@ "description": { "type": "string" }, - "parent_uid": { + "parentUid": { "type": "string" }, "title": { @@ -11808,7 +11835,7 @@ "type": "integer", "format": "int64" }, - "parent_uid": { + "parentUid": { "description": "only used if nested folders are enabled", "type": "string" }, @@ -12573,32 +12600,6 @@ } } }, - "Model": { - "description": "THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.\nEquivalent Go types at stable import paths are provided in https://github.com/grafana/grok.", - "type": "object", - "title": "Model is the Go representation of a playlist.", - "properties": { - "interval": { - "description": "Interval sets the time between switching views in a playlist.\nFIXME: Is this based on a standardized format or what options are available? Can datemath be used?", - "type": "string" - }, - "items": { - "description": "The ordered list of items that the playlist will iterate over.\nFIXME! This should not be optional, but changing it makes the godegen awkward", - "type": "array", - "items": { - "$ref": "#/definitions/PlaylistItem" - } - }, - "name": { - "description": "Name of the playlist.", - "type": "string" - }, - "uid": { - "description": "Unique playlist identifier. Generated on creation, either by the\ncreator of the playlist of by the application.", - "type": "string" - } - } - }, "NavLink": { "type": "object", "properties": { @@ -12852,7 +12853,7 @@ "homeDashboardUID": { "type": "string" }, - "locale": { + "language": { "type": "string" }, "navbar": { @@ -12943,12 +12944,15 @@ "format": "int64" }, "interval": { + "description": "Interval sets the time between switching views in a playlist.\nFIXME: Is this based on a standardized format or what options are available? Can datemath be used?", "type": "string" }, "name": { + "description": "Name of the playlist.", "type": "string" }, "uid": { + "description": "Unique playlist identifier. Generated on creation, either by the\ncreator of the playlist of by the application.", "type": "string" } } @@ -12985,9 +12989,8 @@ } }, "PlaylistItem": { - "description": "THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.\nEquivalent Go types at stable import paths are provided in https://github.com/grafana/grok.", "type": "object", - "title": "PlaylistItem is the Go representation of a playlist.Item.", + "title": "PlaylistItem defines model for playlist.Item.", "properties": { "title": { "description": "Title is an unused property -- it will be removed in the future", @@ -13003,7 +13006,6 @@ } }, "PlaylistItemType": { - "description": "THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.\nEquivalent Go types at stable import paths are provided in https://github.com/grafana/grok.", "type": "string", "title": "Type of the item." }, @@ -13078,7 +13080,7 @@ "homeDashboardUID": { "type": "string" }, - "locale": { + "language": { "type": "string" }, "navbar": { @@ -13329,6 +13331,30 @@ } } }, + "QuotaDTO": { + "type": "object", + "properties": { + "limit": { + "type": "integer", + "format": "int64" + }, + "org_id": { + "type": "integer", + "format": "int64" + }, + "target": { + "type": "string" + }, + "used": { + "type": "integer", + "format": "int64" + }, + "user_id": { + "type": "integer", + "format": "int64" + } + } + }, "RecordingRuleJSON": { "description": "RecordingRuleJSON is the external representation of a recording rule", "type": "object", @@ -14460,6 +14486,9 @@ "UpdateFolderCommand": { "type": "object", "properties": { + "description": { + "type": "string" + }, "overwrite": { "type": "boolean" }, @@ -14506,18 +14535,6 @@ } } }, - "UpdateOrgQuotaCmd": { - "type": "object", - "properties": { - "limit": { - "type": "integer", - "format": "int64" - }, - "target": { - "type": "string" - } - } - }, "UpdateOrgUserCommand": { "type": "object", "properties": { @@ -14563,7 +14580,7 @@ "homeDashboardUID": { "type": "string" }, - "locale": { + "language": { "type": "string" }, "navbar": { @@ -14591,6 +14608,18 @@ } } }, + "UpdateQuotaCmd": { + "type": "object", + "properties": { + "limit": { + "type": "integer", + "format": "int64" + }, + "target": { + "type": "string" + } + } + }, "UpdateRoleCommand": { "type": "object", "properties": { @@ -14683,18 +14712,6 @@ } } }, - "UpdateUserQuotaCmd": { - "type": "object", - "properties": { - "limit": { - "type": "integer", - "format": "int64" - }, - "target": { - "type": "string" - } - } - }, "UserIdDTO": { "type": "object", "properties": { @@ -14799,26 +14816,6 @@ } } }, - "UserQuotaDTO": { - "type": "object", - "properties": { - "limit": { - "type": "integer", - "format": "int64" - }, - "target": { - "type": "string" - }, - "used": { - "type": "integer", - "format": "int64" - }, - "user_id": { - "type": "integer", - "format": "int64" - } - } - }, "UserSearchHitDTO": { "type": "object", "properties": { @@ -15603,7 +15600,7 @@ "getPlaylistResponse": { "description": "", "schema": { - "$ref": "#/definitions/Model" + "$ref": "#/definitions/Playlist" } }, "getPreferencesResponse": { @@ -15641,7 +15638,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/UserQuotaDTO" + "$ref": "#/definitions/QuotaDTO" } } }, @@ -16116,7 +16113,7 @@ "updatePlaylistResponse": { "description": "", "schema": { - "$ref": "#/definitions/Model" + "$ref": "#/definitions/Playlist" } }, "updateServiceAccountResponse": { diff --git a/public/openapi3.json b/public/openapi3.json index ce3ef488c15..31fc5f1bca7 100644 --- a/public/openapi3.json +++ b/public/openapi3.json @@ -954,7 +954,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Model" + "$ref": "#/components/schemas/Playlist" } } }, @@ -1015,7 +1015,7 @@ "application/json": { "schema": { "items": { - "$ref": "#/components/schemas/UserQuotaDTO" + "$ref": "#/components/schemas/QuotaDTO" }, "type": "array" } @@ -1704,7 +1704,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Model" + "$ref": "#/components/schemas/Playlist" } } }, @@ -3032,7 +3032,7 @@ "description": { "type": "string" }, - "parent_uid": { + "parentUid": { "type": "string" }, "title": { @@ -4391,7 +4391,7 @@ "format": "int64", "type": "integer" }, - "parent_uid": { + "parentUid": { "description": "only used if nested folders are enabled", "type": "string" }, @@ -4833,12 +4833,6 @@ }, "GettableNGalertConfig": { "properties": { - "alertmanagers": { - "items": { - "type": "string" - }, - "type": "array" - }, "alertmanagersChoice": { "enum": [ "all", @@ -5771,32 +5765,6 @@ }, "type": "object" }, - "Model": { - "description": "THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.\nEquivalent Go types at stable import paths are provided in https://github.com/grafana/grok.", - "properties": { - "interval": { - "description": "Interval sets the time between switching views in a playlist.\nFIXME: Is this based on a standardized format or what options are available? Can datemath be used?", - "type": "string" - }, - "items": { - "description": "The ordered list of items that the playlist will iterate over.\nFIXME! This should not be optional, but changing it makes the godegen awkward", - "items": { - "$ref": "#/components/schemas/PlaylistItem" - }, - "type": "array" - }, - "name": { - "description": "Name of the playlist.", - "type": "string" - }, - "uid": { - "description": "Unique playlist identifier. Generated on creation, either by the\ncreator of the playlist of by the application.", - "type": "string" - } - }, - "title": "Model is the Go representation of a playlist.", - "type": "object" - }, "MultiStatus": { "type": "object" }, @@ -6299,7 +6267,7 @@ "homeDashboardUID": { "type": "string" }, - "locale": { + "language": { "type": "string" }, "navbar": { @@ -6393,12 +6361,15 @@ "type": "integer" }, "interval": { + "description": "Interval sets the time between switching views in a playlist.\nFIXME: Is this based on a standardized format or what options are available? Can datemath be used?", "type": "string" }, "name": { + "description": "Name of the playlist.", "type": "string" }, "uid": { + "description": "Unique playlist identifier. Generated on creation, either by the\ncreator of the playlist of by the application.", "type": "string" } }, @@ -6436,7 +6407,6 @@ "type": "array" }, "PlaylistItem": { - "description": "THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.\nEquivalent Go types at stable import paths are provided in https://github.com/grafana/grok.", "properties": { "title": { "description": "Title is an unused property -- it will be removed in the future", @@ -6450,11 +6420,10 @@ "type": "string" } }, - "title": "PlaylistItem is the Go representation of a playlist.Item.", + "title": "PlaylistItem defines model for playlist.Item.", "type": "object" }, "PlaylistItemType": { - "description": "THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.\nEquivalent Go types at stable import paths are provided in https://github.com/grafana/grok.", "title": "Type of the item.", "type": "string" }, @@ -6751,12 +6720,6 @@ }, "PostableNGalertConfig": { "properties": { - "alertmanagers": { - "items": { - "type": "string" - }, - "type": "array" - }, "alertmanagersChoice": { "enum": [ "all", @@ -6808,7 +6771,7 @@ "homeDashboardUID": { "type": "string" }, - "locale": { + "language": { "type": "string" }, "navbar": { @@ -7250,6 +7213,30 @@ }, "type": "object" }, + "QuotaDTO": { + "properties": { + "limit": { + "format": "int64", + "type": "integer" + }, + "org_id": { + "format": "int64", + "type": "integer" + }, + "target": { + "type": "string" + }, + "used": { + "format": "int64", + "type": "integer" + }, + "user_id": { + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, "Receiver": { "properties": { "email_configs": { @@ -8924,6 +8911,7 @@ "type": "string" }, "URL": { + "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.", "properties": { "ForceQuery": { "type": "boolean" @@ -8934,9 +8922,6 @@ "Host": { "type": "string" }, - "OmitHost": { - "type": "boolean" - }, "Opaque": { "type": "string" }, @@ -8959,7 +8944,7 @@ "$ref": "#/components/schemas/Userinfo" } }, - "title": "URL is a custom URL type that allows validation at configuration load time.", + "title": "A URL represents a parsed URL (technically, a URI reference).", "type": "object" }, "UpdateAlertNotificationCommand": { @@ -9158,6 +9143,9 @@ }, "UpdateFolderCommand": { "properties": { + "description": { + "type": "string" + }, "overwrite": { "type": "boolean" }, @@ -9205,18 +9193,6 @@ }, "type": "object" }, - "UpdateOrgQuotaCmd": { - "properties": { - "limit": { - "format": "int64", - "type": "integer" - }, - "target": { - "type": "string" - } - }, - "type": "object" - }, "UpdateOrgUserCommand": { "properties": { "role": { @@ -9261,7 +9237,7 @@ "homeDashboardUID": { "type": "string" }, - "locale": { + "language": { "type": "string" }, "navbar": { @@ -9290,6 +9266,18 @@ }, "type": "object" }, + "UpdateQuotaCmd": { + "properties": { + "limit": { + "format": "int64", + "type": "integer" + }, + "target": { + "type": "string" + } + }, + "type": "object" + }, "UpdateRoleCommand": { "properties": { "description": { @@ -9382,18 +9370,6 @@ }, "type": "object" }, - "UpdateUserQuotaCmd": { - "properties": { - "limit": { - "format": "int64", - "type": "integer" - }, - "target": { - "type": "string" - } - }, - "type": "object" - }, "UserIdDTO": { "properties": { "id": { @@ -9498,26 +9474,6 @@ }, "type": "object" }, - "UserQuotaDTO": { - "properties": { - "limit": { - "format": "int64", - "type": "integer" - }, - "target": { - "type": "string" - }, - "used": { - "format": "int64", - "type": "integer" - }, - "user_id": { - "format": "int64", - "type": "integer" - } - }, - "type": "object" - }, "UserSearchHitDTO": { "properties": { "authLabels": { @@ -9763,6 +9719,7 @@ "type": "object" }, "alertGroup": { + "description": "AlertGroup alert group", "properties": { "alerts": { "description": "alerts", @@ -9891,7 +9848,6 @@ "type": "object" }, "gettableAlert": { - "description": "GettableAlert gettable alert", "properties": { "annotations": { "$ref": "#/components/schemas/labelSet" @@ -9954,6 +9910,7 @@ "type": "array" }, "gettableSilence": { + "description": "GettableSilence gettable silence", "properties": { "comment": { "description": "comment", @@ -10002,6 +9959,7 @@ "type": "object" }, "gettableSilences": { + "description": "GettableSilences gettable silences", "items": { "$ref": "#/components/schemas/gettableSilence" }, @@ -10152,6 +10110,7 @@ "type": "array" }, "postableSilence": { + "description": "PostableSilence postable silence", "properties": { "comment": { "description": "comment", @@ -10189,7 +10148,6 @@ "type": "object" }, "receiver": { - "description": "Receiver receiver", "properties": { "active": { "description": "active", @@ -11795,7 +11753,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UpdateUserQuotaCmd" + "$ref": "#/components/schemas/UpdateQuotaCmd" } } }, @@ -12929,6 +12887,15 @@ "/api/v1/provisioning/alert-rules": { "post": { "operationId": "RoutePostAlertRule", + "parameters": [ + { + "in": "header", + "name": "X-Disable-Provenance", + "schema": { + "type": "string" + } + } + ], "requestBody": { "content": { "application/json": { @@ -13035,6 +13002,13 @@ "schema": { "type": "string" } + }, + { + "in": "header", + "name": "X-Disable-Provenance", + "schema": { + "type": "string" + } } ], "requestBody": { @@ -13969,6 +13943,7 @@ }, "/dashboards/id/{DashboardID}/permissions": { "get": { + "deprecated": true, "description": "Please refer to [updated API](#/dashboard_permissions/getDashboardPermissionsListByUID) instead", "operationId": "getDashboardPermissionsListByID", "parameters": [ @@ -14005,6 +13980,7 @@ ] }, "post": { + "deprecated": true, "description": "Please refer to [updated API](#/dashboard_permissions/updateDashboardPermissionsByUID) instead\n\nThis operation will remove existing permissions if they’re not included in the request.", "operationId": "updateDashboardPermissionsByID", "parameters": [ @@ -14057,6 +14033,7 @@ }, "/dashboards/id/{DashboardID}/restore": { "post": { + "deprecated": true, "description": "Please refer to [updated API](#/dashboard_versions/restoreDashboardVersionByUID) instead", "operationId": "restoreDashboardVersionByID", "parameters": [ @@ -14106,6 +14083,7 @@ }, "/dashboards/id/{DashboardID}/versions": { "get": { + "deprecated": true, "description": "Please refer to [updated API](#/dashboard_versions/getDashboardVersionsByUID) instead", "operationId": "getDashboardVersionsByID", "parameters": [ @@ -14144,6 +14122,7 @@ }, "/dashboards/id/{DashboardID}/versions/{DashboardVersionID}": { "get": { + "deprecated": true, "description": "Please refer to [updated API](#/dashboard_versions/getDashboardVersionByUID) instead", "operationId": "getDashboardVersionByID", "parameters": [ @@ -14928,6 +14907,7 @@ }, "/datasources/proxy/{id}/{datasource_proxy_route}": { "delete": { + "deprecated": true, "description": "Proxies all calls to the actual data source.\n\nPlease refer to [updated API](#/datasources/datasourceProxyDELETEByUIDcalls) instead", "operationId": "datasourceProxyDELETEcalls", "parameters": [ @@ -14974,6 +14954,7 @@ ] }, "get": { + "deprecated": true, "description": "Proxies all calls to the actual data source.\n\nPlease refer to [updated API](#/datasources/datasourceProxyGETByUIDcalls) instead", "operationId": "datasourceProxyGETcalls", "parameters": [ @@ -15020,6 +15001,7 @@ ] }, "post": { + "deprecated": true, "description": "Proxies all calls to the actual data source. The data source should support POST methods for the specific path and role as defined\n\nPlease refer to [updated API](#/datasources/datasourceProxyPOSTByUIDcalls) instead", "operationId": "datasourceProxyPOSTcalls", "parameters": [ @@ -15732,6 +15714,7 @@ }, "/datasources/{id}": { "delete": { + "deprecated": true, "description": "If you are running Grafana Enterprise and have Fine-grained access control enabled\nyou need to have a permission with action: `datasources:delete` and scopes: `datasources:*`, `datasources:id:*` and `datasources:id:1` (single data source).\n\nPlease refer to [updated API](#/datasources/deleteDataSourceByUID) instead", "operationId": "deleteDataSourceByID", "parameters": [ @@ -15767,6 +15750,7 @@ ] }, "get": { + "deprecated": true, "description": "If you are running Grafana Enterprise and have Fine-grained access control enabled\nyou need to have a permission with action: `datasources:read` and scopes: `datasources:*`, `datasources:id:*` and `datasources:id:1` (single data source).\n\nPlease refer to [updated API](#/datasources/getDataSourceByUID) instead", "operationId": "getDataSourceByID", "parameters": [ @@ -15805,6 +15789,7 @@ ] }, "put": { + "deprecated": true, "description": "Similar to creating a data source, `password` and `basicAuthPassword` should be defined under\nsecureJsonData in order to be stored securely as an encrypted blob in the database. Then, the\nencrypted fields are listed under secureJsonFields section in the response.\n\nIf you are running Grafana Enterprise and have Fine-grained access control enabled\nyou need to have a permission with action: `datasources:write` and scopes: `datasources:*`, `datasources:id:*` and `datasources:id:1` (single data source).\n\nPlease refer to [updated API](#/datasources/updateDataSourceByUID) instead", "operationId": "updateDataSourceByID", "parameters": [ @@ -15850,6 +15835,7 @@ }, "/datasources/{id}/health": { "get": { + "deprecated": true, "description": "Please refer to [updated API](#/datasources/checkDatasourceHealthWithUID) instead", "operationId": "checkDatasourceHealthByID", "parameters": [ @@ -15887,6 +15873,7 @@ }, "/datasources/{id}/resources/{datasource_proxy_route}": { "get": { + "deprecated": true, "description": "Please refer to [updated API](#/datasources/callDatasourceResourceWithUID) instead", "operationId": "callDatasourceResourceByID", "parameters": [ @@ -17140,6 +17127,33 @@ ] } }, + "/org/quotas": { + "get": { + "description": "If you are running Grafana Enterprise and have Fine-grained access control enabled, you need to have a permission with action `orgs.quotas:read` and scope `org:id:1` (orgIDScope).", + "operationId": "getCurrentOrgQuota", + "responses": { + "200": { + "$ref": "#/components/responses/getQuotaResponse" + }, + "401": { + "$ref": "#/components/responses/unauthorisedError" + }, + "403": { + "$ref": "#/components/responses/forbiddenError" + }, + "404": { + "$ref": "#/components/responses/notFoundError" + }, + "500": { + "$ref": "#/components/responses/internalServerError" + } + }, + "summary": "Fetch Organization quota.", + "tags": [ + "getCurrentOrg" + ] + } + }, "/org/users": { "get": { "description": "Returns all org users within the current organization. Accessible to users with org admin role.\nIf you are running Grafana Enterprise and have Fine-grained access control enabled\nyou need to have a permission with action: `org.users:read` with scope `users:*`.", @@ -17710,7 +17724,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UpdateOrgQuotaCmd" + "$ref": "#/components/schemas/UpdateQuotaCmd" } } }, @@ -18869,6 +18883,7 @@ }, "/reports/render/pdf/{dashboardID}": { "get": { + "deprecated": true, "description": "Please refer to [reports enterprise](#/reports/renderReportPDFs) instead. This will be removed in Grafana 10.", "operationId": "renderReportPDF", "parameters": [ @@ -21070,6 +21085,7 @@ }, "/user/stars/dashboard/{dashboard_id}": { "delete": { + "deprecated": true, "description": "Deletes the starring of the given Dashboard for the actual user.", "operationId": "unstarDashboard", "parameters": [ @@ -21105,6 +21121,7 @@ ] }, "post": { + "deprecated": true, "description": "Stars the given Dashboard for the actual user.", "operationId": "starDashboard", "parameters": [ From 49f91721ec4d76076c2de4eb2321e3039c60571a Mon Sep 17 00:00:00 2001 From: Gilles De Mey Date: Thu, 24 Nov 2022 19:58:02 +0100 Subject: [PATCH 012/131] Alerting: fix "no permissions" alert flashing (#59300) --- .../alerting/unified/ExistingRuleEditor.tsx | 16 ++++++++++++++-- .../alerting/unified/hooks/useIsRuleEditable.ts | 1 - 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/public/app/features/alerting/unified/ExistingRuleEditor.tsx b/public/app/features/alerting/unified/ExistingRuleEditor.tsx index 612b7c345cd..2a704fcff05 100644 --- a/public/app/features/alerting/unified/ExistingRuleEditor.tsx +++ b/public/app/features/alerting/unified/ExistingRuleEditor.tsx @@ -19,9 +19,21 @@ interface ExistingRuleEditorProps { export function ExistingRuleEditor({ identifier }: ExistingRuleEditorProps) { useCleanup((state) => (state.unifiedAlerting.ruleForm.existingRule = initialAsyncRequestState)); - const { loading, result, error, dispatched } = useUnifiedAlertingSelector((state) => state.ruleForm.existingRule); + + const { + loading: loadingAlertRule, + result, + error, + dispatched, + } = useUnifiedAlertingSelector((state) => state.ruleForm.existingRule); + const dispatch = useDispatch(); - const { isEditable } = useIsRuleEditable(ruleId.ruleIdentifierToRuleSourceName(identifier), result?.rule); + const { isEditable, loading: loadingEditable } = useIsRuleEditable( + ruleId.ruleIdentifierToRuleSourceName(identifier), + result?.rule + ); + + const loading = loadingAlertRule || loadingEditable; useEffect(() => { if (!dispatched) { diff --git a/public/app/features/alerting/unified/hooks/useIsRuleEditable.ts b/public/app/features/alerting/unified/hooks/useIsRuleEditable.ts index b2b97cdecfb..150f4ee00ef 100644 --- a/public/app/features/alerting/unified/hooks/useIsRuleEditable.ts +++ b/public/app/features/alerting/unified/hooks/useIsRuleEditable.ts @@ -18,7 +18,6 @@ export function useIsRuleEditable(rulesSourceName: string, rule?: RulerRuleDTO): const folderUID = rule && isGrafanaRulerRule(rule) ? rule.grafana_alert.namespace_uid : undefined; const rulePermission = getRulesPermissions(rulesSourceName); - const { folder, loading } = useFolder(folderUID); if (!rule) { From 28c390bc908b6951c7337369bf1a708fc7bec4b0 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:00:48 +0100 Subject: [PATCH 013/131] Loki: Fix adhoc parsing if multiple stream selectors and some empty (#58658) --- .../datasource/loki/modifyQuery.test.ts | 70 ++++++++++--------- .../plugins/datasource/loki/modifyQuery.ts | 11 ++- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/public/app/plugins/datasource/loki/modifyQuery.test.ts b/public/app/plugins/datasource/loki/modifyQuery.test.ts index d54f2e35a46..171b5b02a57 100644 --- a/public/app/plugins/datasource/loki/modifyQuery.test.ts +++ b/public/app/plugins/datasource/loki/modifyQuery.test.ts @@ -8,40 +8,42 @@ import { describe('addLabelToQuery()', () => { it.each` - query | description | label | operator | value | expectedResult - ${'{x="y"}'} | ${'no label and value'} | ${''} | ${'='} | ${''} | ${''} - ${'{x="yy"}'} | ${'simple query'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="yy", bar="baz"}'} - ${'{x="yy"}'} | ${'simple query'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="yy", bar="baz"}'} - ${'{x="yy"}'} | ${'custom operator'} | ${'bar'} | ${'!='} | ${'baz'} | ${'{x="yy", bar!="baz"}'} - ${'rate({}[1m])'} | ${'do not modify ranges'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[1m])'} - ${'sum by (host) (rate({} [1m]))'} | ${'detect in-order function use'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum by (host) (rate({bar="baz"}[1m]))'} - ${'{instance="my-host.com:9100"}'} | ${'selectors with punctuation'} | ${'bar'} | ${'='} | ${'baz'} | ${'{instance="my-host.com:9100", bar="baz"}'} - ${'{list="a,b,c"}'} | ${'selectors with punctuation'} | ${'bar'} | ${'='} | ${'baz'} | ${'{list="a,b,c", bar="baz"}'} - ${'rate({}[5m]) + rate({}[5m])'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[5m]) + rate({bar="baz"}[5m])'} - ${'avg(rate({x="y"} [$__interval]))+ sum(rate({}[5m]))'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'avg(rate({x="y", bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} - ${'rate({x="yy"}[5m]) * rate({y="zz",a="bb"}[5m]) * rate({}[5m])'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({x="yy", bar="baz"}[5m]) * rate({y="zz", a="bb", bar="baz"}[5m]) * rate({bar="baz"}[5m])'} - ${'{x="yy", bar!="baz"}'} | ${'do not add duplicate labels'} | ${'bar'} | ${'!='} | ${'baz'} | ${'{x="yy", bar!="baz"}'} - ${'rate({bar="baz"}[1m])'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[1m])'} - ${'{list="a,b,c", bar="baz"}'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'{list="a,b,c", bar="baz"}'} - ${'avg(rate({bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'avg(rate({bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} - ${'{x="y"} |="yy"'} | ${'do not remove filters'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} |="yy"'} - ${'{x="y"} |="yy" !~"xx"'} | ${'do not remove filters'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} |="yy" !~"xx"'} - ${'{x="y"} or {}'} | ${'metrics with logical operators'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} or {bar="baz"}'} - ${'{x="y"} and {}'} | ${'metrics with logical operators'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} and {bar="baz"}'} - ${'sum(rate({job="foo"}[2m])) by (value $variable)'} | ${'template variables'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({job="foo", bar="baz"}[2m])) by (value $variable)'} - ${'rate({x="y"}[${__range_s}s])'} | ${'metrics query with range grafana variable'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({x="y", bar="baz"}[${__range_s}s])'} - ${'max by (id, name, type) ({type=~"foo|bar|baz-test"}) * on(id) group_right(id, type, name) sum by (id) (rate({} [5m])) * 1000'} | ${'metrics query with labels in label list with the group modifier'} | ${'bar'} | ${'='} | ${'baz'} | ${'max by (id, name, type) ({type=~"foo|bar|baz-test", bar="baz"}) * on(id) group_right(id, type, name) sum by (id) (rate({bar="baz"}[5m])) * 1000'} - ${'{foo="bar"} | logfmt'} | ${'query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz`'} - ${'{foo="bar"} | logfmt | json'} | ${'query with multiple parsers'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | json | bar=`baz`'} - ${'{foo="bar"} | logfmt | x="y"'} | ${'query with parser and label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | x="y" | bar=`baz`'} - ${'rate({foo="bar"} | logfmt [5m])'} | ${'metrics query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({foo="bar"} | logfmt | bar=`baz` [5m])'} - ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | line_format "{{.status}}" [5m]))'} | ${'metrics query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | bar=`baz` | line_format "{{.status}}" [5m]))'} - ${'{foo="bar"} | logfmt | line_format "{{.status}}"'} | ${'do not add filter to line_format expressions in query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz` | line_format "{{.status}}"'} - ${'{foo="bar"} | logfmt | line_format "{{status}}"'} | ${'do not add filter to line_format expressions in query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz` | line_format "{{status}}"'} - ${'{}'} | ${'query without stream selector'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}'} - ${'{} | logfmt'} | ${'query without stream selector and with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| logfmt'} - ${'{} | x="y"'} | ${'query without stream selector and with label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| x="y"'} - ${'{} | logfmt | x="y"'} | ${'query without stream selector and with parser and label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| logfmt | x="y"'} + query | description | label | operator | value | expectedResult + ${'{x="y"}'} | ${'no label and value'} | ${''} | ${'='} | ${''} | ${''} + ${'{x="yy"}'} | ${'simple query'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="yy", bar="baz"}'} + ${'{x="yy"}'} | ${'simple query'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="yy", bar="baz"}'} + ${'{x="yy"}'} | ${'custom operator'} | ${'bar'} | ${'!='} | ${'baz'} | ${'{x="yy", bar!="baz"}'} + ${'rate({}[1m])'} | ${'do not modify ranges'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[1m])'} + ${'sum by (host) (rate({} [1m]))'} | ${'detect in-order function use'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum by (host) (rate({bar="baz"}[1m]))'} + ${'{instance="my-host.com:9100"}'} | ${'selectors with punctuation'} | ${'bar'} | ${'='} | ${'baz'} | ${'{instance="my-host.com:9100", bar="baz"}'} + ${'{list="a,b,c"}'} | ${'selectors with punctuation'} | ${'bar'} | ${'='} | ${'baz'} | ${'{list="a,b,c", bar="baz"}'} + ${'rate({}[5m]) + rate({}[5m])'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[5m]) + rate({bar="baz"}[5m])'} + ${'avg(rate({x="y"} [$__interval]))+ sum(rate({}[5m]))'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'avg(rate({x="y", bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} + ${'rate({x="yy"}[5m]) * rate({y="zz",a="bb"}[5m]) * rate({}[5m])'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({x="yy", bar="baz"}[5m]) * rate({y="zz", a="bb", bar="baz"}[5m]) * rate({bar="baz"}[5m])'} + ${'{x="yy", bar!="baz"}'} | ${'do not add duplicate labels'} | ${'bar'} | ${'!='} | ${'baz'} | ${'{x="yy", bar!="baz"}'} + ${'rate({bar="baz"}[1m])'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[1m])'} + ${'{list="a,b,c", bar="baz"}'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'{list="a,b,c", bar="baz"}'} + ${'avg(rate({bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'avg(rate({bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} + ${'{x="y"} |="yy"'} | ${'do not remove filters'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} |="yy"'} + ${'{x="y"} |="yy" !~"xx"'} | ${'do not remove filters'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} |="yy" !~"xx"'} + ${'{x="y"} or {}'} | ${'metric with logical operators'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} or {bar="baz"}'} + ${'{x="y"} and {}'} | ${'metric with logical operators'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} and {bar="baz"}'} + ${'sum(rate({job="foo"}[2m])) by (value $variable)'} | ${'template variables'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({job="foo", bar="baz"}[2m])) by (value $variable)'} + ${'rate({x="y"}[${__range_s}s])'} | ${'metric query with range grafana variable'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({x="y", bar="baz"}[${__range_s}s])'} + ${'max by (id, name, type) ({type=~"foo|bar|baz-test"}) * on(id) group_right(id, type, name) sum by (id) (rate({} [5m])) * 1000'} | ${'metric query with labels in label list with the group modifier'} | ${'bar'} | ${'='} | ${'baz'} | ${'max by (id, name, type) ({type=~"foo|bar|baz-test", bar="baz"}) * on(id) group_right(id, type, name) sum by (id) (rate({bar="baz"}[5m])) * 1000'} + ${'{foo="bar"} | logfmt'} | ${'query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz`'} + ${'{foo="bar"} | logfmt | json'} | ${'query with multiple parsers'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | json | bar=`baz`'} + ${'{foo="bar"} | logfmt | x="y"'} | ${'query with parser and label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | x="y" | bar=`baz`'} + ${'rate({foo="bar"} | logfmt [5m])'} | ${'metric query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({foo="bar"} | logfmt | bar=`baz` [5m])'} + ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | line_format "{{.status}}" [5m]))'} | ${'metric query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | bar=`baz` | line_format "{{.status}}" [5m]))'} + ${'{foo="bar"} | logfmt | line_format "{{.status}}"'} | ${'do not add filter to line_format expressions in query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz` | line_format "{{.status}}"'} + ${'{foo="bar"} | logfmt | line_format "{{status}}"'} | ${'do not add filter to line_format expressions in query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz` | line_format "{{status}}"'} + ${'{}'} | ${'query without stream selector'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}'} + ${'{} | logfmt'} | ${'query without stream selector and with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| logfmt'} + ${'{} | x="y"'} | ${'query without stream selector and with label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| x="y"'} + ${'{} | logfmt | x="y"'} | ${'query without stream selector and with parser and label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| logfmt | x="y"'} + ${'sum(rate({x="y"} [5m])) + sum(rate({} | logfmt [5m]))'} | ${'metric query with 1 empty and 1 not empty stream selector with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({x="y", bar="baz"} [5m])) + sum(rate({bar="baz"}| logfmt [5m]))'} + ${'sum(rate({x="y"} | logfmt [5m])) + sum(rate({} [5m]))'} | ${'metric query with 1 non-empty and 1 not empty stream selector with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({x="y", bar="baz"} | logfmt [5m])) + sum(rate({bar="baz"}[5m]))'} `( 'should add label to query: $query, description: $description', ({ query, description, label, operator, value, expectedResult }) => { diff --git a/public/app/plugins/datasource/loki/modifyQuery.ts b/public/app/plugins/datasource/loki/modifyQuery.ts index 332bcc85ce0..2b3b2a3d021 100644 --- a/public/app/plugins/datasource/loki/modifyQuery.ts +++ b/public/app/plugins/datasource/loki/modifyQuery.ts @@ -42,7 +42,13 @@ export function addLabelToQuery(query: string, key: string, operator: string, va } const streamSelectorPositions = getStreamSelectorPositions(query); - const hasStreamSelectorMatchers = getMatcherInStreamPositions(query).length > 0; + const hasStreamSelectorMatchers = getMatcherInStreamPositions(query); + const everyStreamSelectorHasMatcher = streamSelectorPositions.every((streamSelectorPosition) => + hasStreamSelectorMatchers.some( + (matcherPosition) => + matcherPosition.from >= streamSelectorPosition.from && matcherPosition.to <= streamSelectorPosition.to + ) + ); const parserPositions = getParserPositions(query); const labelFilterPositions = getLabelFilterPositions(query); if (!streamSelectorPositions.length) { @@ -51,7 +57,8 @@ export function addLabelToQuery(query: string, key: string, operator: string, va const filter = toLabelFilter(key, value, operator); // If we have non-empty stream selector and parser/label filter, we want to add a new label filter after the last one. - if (hasStreamSelectorMatchers && (labelFilterPositions.length || parserPositions.length)) { + // If some of the stream selectors don't have matchers, we want to add new matcher to the all stream selectors. + if (everyStreamSelectorHasMatcher && (labelFilterPositions.length || parserPositions.length)) { const positionToAdd = findLastPosition([...labelFilterPositions, ...parserPositions]); return addFilterAsLabelFilter(query, [positionToAdd], filter); } else { From f3f1c5aa32e4cc70873f7c5660040d7cd5202a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 25 Nov 2022 14:20:56 +0100 Subject: [PATCH 014/131] Scene: Fixes infinite update loop (#59306) * Scene: Fixes infinite update loop * fixed lint issue --- .../features/scenes/scenes/variablesDemo.tsx | 28 ++----------------- .../variables/sets/SceneVariableSet.test.tsx | 24 ++++++++++++++++ .../scenes/variables/sets/SceneVariableSet.ts | 5 ++++ .../variables/variants/TestVariable.tsx | 6 +++- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/public/app/features/scenes/scenes/variablesDemo.tsx b/public/app/features/scenes/scenes/variablesDemo.tsx index aae519e72b7..2e71cf55865 100644 --- a/public/app/features/scenes/scenes/variablesDemo.tsx +++ b/public/app/features/scenes/scenes/variablesDemo.tsx @@ -7,7 +7,6 @@ import { SceneFlexLayout } from '../components/layout/SceneFlexLayout'; import { SceneTimeRange } from '../core/SceneTimeRange'; import { VariableValueSelectors } from '../variables/components/VariableValueSelectors'; import { SceneVariableSet } from '../variables/sets/SceneVariableSet'; -import { ConstantVariable } from '../variables/variants/ConstantVariable'; import { CustomVariable } from '../variables/variants/CustomVariable'; import { DataSourceVariable } from '../variables/variants/DataSourceVariable'; import { TestVariable } from '../variables/variants/TestVariable'; @@ -45,37 +44,14 @@ export function getVariablesDemo(): Scene { text: '', options: [], }), - new ConstantVariable({ - name: 'constant', - value: 'slow', - }), new CustomVariable({ - name: 'Single Custom', + name: 'custom', query: 'A : 10,B : 20', }), - new CustomVariable({ - name: 'Multi Custom', - query: 'A : 10,B : 20', - isMulti: true, - }), new DataSourceVariable({ - name: 'DataSource', + name: 'ds', query: 'testdata', }), - new DataSourceVariable({ - name: 'DataSource', - query: 'prometheus', - }), - new DataSourceVariable({ - name: 'DataSource multi', - query: 'prometheus', - isMulti: true, - }), - new DataSourceVariable({ - name: 'Datasource w/ regex and using $constant', - query: 'prometheus', - regex: '.*$constant.*', - }), ], }), layout: new SceneFlexLayout({ diff --git a/public/app/features/scenes/variables/sets/SceneVariableSet.test.tsx b/public/app/features/scenes/variables/sets/SceneVariableSet.test.tsx index c5898267f9b..6c41b10ced9 100644 --- a/public/app/features/scenes/variables/sets/SceneVariableSet.test.tsx +++ b/public/app/features/scenes/variables/sets/SceneVariableSet.test.tsx @@ -134,4 +134,28 @@ describe('SceneVariableList', () => { }); }); }); + + describe('When activated with variables update at the same time', () => { + it('Should not start variables multiple times', async () => { + const A = new TestVariable({ name: 'A', query: 'A.*', value: '', text: '', options: [] }); + const B = new TestVariable({ name: 'B', query: 'B.*', value: '', text: '', options: [] }); + + const scene = new TestScene({ + $variables: new SceneVariableSet({ variables: [A, B] }), + }); + + scene.activate(); + + // Should start variables + expect(A.state.loading).toBe(true); + expect(B.state.loading).toBe(true); + expect(A.getValueOptionsCount).toBe(1); + + // Complete the second one + B.signalUpdateCompleted(); + + // When B complete should not start another instance of A + expect(A.getValueOptionsCount).toBe(1); + }); + }); }); diff --git a/public/app/features/scenes/variables/sets/SceneVariableSet.ts b/public/app/features/scenes/variables/sets/SceneVariableSet.ts index 58ad9bedf80..32388dc83c9 100644 --- a/public/app/features/scenes/variables/sets/SceneVariableSet.ts +++ b/public/app/features/scenes/variables/sets/SceneVariableSet.ts @@ -60,6 +60,11 @@ export class SceneVariableSet extends SceneObjectBase imp throw new Error('Variable added to variablesToUpdate but does not have validateAndUpdate'); } + // Ignore it if it's already started + if (this.updating.has(variable)) { + continue; + } + // Wait for variables that has dependencies that also needs updates if (this.hasDependendencyInUpdateQueue(variable)) { continue; diff --git a/public/app/features/scenes/variables/variants/TestVariable.tsx b/public/app/features/scenes/variables/variants/TestVariable.tsx index a9ad24ec298..10601be6e9e 100644 --- a/public/app/features/scenes/variables/variants/TestVariable.tsx +++ b/public/app/features/scenes/variables/variants/TestVariable.tsx @@ -23,6 +23,7 @@ export interface TestVariableState extends MultiValueVariableState { export class TestVariable extends MultiValueVariable { private completeUpdate = new Subject(); public isGettingValues = true; + public getValueOptionsCount = 0; protected _variableDependency = new VariableDependencyConfig(this, { statePaths: ['query'], @@ -42,10 +43,12 @@ export class TestVariable extends MultiValueVariable { public getValueOptions(args: VariableGetOptionsArgs): Observable { const { delayMs } = this.state; + this.getValueOptionsCount += 1; + return new Observable((observer) => { this.setState({ loading: true }); - this.completeUpdate.subscribe({ + const sub = this.completeUpdate.subscribe({ next: () => { observer.next(this.issueQuery()); }, @@ -60,6 +63,7 @@ export class TestVariable extends MultiValueVariable { this.isGettingValues = true; return () => { + sub.unsubscribe(); clearTimeout(timeout); this.isGettingValues = false; }; From 7ba86dc1dc7133ef0c1b8ad9e59d4b9e9d5a0bb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Farkas?= Date: Mon, 28 Nov 2022 08:57:56 +0100 Subject: [PATCH 015/131] Elastic: always return DataFrames from the datasource's query method (#58636) * elastic: response parsing: convert TableModel to DataFrame * elastic: response parsing: convert JSONDocumentData to DataFrame * elastic: response parsing: convert TimeSeries to DataFrame * elastic: unit tests: type fixes * elastic: fixed unit test --- .../elasticsearch/ElasticResponse.test.ts | 270 ++++++++++-------- .../elasticsearch/ElasticResponse.ts | 6 +- .../elasticsearch/datasource.test.ts | 23 +- 3 files changed, 169 insertions(+), 130 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/ElasticResponse.test.ts b/public/app/plugins/datasource/elasticsearch/ElasticResponse.test.ts index 2005a0a5c66..f44f8a1b1ac 100644 --- a/public/app/plugins/datasource/elasticsearch/ElasticResponse.test.ts +++ b/public/app/plugins/datasource/elasticsearch/ElasticResponse.test.ts @@ -1,17 +1,33 @@ -import { Column, DataFrameView, Field, FieldCache, KeyValue, MutableDataFrame } from '@grafana/data'; +import { DataFrame, DataFrameView, Field, FieldCache, FieldType, KeyValue, MutableDataFrame } from '@grafana/data'; import flatten from 'app/core/utils/flatten'; import { ElasticResponse } from './ElasticResponse'; import { highlightTags } from './queryDef'; import { ElasticsearchQuery } from './types'; +function getTimeField(frame: DataFrame): Field { + const field = frame.fields[0]; + if (field.type !== FieldType.time) { + throw new Error('first field should be the time-field'); + } + return field; +} + +function getValueField(frame: DataFrame): Field { + const field = frame.fields[1]; + if (field.type !== FieldType.number) { + throw new Error('second field should be the number-field'); + } + return field; +} + describe('ElasticResponse', () => { let targets: ElasticsearchQuery[]; let response: { responses: unknown[]; }; let result: { - data: MockedResultData[]; + data: DataFrame[]; }; describe('refId matching', () => { @@ -296,16 +312,17 @@ describe('ElasticResponse', () => { it('should return 1 series', () => { expect(result.data.length).toBe(1); - expect(result.data[0].target).toBe('Count'); - expect(result.data[0].datapoints.length).toBe(2); - expect(result.data[0].datapoints[0][0]).toBe(10); - expect(result.data[0].datapoints[0][1]).toBe(1000); + const frame = result.data[0]; + expect(frame.name).toBe('Count'); + expect(frame.length).toBe(2); + expect(getTimeField(frame).values.get(0)).toBe(1000); + expect(getValueField(frame).values.get(0)).toBe(10); }); }); describe('simple query count & avg aggregation', () => { let result: { - data: MockedResultData[]; + data: DataFrame[]; }; beforeEach(() => { @@ -347,19 +364,20 @@ describe('ElasticResponse', () => { it('should return 2 series', () => { expect(result.data.length).toBe(2); - expect(result.data[0].datapoints.length).toBe(2); - expect(result.data[0].datapoints[0][0]).toBe(10); - expect(result.data[0].datapoints[0][1]).toBe(1000); + const frame1 = result.data[0]; + const frame2 = result.data[1]; + expect(frame1.length).toBe(2); + expect(getValueField(frame1).values.get(0)).toBe(10); + expect(getTimeField(frame1).values.get(0)).toBe(1000); - expect(result.data[1].target).toBe('Average value'); - expect(result.data[1].datapoints[0][0]).toBe(88); - expect(result.data[1].datapoints[1][0]).toBe(99); + expect(frame2.name).toBe('Average value'); + expect(getValueField(frame2).values.toArray()).toStrictEqual([88, 99]); }); }); describe('single group by query one metric', () => { let result: { - data: MockedResultData[]; + data: DataFrame[]; }; beforeEach(() => { @@ -411,15 +429,17 @@ describe('ElasticResponse', () => { it('should return 2 series', () => { expect(result.data.length).toBe(2); - expect(result.data[0].datapoints.length).toBe(2); - expect(result.data[0].target).toBe('server1'); - expect(result.data[1].target).toBe('server2'); + const frame1 = result.data[0]; + const frame2 = result.data[1]; + expect(frame1.length).toBe(2); + expect(frame1.name).toBe('server1'); + expect(frame2.name).toBe('server2'); }); }); describe('single group by query two metrics', () => { let result: { - data: MockedResultData[]; + data: DataFrame[]; }; beforeEach(() => { @@ -474,17 +494,17 @@ describe('ElasticResponse', () => { it('should return 2 series', () => { expect(result.data.length).toBe(4); - expect(result.data[0].datapoints.length).toBe(2); - expect(result.data[0].target).toBe('server1 Count'); - expect(result.data[1].target).toBe('server1 Average @value'); - expect(result.data[2].target).toBe('server2 Count'); - expect(result.data[3].target).toBe('server2 Average @value'); + expect(result.data[0].length).toBe(2); + expect(result.data[0].name).toBe('server1 Count'); + expect(result.data[1].name).toBe('server1 Average @value'); + expect(result.data[2].name).toBe('server2 Count'); + expect(result.data[3].name).toBe('server2 Average @value'); }); }); describe('with percentiles ', () => { let result: { - data: MockedResultData[]; + data: DataFrame[]; }; beforeEach(() => { @@ -523,18 +543,18 @@ describe('ElasticResponse', () => { it('should return 2 series', () => { expect(result.data.length).toBe(2); - expect(result.data[0].datapoints.length).toBe(2); - expect(result.data[0].target).toBe('p75 @value'); - expect(result.data[1].target).toBe('p90 @value'); - expect(result.data[0].datapoints[0][0]).toBe(3.3); - expect(result.data[0].datapoints[0][1]).toBe(1000); - expect(result.data[1].datapoints[1][0]).toBe(4.5); + expect(result.data[0].length).toBe(2); + expect(result.data[0].name).toBe('p75 @value'); + expect(result.data[1].name).toBe('p90 @value'); + expect(getValueField(result.data[0]).values.get(0)).toBe(3.3); + expect(getTimeField(result.data[0]).values.get(0)).toBe(1000); + expect(getValueField(result.data[1]).values.get(1)).toBe(4.5); }); }); describe('with extended_stats', () => { let result: { - data: MockedResultData[]; + data: DataFrame[]; }; beforeEach(() => { @@ -605,12 +625,12 @@ describe('ElasticResponse', () => { it('should return 4 series', () => { expect(result.data.length).toBe(4); - expect(result.data[0].datapoints.length).toBe(1); - expect(result.data[0].target).toBe('server1 Max @value'); - expect(result.data[1].target).toBe('server1 Std Dev Upper @value'); + expect(result.data[0].length).toBe(1); + expect(result.data[0].name).toBe('server1 Max @value'); + expect(result.data[1].name).toBe('server1 Std Dev Upper @value'); - expect(result.data[0].datapoints[0][0]).toBe(10.2); - expect(result.data[1].datapoints[0][0]).toBe(3); + expect(getValueField(result.data[0]).values.get(0)).toBe(10.2); + expect(getValueField(result.data[1]).values.get(0)).toBe(3); }); }); @@ -666,26 +686,28 @@ describe('ElasticResponse', () => { expect(result.data.length).toBe(2); const firstSeries = result.data[0]; - expect(firstSeries.target).toBe('Top Metrics @value'); - expect(firstSeries.datapoints.length).toBe(2); - expect(firstSeries.datapoints).toEqual([ - [1, new Date('2021-01-01T00:00:00.000Z').valueOf()], - [1, new Date('2021-01-01T00:00:10.000Z').valueOf()], + expect(firstSeries.name).toBe('Top Metrics @value'); + expect(firstSeries.length).toBe(2); + expect(getTimeField(firstSeries).values.toArray()).toStrictEqual([ + new Date('2021-01-01T00:00:00.000Z').valueOf(), + new Date('2021-01-01T00:00:10.000Z').valueOf(), ]); + expect(getValueField(firstSeries).values.toArray()).toStrictEqual([1, 1]); const secondSeries = result.data[1]; - expect(secondSeries.target).toBe('Top Metrics @anotherValue'); - expect(secondSeries.datapoints.length).toBe(2); - expect(secondSeries.datapoints).toEqual([ - [2, new Date('2021-01-01T00:00:00.000Z').valueOf()], - [2, new Date('2021-01-01T00:00:10.000Z').valueOf()], + expect(secondSeries.name).toBe('Top Metrics @anotherValue'); + expect(secondSeries.length).toBe(2); + expect(getTimeField(secondSeries).values.toArray()).toStrictEqual([ + new Date('2021-01-01T00:00:00.000Z').valueOf(), + new Date('2021-01-01T00:00:10.000Z').valueOf(), ]); + expect(getValueField(secondSeries).values.toArray()).toStrictEqual([2, 2]); }); }); describe('single group by with alias pattern', () => { let result: { - data: MockedResultData[]; + data: DataFrame[]; }; beforeEach(() => { @@ -748,16 +770,16 @@ describe('ElasticResponse', () => { it('should return 2 series', () => { expect(result.data.length).toBe(3); - expect(result.data[0].datapoints.length).toBe(2); - expect(result.data[0].target).toBe('server1 Count and {{not_exist}} server1'); - expect(result.data[1].target).toBe('server2 Count and {{not_exist}} server2'); - expect(result.data[2].target).toBe('0 Count and {{not_exist}} 0'); + expect(result.data[0].length).toBe(2); + expect(result.data[0].name).toBe('server1 Count and {{not_exist}} server1'); + expect(result.data[1].name).toBe('server2 Count and {{not_exist}} server2'); + expect(result.data[2].name).toBe('0 Count and {{not_exist}} 0'); }); }); describe('histogram response', () => { let result: { - data: MockedResultData[]; + data: DataFrame[]; }; beforeEach(() => { @@ -787,15 +809,20 @@ describe('ElasticResponse', () => { result = new ElasticResponse(targets, response).getTimeSeries(); }); - it('should return table with byte and count', () => { - expect(result.data[0].rows.length).toBe(3); - expect(result.data[0].columns).toEqual([{ text: 'bytes', filterable: true }, { text: 'Count' }]); + it('should return dataframe with byte and count', () => { + expect(result.data[0].length).toBe(3); + const { fields } = result.data[0]; + expect(fields.length).toBe(2); + expect(fields[0].name).toBe('bytes'); + expect(fields[0].config).toStrictEqual({ filterable: true }); + expect(fields[1].name).toBe('Count'); + expect(fields[1].config).toStrictEqual({}); }); }); describe('with two filters agg', () => { let result: { - data: MockedResultData[]; + data: DataFrame[]; }; beforeEach(() => { @@ -852,9 +879,9 @@ describe('ElasticResponse', () => { it('should return 2 series', () => { expect(result.data.length).toBe(2); - expect(result.data[0].datapoints.length).toBe(2); - expect(result.data[0].target).toBe('@metric:cpu'); - expect(result.data[1].target).toBe('@metric:logins.count'); + expect(result.data[0].length).toBe(2); + expect(result.data[0].name).toBe('@metric:cpu'); + expect(result.data[1].name).toBe('@metric:logins.count'); }); }); @@ -911,7 +938,7 @@ describe('ElasticResponse', () => { it('should remove first and last value', () => { expect(result.data.length).toBe(2); - expect(result.data[0].datapoints.length).toBe(1); + expect(result.data[0].length).toBe(1); }); }); @@ -954,22 +981,23 @@ describe('ElasticResponse', () => { result = new ElasticResponse(targets, response).getTimeSeries(); }); - it('should return table', () => { + it('should return dataframe', () => { expect(result.data.length).toBe(1); - expect(result.data[0].type).toBe('table'); - expect(result.data[0].rows.length).toBe(2); - expect(result.data[0].rows[0][0]).toBe('server-1'); - expect(result.data[0].rows[0][1]).toBe(1000); - expect(result.data[0].rows[0][2]).toBe(369); + expect(result.data[0].length).toBe(2); + expect(result.data[0].fields.length).toBe(3); + const field1 = result.data[0].fields[0]; + const field2 = result.data[0].fields[1]; + const field3 = result.data[0].fields[2]; - expect(result.data[0].rows[1][0]).toBe('server-2'); - expect(result.data[0].rows[1][1]).toBe(2000); + expect(field1.values.toArray()).toStrictEqual(['server-1', 'server-2']); + expect(field2.values.toArray()).toStrictEqual([1000, 2000]); + expect(field3.values.toArray()).toStrictEqual([369, 200]); }); }); describe('No group by time with percentiles ', () => { let result: { - data: MockedResultData[]; + data: DataFrame[]; }; beforeEach(() => { @@ -1006,19 +1034,19 @@ describe('ElasticResponse', () => { result = new ElasticResponse(targets, response).getTimeSeries(); }); - it('should return table', () => { + it('should return dataframe', () => { expect(result.data.length).toBe(1); - expect(result.data[0].type).toBe('table'); - expect(result.data[0].columns[0].text).toBe('id'); - expect(result.data[0].columns[1].text).toBe('p75 value'); - expect(result.data[0].columns[2].text).toBe('p90 value'); - expect(result.data[0].rows.length).toBe(2); - expect(result.data[0].rows[0][0]).toBe('id1'); - expect(result.data[0].rows[0][1]).toBe(3.3); - expect(result.data[0].rows[0][2]).toBe(5.5); - expect(result.data[0].rows[1][0]).toBe('id2'); - expect(result.data[0].rows[1][1]).toBe(2.3); - expect(result.data[0].rows[1][2]).toBe(4.5); + expect(result.data[0].length).toBe(2); + const field1 = result.data[0].fields[0]; + const field2 = result.data[0].fields[1]; + const field3 = result.data[0].fields[2]; + expect(field1.name).toBe('id'); + expect(field2.name).toBe('p75 value'); + expect(field3.name).toBe('p90 value'); + + expect(field1.values.toArray()).toStrictEqual(['id1', 'id2']); + expect(field2.values.toArray()).toStrictEqual([3.3, 2.3]); + expect(field3.values.toArray()).toStrictEqual([5.5, 4.5]); }); }); @@ -1058,15 +1086,17 @@ describe('ElasticResponse', () => { }); it('should include field in metric name', () => { - expect(result.data[0].type).toBe('table'); - expect(result.data[0].rows[0][1]).toBe(1000); - expect(result.data[0].rows[0][2]).toBe(3000); + expect(result.data[0].length).toBe(1); + expect(result.data[0].fields.length).toBe(3); + expect(result.data[0].fields[0].values.toArray()).toStrictEqual(['server-1']); + expect(result.data[0].fields[1].values.toArray()).toStrictEqual([1000]); + expect(result.data[0].fields[2].values.toArray()).toStrictEqual([3000]); }); }); describe('Raw documents query', () => { let result: { - data: Array>; + data: DataFrame[]; }; beforeEach(() => { targets = [ @@ -1102,19 +1132,23 @@ describe('ElasticResponse', () => { result = new ElasticResponse(targets, response).getTimeSeries(); }); - it('should return docs', () => { + it('should return raw_document formatted data', () => { expect(result.data.length).toBe(1); - expect(result.data[0].type).toBe('docs'); - expect(result.data[0].total).toBe(100); - expect(result.data[0].datapoints.length).toBe(2); - expect(result.data[0].datapoints[0].sourceProp).toBe('asd'); - expect(result.data[0].datapoints[0].fieldProp).toBe('field'); + const frame = result.data[0]; + const { fields } = frame; + expect(fields.length).toBe(1); + const field = fields[0]; + expect(field.type === FieldType.other); + const values = field.values.toArray(); + expect(values.length).toBe(2); + expect(values[0].sourceProp).toBe('asd'); + expect(values[0].fieldProp).toBe('field'); }); }); describe('with bucket_script ', () => { let result: { - data: MockedResultData[]; + data: DataFrame[]; }; beforeEach(() => { @@ -1168,22 +1202,22 @@ describe('ElasticResponse', () => { }); it('should return 3 series', () => { expect(result.data.length).toBe(3); - expect(result.data[0].datapoints.length).toBe(2); - expect(result.data[0].target).toBe('Sum @value'); - expect(result.data[1].target).toBe('Max @value'); - expect(result.data[2].target).toBe('Sum @value * Max @value'); - expect(result.data[0].datapoints[0][0]).toBe(2); - expect(result.data[1].datapoints[0][0]).toBe(3); - expect(result.data[2].datapoints[0][0]).toBe(6); - expect(result.data[0].datapoints[1][0]).toBe(3); - expect(result.data[1].datapoints[1][0]).toBe(4); - expect(result.data[2].datapoints[1][0]).toBe(12); + expect(result.data[0].length).toBe(2); + expect(result.data[0].name).toBe('Sum @value'); + expect(result.data[1].name).toBe('Max @value'); + expect(result.data[2].name).toBe('Sum @value * Max @value'); + expect(getValueField(result.data[0]).values.get(0)).toBe(2); + expect(getValueField(result.data[1]).values.get(0)).toBe(3); + expect(getValueField(result.data[2]).values.get(0)).toBe(6); + expect(getValueField(result.data[0]).values.get(1)).toBe(3); + expect(getValueField(result.data[1]).values.get(1)).toBe(4); + expect(getValueField(result.data[2]).values.get(1)).toBe(12); }); }); describe('terms with bucket_script and two scripts', () => { let result: { - data: MockedResultData[]; + data: DataFrame[]; }; beforeEach(() => { @@ -1248,16 +1282,15 @@ describe('ElasticResponse', () => { }); it('should return 2 rows with 5 columns', () => { - expect(result.data[0].columns.length).toBe(5); - expect(result.data[0].rows.length).toBe(2); - expect(result.data[0].rows[0][1]).toBe(2); - expect(result.data[0].rows[0][2]).toBe(3); - expect(result.data[0].rows[0][3]).toBe(6); - expect(result.data[0].rows[0][4]).toBe(24); - expect(result.data[0].rows[1][1]).toBe(3); - expect(result.data[0].rows[1][2]).toBe(4); - expect(result.data[0].rows[1][3]).toBe(12); - expect(result.data[0].rows[1][4]).toBe(48); + const frame = result.data[0]; + expect(frame.length).toBe(2); + const { fields } = frame; + expect(fields.length).toBe(5); + expect(fields[0].values.toArray()).toStrictEqual([1000, 2000]); + expect(fields[1].values.toArray()).toStrictEqual([2, 3]); + expect(fields[2].values.toArray()).toStrictEqual([3, 4]); + expect(fields[3].values.toArray()).toStrictEqual([6, 12]); + expect(fields[4].values.toArray()).toStrictEqual([24, 48]); }); }); @@ -1516,14 +1549,3 @@ interface MockedQueryData { target: ElasticsearchQuery; response: MockedElasticResponse; } - -interface MockedResultData { - refId: string; - target: string; - datapoints: T[]; - type: string; - rows: number[][]; - total: number; - fields: Field[]; - columns: Column[]; -} diff --git a/public/app/plugins/datasource/elasticsearch/ElasticResponse.ts b/public/app/plugins/datasource/elasticsearch/ElasticResponse.ts index ca004dca9ca..665d5ab3c33 100644 --- a/public/app/plugins/datasource/elasticsearch/ElasticResponse.ts +++ b/public/app/plugins/datasource/elasticsearch/ElasticResponse.ts @@ -484,7 +484,11 @@ export class ElasticResponse { if (this.targets.some((target) => queryDef.hasMetricOfType(target, 'raw_data'))) { return this.processResponseToDataFrames(false); } - return this.processResponseToSeries(); + const result = this.processResponseToSeries(); + return { + ...result, + data: result.data.map((item) => toDataFrame(item)), + }; } getLogs(logMessageField?: string, logLevelField?: string): DataQueryResponse { diff --git a/public/app/plugins/datasource/elasticsearch/datasource.test.ts b/public/app/plugins/datasource/elasticsearch/datasource.test.ts index aa7670c794c..2c582baca57 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.test.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.test.ts @@ -12,6 +12,7 @@ import { DateTime, dateTime, Field, + FieldType, MutableDataFrame, RawTimeRange, TimeRange, @@ -223,11 +224,23 @@ describe('ElasticDatasource', () => { expect(received[0]).toEqual({ data: [ { - datapoints: [[10, 1000]], - metric: 'count', - props: {}, + name: 'resolvedVariable', refId: 'test', - target: 'resolvedVariable', + fields: [ + { + name: 'Time', + type: FieldType.time, + config: {}, + values: new ArrayVector([1000]), + }, + { + name: 'Value', + type: FieldType.number, + config: {}, + values: new ArrayVector([10]), + }, + ], + length: 1, }, ], }); @@ -255,7 +268,7 @@ describe('ElasticDatasource', () => { it('should resolve the alias variable for the alias/target in the result', async () => { const { result } = await runScenario(); - expect(result.data[0].target).toEqual('resolvedVariable'); + expect(result.data[0].name).toEqual('resolvedVariable'); }); it('should json escape lucene query', async () => { From 16af756d50f0146ad60538f3e09f74bd2dcde372 Mon Sep 17 00:00:00 2001 From: Victor Marin <36818606+mdvictor@users.noreply.github.com> Date: Mon, 28 Nov 2022 10:16:35 +0200 Subject: [PATCH 016/131] TablePanel: Add support for Count calculation per column or per entire dataset (#58134) * WIP * TablePanel: Add support for Count calculation per column or per entire dataset * refactor * refactor * refactor + fixes * refactor + tests * Docs and cue model fix --- .../visualizations/table/index.md | 12 ++ .../src/transformations/fieldReducer.ts | 2 +- .../src/components/Table/FooterRow.tsx | 15 ++- .../src/components/Table/Table.test.tsx | 115 ++++++++++++++++++ .../grafana-ui/src/components/Table/Table.tsx | 50 ++++++-- .../grafana-ui/src/components/Table/types.ts | 1 + .../grafana-ui/src/components/Table/utils.tsx | 5 +- public/app/plugins/panel/table/models.gen.ts | 1 + public/app/plugins/panel/table/module.tsx | 12 +- 9 files changed, 192 insertions(+), 21 deletions(-) diff --git a/docs/sources/panels-visualizations/visualizations/table/index.md b/docs/sources/panels-visualizations/visualizations/table/index.md index f795848e51b..914e09ba23c 100644 --- a/docs/sources/panels-visualizations/visualizations/table/index.md +++ b/docs/sources/panels-visualizations/visualizations/table/index.md @@ -160,3 +160,15 @@ Columns with filters applied have a blue funnel displayed next to the title. {{< figure src="/static/img/docs/tables/filtered-column.png" max-width="500px" caption="Filtered column" class="docs-image--no-shadow" >}} To remove the filter, click the blue funnel icon and then click **Clear filter**. + +## Table footer + +You can use the table footer to show [calculations]({{< relref "../../calculation-types/" >}}) on fields. + +After enabling the table footer, you can select your **Calculation** and select the **Fields** that should be calculated. Not selecting any field apply the calculation to all numeric fields. + +### Count rows + +On selecting the **Count** calculation, you will see the **Count rows** switch. + +By enabling this option the footer will show the number of rows in the dataset instead of the number of values in the selected fields. diff --git a/packages/grafana-data/src/transformations/fieldReducer.ts b/packages/grafana-data/src/transformations/fieldReducer.ts index 59691aaf81c..4ce873ab34c 100644 --- a/packages/grafana-data/src/transformations/fieldReducer.ts +++ b/packages/grafana-data/src/transformations/fieldReducer.ts @@ -291,7 +291,7 @@ export function doStandardCalcs(field: Field, ignoreNulls: boolean, nullAsZero: } as FieldCalcs; const data = field.values; - calcs.count = data.length; + calcs.count = ignoreNulls ? data.length : data.toArray().filter((val) => val != null).length; const isNumberField = field.type === FieldType.number || FieldType.time; diff --git a/packages/grafana-ui/src/components/Table/FooterRow.tsx b/packages/grafana-ui/src/components/Table/FooterRow.tsx index 59a4659f3d8..8e9cc1e423c 100644 --- a/packages/grafana-ui/src/components/Table/FooterRow.tsx +++ b/packages/grafana-ui/src/components/Table/FooterRow.tsx @@ -40,9 +40,7 @@ export const FooterRow = (props: FooterRowProps) => { data-testid={e2eSelectorsTable.footer} style={height ? { height: `${height}px` } : undefined} > - {footerGroup.headers.map((column: ColumnInstance, index: number) => - renderFooterCell(column, tableStyles, height) - )} + {footerGroup.headers.map((column: ColumnInstance) => renderFooterCell(column, tableStyles, height))} ); })} @@ -71,10 +69,19 @@ function renderFooterCell(column: ColumnInstance, tableStyles: TableStyles, heig ); } -export function getFooterValue(index: number, footerValues?: FooterItem[]) { +export function getFooterValue(index: number, footerValues?: FooterItem[], isCountRowsSet?: boolean) { if (footerValues === undefined) { return EmptyCell; } + if (isCountRowsSet) { + const count = footerValues[index]; + if (typeof count !== 'string') { + return EmptyCell; + } + + return FooterCell({ value: [{ Count: count }] }); + } + return FooterCell({ value: footerValues[index] }); } diff --git a/packages/grafana-ui/src/components/Table/Table.test.tsx b/packages/grafana-ui/src/components/Table/Table.test.tsx index b6ce1f0d003..95ca40bf7a5 100644 --- a/packages/grafana-ui/src/components/Table/Table.test.tsx +++ b/packages/grafana-ui/src/components/Table/Table.test.tsx @@ -399,6 +399,121 @@ describe('Table', () => { }); }); + describe('on table footer enabled and count calculation selected', () => { + it('should show count of non-null values', async () => { + getTestContext({ + footerOptions: { show: true, reducer: ['count'] }, + data: toDataFrame({ + name: 'A', + fields: [ + { + name: 'number', + type: FieldType.number, + values: [1, 1, 1, 2, null], + config: { + custom: { + filterable: true, + }, + }, + }, + ], + }), + }); + + expect(within(getFooter()).getByRole('columnheader').getElementsByTagName('span')[0].textContent).toEqual('4'); + }); + + it('should show count of rows when `count rows` is selected', async () => { + getTestContext({ + footerOptions: { show: true, reducer: ['count'], countRows: true }, + data: toDataFrame({ + name: 'A', + fields: [ + { + name: 'number1', + type: FieldType.number, + values: [1, 1, 1, 2, null], + config: { + custom: { + filterable: true, + }, + }, + }, + ], + }), + }); + + expect(within(getFooter()).getByRole('columnheader').getElementsByTagName('span')[0].textContent).toEqual( + 'Count:' + ); + expect(within(getFooter()).getByRole('columnheader').getElementsByTagName('span')[1].textContent).toEqual('5'); + }); + + it('should show correct counts when turning `count rows` on and off', async () => { + const { rerender } = getTestContext({ + footerOptions: { show: true, reducer: ['count'], countRows: true }, + data: toDataFrame({ + name: 'A', + fields: [ + { + name: 'number1', + type: FieldType.number, + values: [1, 1, 1, 2, null], + config: { + custom: { + filterable: true, + }, + }, + }, + ], + }), + }); + + expect(within(getFooter()).getByRole('columnheader').getElementsByTagName('span')[0].textContent).toEqual( + 'Count:' + ); + expect(within(getFooter()).getByRole('columnheader').getElementsByTagName('span')[1].textContent).toEqual('5'); + + const onSortByChange = jest.fn(); + const onCellFilterAdded = jest.fn(); + const onColumnResize = jest.fn(); + const props: Props = { + ariaLabel: 'aria-label', + data: getDefaultDataFrame(), + height: 600, + width: 800, + onSortByChange, + onCellFilterAdded, + onColumnResize, + }; + + const propOverrides = { + footerOptions: { show: true, reducer: ['count'], countRows: false }, + data: toDataFrame({ + name: 'A', + fields: [ + { + name: 'number', + type: FieldType.number, + values: [1, 1, 1, 2, null], + config: { + custom: { + filterable: true, + }, + }, + }, + ], + }), + }; + + Object.assign(props, propOverrides); + + rerender(); + + expect(within(getFooter()).getByRole('columnheader').getElementsByTagName('span')[0].textContent).toEqual('4'); + }); + }); + describe('when mounted with data and sub-data', () => { it('then correct rows should be rendered and new table is rendered when expander is clicked', () => { getTestContext({ diff --git a/packages/grafana-ui/src/components/Table/Table.tsx b/packages/grafana-ui/src/components/Table/Table.tsx index dc3103067ec..792d9a027fb 100644 --- a/packages/grafana-ui/src/components/Table/Table.tsx +++ b/packages/grafana-ui/src/components/Table/Table.tsx @@ -12,7 +12,7 @@ import { import usePrevious from 'react-use/lib/usePrevious'; import { VariableSizeList } from 'react-window'; -import { DataFrame, getFieldDisplayName, Field } from '@grafana/data'; +import { DataFrame, getFieldDisplayName, Field, ReducerID } from '@grafana/data'; import { useStyles2, useTheme2 } from '../../themes'; import { CustomScrollbar } from '../CustomScrollbar/CustomScrollbar'; @@ -188,10 +188,27 @@ export const Table = memo((props: Props) => { return Array(data.length).fill(0); }, [data]); + const isCountRowsSet = Boolean( + footerOptions?.countRows && + footerOptions.reducer && + footerOptions.reducer.length && + footerOptions.reducer[0] === ReducerID.count + ); + // React-table column definitions const memoizedColumns = useMemo( - () => getColumns(data, width, columnMinWidth, expandedIndexes, setExpandedIndexes, !!subData?.length, footerItems), - [data, width, columnMinWidth, footerItems, subData, expandedIndexes] + () => + getColumns( + data, + width, + columnMinWidth, + expandedIndexes, + setExpandedIndexes, + !!subData?.length, + footerItems, + isCountRowsSet + ), + [data, width, columnMinWidth, footerItems, subData, expandedIndexes, isCountRowsSet] ); // Internal react table state reducer @@ -244,17 +261,24 @@ export const Table = memo((props: Props) => { return; } - if (footerOptions.show) { - setFooterItems( - getFooterItems( - headerGroups[0].headers as unknown as Array<{ field: Field }>, - createFooterCalculationValues(rows), - footerOptions, - theme - ) - ); - } else { + if (!footerOptions.show) { setFooterItems(undefined); + return; + } + + const footerItems = getFooterItems( + headerGroups[0].headers as unknown as Array<{ field: Field }>, + createFooterCalculationValues(rows), + footerOptions, + theme + ); + + if (isCountRowsSet) { + const footerItemsCountRows: FooterItem[] = new Array(footerItems.length).fill(undefined); + footerItemsCountRows[0] = data.length.toString(); + setFooterItems(footerItemsCountRows); + } else { + setFooterItems(footerItems); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [footerOptions, theme, state.filters, data]); diff --git a/packages/grafana-ui/src/components/Table/types.ts b/packages/grafana-ui/src/components/Table/types.ts index 0f613eda42e..6942ec7594b 100644 --- a/packages/grafana-ui/src/components/Table/types.ts +++ b/packages/grafana-ui/src/components/Table/types.ts @@ -50,4 +50,5 @@ export interface TableFooterCalc { reducer: string[]; // actually 1 value fields?: string[]; enablePagination?: boolean; + countRows?: boolean; } diff --git a/packages/grafana-ui/src/components/Table/utils.tsx b/packages/grafana-ui/src/components/Table/utils.tsx index fdce74d6853..fd5f1b46c88 100644 --- a/packages/grafana-ui/src/components/Table/utils.tsx +++ b/packages/grafana-ui/src/components/Table/utils.tsx @@ -68,7 +68,8 @@ export function getColumns( expandedIndexes: Set, setExpandedIndexes: (indexes: Set) => void, expander: boolean, - footerValues?: FooterItem[] + footerValues?: FooterItem[], + isCountRowsSet?: boolean ): GrafanaTableColumn[] { const columns: GrafanaTableColumn[] = expander ? [ @@ -134,7 +135,7 @@ export function getColumns( minWidth: fieldTableOptions.minWidth ?? columnMinWidth, filter: memoizeOne(filterByValue(field)), justifyContent: getTextAlign(field), - Footer: getFooterValue(fieldIndex, footerValues), + Footer: getFooterValue(fieldIndex, footerValues, isCountRowsSet), }); } diff --git a/public/app/plugins/panel/table/models.gen.ts b/public/app/plugins/panel/table/models.gen.ts index cc01543b6d9..c4d60a02e10 100644 --- a/public/app/plugins/panel/table/models.gen.ts +++ b/public/app/plugins/panel/table/models.gen.ts @@ -27,6 +27,7 @@ export const defaultPanelOptions: PanelOptions = { footer: { show: false, reducer: [], + countRows: false, }, }; diff --git a/public/app/plugins/panel/table/module.tsx b/public/app/plugins/panel/table/module.tsx index ad19eba30a9..5872e486f2b 100644 --- a/public/app/plugins/panel/table/module.tsx +++ b/public/app/plugins/panel/table/module.tsx @@ -131,6 +131,14 @@ export const plugin = new PanelPlugin(TablePane defaultValue: [ReducerID.sum], showIf: (cfg) => cfg.footer?.show, }) + .addBooleanSwitch({ + path: 'footer.countRows', + category: [footerCategory], + name: 'Count rows', + description: 'Display a single count for all data rows', + defaultValue: defaultPanelOptions.footer?.countRows, + showIf: (cfg) => cfg.footer?.reducer?.length === 1 && cfg.footer?.reducer[0] === ReducerID.count, + }) .addMultiSelect({ path: 'footer.fields', category: [footerCategory], @@ -156,7 +164,9 @@ export const plugin = new PanelPlugin(TablePane }, }, defaultValue: '', - showIf: (cfg) => cfg.footer?.show, + showIf: (cfg) => + (cfg.footer?.show && !cfg.footer?.countRows) || + (cfg.footer?.reducer?.length === 1 && cfg.footer?.reducer[0] !== ReducerID.count), }) .addCustomEditor({ id: 'footer.enablePagination', From 64143ea7d0f6d7a85e06147ee29e2fb063873049 Mon Sep 17 00:00:00 2001 From: Andres Martinez Gotor Date: Mon, 28 Nov 2022 09:17:01 +0100 Subject: [PATCH 017/131] GoogleCloudMonitoring: Refactor types (#58644) --- pkg/tsdb/cloudmonitoring/cloudmonitoring.go | 169 ++++--- .../cloudmonitoring/cloudmonitoring_test.go | 455 +++++++++++------- .../cloudmonitoring/time_series_filter.go | 66 +-- .../time_series_filter_test.go | 112 +++-- pkg/tsdb/cloudmonitoring/time_series_query.go | 62 ++- .../cloudmonitoring/time_series_query_test.go | 40 +- pkg/tsdb/cloudmonitoring/types.go | 121 ++--- pkg/tsdb/cloudmonitoring/utils.go | 7 + 8 files changed, 624 insertions(+), 408 deletions(-) diff --git a/pkg/tsdb/cloudmonitoring/cloudmonitoring.go b/pkg/tsdb/cloudmonitoring/cloudmonitoring.go index a690c78dc4e..bfab20afd32 100644 --- a/pkg/tsdb/cloudmonitoring/cloudmonitoring.go +++ b/pkg/tsdb/cloudmonitoring/cloudmonitoring.go @@ -62,7 +62,6 @@ const ( annotationQueryType = "annotation" metricQueryType = "metrics" sloQueryType = "slo" - mqlEditorMode = "mql" crossSeriesReducerDefault = "REDUCE_NONE" perSeriesAlignerDefault = "ALIGN_MEAN" ) @@ -217,16 +216,20 @@ func migrateRequest(req *backend.QueryDataRequest) error { if rawQuery["metricQuery"] == nil { // migrate legacy query - var mq metricQuery + var mq timeSeriesList err = json.Unmarshal(q.JSON, &mq) if err != nil { return err } + q.QueryType = metricQueryType + gq := grafanaQuery{ + TimeSeriesList: &mq, + } + if rawQuery["aliasBy"] != nil { + gq.AliasBy = rawQuery["aliasBy"].(string) + } - b, err := json.Marshal(grafanaQuery{ - QueryType: metricQueryType, - MetricQuery: mq, - }) + b, err := json.Marshal(gq) if err != nil { return err } @@ -237,6 +240,62 @@ func migrateRequest(req *backend.QueryDataRequest) error { if rawQuery["type"] != nil && rawQuery["type"].(string) == "annotationQuery" { q.QueryType = annotationQueryType } + if rawQuery["queryType"] != nil { + q.QueryType = rawQuery["queryType"].(string) + } + + // Metric query was divided between timeSeriesList and timeSeriesQuery API calls + if rawQuery["metricQuery"] != nil { + metricQuery := rawQuery["metricQuery"].(map[string]interface{}) + + if metricQuery["editorMode"] != nil && toString(metricQuery["editorMode"]) == "mql" { + rawQuery["timeSeriesQuery"] = &timeSeriesQuery{ + ProjectName: toString(metricQuery["projectName"]), + Query: toString(metricQuery["query"]), + GraphPeriod: toString(metricQuery["graphPeriod"]), + } + } else { + rawQuery["timeSeriesList"] = metricQuery + } + if metricQuery["aliasBy"] != nil { + rawQuery["aliasBy"] = metricQuery["aliasBy"] + } + b, err := json.Marshal(rawQuery) + if err != nil { + return err + } + if q.QueryType == "" { + q.QueryType = metricQueryType + } + q.JSON = b + } + + // SloQuery was merged into timeSeriesList + if rawQuery["sloQuery"] != nil { + if rawQuery["timeSeriesList"] == nil { + rawQuery["timeSeriesList"] = map[string]interface{}{} + } + tsl := rawQuery["timeSeriesList"].(map[string]interface{}) + sloq := rawQuery["sloQuery"].(map[string]interface{}) + if sloq["projectName"] != nil { + tsl["projectName"] = sloq["projectName"] + } + if sloq["alignmentPeriod"] != nil { + tsl["alignmentPeriod"] = sloq["alignmentPeriod"] + } + if sloq["perSeriesAligner"] != nil { + tsl["perSeriesAligner"] = sloq["perSeriesAligner"] + } + rawQuery["timeSeriesList"] = tsl + b, err := json.Marshal(rawQuery) + if err != nil { + return err + } + if q.QueryType == "" { + q.QueryType = sloQueryType + } + q.JSON = b + } req.Queries[i] = q } @@ -315,58 +374,49 @@ func (s *Service) buildQueryExecutors(logger log.Logger, req *backend.QueryDataR return nil, fmt.Errorf("could not unmarshal CloudMonitoringQuery json: %w", err) } - q.MetricQuery.PreprocessorType = toPreprocessorType(q.MetricQuery.Preprocessor) - var target string params := url.Values{} params.Add("interval.startTime", startTime.UTC().Format(time.RFC3339)) params.Add("interval.endTime", endTime.UTC().Format(time.RFC3339)) var queryInterface cloudMonitoringQueryExecutor - cmtsf := &cloudMonitoringTimeSeriesFilter{ - RefID: query.RefID, - GroupBys: []string{}, - logger: logger, + cmtsf := &cloudMonitoringTimeSeriesList{ + refID: query.RefID, + logger: logger, + aliasBy: q.AliasBy, } - switch q.QueryType { + switch query.QueryType { case metricQueryType, annotationQueryType: - if q.MetricQuery.EditorMode == mqlEditorMode { + if q.TimeSeriesQuery != nil { queryInterface = &cloudMonitoringTimeSeriesQuery{ - RefID: query.RefID, - ProjectName: q.MetricQuery.ProjectName, - Query: q.MetricQuery.Query, - IntervalMS: query.Interval.Milliseconds(), - AliasBy: q.MetricQuery.AliasBy, - timeRange: req.Queries[0].TimeRange, - GraphPeriod: q.MetricQuery.GraphPeriod, + refID: query.RefID, + aliasBy: q.AliasBy, + parameters: q.TimeSeriesQuery, + IntervalMS: query.Interval.Milliseconds(), + timeRange: req.Queries[0].TimeRange, } - } else { - cmtsf.AliasBy = q.MetricQuery.AliasBy - cmtsf.ProjectName = q.MetricQuery.ProjectName - cmtsf.GroupBys = append(cmtsf.GroupBys, q.MetricQuery.GroupBys...) - if q.MetricQuery.View == "" { - q.MetricQuery.View = "FULL" + } else if q.TimeSeriesList != nil { + if q.TimeSeriesList.View == "" { + q.TimeSeriesList.View = "FULL" } - params.Add("filter", buildFilterString(q.MetricQuery.MetricType, q.MetricQuery.Filters)) - params.Add("view", q.MetricQuery.View) - setMetricAggParams(¶ms, &q.MetricQuery, durationSeconds, query.Interval.Milliseconds()) + cmtsf.parameters = q.TimeSeriesList + params.Add("filter", buildFilterString(q.TimeSeriesList.MetricType, q.TimeSeriesList.Filters)) + params.Add("view", q.TimeSeriesList.View) + setMetricAggParams(¶ms, q.TimeSeriesList, durationSeconds, query.Interval.Milliseconds()) queryInterface = cmtsf + } else { + return nil, fmt.Errorf("missing query info") } case sloQueryType: - cmtsf.AliasBy = q.SloQuery.AliasBy - cmtsf.ProjectName = q.SloQuery.ProjectName - cmtsf.Selector = q.SloQuery.SelectorName - cmtsf.Service = q.SloQuery.ServiceId - cmtsf.Slo = q.SloQuery.SloId - params.Add("filter", buildSLOFilterExpression(q.SloQuery)) - setSloAggParams(¶ms, &q.SloQuery, durationSeconds, query.Interval.Milliseconds()) + cmtsf.sloQ = q.SloQuery + cmtsf.parameters = q.TimeSeriesList + params.Add("filter", buildSLOFilterExpression(q.TimeSeriesList.ProjectName, q.SloQuery)) + setSloAggParams(¶ms, q.SloQuery, q.TimeSeriesList.AlignmentPeriod, durationSeconds, query.Interval.Milliseconds()) queryInterface = cmtsf default: - return nil, fmt.Errorf("unrecognized query type %q", q.QueryType) + return nil, fmt.Errorf("unrecognized query type %q", query.QueryType) } - target = params.Encode() - cmtsf.Target = target - cmtsf.Params = params + cmtsf.params = params if setting.Env == setting.Dev { logger.Debug("CloudMonitoring request", "params", params) @@ -428,8 +478,8 @@ func buildFilterString(metricType string, filterParts []string) string { return strings.Trim(fmt.Sprintf(`metric.type="%s" %s`, metricType, filterString), " ") } -func buildSLOFilterExpression(q sloQuery) string { - sloName := fmt.Sprintf("projects/%s/services/%s/serviceLevelObjectives/%s", q.ProjectName, q.ServiceId, q.SloId) +func buildSLOFilterExpression(projectName string, q *sloQuery) string { + sloName := fmt.Sprintf("projects/%s/services/%s/serviceLevelObjectives/%s", projectName, q.ServiceId, q.SloId) if q.SelectorName == "select_slo_burn_rate" { return fmt.Sprintf(`%s("%s", "%s")`, q.SelectorName, sloName, q.LookbackPeriod) @@ -438,7 +488,7 @@ func buildSLOFilterExpression(q sloQuery) string { } } -func setMetricAggParams(params *url.Values, query *metricQuery, durationSeconds int, intervalMs int64) { +func setMetricAggParams(params *url.Values, query *timeSeriesList, durationSeconds int, intervalMs int64) { if query.CrossSeriesReducer == "" { query.CrossSeriesReducer = crossSeriesReducerDefault } @@ -452,7 +502,8 @@ func setMetricAggParams(params *url.Values, query *metricQuery, durationSeconds // In case a preprocessor is defined, the preprocessor becomes the primary aggregation // and the aggregation that is specified in the UI becomes the secondary aggregation // Rules are specified in this issue: https://github.com/grafana/grafana/issues/30866 - if query.PreprocessorType != PreprocessorTypeNone { + t := toPreprocessorType(query.Preprocessor) + if t != PreprocessorTypeNone { params.Add("secondaryAggregation.alignmentPeriod", alignmentPeriod) params.Add("secondaryAggregation.crossSeriesReducer", query.CrossSeriesReducer) params.Add("secondaryAggregation.perSeriesAligner", query.PerSeriesAligner) @@ -464,7 +515,7 @@ func setMetricAggParams(params *url.Values, query *metricQuery, durationSeconds params.Add("aggregation.crossSeriesReducer", primaryCrossSeriesReducer) aligner := "ALIGN_RATE" - if query.PreprocessorType == PreprocessorTypeDelta { + if t == PreprocessorTypeDelta { aligner = "ALIGN_DELTA" } params.Add("aggregation.perSeriesAligner", aligner) @@ -484,8 +535,8 @@ func setMetricAggParams(params *url.Values, query *metricQuery, durationSeconds } } -func setSloAggParams(params *url.Values, query *sloQuery, durationSeconds int, intervalMs int64) { - params.Add("aggregation.alignmentPeriod", calculateAlignmentPeriod(query.AlignmentPeriod, intervalMs, durationSeconds)) +func setSloAggParams(params *url.Values, query *sloQuery, alignmentPeriod string, durationSeconds int, intervalMs int64) { + params.Add("aggregation.alignmentPeriod", calculateAlignmentPeriod(alignmentPeriod, intervalMs, durationSeconds)) if query.SelectorName == "select_slo_health" { params.Add("aggregation.perSeriesAligner", "ALIGN_MEAN") } else { @@ -515,12 +566,12 @@ func calculateAlignmentPeriod(alignmentPeriod string, intervalMs int64, duration } func formatLegendKeys(metricType string, defaultMetricName string, labels map[string]string, - additionalLabels map[string]string, query *cloudMonitoringTimeSeriesFilter) string { - if query.AliasBy == "" { + additionalLabels map[string]string, query *cloudMonitoringTimeSeriesList) string { + if query.aliasBy == "" { return defaultMetricName } - result := legendKeyFormat.ReplaceAllFunc([]byte(query.AliasBy), func(in []byte) []byte { + result := legendKeyFormat.ReplaceAllFunc([]byte(query.aliasBy), func(in []byte) []byte { metaPartName := strings.Replace(string(in), "{{", "", 1) metaPartName = strings.Replace(metaPartName, "}}", "", 1) metaPartName = strings.TrimSpace(metaPartName) @@ -543,20 +594,20 @@ func formatLegendKeys(metricType string, defaultMetricName string, labels map[st return []byte(val) } - if metaPartName == "project" && query.ProjectName != "" { - return []byte(query.ProjectName) + if metaPartName == "project" && query.parameters.ProjectName != "" { + return []byte(query.parameters.ProjectName) } - if metaPartName == "service" && query.Service != "" { - return []byte(query.Service) + if metaPartName == "service" && query.sloQ.ServiceId != "" { + return []byte(query.sloQ.ServiceId) } - if metaPartName == "slo" && query.Slo != "" { - return []byte(query.Slo) + if metaPartName == "slo" && query.sloQ.SloId != "" { + return []byte(query.sloQ.SloId) } - if metaPartName == "selector" && query.Selector != "" { - return []byte(query.Selector) + if metaPartName == "selector" && query.sloQ.SelectorName != "" { + return []byte(query.sloQ.SelectorName) } return in diff --git a/pkg/tsdb/cloudmonitoring/cloudmonitoring_test.go b/pkg/tsdb/cloudmonitoring/cloudmonitoring_test.go index ad01588cb56..939fe578061 100644 --- a/pkg/tsdb/cloudmonitoring/cloudmonitoring_test.go +++ b/pkg/tsdb/cloudmonitoring/cloudmonitoring_test.go @@ -50,31 +50,60 @@ func TestNewInstanceSettings(t *testing.T) { func TestCloudMonitoring(t *testing.T) { service := &Service{} + t.Run("parses a time series list query", func(t *testing.T) { + req := baseTimeSeriesList() + qes, err := service.buildQueryExecutors(slog, req) + require.NoError(t, err) + queries := getCloudMonitoringListFromInterface(t, qes) + + require.Len(t, queries, 1) + assert.Equal(t, "A", queries[0].refID) + assert.Equal(t, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_NONE&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL", queries[0].params.Encode()) + assert.Equal(t, 7, len(queries[0].params)) + assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].params["interval.startTime"][0]) + assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].params["interval.endTime"][0]) + assert.Equal(t, "ALIGN_MEAN", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, "metric.type=\"a/metric/type\"", queries[0].params["filter"][0]) + assert.Equal(t, "FULL", queries[0].params["view"][0]) + assert.Equal(t, "testalias", queries[0].aliasBy) + }) + + t.Run("parses a time series query", func(t *testing.T) { + req := baseTimeSeriesQuery() + qes, err := service.buildQueryExecutors(slog, req) + require.NoError(t, err) + queries := getCloudMonitoringQueryFromInterface(t, qes) + + require.Len(t, queries, 1) + assert.Equal(t, "A", queries[0].refID) + assert.Equal(t, "foo", queries[0].parameters.Query) + assert.Equal(t, "testalias", queries[0].aliasBy) + }) + t.Run("Parse migrated queries from frontend and build Google Cloud Monitoring API queries", func(t *testing.T) { t.Run("and query has no aggregation set", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() err := migrateRequest(req) require.NoError(t, err) - require.NoError(t, err) qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) require.Len(t, queries, 1) - assert.Equal(t, "A", queries[0].RefID) - assert.Equal(t, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_NONE&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL", queries[0].Target) - assert.Equal(t, 7, len(queries[0].Params)) - assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].Params["interval.startTime"][0]) - assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].Params["interval.endTime"][0]) - assert.Equal(t, "ALIGN_MEAN", queries[0].Params["aggregation.perSeriesAligner"][0]) - assert.Equal(t, "metric.type=\"a/metric/type\"", queries[0].Params["filter"][0]) - assert.Equal(t, "FULL", queries[0].Params["view"][0]) - assert.Equal(t, "testalias", queries[0].AliasBy) + assert.Equal(t, "A", queries[0].refID) + assert.Equal(t, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_NONE&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL", queries[0].params.Encode()) + assert.Equal(t, 7, len(queries[0].params)) + assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].params["interval.startTime"][0]) + assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].params["interval.endTime"][0]) + assert.Equal(t, "ALIGN_MEAN", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, "metric.type=\"a/metric/type\"", queries[0].params["filter"][0]) + assert.Equal(t, "FULL", queries[0].params["view"][0]) + assert.Equal(t, "testalias", queries[0].aliasBy) t.Run("and generated deep link has correct parameters", func(t *testing.T) { // assign resource type to query parameters to be included in the deep link filter // in the actual workflow this information comes from the response of the Monitoring API - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -91,7 +120,7 @@ func TestCloudMonitoring(t *testing.T) { }) t.Run("and query has filters", func(t *testing.T) { - query := baseReq() + query := deprecatedReq() query.Queries[0].JSON = json.RawMessage(`{ "metricType": "a/metric/type", "filters": ["key", "=", "value", "AND", "key2", "=", "value2", "AND", "resource.type", "=", "another/resource/type"] @@ -101,14 +130,14 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, query) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) assert.Equal(t, 1, len(queries)) - assert.Equal(t, `metric.type="a/metric/type" key="value" key2="value2" resource.type="another/resource/type"`, queries[0].Params["filter"][0]) + assert.Equal(t, `metric.type="a/metric/type" key="value" key2="value2" resource.type="another/resource/type"`, queries[0].params["filter"][0]) // assign a resource type to query parameters // in the actual workflow this information comes from the response of the Monitoring API // the deep link should not contain this resource type since another resource type is included in the query filters - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -124,7 +153,7 @@ func TestCloudMonitoring(t *testing.T) { t.Run("and alignmentPeriod is set to grafana-auto", func(t *testing.T) { t.Run("and IntervalMS is larger than 60000", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].Interval = 1000000 * time.Millisecond req.Queries[0].JSON = json.RawMessage(`{ "alignmentPeriod": "grafana-auto", @@ -135,12 +164,12 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `+1000s`, queries[0].Params["aggregation.alignmentPeriod"][0]) + queries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `+1000s`, queries[0].params["aggregation.alignmentPeriod"][0]) // assign resource type to query parameters to be included in the deep link filter // in the actual workflow this information comes from the response of the Monitoring API - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -154,7 +183,7 @@ func TestCloudMonitoring(t *testing.T) { verifyDeepLink(t, dl, expectedTimeSelection, expectedTimeSeriesFilter) }) t.Run("and IntervalMS is less than 60000", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].Interval = 30000 * time.Millisecond req.Queries[0].JSON = json.RawMessage(`{ "alignmentPeriod": "grafana-auto", @@ -165,12 +194,12 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `+60s`, queries[0].Params["aggregation.alignmentPeriod"][0]) + queries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `+60s`, queries[0].params["aggregation.alignmentPeriod"][0]) // assign resource type to query parameters to be included in the deep link filter // in the actual workflow this information comes from the response of the Monitoring API - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -189,7 +218,7 @@ func TestCloudMonitoring(t *testing.T) { now := time.Now().UTC() t.Run("and range is two hours", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].TimeRange.From = now.Add(-(time.Hour * 2)) req.Queries[0].TimeRange.To = now req.Queries[0].JSON = json.RawMessage(`{ @@ -201,12 +230,12 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `+60s`, queries[0].Params["aggregation.alignmentPeriod"][0]) + queries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `+60s`, queries[0].params["aggregation.alignmentPeriod"][0]) }) t.Run("and range is 22 hours", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].TimeRange.From = now.Add(-(time.Hour * 22)) req.Queries[0].TimeRange.To = now req.Queries[0].JSON = json.RawMessage(`{ @@ -218,12 +247,12 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `+60s`, queries[0].Params["aggregation.alignmentPeriod"][0]) + queries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `+60s`, queries[0].params["aggregation.alignmentPeriod"][0]) }) t.Run("and range is 23 hours", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].TimeRange.From = now.Add(-(time.Hour * 23)) req.Queries[0].TimeRange.To = now req.Queries[0].JSON = json.RawMessage(`{ @@ -235,12 +264,12 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `+300s`, queries[0].Params["aggregation.alignmentPeriod"][0]) + queries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `+300s`, queries[0].params["aggregation.alignmentPeriod"][0]) }) t.Run("and range is 7 days", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].TimeRange.From = now req.Queries[0].TimeRange.To = now.AddDate(0, 0, 7) req.Queries[0].JSON = json.RawMessage(`{ @@ -252,8 +281,8 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `+3600s`, queries[0].Params["aggregation.alignmentPeriod"][0]) + queries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `+3600s`, queries[0].params["aggregation.alignmentPeriod"][0]) }) }) @@ -261,7 +290,7 @@ func TestCloudMonitoring(t *testing.T) { now := time.Now().UTC() t.Run("and range is two hours", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].TimeRange.From = now.Add(-(time.Hour * 2)) req.Queries[0].TimeRange.To = now req.Queries[0].JSON = json.RawMessage(`{ @@ -273,12 +302,12 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `+60s`, queries[0].Params["aggregation.alignmentPeriod"][0]) + queries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `+60s`, queries[0].params["aggregation.alignmentPeriod"][0]) // assign resource type to query parameters to be included in the deep link filter // in the actual workflow this information comes from the response of the Monitoring API - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -293,7 +322,7 @@ func TestCloudMonitoring(t *testing.T) { }) t.Run("and range is 22 hours", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].TimeRange.From = now.Add(-(time.Hour * 22)) req.Queries[0].TimeRange.To = now req.Queries[0].JSON = json.RawMessage(`{ @@ -305,12 +334,12 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `+60s`, queries[0].Params["aggregation.alignmentPeriod"][0]) + queries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `+60s`, queries[0].params["aggregation.alignmentPeriod"][0]) // assign resource type to query parameters to be included in the deep link filter // in the actual workflow this information comes from the response of the Monitoring API - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -325,7 +354,7 @@ func TestCloudMonitoring(t *testing.T) { }) t.Run("and range is 23 hours", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].TimeRange.From = now.Add(-(time.Hour * 23)) req.Queries[0].TimeRange.To = now req.Queries[0].JSON = json.RawMessage(`{ @@ -337,12 +366,12 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `+300s`, queries[0].Params["aggregation.alignmentPeriod"][0]) + queries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `+300s`, queries[0].params["aggregation.alignmentPeriod"][0]) // assign resource type to query parameters to be included in the deep link filter // in the actual workflow this information comes from the response of the Monitoring API - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -357,7 +386,7 @@ func TestCloudMonitoring(t *testing.T) { }) t.Run("and range is 7 days", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].TimeRange.From = now.AddDate(0, 0, -7) req.Queries[0].TimeRange.To = now req.Queries[0].JSON = json.RawMessage(`{ @@ -369,12 +398,12 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `+3600s`, queries[0].Params["aggregation.alignmentPeriod"][0]) + queries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `+3600s`, queries[0].params["aggregation.alignmentPeriod"][0]) // assign resource type to query parameters to be included in the deep link filter // in the actual workflow this information comes from the response of the Monitoring API - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -391,7 +420,7 @@ func TestCloudMonitoring(t *testing.T) { t.Run("and alignmentPeriod is set in frontend", func(t *testing.T) { t.Run("and alignment period is within accepted range", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].Interval = 1000 req.Queries[0].JSON = json.RawMessage(`{ "alignmentPeriod": "+600s" @@ -401,12 +430,12 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `+600s`, queries[0].Params["aggregation.alignmentPeriod"][0]) + queries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `+600s`, queries[0].params["aggregation.alignmentPeriod"][0]) // assign resource type to query parameters to be included in the deep link filter // in the actual workflow this information comes from the response of the Monitoring API - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -422,7 +451,7 @@ func TestCloudMonitoring(t *testing.T) { }) t.Run("and query has aggregation mean set", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].JSON = json.RawMessage(`{ "metricType": "a/metric/type", "crossSeriesReducer": "REDUCE_SUM", @@ -433,23 +462,23 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) assert.Equal(t, 1, len(queries)) - assert.Equal(t, "A", queries[0].RefID) - assert.Equal(t, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_SUM&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL", queries[0].Target) - assert.Equal(t, 7, len(queries[0].Params)) - assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].Params["interval.startTime"][0]) - assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].Params["interval.endTime"][0]) - assert.Equal(t, "REDUCE_SUM", queries[0].Params["aggregation.crossSeriesReducer"][0]) - assert.Equal(t, "ALIGN_MEAN", queries[0].Params["aggregation.perSeriesAligner"][0]) - assert.Equal(t, "+60s", queries[0].Params["aggregation.alignmentPeriod"][0]) - assert.Equal(t, "metric.type=\"a/metric/type\"", queries[0].Params["filter"][0]) - assert.Equal(t, "FULL", queries[0].Params["view"][0]) + assert.Equal(t, "A", queries[0].refID) + assert.Equal(t, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_SUM&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL", queries[0].params.Encode()) + assert.Equal(t, 7, len(queries[0].params)) + assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].params["interval.startTime"][0]) + assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].params["interval.endTime"][0]) + assert.Equal(t, "REDUCE_SUM", queries[0].params["aggregation.crossSeriesReducer"][0]) + assert.Equal(t, "ALIGN_MEAN", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, "+60s", queries[0].params["aggregation.alignmentPeriod"][0]) + assert.Equal(t, "metric.type=\"a/metric/type\"", queries[0].params["filter"][0]) + assert.Equal(t, "FULL", queries[0].params["view"][0]) // assign resource type to query parameters to be included in the deep link filter // in the actual workflow this information comes from the response of the Monitoring API - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -467,7 +496,7 @@ func TestCloudMonitoring(t *testing.T) { }) t.Run("and query has group bys", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].JSON = json.RawMessage(`{ "metricType": "a/metric/type", "crossSeriesReducer": "REDUCE_NONE", @@ -479,23 +508,23 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) assert.Equal(t, 1, len(queries)) - assert.Equal(t, "A", queries[0].RefID) - assert.Equal(t, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_NONE&aggregation.groupByFields=metric.label.group1&aggregation.groupByFields=metric.label.group2&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL", queries[0].Target) - assert.Equal(t, 8, len(queries[0].Params)) - assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].Params["interval.startTime"][0]) - assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].Params["interval.endTime"][0]) - assert.Equal(t, "ALIGN_MEAN", queries[0].Params["aggregation.perSeriesAligner"][0]) - assert.Equal(t, "metric.label.group1", queries[0].Params["aggregation.groupByFields"][0]) - assert.Equal(t, "metric.label.group2", queries[0].Params["aggregation.groupByFields"][1]) - assert.Equal(t, "metric.type=\"a/metric/type\"", queries[0].Params["filter"][0]) - assert.Equal(t, "FULL", queries[0].Params["view"][0]) + assert.Equal(t, "A", queries[0].refID) + assert.Equal(t, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_NONE&aggregation.groupByFields=metric.label.group1&aggregation.groupByFields=metric.label.group2&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL", queries[0].params.Encode()) + assert.Equal(t, 8, len(queries[0].params)) + assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].params["interval.startTime"][0]) + assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].params["interval.endTime"][0]) + assert.Equal(t, "ALIGN_MEAN", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, "metric.label.group1", queries[0].params["aggregation.groupByFields"][0]) + assert.Equal(t, "metric.label.group2", queries[0].params["aggregation.groupByFields"][1]) + assert.Equal(t, "metric.type=\"a/metric/type\"", queries[0].params["filter"][0]) + assert.Equal(t, "FULL", queries[0].params["view"][0]) // assign resource type to query parameters to be included in the deep link filter // in the actual workflow this information comes from the response of the Monitoring API - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -536,28 +565,31 @@ func TestCloudMonitoring(t *testing.T) { }, }, } + err := migrateRequest(req) + require.NoError(t, err) + t.Run("and query type is metrics", func(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) assert.Equal(t, 1, len(queries)) - assert.Equal(t, "A", queries[0].RefID) - assert.Equal(t, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_NONE&aggregation.groupByFields=metric.label.group1&aggregation.groupByFields=metric.label.group2&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL", queries[0].Target) - assert.Equal(t, 8, len(queries[0].Params)) - assert.Equal(t, "metric.label.group1", queries[0].Params["aggregation.groupByFields"][0]) - assert.Equal(t, "metric.label.group2", queries[0].Params["aggregation.groupByFields"][1]) - assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].Params["interval.startTime"][0]) - assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].Params["interval.endTime"][0]) - assert.Equal(t, "ALIGN_MEAN", queries[0].Params["aggregation.perSeriesAligner"][0]) - assert.Equal(t, "metric.type=\"a/metric/type\"", queries[0].Params["filter"][0]) - assert.Equal(t, "FULL", queries[0].Params["view"][0]) - assert.Equal(t, "testalias", queries[0].AliasBy) - assert.Equal(t, []string{"metric.label.group1", "metric.label.group2"}, queries[0].GroupBys) + assert.Equal(t, "A", queries[0].refID) + assert.Equal(t, "aggregation.alignmentPeriod=%2B60s&aggregation.crossSeriesReducer=REDUCE_NONE&aggregation.groupByFields=metric.label.group1&aggregation.groupByFields=metric.label.group2&aggregation.perSeriesAligner=ALIGN_MEAN&filter=metric.type%3D%22a%2Fmetric%2Ftype%22&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z&view=FULL", queries[0].params.Encode()) + assert.Equal(t, 8, len(queries[0].params)) + assert.Equal(t, "metric.label.group1", queries[0].params["aggregation.groupByFields"][0]) + assert.Equal(t, "metric.label.group2", queries[0].params["aggregation.groupByFields"][1]) + assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].params["interval.startTime"][0]) + assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].params["interval.endTime"][0]) + assert.Equal(t, "ALIGN_MEAN", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, "metric.type=\"a/metric/type\"", queries[0].params["filter"][0]) + assert.Equal(t, "FULL", queries[0].params["view"][0]) + assert.Equal(t, "testalias", queries[0].aliasBy) + assert.Equal(t, []string{"metric.label.group1", "metric.label.group2"}, queries[0].parameters.GroupBys) // assign resource type to query parameters to be included in the deep link filter // in the actual workflow this information comes from the response of the Monitoring API - queries[0].Params.Set("resourceType", "a/resource/type") + queries[0].params.Set("resourceType", "a/resource/type") dl := queries[0].buildDeepLink() expectedTimeSelection := map[string]string{ @@ -583,21 +615,18 @@ func TestCloudMonitoring(t *testing.T) { }, "sloQuery": {} }`) + err = migrateRequest(req) + require.NoError(t, err) qes, err = service.buildQueryExecutors(slog, req) require.NoError(t, err) - tqueries := make([]*cloudMonitoringTimeSeriesQuery, 0) - for _, qi := range qes { - q, ok := qi.(*cloudMonitoringTimeSeriesQuery) - assert.True(t, ok) - tqueries = append(tqueries, q) - } + tqueries := getCloudMonitoringQueryFromInterface(t, qes) assert.Equal(t, 1, len(tqueries)) - assert.Equal(t, "A", tqueries[0].RefID) - assert.Equal(t, "test-proj", tqueries[0].ProjectName) - assert.Equal(t, "test-query", tqueries[0].Query) - assert.Equal(t, "test-alias", tqueries[0].AliasBy) + assert.Equal(t, "A", tqueries[0].refID) + assert.Equal(t, "test-proj", tqueries[0].parameters.ProjectName) + assert.Equal(t, "test-query", tqueries[0].parameters.Query) + assert.Equal(t, "test-alias", tqueries[0].aliasBy) }) t.Run("and query type is SLOs", func(t *testing.T) { @@ -614,20 +643,22 @@ func TestCloudMonitoring(t *testing.T) { }, "metricQuery": {} }`) + err := migrateRequest(req) + require.NoError(t, err) qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) assert.Equal(t, 1, len(queries)) - assert.Equal(t, "A", queries[0].RefID) - assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].Params["interval.startTime"][0]) - assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].Params["interval.endTime"][0]) - assert.Equal(t, `+60s`, queries[0].Params["aggregation.alignmentPeriod"][0]) - assert.Equal(t, "", queries[0].AliasBy) - assert.Equal(t, "ALIGN_MEAN", queries[0].Params["aggregation.perSeriesAligner"][0]) - assert.Equal(t, `aggregation.alignmentPeriod=%2B60s&aggregation.perSeriesAligner=ALIGN_MEAN&filter=select_slo_health%28%22projects%2Ftest-proj%2Fservices%2Ftest-service%2FserviceLevelObjectives%2Ftest-slo%22%29&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z`, queries[0].Target) - assert.Equal(t, 5, len(queries[0].Params)) + assert.Equal(t, "A", queries[0].refID) + assert.Equal(t, "2018-03-15T13:00:00Z", queries[0].params["interval.startTime"][0]) + assert.Equal(t, "2018-03-15T13:34:00Z", queries[0].params["interval.endTime"][0]) + assert.Equal(t, `+60s`, queries[0].params["aggregation.alignmentPeriod"][0]) + assert.Equal(t, "", queries[0].aliasBy) + assert.Equal(t, "ALIGN_MEAN", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, `aggregation.alignmentPeriod=%2B60s&aggregation.perSeriesAligner=ALIGN_MEAN&filter=select_slo_health%28%22projects%2Ftest-proj%2Fservices%2Ftest-service%2FserviceLevelObjectives%2Ftest-slo%22%29&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z`, queries[0].params.Encode()) + assert.Equal(t, 5, len(queries[0].params)) req.Queries[0].JSON = json.RawMessage(`{ "queryType": "slo", @@ -642,11 +673,13 @@ func TestCloudMonitoring(t *testing.T) { }, "metricQuery": {} }`) + err = migrateRequest(req) + require.NoError(t, err) qes, err = service.buildQueryExecutors(slog, req) require.NoError(t, err) - qqueries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, "ALIGN_NEXT_OLDER", qqueries[0].Params["aggregation.perSeriesAligner"][0]) + qqueries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, "ALIGN_NEXT_OLDER", qqueries[0].params["aggregation.perSeriesAligner"][0]) dl := qqueries[0].buildDeepLink() assert.Empty(t, dl) @@ -665,11 +698,13 @@ func TestCloudMonitoring(t *testing.T) { }, "metricQuery": {} }`) + err = migrateRequest(req) + require.NoError(t, err) qes, err = service.buildQueryExecutors(slog, req) require.NoError(t, err) - qqqueries := getCloudMonitoringQueriesFromInterface(t, qes) - assert.Equal(t, `aggregation.alignmentPeriod=%2B60s&aggregation.perSeriesAligner=ALIGN_NEXT_OLDER&filter=select_slo_burn_rate%28%22projects%2Ftest-proj%2Fservices%2Ftest-service%2FserviceLevelObjectives%2Ftest-slo%22%2C+%221h%22%29&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z`, qqqueries[0].Target) + qqqueries := getCloudMonitoringListFromInterface(t, qes) + assert.Equal(t, `aggregation.alignmentPeriod=%2B60s&aggregation.perSeriesAligner=ALIGN_NEXT_OLDER&filter=select_slo_burn_rate%28%22projects%2Ftest-proj%2Fservices%2Ftest-service%2FserviceLevelObjectives%2Ftest-slo%22%2C+%221h%22%29&interval.endTime=2018-03-15T13%3A34%3A00Z&interval.startTime=2018-03-15T13%3A00%3A00Z`, qqqueries[0].params.Encode()) }) }) @@ -761,7 +796,7 @@ func TestCloudMonitoring(t *testing.T) { }) t.Run("and query preprocessor is not defined", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].JSON = json.RawMessage(`{ "metricType": "a/metric/type", "crossSeriesReducer": "REDUCE_MIN", @@ -775,22 +810,22 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) assert.Equal(t, 1, len(queries)) - assert.Equal(t, "REDUCE_MIN", queries[0].Params["aggregation.crossSeriesReducer"][0]) - assert.Equal(t, "REDUCE_SUM", queries[0].Params["aggregation.perSeriesAligner"][0]) - assert.Equal(t, "+60s", queries[0].Params["aggregation.alignmentPeriod"][0]) - assert.Equal(t, "labelname", queries[0].Params["aggregation.groupByFields"][0]) + assert.Equal(t, "REDUCE_MIN", queries[0].params["aggregation.crossSeriesReducer"][0]) + assert.Equal(t, "REDUCE_SUM", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, "+60s", queries[0].params["aggregation.alignmentPeriod"][0]) + assert.Equal(t, "labelname", queries[0].params["aggregation.groupByFields"][0]) - assert.NotContains(t, queries[0].Params, "secondaryAggregation.crossSeriesReducer") - assert.NotContains(t, "REDUCE_SUM", queries[0].Params, "secondaryAggregation.perSeriesAligner") - assert.NotContains(t, "+60s", queries[0].Params, "secondaryAggregation.alignmentPeriod") - assert.NotContains(t, "labelname", queries[0].Params, "secondaryAggregation.groupByFields") + assert.NotContains(t, queries[0].params, "secondaryAggregation.crossSeriesReducer") + assert.NotContains(t, "REDUCE_SUM", queries[0].params, "secondaryAggregation.perSeriesAligner") + assert.NotContains(t, "+60s", queries[0].params, "secondaryAggregation.alignmentPeriod") + assert.NotContains(t, "labelname", queries[0].params, "secondaryAggregation.groupByFields") }) t.Run("and query preprocessor is set to none", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].JSON = json.RawMessage(`{ "metricType": "a/metric/type", "crossSeriesReducer": "REDUCE_MIN", @@ -805,22 +840,22 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) assert.Equal(t, 1, len(queries)) - assert.Equal(t, "REDUCE_MIN", queries[0].Params["aggregation.crossSeriesReducer"][0]) - assert.Equal(t, "REDUCE_SUM", queries[0].Params["aggregation.perSeriesAligner"][0]) - assert.Equal(t, "+60s", queries[0].Params["aggregation.alignmentPeriod"][0]) - assert.Equal(t, "labelname", queries[0].Params["aggregation.groupByFields"][0]) + assert.Equal(t, "REDUCE_MIN", queries[0].params["aggregation.crossSeriesReducer"][0]) + assert.Equal(t, "REDUCE_SUM", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, "+60s", queries[0].params["aggregation.alignmentPeriod"][0]) + assert.Equal(t, "labelname", queries[0].params["aggregation.groupByFields"][0]) - assert.NotContains(t, queries[0].Params, "secondaryAggregation.crossSeriesReducer") - assert.NotContains(t, "REDUCE_SUM", queries[0].Params, "secondaryAggregation.perSeriesAligner") - assert.NotContains(t, "+60s", queries[0].Params, "secondaryAggregation.alignmentPeriod") - assert.NotContains(t, "labelname", queries[0].Params, "secondaryAggregation.groupByFields") + assert.NotContains(t, queries[0].params, "secondaryAggregation.crossSeriesReducer") + assert.NotContains(t, "REDUCE_SUM", queries[0].params, "secondaryAggregation.perSeriesAligner") + assert.NotContains(t, "+60s", queries[0].params, "secondaryAggregation.alignmentPeriod") + assert.NotContains(t, "labelname", queries[0].params, "secondaryAggregation.groupByFields") }) t.Run("and query preprocessor is set to rate and there's no group bys", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].JSON = json.RawMessage(`{ "metricType": "a/metric/type", "crossSeriesReducer": "REDUCE_SUM", @@ -835,20 +870,20 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) assert.Equal(t, 1, len(queries)) - assert.Equal(t, "REDUCE_NONE", queries[0].Params["aggregation.crossSeriesReducer"][0]) - assert.Equal(t, "ALIGN_RATE", queries[0].Params["aggregation.perSeriesAligner"][0]) - assert.Equal(t, "+60s", queries[0].Params["aggregation.alignmentPeriod"][0]) + assert.Equal(t, "REDUCE_NONE", queries[0].params["aggregation.crossSeriesReducer"][0]) + assert.Equal(t, "ALIGN_RATE", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, "+60s", queries[0].params["aggregation.alignmentPeriod"][0]) - assert.Equal(t, "REDUCE_SUM", queries[0].Params["secondaryAggregation.crossSeriesReducer"][0]) - assert.Equal(t, "REDUCE_MIN", queries[0].Params["secondaryAggregation.perSeriesAligner"][0]) - assert.Equal(t, "+60s", queries[0].Params["secondaryAggregation.alignmentPeriod"][0]) + assert.Equal(t, "REDUCE_SUM", queries[0].params["secondaryAggregation.crossSeriesReducer"][0]) + assert.Equal(t, "REDUCE_MIN", queries[0].params["secondaryAggregation.perSeriesAligner"][0]) + assert.Equal(t, "+60s", queries[0].params["secondaryAggregation.alignmentPeriod"][0]) }) t.Run("and query preprocessor is set to rate and group bys exist", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].JSON = json.RawMessage(`{ "metricType": "a/metric/type", "crossSeriesReducer": "REDUCE_SUM", @@ -863,22 +898,22 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) assert.Equal(t, 1, len(queries)) - assert.Equal(t, "REDUCE_SUM", queries[0].Params["aggregation.crossSeriesReducer"][0]) - assert.Equal(t, "ALIGN_RATE", queries[0].Params["aggregation.perSeriesAligner"][0]) - assert.Equal(t, "+60s", queries[0].Params["aggregation.alignmentPeriod"][0]) - assert.Equal(t, "labelname", queries[0].Params["aggregation.groupByFields"][0]) + assert.Equal(t, "REDUCE_SUM", queries[0].params["aggregation.crossSeriesReducer"][0]) + assert.Equal(t, "ALIGN_RATE", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, "+60s", queries[0].params["aggregation.alignmentPeriod"][0]) + assert.Equal(t, "labelname", queries[0].params["aggregation.groupByFields"][0]) - assert.Equal(t, "REDUCE_SUM", queries[0].Params["secondaryAggregation.crossSeriesReducer"][0]) - assert.Equal(t, "REDUCE_MIN", queries[0].Params["secondaryAggregation.perSeriesAligner"][0]) - assert.Equal(t, "+60s", queries[0].Params["secondaryAggregation.alignmentPeriod"][0]) - assert.Equal(t, "labelname", queries[0].Params["secondaryAggregation.groupByFields"][0]) + assert.Equal(t, "REDUCE_SUM", queries[0].params["secondaryAggregation.crossSeriesReducer"][0]) + assert.Equal(t, "REDUCE_MIN", queries[0].params["secondaryAggregation.perSeriesAligner"][0]) + assert.Equal(t, "+60s", queries[0].params["secondaryAggregation.alignmentPeriod"][0]) + assert.Equal(t, "labelname", queries[0].params["secondaryAggregation.groupByFields"][0]) }) t.Run("and query preprocessor is set to delta and there's no group bys", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].JSON = json.RawMessage(`{ "metricType": "a/metric/type", "crossSeriesReducer": "REDUCE_MIN", @@ -893,20 +928,20 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) assert.Equal(t, 1, len(queries)) - assert.Equal(t, "REDUCE_NONE", queries[0].Params["aggregation.crossSeriesReducer"][0]) - assert.Equal(t, "ALIGN_DELTA", queries[0].Params["aggregation.perSeriesAligner"][0]) - assert.Equal(t, "+60s", queries[0].Params["aggregation.alignmentPeriod"][0]) + assert.Equal(t, "REDUCE_NONE", queries[0].params["aggregation.crossSeriesReducer"][0]) + assert.Equal(t, "ALIGN_DELTA", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, "+60s", queries[0].params["aggregation.alignmentPeriod"][0]) - assert.Equal(t, "REDUCE_MIN", queries[0].Params["secondaryAggregation.crossSeriesReducer"][0]) - assert.Equal(t, "REDUCE_SUM", queries[0].Params["secondaryAggregation.perSeriesAligner"][0]) - assert.Equal(t, "+60s", queries[0].Params["secondaryAggregation.alignmentPeriod"][0]) + assert.Equal(t, "REDUCE_MIN", queries[0].params["secondaryAggregation.crossSeriesReducer"][0]) + assert.Equal(t, "REDUCE_SUM", queries[0].params["secondaryAggregation.perSeriesAligner"][0]) + assert.Equal(t, "+60s", queries[0].params["secondaryAggregation.alignmentPeriod"][0]) }) t.Run("and query preprocessor is set to delta and group bys exist", func(t *testing.T) { - req := baseReq() + req := deprecatedReq() req.Queries[0].JSON = json.RawMessage(`{ "metricType": "a/metric/type", "crossSeriesReducer": "REDUCE_MIN", @@ -921,27 +956,39 @@ func TestCloudMonitoring(t *testing.T) { qes, err := service.buildQueryExecutors(slog, req) require.NoError(t, err) - queries := getCloudMonitoringQueriesFromInterface(t, qes) + queries := getCloudMonitoringListFromInterface(t, qes) assert.Equal(t, 1, len(queries)) - assert.Equal(t, "REDUCE_MIN", queries[0].Params["aggregation.crossSeriesReducer"][0]) - assert.Equal(t, "ALIGN_DELTA", queries[0].Params["aggregation.perSeriesAligner"][0]) - assert.Equal(t, "+60s", queries[0].Params["aggregation.alignmentPeriod"][0]) - assert.Equal(t, "labelname", queries[0].Params["aggregation.groupByFields"][0]) + assert.Equal(t, "REDUCE_MIN", queries[0].params["aggregation.crossSeriesReducer"][0]) + assert.Equal(t, "ALIGN_DELTA", queries[0].params["aggregation.perSeriesAligner"][0]) + assert.Equal(t, "+60s", queries[0].params["aggregation.alignmentPeriod"][0]) + assert.Equal(t, "labelname", queries[0].params["aggregation.groupByFields"][0]) - assert.Equal(t, "REDUCE_MIN", queries[0].Params["secondaryAggregation.crossSeriesReducer"][0]) - assert.Equal(t, "REDUCE_SUM", queries[0].Params["secondaryAggregation.perSeriesAligner"][0]) - assert.Equal(t, "+60s", queries[0].Params["secondaryAggregation.alignmentPeriod"][0]) - assert.Equal(t, "labelname", queries[0].Params["secondaryAggregation.groupByFields"][0]) + assert.Equal(t, "REDUCE_MIN", queries[0].params["secondaryAggregation.crossSeriesReducer"][0]) + assert.Equal(t, "REDUCE_SUM", queries[0].params["secondaryAggregation.perSeriesAligner"][0]) + assert.Equal(t, "+60s", queries[0].params["secondaryAggregation.alignmentPeriod"][0]) + assert.Equal(t, "labelname", queries[0].params["secondaryAggregation.groupByFields"][0]) }) } -func getCloudMonitoringQueriesFromInterface(t *testing.T, qes []cloudMonitoringQueryExecutor) []*cloudMonitoringTimeSeriesFilter { +func getCloudMonitoringListFromInterface(t *testing.T, qes []cloudMonitoringQueryExecutor) []*cloudMonitoringTimeSeriesList { t.Helper() - queries := make([]*cloudMonitoringTimeSeriesFilter, 0) + queries := make([]*cloudMonitoringTimeSeriesList, 0) for _, qi := range qes { - q, ok := qi.(*cloudMonitoringTimeSeriesFilter) + q, ok := qi.(*cloudMonitoringTimeSeriesList) + require.Truef(t, ok, "Received wrong type %T", qi) + queries = append(queries, q) + } + return queries +} + +func getCloudMonitoringQueryFromInterface(t *testing.T, qes []cloudMonitoringQueryExecutor) []*cloudMonitoringTimeSeriesQuery { + t.Helper() + + queries := make([]*cloudMonitoringTimeSeriesQuery, 0) + for _, qi := range qes { + q, ok := qi.(*cloudMonitoringTimeSeriesQuery) require.Truef(t, ok, "Received wrong type %T", qi) queries = append(queries, q) } @@ -1009,7 +1056,7 @@ func verifyDeepLink(t *testing.T, dl string, expectedTimeSelection map[string]st } } -func baseReq() *backend.QueryDataRequest { +func deprecatedReq() *backend.QueryDataRequest { fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) query := &backend.QueryDataRequest{ Queries: []backend.DataQuery{ @@ -1031,6 +1078,54 @@ func baseReq() *backend.QueryDataRequest { return query } +func baseTimeSeriesList() *backend.QueryDataRequest { + fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) + query := &backend.QueryDataRequest{ + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{ + From: fromStart, + To: fromStart.Add(34 * time.Minute), + }, + QueryType: "metrics", + JSON: json.RawMessage(`{ + "timeSeriesList": { + "metricType": "a/metric/type", + "view": "FULL" + }, + "aliasBy": "testalias" + }`), + }, + }, + } + return query +} + +func baseTimeSeriesQuery() *backend.QueryDataRequest { + fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) + query := &backend.QueryDataRequest{ + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{ + From: fromStart, + To: fromStart.Add(34 * time.Minute), + }, + QueryType: "metrics", + JSON: json.RawMessage(`{ + "queryType": "metrics", + "timeSeriesQuery": { + "query": "foo" + }, + "aliasBy": "testalias" + }`), + }, + }, + } + return query +} + func TestCheckHealth(t *testing.T) { t.Run("and using GCE authentation should return proper error", func(t *testing.T) { im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { diff --git a/pkg/tsdb/cloudmonitoring/time_series_filter.go b/pkg/tsdb/cloudmonitoring/time_series_filter.go index 011d9b1a3f4..b2878e3230e 100644 --- a/pkg/tsdb/cloudmonitoring/time_series_filter.go +++ b/pkg/tsdb/cloudmonitoring/time_series_filter.go @@ -18,8 +18,8 @@ import ( "github.com/grafana/grafana/pkg/infra/tracing" ) -func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) doRequestFilterPage(ctx context.Context, r *http.Request, dsInfo datasourceInfo) (cloudMonitoringResponse, error) { - r.URL.RawQuery = timeSeriesFilter.Params.Encode() +func (timeSeriesFilter *cloudMonitoringTimeSeriesList) doRequestFilterPage(ctx context.Context, r *http.Request, dsInfo datasourceInfo) (cloudMonitoringResponse, error) { + r.URL.RawQuery = timeSeriesFilter.params.Encode() r = r.WithContext(ctx) res, err := dsInfo.services[cloudMonitor].client.Do(r) if err != nil { @@ -34,10 +34,10 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) doRequestFilterPage(ctx return dnext, nil } -func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) run(ctx context.Context, req *backend.QueryDataRequest, +func (timeSeriesFilter *cloudMonitoringTimeSeriesList) run(ctx context.Context, req *backend.QueryDataRequest, s *Service, dsInfo datasourceInfo, tracer tracing.Tracer) (*backend.DataResponse, cloudMonitoringResponse, string, error) { dr := &backend.DataResponse{} - projectName := timeSeriesFilter.ProjectName + projectName := timeSeriesFilter.parameters.ProjectName if projectName == "" { var err error projectName, err = s.getDefaultProject(ctx, dsInfo) @@ -71,7 +71,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) run(ctx context.Context } ctx, span := tracer.Start(ctx, "cloudMonitoring query") - span.SetAttributes("target", timeSeriesFilter.Target, attribute.Key("target").String(timeSeriesFilter.Target)) + span.SetAttributes("target", timeSeriesFilter.params.Encode(), attribute.Key("target").String(timeSeriesFilter.params.Encode())) span.SetAttributes("from", req.Queries[0].TimeRange.From, attribute.Key("from").String(req.Queries[0].TimeRange.From.String())) span.SetAttributes("until", req.Queries[0].TimeRange.To, attribute.Key("until").String(req.Queries[0].TimeRange.To.String())) span.SetAttributes("datasource_id", dsInfo.id, attribute.Key("datasource_id").Int64(dsInfo.id)) @@ -86,7 +86,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) run(ctx context.Context } nextPageToken := d.NextPageToken for nextPageToken != "" { - timeSeriesFilter.Params["pageToken"] = []string{d.NextPageToken} + timeSeriesFilter.params["pageToken"] = []string{d.NextPageToken} nextPage, err := timeSeriesFilter.doRequestFilterPage(ctx, r, dsInfo) if err != nil { dr.Error = err @@ -100,7 +100,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) run(ctx context.Context } //nolint:gocyclo -func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes *backend.DataResponse, +func (timeSeriesFilter *cloudMonitoringTimeSeriesList) parseResponse(queryRes *backend.DataResponse, response cloudMonitoringResponse, executedQueryString string) error { frames := data.Frames{} @@ -112,7 +112,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes seriesLabels["resource.type"] = series.Resource.Type frame := data.NewFrameOfFieldTypes("", len(series.Points), data.FieldTypeTime, data.FieldTypeFloat64) - frame.RefID = timeSeriesFilter.RefID + frame.RefID = timeSeriesFilter.refID frame.Meta = &data.FrameMeta{ ExecutedQueryString: executedQueryString, } @@ -121,7 +121,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes labels["metric.label."+key] = value seriesLabels["metric.label."+key] = value - if len(timeSeriesFilter.GroupBys) == 0 || containsLabel(timeSeriesFilter.GroupBys, "metric.label."+key) { + if len(timeSeriesFilter.parameters.GroupBys) == 0 || containsLabel(timeSeriesFilter.parameters.GroupBys, "metric.label."+key) { defaultMetricName += " " + value } } @@ -130,7 +130,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes labels["resource.label."+key] = value seriesLabels["resource.label."+key] = value - if containsLabel(timeSeriesFilter.GroupBys, "resource.label."+key) { + if containsLabel(timeSeriesFilter.parameters.GroupBys, "resource.label."+key) { defaultMetricName += " " + value } } @@ -161,10 +161,10 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes } customFrameMeta := map[string]interface{}{} - customFrameMeta["alignmentPeriod"] = timeSeriesFilter.Params.Get("aggregation.alignmentPeriod") - customFrameMeta["perSeriesAligner"] = timeSeriesFilter.Params.Get("aggregation.perSeriesAligner") + customFrameMeta["alignmentPeriod"] = timeSeriesFilter.params.Get("aggregation.alignmentPeriod") + customFrameMeta["perSeriesAligner"] = timeSeriesFilter.params.Get("aggregation.perSeriesAligner") customFrameMeta["labels"] = labels - customFrameMeta["groupBys"] = timeSeriesFilter.GroupBys + customFrameMeta["groupBys"] = timeSeriesFilter.parameters.GroupBys if frame.Meta != nil { frame.Meta.Custom = customFrameMeta } else { @@ -208,7 +208,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes timeField, valueField, }, - RefID: timeSeriesFilter.RefID, + RefID: timeSeriesFilter.refID, Meta: &data.FrameMeta{ ExecutedQueryString: executedQueryString, }, @@ -227,7 +227,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes } if len(response.TimeSeries) > 0 { dl := timeSeriesFilter.buildDeepLink() - frames = addConfigData(frames, dl, response.Unit, timeSeriesFilter.Params.Get("aggregation.alignmentPeriod")) + frames = addConfigData(frames, dl, response.Unit, timeSeriesFilter.params.Get("aggregation.alignmentPeriod")) } queryRes.Frames = frames @@ -235,7 +235,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes return nil } -func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) handleNonDistributionSeries(series timeSeries, +func (timeSeriesFilter *cloudMonitoringTimeSeriesList) handleNonDistributionSeries(series timeSeries, defaultMetricName string, seriesLabels map[string]string, frame *data.Frame) { for i := 0; i < len(series.Points); i++ { point := series.Points[i] @@ -265,14 +265,14 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) handleNonDistributionSe setDisplayNameAsFieldName(dataField) } -func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) buildDeepLink() string { - if timeSeriesFilter.Slo != "" { +func (timeSeriesFilter *cloudMonitoringTimeSeriesList) buildDeepLink() string { + if timeSeriesFilter.sloQ != nil && timeSeriesFilter.sloQ.SloId != "" { return "" } - filter := timeSeriesFilter.Params.Get("filter") + filter := timeSeriesFilter.params.Get("filter") if !strings.Contains(filter, "resource.type=") { - resourceType := timeSeriesFilter.Params.Get("resourceType") + resourceType := timeSeriesFilter.params.Get("resourceType") if resourceType != "" { filter = fmt.Sprintf(`resource.type="%s" %s`, resourceType, filter) } @@ -281,12 +281,12 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) buildDeepLink() string u, err := url.Parse("https://console.cloud.google.com/monitoring/metrics-explorer") if err != nil { slog.Error("Failed to generate deep link: unable to parse metrics explorer URL", "ProjectName", - timeSeriesFilter.ProjectName, "query", timeSeriesFilter.RefID) + timeSeriesFilter.parameters.ProjectName, "query", timeSeriesFilter.refID) return "" } rawQuery := u.Query() - rawQuery.Set("project", timeSeriesFilter.ProjectName) + rawQuery.Set("project", timeSeriesFilter.parameters.ProjectName) rawQuery.Set("Grafana_deeplink", "true") pageState := map[string]interface{}{ @@ -296,11 +296,11 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) buildDeepLink() string { "timeSeriesFilter": map[string]interface{}{ "aggregations": []string{}, - "crossSeriesReducer": timeSeriesFilter.Params.Get("aggregation.crossSeriesReducer"), + "crossSeriesReducer": timeSeriesFilter.params.Get("aggregation.crossSeriesReducer"), "filter": filter, - "groupByFields": timeSeriesFilter.Params["aggregation.groupByFields"], - "minAlignmentPeriod": strings.TrimPrefix(timeSeriesFilter.Params.Get("aggregation.alignmentPeriod"), "+"), // get rid of leading + - "perSeriesAligner": timeSeriesFilter.Params.Get("aggregation.perSeriesAligner"), + "groupByFields": timeSeriesFilter.params["aggregation.groupByFields"], + "minAlignmentPeriod": strings.TrimPrefix(timeSeriesFilter.params.Get("aggregation.alignmentPeriod"), "+"), // get rid of leading + + "perSeriesAligner": timeSeriesFilter.params.Get("aggregation.perSeriesAligner"), "secondaryGroupByFields": []string{}, "unitOverride": "1", }, @@ -314,15 +314,15 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) buildDeepLink() string }, "timeSelection": map[string]string{ "timeRange": "custom", - "start": timeSeriesFilter.Params.Get("interval.startTime"), - "end": timeSeriesFilter.Params.Get("interval.endTime"), + "start": timeSeriesFilter.params.Get("interval.startTime"), + "end": timeSeriesFilter.params.Get("interval.endTime"), }, } blob, err := json.Marshal(pageState) if err != nil { - slog.Error("Failed to generate deep link", "pageState", pageState, "ProjectName", timeSeriesFilter.ProjectName, - "query", timeSeriesFilter.RefID) + slog.Error("Failed to generate deep link", "pageState", pageState, "ProjectName", timeSeriesFilter.parameters.ProjectName, + "query", timeSeriesFilter.refID) return "" } @@ -332,7 +332,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) buildDeepLink() string accountChooserURL, err := url.Parse("https://accounts.google.com/AccountChooser") if err != nil { slog.Error("Failed to generate deep link: unable to parse account chooser URL", "ProjectName", - timeSeriesFilter.ProjectName, "query", timeSeriesFilter.RefID) + timeSeriesFilter.parameters.ProjectName, "query", timeSeriesFilter.refID) return "" } accountChooserQuery := accountChooserURL.Query() @@ -349,6 +349,6 @@ func setDisplayNameAsFieldName(f *data.Field) { f.Config.DisplayNameFromDS = f.Name } -func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) getRefID() string { - return timeSeriesFilter.RefID +func (timeSeriesFilter *cloudMonitoringTimeSeriesList) getRefID() string { + return timeSeriesFilter.refID } diff --git a/pkg/tsdb/cloudmonitoring/time_series_filter_test.go b/pkg/tsdb/cloudmonitoring/time_series_filter_test.go index 20aa99c02c9..e73ceae0c70 100644 --- a/pkg/tsdb/cloudmonitoring/time_series_filter_test.go +++ b/pkg/tsdb/cloudmonitoring/time_series_filter_test.go @@ -24,7 +24,7 @@ func TestTimeSeriesFilter(t *testing.T) { assert.Equal(t, 1, len(data.TimeSeries)) res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}} + query := &cloudMonitoringTimeSeriesList{params: url.Values{}, parameters: &timeSeriesList{}} err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -47,7 +47,7 @@ func TestTimeSeriesFilter(t *testing.T) { require.NoError(t, err) assert.Equal(t, 3, len(data.TimeSeries)) res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}} + query := &cloudMonitoringTimeSeriesList{params: url.Values{}, parameters: &timeSeriesList{}} err = query.parseResponse(res, data, "") require.NoError(t, err) @@ -87,9 +87,9 @@ func TestTimeSeriesFilter(t *testing.T) { require.NoError(t, err) assert.Equal(t, 3, len(data.TimeSeries)) res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}, GroupBys: []string{ + query := &cloudMonitoringTimeSeriesList{params: url.Values{}, parameters: &timeSeriesList{GroupBys: []string{ "metric.label.instance_name", "resource.label.zone", - }} + }}} err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -108,7 +108,13 @@ func TestTimeSeriesFilter(t *testing.T) { res := &backend.DataResponse{} t.Run("and the alias pattern is for metric type, a metric label and a resource label", func(t *testing.T) { - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}, AliasBy: "{{metric.type}} - {{metric.label.instance_name}} - {{resource.label.zone}}", GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}} + query := &cloudMonitoringTimeSeriesList{ + params: url.Values{}, + parameters: &timeSeriesList{ + GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}, + }, + aliasBy: "{{metric.type}} - {{metric.label.instance_name}} - {{resource.label.zone}}", + } err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -121,7 +127,11 @@ func TestTimeSeriesFilter(t *testing.T) { }) t.Run("and the alias pattern is for metric name", func(t *testing.T) { - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}, AliasBy: "metric {{metric.name}} service {{metric.service}}", GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}} + query := &cloudMonitoringTimeSeriesList{ + params: url.Values{}, + parameters: &timeSeriesList{GroupBys: []string{"metric.label.instance_name", "resource.label.zone"}}, + aliasBy: "metric {{metric.name}} service {{metric.service}}", + } err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -139,7 +149,11 @@ func TestTimeSeriesFilter(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, len(data.TimeSeries)) res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}, AliasBy: "{{bucket}}"} + query := &cloudMonitoringTimeSeriesList{ + params: url.Values{}, + parameters: &timeSeriesList{}, + aliasBy: "{{bucket}}", + } err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -180,7 +194,11 @@ func TestTimeSeriesFilter(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, len(data.TimeSeries)) res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}, AliasBy: "{{bucket}}"} + query := &cloudMonitoringTimeSeriesList{ + params: url.Values{}, + parameters: &timeSeriesList{}, + aliasBy: "{{bucket}}", + } err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -214,7 +232,11 @@ func TestTimeSeriesFilter(t *testing.T) { require.NoError(t, err) assert.Equal(t, 3, len(data.TimeSeries)) res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}, AliasBy: "{{bucket}}"} + query := &cloudMonitoringTimeSeriesList{ + params: url.Values{}, + parameters: &timeSeriesList{}, + aliasBy: "{{bucket}}", + } err = query.parseResponse(res, data, "") require.NoError(t, err) require.NoError(t, err) @@ -250,7 +272,11 @@ func TestTimeSeriesFilter(t *testing.T) { t.Run("and systemlabel contains key with array of string", func(t *testing.T) { res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}, AliasBy: "{{metadata.system_labels.test}}"} + query := &cloudMonitoringTimeSeriesList{ + params: url.Values{}, + parameters: &timeSeriesList{}, + aliasBy: "{{metadata.system_labels.test}}", + } err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -264,7 +290,11 @@ func TestTimeSeriesFilter(t *testing.T) { t.Run("and systemlabel contains key with array of string2", func(t *testing.T) { res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}, AliasBy: "{{metadata.system_labels.test2}}"} + query := &cloudMonitoringTimeSeriesList{ + params: url.Values{}, + parameters: &timeSeriesList{}, + aliasBy: "{{metadata.system_labels.test2}}", + } err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -281,13 +311,17 @@ func TestTimeSeriesFilter(t *testing.T) { t.Run("and alias by is expanded", func(t *testing.T) { res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{ - Params: url.Values{}, - ProjectName: "test-proj", - Selector: "select_slo_compliance", - Service: "test-service", - Slo: "test-slo", - AliasBy: "{{project}} - {{service}} - {{slo}} - {{selector}}", + query := &cloudMonitoringTimeSeriesList{ + params: url.Values{}, + parameters: &timeSeriesList{ + ProjectName: "test-proj", + }, + aliasBy: "{{project}} - {{service}} - {{slo}} - {{selector}}", + sloQ: &sloQuery{ + SelectorName: "select_slo_compliance", + ServiceId: "test-service", + SloId: "test-slo", + }, } err = query.parseResponse(res, data, "") require.NoError(t, err) @@ -304,12 +338,16 @@ func TestTimeSeriesFilter(t *testing.T) { t.Run("and alias by is expanded", func(t *testing.T) { res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{ - Params: url.Values{}, - ProjectName: "test-proj", - Selector: "select_slo_compliance", - Service: "test-service", - Slo: "test-slo", + query := &cloudMonitoringTimeSeriesList{ + params: url.Values{}, + parameters: &timeSeriesList{ + ProjectName: "test-proj", + }, + sloQ: &sloQuery{ + SelectorName: "select_slo_compliance", + ServiceId: "test-service", + SloId: "test-slo", + }, } err = query.parseResponse(res, data, "") require.NoError(t, err) @@ -325,7 +363,7 @@ func TestTimeSeriesFilter(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, len(data.TimeSeries)) res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}} + query := &cloudMonitoringTimeSeriesList{params: url.Values{}, parameters: &timeSeriesList{}} err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -338,7 +376,7 @@ func TestTimeSeriesFilter(t *testing.T) { require.NoError(t, err) assert.Equal(t, 3, len(data.TimeSeries)) res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}} + query := &cloudMonitoringTimeSeriesList{params: url.Values{}, parameters: &timeSeriesList{}} err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -358,9 +396,11 @@ func TestTimeSeriesFilter(t *testing.T) { res := &backend.DataResponse{} query := &cloudMonitoringTimeSeriesQuery{ - ProjectName: "test-proj", - Query: "test-query", - AliasBy: "{{project}} - {{resource.label.zone}} - {{resource.label.instance_id}} - {{metric.label.response_code_class}}", + parameters: &timeSeriesQuery{ + ProjectName: "test-proj", + Query: "test-query", + }, + aliasBy: "{{project}} - {{resource.label.zone}} - {{resource.label.instance_id}} - {{metric.label.response_code_class}}", timeRange: backend.TimeRange{ From: fromStart, To: fromStart.Add(34 * time.Minute), @@ -378,7 +418,7 @@ func TestTimeSeriesFilter(t *testing.T) { require.NoError(t, err) assert.Equal(t, 3, len(data.TimeSeries)) res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}} + query := &cloudMonitoringTimeSeriesList{params: url.Values{}, parameters: &timeSeriesList{}} err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -394,9 +434,9 @@ func TestTimeSeriesFilter(t *testing.T) { require.NoError(t, err) assert.Equal(t, 3, len(data.TimeSeries)) res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{ + query := &cloudMonitoringTimeSeriesList{params: url.Values{ "aggregation.alignmentPeriod": []string{"+60s"}, - }} + }, parameters: &timeSeriesList{}} err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames @@ -411,7 +451,7 @@ func TestTimeSeriesFilter(t *testing.T) { assert.Equal(t, 1, len(data.TimeSeries)) res := &backend.DataResponse{} - require.NoError(t, (&cloudMonitoringTimeSeriesFilter{GroupBys: []string{"test_group_by"}}).parseResponse(res, data, "test_query")) + require.NoError(t, (&cloudMonitoringTimeSeriesList{parameters: &timeSeriesList{GroupBys: []string{"test_group_by"}}}).parseResponse(res, data, "test_query")) require.NotNil(t, res.Frames[0].Meta) assert.Equal(t, sdkdata.FrameMeta{ @@ -434,7 +474,7 @@ func TestTimeSeriesFilter(t *testing.T) { assert.Equal(t, 1, len(data.TimeSeries)) res := &backend.DataResponse{} - require.NoError(t, (&cloudMonitoringTimeSeriesFilter{GroupBys: []string{"test_group_by"}}).parseResponse(res, data, "test_query")) + require.NoError(t, (&cloudMonitoringTimeSeriesList{parameters: &timeSeriesList{GroupBys: []string{"test_group_by"}}}).parseResponse(res, data, "test_query")) require.NotNil(t, res.Frames[0].Meta) assert.Equal(t, sdkdata.FrameMeta{ @@ -457,7 +497,7 @@ func TestTimeSeriesFilter(t *testing.T) { assert.Equal(t, 1, len(data.TimeSeries)) res := &backend.DataResponse{} - require.NoError(t, (&cloudMonitoringTimeSeriesFilter{GroupBys: []string{"test_group_by"}}).parseResponse(res, data, "test_query")) + require.NoError(t, (&cloudMonitoringTimeSeriesList{parameters: &timeSeriesList{GroupBys: []string{"test_group_by"}}}).parseResponse(res, data, "test_query")) require.NotNil(t, res.Frames[0].Meta) assert.Equal(t, sdkdata.FrameMeta{ @@ -481,7 +521,7 @@ func TestTimeSeriesFilter(t *testing.T) { assert.Equal(t, 1, len(data.TimeSeries)) res := &backend.DataResponse{} - query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}, Slo: "yes"} + query := &cloudMonitoringTimeSeriesList{params: url.Values{}, sloQ: &sloQuery{SloId: "yes"}, parameters: &timeSeriesList{}} err = query.parseResponse(res, data, "") require.NoError(t, err) frames := res.Frames diff --git a/pkg/tsdb/cloudmonitoring/time_series_query.go b/pkg/tsdb/cloudmonitoring/time_series_query.go index 05ca9505f51..c3181d45149 100644 --- a/pkg/tsdb/cloudmonitoring/time_series_query.go +++ b/pkg/tsdb/cloudmonitoring/time_series_query.go @@ -25,13 +25,13 @@ import ( func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) appendGraphPeriod(req *backend.QueryDataRequest) string { // GraphPeriod needs to be explicitly disabled. // If not set, the default behavior is to set an automatic value - if timeSeriesQuery.GraphPeriod != "disabled" { - if timeSeriesQuery.GraphPeriod == "auto" || timeSeriesQuery.GraphPeriod == "" { + if timeSeriesQuery.parameters.GraphPeriod != "disabled" { + if timeSeriesQuery.parameters.GraphPeriod == "auto" || timeSeriesQuery.parameters.GraphPeriod == "" { intervalCalculator := intervalv2.NewCalculator(intervalv2.CalculatorOptions{}) interval := intervalCalculator.Calculate(req.Queries[0].TimeRange, time.Duration(timeSeriesQuery.IntervalMS/1000)*time.Second, req.Queries[0].MaxDataPoints) - timeSeriesQuery.GraphPeriod = interval.Text + timeSeriesQuery.parameters.GraphPeriod = interval.Text } - return fmt.Sprintf(" | graph_period %s", timeSeriesQuery.GraphPeriod) + return fmt.Sprintf(" | graph_period %s", timeSeriesQuery.parameters.GraphPeriod) } return "" } @@ -57,7 +57,7 @@ func doRequestQueryPage(log log.Logger, requestBody map[string]interface{}, r *h func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) run(ctx context.Context, req *backend.QueryDataRequest, s *Service, dsInfo datasourceInfo, tracer tracing.Tracer) (*backend.DataResponse, cloudMonitoringResponse, string, error) { dr := &backend.DataResponse{} - projectName := timeSeriesQuery.ProjectName + projectName := timeSeriesQuery.parameters.ProjectName if projectName == "" { var err error @@ -69,21 +69,21 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) run(ctx context.Context, timeSeriesQuery.logger.Info("No project name set on query, using project name from datasource", "projectName", projectName) } - timeSeriesQuery.Query += timeSeriesQuery.appendGraphPeriod(req) + timeSeriesQuery.parameters.Query += timeSeriesQuery.appendGraphPeriod(req) from := req.Queries[0].TimeRange.From to := req.Queries[0].TimeRange.To timeFormat := "2006/01/02-15:04:05" - timeSeriesQuery.Query += fmt.Sprintf(" | within d'%s', d'%s'", from.UTC().Format(timeFormat), to.UTC().Format(timeFormat)) + timeSeriesQuery.parameters.Query += fmt.Sprintf(" | within d'%s', d'%s'", from.UTC().Format(timeFormat), to.UTC().Format(timeFormat)) p := path.Join("/v3/projects", projectName, "timeSeries:query") ctx, span := tracer.Start(ctx, "cloudMonitoring MQL query") - span.SetAttributes("query", timeSeriesQuery.Query, attribute.Key("query").String(timeSeriesQuery.Query)) + span.SetAttributes("query", timeSeriesQuery.parameters.Query, attribute.Key("query").String(timeSeriesQuery.parameters.Query)) span.SetAttributes("from", req.Queries[0].TimeRange.From, attribute.Key("from").String(req.Queries[0].TimeRange.From.String())) span.SetAttributes("until", req.Queries[0].TimeRange.To, attribute.Key("until").String(req.Queries[0].TimeRange.To.String())) defer span.End() requestBody := map[string]interface{}{ - "query": timeSeriesQuery.Query, + "query": timeSeriesQuery.parameters.Query, } r, err := s.createRequest(timeSeriesQuery.logger, &dsInfo, p, bytes.NewBuffer([]byte{})) if err != nil { @@ -100,7 +100,7 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) run(ctx context.Context, } for d.NextPageToken != "" { requestBody := map[string]interface{}{ - "query": timeSeriesQuery.Query, + "query": timeSeriesQuery.parameters.Query, "pageToken": d.NextPageToken, } nextPage, err := doRequestQueryPage(timeSeriesQuery.logger, requestBody, r, dsInfo) @@ -112,7 +112,7 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) run(ctx context.Context, d.NextPageToken = nextPage.NextPageToken } - return dr, d, timeSeriesQuery.Query, nil + return dr, d, timeSeriesQuery.parameters.Query, nil } func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *backend.DataResponse, @@ -122,7 +122,7 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *b for _, series := range response.TimeSeriesData { seriesLabels := make(map[string]string) frame := data.NewFrameOfFieldTypes("", len(series.PointData), data.FieldTypeTime, data.FieldTypeFloat64) - frame.RefID = timeSeriesQuery.RefID + frame.RefID = timeSeriesQuery.refID frame.Meta = &data.FrameMeta{ ExecutedQueryString: executedQueryString, } @@ -184,8 +184,12 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *b } metricName := formatLegendKeys(d.Key, defaultMetricName, seriesLabels, nil, - &cloudMonitoringTimeSeriesFilter{ - ProjectName: timeSeriesQuery.ProjectName, AliasBy: timeSeriesQuery.AliasBy, logger: timeSeriesQuery.logger, + &cloudMonitoringTimeSeriesList{ + parameters: &timeSeriesList{ + ProjectName: timeSeriesQuery.parameters.ProjectName, + }, + aliasBy: timeSeriesQuery.aliasBy, + logger: timeSeriesQuery.logger, }) dataField := frame.Fields[1] dataField.Name = metricName @@ -219,7 +223,11 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *b timeField := data.NewField(data.TimeSeriesTimeFieldName, nil, []time.Time{}) valueField := data.NewField(data.TimeSeriesValueFieldName, nil, []float64{}) - frameName := formatLegendKeys(d.Key, defaultMetricName, nil, additionalLabels, &cloudMonitoringTimeSeriesFilter{ProjectName: timeSeriesQuery.ProjectName, AliasBy: timeSeriesQuery.AliasBy, logger: timeSeriesQuery.logger}) + frameName := formatLegendKeys(d.Key, defaultMetricName, nil, additionalLabels, &cloudMonitoringTimeSeriesList{ + parameters: &timeSeriesList{ProjectName: timeSeriesQuery.parameters.ProjectName}, + aliasBy: timeSeriesQuery.aliasBy, + logger: timeSeriesQuery.logger, + }) valueField.Name = frameName valueField.Labels = seriesLabels setDisplayNameAsFieldName(valueField) @@ -230,7 +238,7 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *b timeField, valueField, }, - RefID: timeSeriesQuery.RefID, + RefID: timeSeriesQuery.refID, } if maxKey < i { @@ -247,7 +255,11 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *b additionalLabels := data.Labels{"bucket": bucketBound} timeField := data.NewField(data.TimeSeriesTimeFieldName, nil, []time.Time{}) valueField := data.NewField(data.TimeSeriesValueFieldName, nil, []float64{}) - frameName := formatLegendKeys(d.Key, defaultMetricName, seriesLabels, additionalLabels, &cloudMonitoringTimeSeriesFilter{ProjectName: timeSeriesQuery.ProjectName, AliasBy: timeSeriesQuery.AliasBy, logger: timeSeriesQuery.logger}) + frameName := formatLegendKeys(d.Key, defaultMetricName, seriesLabels, additionalLabels, &cloudMonitoringTimeSeriesList{ + parameters: &timeSeriesList{ProjectName: timeSeriesQuery.parameters.ProjectName}, + aliasBy: timeSeriesQuery.aliasBy, + logger: timeSeriesQuery.logger, + }) valueField.Name = frameName valueField.Labels = seriesLabels setDisplayNameAsFieldName(valueField) @@ -258,7 +270,7 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *b timeField, valueField, }, - RefID: timeSeriesQuery.RefID, + RefID: timeSeriesQuery.refID, } } } @@ -278,7 +290,7 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *b } if len(response.TimeSeriesData) > 0 { dl := timeSeriesQuery.buildDeepLink() - frames = addConfigData(frames, dl, response.Unit, timeSeriesQuery.GraphPeriod) + frames = addConfigData(frames, dl, response.Unit, timeSeriesQuery.parameters.GraphPeriod) } queryRes.Frames = frames @@ -289,12 +301,12 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *b func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) buildDeepLink() string { u, err := url.Parse("https://console.cloud.google.com/monitoring/metrics-explorer") if err != nil { - timeSeriesQuery.logger.Error("Failed to generate deep link: unable to parse metrics explorer URL", "projectName", timeSeriesQuery.ProjectName, "query", timeSeriesQuery.RefID) + timeSeriesQuery.logger.Error("Failed to generate deep link: unable to parse metrics explorer URL", "projectName", timeSeriesQuery.parameters.ProjectName, "query", timeSeriesQuery.refID) return "" } q := u.Query() - q.Set("project", timeSeriesQuery.ProjectName) + q.Set("project", timeSeriesQuery.parameters.ProjectName) q.Set("Grafana_deeplink", "true") pageState := map[string]interface{}{ @@ -302,7 +314,7 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) buildDeepLink() string { "constantLines": []string{}, "dataSets": []map[string]interface{}{ { - "timeSeriesQuery": timeSeriesQuery.Query, + "timeSeriesQuery": timeSeriesQuery.parameters.Query, "targetAxis": "Y1", "plotType": "LINE", }, @@ -322,7 +334,7 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) buildDeepLink() string { blob, err := json.Marshal(pageState) if err != nil { - timeSeriesQuery.logger.Error("Failed to generate deep link", "pageState", pageState, "ProjectName", timeSeriesQuery.ProjectName, "query", timeSeriesQuery.RefID) + timeSeriesQuery.logger.Error("Failed to generate deep link", "pageState", pageState, "ProjectName", timeSeriesQuery.parameters.ProjectName, "query", timeSeriesQuery.refID) return "" } @@ -331,7 +343,7 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) buildDeepLink() string { accountChooserURL, err := url.Parse("https://accounts.google.com/AccountChooser") if err != nil { - timeSeriesQuery.logger.Error("Failed to generate deep link: unable to parse account chooser URL", "ProjectName", timeSeriesQuery.ProjectName, "query", timeSeriesQuery.RefID) + timeSeriesQuery.logger.Error("Failed to generate deep link: unable to parse account chooser URL", "ProjectName", timeSeriesQuery.parameters.ProjectName, "query", timeSeriesQuery.refID) return "" } accountChooserQuery := accountChooserURL.Query() @@ -342,5 +354,5 @@ func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) buildDeepLink() string { } func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) getRefID() string { - return timeSeriesQuery.RefID + return timeSeriesQuery.refID } diff --git a/pkg/tsdb/cloudmonitoring/time_series_query_test.go b/pkg/tsdb/cloudmonitoring/time_series_query_test.go index 2e983dcaa3d..025e8cde225 100644 --- a/pkg/tsdb/cloudmonitoring/time_series_query_test.go +++ b/pkg/tsdb/cloudmonitoring/time_series_query_test.go @@ -21,8 +21,10 @@ func TestTimeSeriesQuery(t *testing.T) { t.Run("and alias template is not specified", func(t *testing.T) { res := &backend.DataResponse{} query := &cloudMonitoringTimeSeriesQuery{ - ProjectName: "test-proj", - Query: "test-query", + parameters: &timeSeriesQuery{ + ProjectName: "test-proj", + Query: "test-query", + }, timeRange: backend.TimeRange{ From: fromStart, To: fromStart.Add(34 * time.Minute), @@ -37,9 +39,11 @@ func TestTimeSeriesQuery(t *testing.T) { t.Run("and alias template is specified", func(t *testing.T) { res := &backend.DataResponse{} query := &cloudMonitoringTimeSeriesQuery{ - ProjectName: "test-proj", - Query: "test-query", - AliasBy: "{{project}} - {{resource.label.zone}} - {{resource.label.instance_id}} - {{metric.label.response_code_class}}", + parameters: &timeSeriesQuery{ + ProjectName: "test-proj", + Query: "test-query", + }, + aliasBy: "{{project}} - {{resource.label.zone}} - {{resource.label.instance_id}} - {{metric.label.response_code_class}}", timeRange: backend.TimeRange{ From: fromStart, To: fromStart.Add(34 * time.Minute), @@ -62,9 +66,11 @@ func TestTimeSeriesQuery(t *testing.T) { res := &backend.DataResponse{} query := &cloudMonitoringTimeSeriesQuery{ - ProjectName: "test-proj", - Query: "test-query", - AliasBy: "{{project}} - {{resource.label.zone}} - {{resource.label.instance_id}} - {{metric.label.response_code_class}}", + parameters: &timeSeriesQuery{ + ProjectName: "test-proj", + Query: "test-query", + }, + aliasBy: "{{project}} - {{resource.label.zone}} - {{resource.label.instance_id}} - {{metric.label.response_code_class}}", timeRange: backend.TimeRange{ From: fromStart, To: fromStart.Add(34 * time.Minute), @@ -85,8 +91,10 @@ func TestTimeSeriesQuery(t *testing.T) { fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) res := &backend.DataResponse{} query := &cloudMonitoringTimeSeriesQuery{ - ProjectName: "test-proj", - Query: "test-query", + parameters: &timeSeriesQuery{ + ProjectName: "test-proj", + Query: "test-query", + }, timeRange: backend.TimeRange{ From: fromStart, To: fromStart.Add(34 * time.Minute), @@ -109,13 +117,15 @@ func TestTimeSeriesQuery(t *testing.T) { fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) res := &backend.DataResponse{} query := &cloudMonitoringTimeSeriesQuery{ - ProjectName: "test-proj", - Query: "test-query", + parameters: &timeSeriesQuery{ + ProjectName: "test-proj", + Query: "test-query", + GraphPeriod: "60s", + }, timeRange: backend.TimeRange{ From: fromStart, To: fromStart.Add(34 * time.Minute), }, - GraphPeriod: "60s", } err = query.parseResponse(res, data, "") require.NoError(t, err) @@ -125,12 +135,12 @@ func TestTimeSeriesQuery(t *testing.T) { }) t.Run("appends graph_period to the query", func(t *testing.T) { - query := &cloudMonitoringTimeSeriesQuery{} + query := &cloudMonitoringTimeSeriesQuery{parameters: &timeSeriesQuery{}} assert.Equal(t, query.appendGraphPeriod(&backend.QueryDataRequest{Queries: []backend.DataQuery{{}}}), " | graph_period 1ms") }) t.Run("skips graph_period if disabled", func(t *testing.T) { - query := &cloudMonitoringTimeSeriesQuery{GraphPeriod: "disabled"} + query := &cloudMonitoringTimeSeriesQuery{parameters: &timeSeriesQuery{GraphPeriod: "disabled"}} assert.Equal(t, query.appendGraphPeriod(&backend.QueryDataRequest{Queries: []backend.DataQuery{{}}}), "") }) } diff --git a/pkg/tsdb/cloudmonitoring/types.go b/pkg/tsdb/cloudmonitoring/types.go index 1f939b73b88..d91fd81cddf 100644 --- a/pkg/tsdb/cloudmonitoring/types.go +++ b/pkg/tsdb/cloudmonitoring/types.go @@ -20,67 +20,68 @@ type ( getRefID() string } - // Used to build time series filters - cloudMonitoringTimeSeriesFilter struct { - Target string - Params url.Values - RefID string - GroupBys []string - AliasBy string - ProjectName string - Selector string - Service string - Slo string - logger log.Logger - } - - // Used to build MQL queries - cloudMonitoringTimeSeriesQuery struct { - RefID string - ProjectName string - Query string - IntervalMS int64 - AliasBy string - timeRange backend.TimeRange - GraphPeriod string - logger log.Logger - } - - metricQuery struct { - ProjectName string - MetricType string - CrossSeriesReducer string - AlignmentPeriod string - PerSeriesAligner string - GroupBys []string - Filters []string - AliasBy string - View string - EditorMode string - Query string - Preprocessor string - PreprocessorType preprocessorType - GraphPeriod string - } - - sloQuery struct { - ProjectName string - AlignmentPeriod string - PerSeriesAligner string - AliasBy string - SelectorName string - ServiceId string - SloId string - LookbackPeriod string - } - + // Plugin API query data request used to generate + // a cloudMonitoringTimeSeriesList or cloudMonitoringTimeSeriesQuery grafanaQuery struct { - DatasourceId int - RefId string - QueryType string - MetricQuery metricQuery - SloQuery sloQuery - Type string + AliasBy string `json:"aliasBy"` + TimeSeriesList *timeSeriesList `json:"timeSeriesList,omitempty"` + TimeSeriesQuery *timeSeriesQuery `json:"timeSeriesQuery,omitempty"` + // TODO: Merge SloQuery into TimeSeriesList + SloQuery *sloQuery `json:"sloQuery,omitempty"` + } + + // These should reflect GCM APIs + // timeSeries.list https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.timeSeries/list + timeSeriesList struct { + ProjectName string `json:"projectName"` + CrossSeriesReducer string `json:"crossSeriesReducer"` + AlignmentPeriod string `json:"alignmentPeriod"` + PerSeriesAligner string `json:"perSeriesAligner"` + GroupBys []string `json:"groupBys"` + Filters []string `json:"filters"` + View string `json:"view"` + // Not part of the GCM API + // TODO: Use API fields instead + MetricType string `json:"metricType"` + Preprocessor string `json:"preprocessor"` + } + // TODO: sloQuery can be specified as timeSeriesList parameters + sloQuery struct { + SelectorName string `json:"selectorName"` + ServiceId string `json:"serviceId"` + SloId string `json:"sloId"` + LookbackPeriod string `json:"lookbackPeriod"` + } + + // timeSeries.query https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.timeSeries/query + timeSeriesQuery struct { + ProjectName string `json:"projectName"` + Query string `json:"query"` + // Not part of the GCM API, will be added to Query + GraphPeriod string `json:"graphPeriod"` + } + + // Internal structs. Include computed values + // cloudMonitoringTimeSeriesList is used to build time series with a filter + cloudMonitoringTimeSeriesList struct { + refID string + aliasBy string + logger log.Logger + parameters *timeSeriesList + // TODO: Merge SloQuery into TimeSeriesList + sloQ *sloQuery + // Processed properties + params url.Values + } + // cloudMonitoringTimeSeriesQuery is used to build MQL queries + cloudMonitoringTimeSeriesQuery struct { + refID string + aliasBy string + logger log.Logger + parameters *timeSeriesQuery + // Processed properties + timeRange backend.TimeRange + IntervalMS int64 } cloudMonitoringBucketOptions struct { diff --git a/pkg/tsdb/cloudmonitoring/utils.go b/pkg/tsdb/cloudmonitoring/utils.go index bcaa7550f94..67eb54df84e 100644 --- a/pkg/tsdb/cloudmonitoring/utils.go +++ b/pkg/tsdb/cloudmonitoring/utils.go @@ -45,3 +45,10 @@ func addInterval(period string, field *data.Field) error { } return nil } + +func toString(v interface{}) string { + if v == nil { + return "" + } + return v.(string) +} From 92e3ee7d8906ccc6f838e084b0f28bf7ae7a5e02 Mon Sep 17 00:00:00 2001 From: Selene Date: Mon, 28 Nov 2022 11:40:49 +0100 Subject: [PATCH 018/131] StatsColelctor: Check if result is nil before set the metrics (#59372) Check if result is nil before set the metrics --- pkg/infra/usagestats/statscollector/service.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/infra/usagestats/statscollector/service.go b/pkg/infra/usagestats/statscollector/service.go index ae503063342..e173cce9258 100644 --- a/pkg/infra/usagestats/statscollector/service.go +++ b/pkg/infra/usagestats/statscollector/service.go @@ -321,6 +321,11 @@ func (s *Service) updateTotalStats(ctx context.Context) bool { return false } + if statsQuery.Result == nil { + s.log.Error("Cannot retrieve system stats") + return false + } + metrics.MStatTotalDashboards.Set(float64(statsQuery.Result.Dashboards)) metrics.MStatTotalFolders.Set(float64(statsQuery.Result.Folders)) metrics.MStatTotalUsers.Set(float64(statsQuery.Result.Users)) From 5b861faec39c16cee2f5d0a9cdd8fb890b42558a Mon Sep 17 00:00:00 2001 From: Serge Zaitsev Date: Mon, 28 Nov 2022 12:05:46 +0100 Subject: [PATCH 019/131] Chore: Reduce the usage of sqlstore.createorg and use orgservice instead (#59356) * remove legacy createorg from org service * remove another createorg from orgimpl * remove createorg from api pref tests * remove createorg from api org tests * fix tests * remove createorg from annotations test * remove createorg from team tests * remove createorg from service accounts * remove createorg from accesscontrol tests * remove createorg from provisioning * Use quotaservice from sc.hs --- pkg/api/annotations_test.go | 4 +- pkg/api/org_test.go | 22 +++--- pkg/api/org_users_test.go | 5 +- pkg/api/preferences_test.go | 13 ++-- pkg/api/team_members_test.go | 38 ++++++---- .../resourcepermissions/store_test.go | 25 ++++--- pkg/services/org/orgimpl/store_test.go | 69 ++++++++----------- .../dashboards/config_reader_test.go | 10 +-- .../notifiers/config_reader_test.go | 23 +++---- pkg/services/serviceaccounts/api/api_test.go | 4 +- .../serviceaccounts/database/database_test.go | 13 ++-- 11 files changed, 112 insertions(+), 114 deletions(-) diff --git a/pkg/api/annotations_test.go b/pkg/api/annotations_test.go index 6abdceba3f2..0064cd7eb30 100644 --- a/pkg/api/annotations_test.go +++ b/pkg/api/annotations_test.go @@ -385,7 +385,7 @@ func deleteAnnotationsScenario(t *testing.T, desc string, url string, routePatte func TestAPI_Annotations_AccessControl(t *testing.T) { sc := setupHTTPServer(t, true) setInitCtxSignedInEditor(sc.initCtx) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) dashboardAnnotation := &annotations.Item{Id: 1, DashboardId: 1} @@ -787,7 +787,7 @@ func TestService_AnnotationTypeScopeResolver(t *testing.T) { func TestAPI_MassDeleteAnnotations_AccessControl(t *testing.T) { sc := setupHTTPServer(t, true) setInitCtxSignedInEditor(sc.initCtx) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) type args struct { diff --git a/pkg/api/org_test.go b/pkg/api/org_test.go index 4dcd2fe2e3e..37b36dd8bde 100644 --- a/pkg/api/org_test.go +++ b/pkg/api/org_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/org/orgimpl" "github.com/grafana/grafana/pkg/services/quota/quotatest" "github.com/grafana/grafana/pkg/services/sqlstore" @@ -51,7 +51,7 @@ func TestAPIEndpoint_GetCurrentOrg_LegacyAccessControl(t *testing.T) { sc := setupHTTPServerWithCfg(t, true, cfg) setInitCtxSignedInViewer(sc.initCtx) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) t.Run("Viewer can view CurrentOrg", func(t *testing.T) { @@ -70,7 +70,7 @@ func TestAPIEndpoint_GetCurrentOrg_AccessControl(t *testing.T) { sc := setupHTTPServer(t, true) setInitCtxSignedInViewer(sc.initCtx) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) t.Run("AccessControl allows viewing CurrentOrg with correct permissions", func(t *testing.T) { @@ -91,11 +91,14 @@ func TestAPIEndpoint_GetCurrentOrg_AccessControl(t *testing.T) { } func TestAPIEndpoint_PutCurrentOrg_LegacyAccessControl(t *testing.T) { + var err error cfg := setting.NewCfg() cfg.RBACEnabled = false sc := setupHTTPServerWithCfg(t, true, cfg) + sc.hs.orgService, err = orgimpl.ProvideService(sc.db, sc.cfg, sc.hs.QuotaService) + require.NoError(t, err) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err = sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) input := strings.NewReader(testUpdateOrgNameForm) @@ -116,13 +119,14 @@ func TestAPIEndpoint_PutCurrentOrg_LegacyAccessControl(t *testing.T) { } func TestAPIEndpoint_PutCurrentOrg_AccessControl(t *testing.T) { + var err error sc := setupHTTPServer(t, true) setInitCtxSignedInViewer(sc.initCtx) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: sc.initCtx.UserID}) + sc.hs.orgService, err = orgimpl.ProvideService(sc.db, sc.cfg, sc.hs.QuotaService) require.NoError(t, err) - sc.hs.orgService, err = orgimpl.ProvideService(sc.db, sc.cfg, quotatest.New(false, nil)) + _, err = sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: sc.initCtx.UserID}) require.NoError(t, err) input := strings.NewReader(testUpdateOrgNameForm) @@ -150,7 +154,7 @@ func TestAPIEndpoint_PutCurrentOrgAddress_LegacyAccessControl(t *testing.T) { cfg.RBACEnabled = false sc := setupHTTPServerWithCfg(t, true, cfg) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) input := strings.NewReader(testUpdateOrgAddressForm) @@ -172,7 +176,7 @@ func TestAPIEndpoint_PutCurrentOrgAddress_AccessControl(t *testing.T) { sc := setupHTTPServer(t, true) setInitCtxSignedInViewer(sc.initCtx) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) input := strings.NewReader(testUpdateOrgAddressForm) @@ -208,7 +212,7 @@ func setupOrgsDBForAccessControlTests(t *testing.T, db *sqlstore.SQLStore, c acc // Create `orgsCount` orgs for i := 1; i <= int(orgID); i++ { - err := db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: fmt.Sprintf("TestOrg%v", i), UserId: 0}) + _, err := c.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: fmt.Sprintf("TestOrg%v", i), UserID: 0}) require.NoError(t, err) } } diff --git a/pkg/api/org_users_test.go b/pkg/api/org_users_test.go index caf88809c64..291d2c3988d 100644 --- a/pkg/api/org_users_test.go +++ b/pkg/api/org_users_test.go @@ -341,9 +341,9 @@ func setupOrgUsersDBForAccessControlTests(t *testing.T, db *sqlstore.SQLStore, o require.NoError(t, err) // Create both orgs with server admin - err = db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: testServerAdminViewer.OrgName, UserId: testServerAdminViewer.UserID}) + _, err = orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: testServerAdminViewer.OrgName, UserID: testServerAdminViewer.UserID}) require.NoError(t, err) - err = db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: testAdminOrg2.OrgName, UserId: testServerAdminViewer.UserID}) + _, err = orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: testAdminOrg2.OrgName, UserID: testServerAdminViewer.UserID}) require.NoError(t, err) err = orgService.AddOrgUser(context.Background(), &org.AddOrgUserCommand{LoginOrEmail: testAdminOrg2.Login, Role: testAdminOrg2.OrgRole, OrgID: testAdminOrg2.OrgID, UserID: testAdminOrg2.UserID}) @@ -728,6 +728,7 @@ func TestOrgUsersAPIEndpointWithSetPerms_AccessControl(t *testing.T) { var err error sc := setupHTTPServer(t, true, func(hs *HTTPServer) { hs.tempUserService = tempuserimpl.ProvideService(hs.SQLStore) + hs.orgService, err = orgimpl.ProvideService(hs.SQLStore, setting.NewCfg(), quotatest.New(false, nil)) hs.userService, err = userimpl.ProvideService( hs.SQLStore, nil, setting.NewCfg(), teamimpl.ProvideService(hs.SQLStore.(*sqlstore.SQLStore), setting.NewCfg()), localcache.ProvideService(), quotatest.New(false, nil)) require.NoError(t, err) diff --git a/pkg/api/preferences_test.go b/pkg/api/preferences_test.go index 2edd6d75951..751f23f9a5a 100644 --- a/pkg/api/preferences_test.go +++ b/pkg/api/preferences_test.go @@ -17,6 +17,7 @@ import ( "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/org" pref "github.com/grafana/grafana/pkg/services/preference" "github.com/grafana/grafana/pkg/services/preference/preftest" ) @@ -51,7 +52,7 @@ func TestAPIEndpoint_GetCurrentOrgPreferences_LegacyAccessControl(t *testing.T) prefService.ExpectedPreference = &pref.Preference{HomeDashboardID: 1, Theme: "dark"} sc.hs.preferenceService = prefService - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) setInitCtxSignedInViewer(sc.initCtx) @@ -80,7 +81,7 @@ func TestAPIEndpoint_GetCurrentOrgPreferences_AccessControl(t *testing.T) { prefService.ExpectedPreference = &pref.Preference{HomeDashboardID: 1, Theme: "dark"} sc.hs.preferenceService = prefService - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) t.Run("AccessControl allows getting org preferences with correct permissions", func(t *testing.T) { @@ -105,7 +106,7 @@ func TestAPIEndpoint_PutCurrentOrgPreferences_LegacyAccessControl(t *testing.T) cfg.RBACEnabled = false sc := setupHTTPServerWithCfg(t, true, cfg) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) setInitCtxSignedInViewer(sc.initCtx) @@ -127,7 +128,7 @@ func TestAPIEndpoint_PutCurrentOrgPreferences_AccessControl(t *testing.T) { sc := setupHTTPServer(t, true) setInitCtxSignedInViewer(sc.initCtx) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) input := strings.NewReader(testUpdateOrgPreferencesCmd) @@ -157,7 +158,7 @@ func TestAPIEndpoint_PatchUserPreferences(t *testing.T) { cfg.RBACEnabled = false sc := setupHTTPServerWithCfg(t, true, cfg) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) setInitCtxSignedInOrgAdmin(sc.initCtx) @@ -190,7 +191,7 @@ func TestAPIEndpoint_PatchOrgPreferences(t *testing.T) { cfg.RBACEnabled = false sc := setupHTTPServerWithCfg(t, true, cfg) - err := sc.db.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "TestOrg", UserId: testUserID}) + _, err := sc.hs.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "TestOrg", UserID: testUserID}) require.NoError(t, err) setInitCtxSignedInOrgAdmin(sc.initCtx) diff --git a/pkg/api/team_members_test.go b/pkg/api/team_members_test.go index 41128c9cf37..5336cdb0768 100644 --- a/pkg/api/team_members_test.go +++ b/pkg/api/team_members_test.go @@ -17,6 +17,8 @@ import ( ac "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/licensing" "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/services/org/orgimpl" + "github.com/grafana/grafana/pkg/services/quota/quotatest" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/sqlstore/mockstore" "github.com/grafana/grafana/pkg/services/team/teamimpl" @@ -123,27 +125,26 @@ func createUser(db sqlstore.Store, orgId int64, t *testing.T) int64 { return user.ID } -func setupTeamTestScenario(userCount int, db *sqlstore.SQLStore, t *testing.T) int64 { +func setupTeamTestScenario(userCount int, db *sqlstore.SQLStore, orgService org.Service, t *testing.T) int64 { teamService := teamimpl.ProvideService(db, setting.NewCfg()) // FIXME user, err := db.CreateUser(context.Background(), user.CreateUserCommand{SkipOrgSetup: true, Login: testUserLogin}) require.NoError(t, err) - cmd := &models.CreateOrgCommand{Name: "TestOrg", UserId: user.ID} - err = db.CreateOrg(context.Background(), cmd) + cmd := &org.CreateOrgCommand{Name: "TestOrg", UserID: user.ID} + testOrg, err := orgService.CreateWithMember(context.Background(), cmd) require.NoError(t, err) - testOrg := cmd.Result - team, err := teamService.CreateTeam("test", "test@test.com", testOrg.Id) + team, err := teamService.CreateTeam("test", "test@test.com", testOrg.ID) require.NoError(t, err) for i := 0; i < userCount; i++ { - userId := createUser(db, testOrg.Id, t) + userId := createUser(db, testOrg.ID, t) require.NoError(t, err) - err = teamService.AddTeamMember(userId, testOrg.Id, team.Id, false, 0) + err = teamService.AddTeamMember(userId, testOrg.ID, team.Id, false, 0) require.NoError(t, err) } - return testOrg.Id + return testOrg.ID } var ( @@ -160,11 +161,12 @@ func TestAddTeamMembersAPIEndpoint_LegacyAccessControl(t *testing.T) { cfg.RBACEnabled = false cfg.EditorsCanAdmin = true sc := setupHTTPServerWithCfg(t, true, cfg) + sc.hs.orgService, _ = orgimpl.ProvideService(sc.db, cfg, quotatest.New(false, nil)) guardian := manager.ProvideService(database.ProvideTeamGuardianStore(sc.db, sc.teamService)) sc.hs.teamGuardian = guardian teamMemberCount := 3 - testOrgId := setupTeamTestScenario(teamMemberCount, sc.db, t) + testOrgId := setupTeamTestScenario(teamMemberCount, sc.db, sc.hs.orgService, t) setInitCtxSignedInOrgAdmin(sc.initCtx) newUserId := createUser(sc.db, testOrgId, t) @@ -207,11 +209,12 @@ func TestAddTeamMembersAPIEndpoint_LegacyAccessControl(t *testing.T) { func TestGetTeamMembersAPIEndpoint_RBAC(t *testing.T) { sc := setupHTTPServer(t, true) + sc.hs.orgService, _ = orgimpl.ProvideService(sc.db, sc.cfg, quotatest.New(false, nil)) sc.hs.License = &licensing.OSSLicensingService{} teamMemberCount := 3 // setupTeamTestScenario sets up 3 user (id: 2,3,4) in the team (id: 1) - testOrgId := setupTeamTestScenario(teamMemberCount, sc.db, t) + testOrgId := setupTeamTestScenario(teamMemberCount, sc.db, sc.hs.orgService, t) setInitCtxSignedInViewer(sc.initCtx) t.Run("Access control allows getting a team members with the right permissions", func(t *testing.T) { @@ -261,10 +264,11 @@ func TestGetTeamMembersAPIEndpoint_RBAC(t *testing.T) { func TestAddTeamMembersAPIEndpoint_RBAC(t *testing.T) { sc := setupHTTPServer(t, true) + sc.hs.orgService, _ = orgimpl.ProvideService(sc.db, sc.cfg, quotatest.New(false, nil)) sc.hs.License = &licensing.OSSLicensingService{} teamMemberCount := 3 - testOrgId := setupTeamTestScenario(teamMemberCount, sc.db, t) + testOrgId := setupTeamTestScenario(teamMemberCount, sc.db, sc.hs.orgService, t) setInitCtxSignedInViewer(sc.initCtx) newUserId := createUser(sc.db, testOrgId, t) @@ -297,11 +301,12 @@ func TestUpdateTeamMembersAPIEndpoint_LegacyAccessControl(t *testing.T) { cfg.RBACEnabled = false cfg.EditorsCanAdmin = true sc := setupHTTPServerWithCfg(t, true, cfg) + sc.hs.orgService, _ = orgimpl.ProvideService(sc.db, cfg, quotatest.New(false, nil)) guardian := manager.ProvideService(database.ProvideTeamGuardianStore(sc.db, sc.teamService)) sc.hs.teamGuardian = guardian teamMemberCount := 3 - setupTeamTestScenario(teamMemberCount, sc.db, t) + setupTeamTestScenario(teamMemberCount, sc.db, sc.hs.orgService, t) setInitCtxSignedInOrgAdmin(sc.initCtx) input := strings.NewReader(fmt.Sprintf(updateTeamMemberCmd, models.PERMISSION_ADMIN)) @@ -342,10 +347,11 @@ func TestUpdateTeamMembersAPIEndpoint_LegacyAccessControl(t *testing.T) { func TestUpdateTeamMembersAPIEndpoint_RBAC(t *testing.T) { sc := setupHTTPServer(t, true) + sc.hs.orgService, _ = orgimpl.ProvideService(sc.db, sc.cfg, quotatest.New(false, nil)) sc.hs.License = &licensing.OSSLicensingService{} teamMemberCount := 3 - setupTeamTestScenario(teamMemberCount, sc.db, t) + setupTeamTestScenario(teamMemberCount, sc.db, sc.hs.orgService, t) setInitCtxSignedInViewer(sc.initCtx) input := strings.NewReader(fmt.Sprintf(updateTeamMemberCmd, models.PERMISSION_ADMIN)) @@ -376,11 +382,12 @@ func TestDeleteTeamMembersAPIEndpoint_LegacyAccessControl(t *testing.T) { cfg.RBACEnabled = false cfg.EditorsCanAdmin = true sc := setupHTTPServerWithCfg(t, true, cfg) + sc.hs.orgService, _ = orgimpl.ProvideService(sc.db, sc.cfg, quotatest.New(false, nil)) guardian := manager.ProvideService(database.ProvideTeamGuardianStore(sc.db, sc.teamService)) sc.hs.teamGuardian = guardian teamMemberCount := 3 - setupTeamTestScenario(teamMemberCount, sc.db, t) + setupTeamTestScenario(teamMemberCount, sc.db, sc.hs.orgService, t) setInitCtxSignedInOrgAdmin(sc.initCtx) t.Run("Organisation admins can remove a team member", func(t *testing.T) { @@ -417,10 +424,11 @@ func TestDeleteTeamMembersAPIEndpoint_LegacyAccessControl(t *testing.T) { func TestDeleteTeamMembersAPIEndpoint_RBAC(t *testing.T) { sc := setupHTTPServer(t, true) + sc.hs.orgService, _ = orgimpl.ProvideService(sc.db, sc.cfg, quotatest.New(false, nil)) sc.hs.License = &licensing.OSSLicensingService{} teamMemberCount := 3 - setupTeamTestScenario(teamMemberCount, sc.db, t) + setupTeamTestScenario(teamMemberCount, sc.db, sc.hs.orgService, t) setInitCtxSignedInViewer(sc.initCtx) t.Run("Access control allows removing a team member with the right permissions", func(t *testing.T) { diff --git a/pkg/services/accesscontrol/resourcepermissions/store_test.go b/pkg/services/accesscontrol/resourcepermissions/store_test.go index a852d79e682..81affa327ab 100644 --- a/pkg/services/accesscontrol/resourcepermissions/store_test.go +++ b/pkg/services/accesscontrol/resourcepermissions/store_test.go @@ -10,8 +10,10 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/infra/db" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/services/org/orgimpl" + "github.com/grafana/grafana/pkg/services/quota/quotatest" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/user" ) @@ -435,8 +437,10 @@ func TestIntegrationStore_GetResourcePermissions(t *testing.T) { for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { store, sql := setupTestEnv(t) + orgService, err := orgimpl.ProvideService(sql, sql.Cfg, quotatest.New(false, nil)) + require.NoError(t, err) - err := sql.WithDbSession(context.Background(), func(sess *db.Session) error { + err = sql.WithDbSession(context.Background(), func(sess *db.Session) error { role := &accesscontrol.Role{ OrgID: tt.user.OrgID, UID: "seeded", @@ -471,7 +475,7 @@ func TestIntegrationStore_GetResourcePermissions(t *testing.T) { }) require.NoError(t, err) - seedResourcePermissions(t, store, sql, tt.query.Actions, tt.query.Resource, tt.query.ResourceID, tt.query.ResourceAttribute, tt.numUsers) + seedResourcePermissions(t, store, sql, orgService, tt.query.Actions, tt.query.Resource, tt.query.ResourceID, tt.query.ResourceAttribute, tt.numUsers) tt.query.User = tt.user permissions, err := store.GetResourcePermissions(context.Background(), tt.user.OrgID, tt.query) @@ -481,21 +485,20 @@ func TestIntegrationStore_GetResourcePermissions(t *testing.T) { } } -func seedResourcePermissions(t *testing.T, store *store, sql *sqlstore.SQLStore, actions []string, resource, resourceID, resourceAttribute string, numUsers int) { +func seedResourcePermissions(t *testing.T, store *store, sql *sqlstore.SQLStore, orgService org.Service, actions []string, resource, resourceID, resourceAttribute string, numUsers int) { t.Helper() - var org *models.Org + var orgModel *org.Org for i := 0; i < numUsers; i++ { - if org == nil { - cmd := &models.CreateOrgCommand{Name: "test", UserId: int64(i)} - err := sql.CreateOrg(context.Background(), cmd) + if orgModel == nil { + cmd := &org.CreateOrgCommand{Name: "test", UserID: int64(i)} + addedOrg, err := orgService.CreateWithMember(context.Background(), cmd) require.NoError(t, err) - addedOrg := cmd.Result - org = &addedOrg + orgModel = addedOrg } u, err := sql.CreateUser(context.Background(), user.CreateUserCommand{ Login: fmt.Sprintf("user:%s%d", resourceID, i), - OrgID: org.Id, + OrgID: orgModel.ID, }) require.NoError(t, err) diff --git a/pkg/services/org/orgimpl/store_test.go b/pkg/services/org/orgimpl/store_test.go index e14d60369f2..ecaf5b34297 100644 --- a/pkg/services/org/orgimpl/store_test.go +++ b/pkg/services/org/orgimpl/store_test.go @@ -173,21 +173,21 @@ func TestIntegrationOrgDataAccess(t *testing.T) { }) t.Run("Testing Account DB Access", func(t *testing.T) { - sqlStore := db.InitTestDB(t) + ss := db.InitTestDB(t) + orgStore = sqlStore{ + db: ss, + dialect: ss.GetDialect(), + } + ids := []int64{} + + for i := 1; i < 4; i++ { + cmd := &org.CreateOrgCommand{Name: fmt.Sprint("Org #", i)} + res, err := orgStore.CreateWithMember(context.Background(), cmd) + require.NoError(t, err) + ids = append(ids, res.ID) + } t.Run("Given we have organizations, we can query them by IDs", func(t *testing.T) { - var err error - var cmd *models.CreateOrgCommand - ids := []int64{} - - for i := 1; i < 4; i++ { - cmd = &models.CreateOrgCommand{Name: fmt.Sprint("Org #", i)} - err = sqlStore.CreateOrg(context.Background(), cmd) - require.NoError(t, err) - - ids = append(ids, cmd.Result.Id) - } - query := &org.SearchOrgsQuery{IDs: ids} queryResult, err := orgStore.Search(context.Background(), query) @@ -195,37 +195,28 @@ func TestIntegrationOrgDataAccess(t *testing.T) { require.Equal(t, len(queryResult), 3) }) - t.Run("Given we have organizations, we can limit and paginate search", func(t *testing.T) { - sqlStore = db.InitTestDB(t) - for i := 1; i < 4; i++ { - cmd := &models.CreateOrgCommand{Name: fmt.Sprint("Org #", i)} - err := sqlStore.CreateOrg(context.Background(), cmd) - require.NoError(t, err) - } + t.Run("Should be able to search with defaults", func(t *testing.T) { + query := &org.SearchOrgsQuery{} + queryResult, err := orgStore.Search(context.Background(), query) - t.Run("Should be able to search with defaults", func(t *testing.T) { - query := &org.SearchOrgsQuery{} - queryResult, err := orgStore.Search(context.Background(), query) + require.NoError(t, err) + require.Equal(t, len(queryResult), 3) + }) - require.NoError(t, err) - require.Equal(t, len(queryResult), 3) - }) + t.Run("Should be able to limit search", func(t *testing.T) { + query := &org.SearchOrgsQuery{Limit: 1} + queryResult, err := orgStore.Search(context.Background(), query) - t.Run("Should be able to limit search", func(t *testing.T) { - query := &org.SearchOrgsQuery{Limit: 1} - queryResult, err := orgStore.Search(context.Background(), query) + require.NoError(t, err) + require.Equal(t, len(queryResult), 1) + }) - require.NoError(t, err) - require.Equal(t, len(queryResult), 1) - }) + t.Run("Should be able to limit and paginate search", func(t *testing.T) { + query := &org.SearchOrgsQuery{Limit: 2, Page: 1} + queryResult, err := orgStore.Search(context.Background(), query) - t.Run("Should be able to limit and paginate search", func(t *testing.T) { - query := &org.SearchOrgsQuery{Limit: 2, Page: 1} - queryResult, err := orgStore.Search(context.Background(), query) - - require.NoError(t, err) - require.Equal(t, len(queryResult), 1) - }) + require.NoError(t, err) + require.Equal(t, len(queryResult), 1) }) }) } diff --git a/pkg/services/provisioning/dashboards/config_reader_test.go b/pkg/services/provisioning/dashboards/config_reader_test.go index 468cff28ed6..0e5676e0542 100644 --- a/pkg/services/provisioning/dashboards/config_reader_test.go +++ b/pkg/services/provisioning/dashboards/config_reader_test.go @@ -3,14 +3,12 @@ package dashboards import ( "context" "errors" - "fmt" "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/org/orgtest" @@ -26,7 +24,7 @@ var ( func TestDashboardsAsConfig(t *testing.T) { t.Run("Dashboards as configuration", func(t *testing.T) { logger := log.New("test-logger") - store := db.InitTestDB(t) + // store := db.InitTestDB(t) orgFake := orgtest.NewOrgServiceFake() t.Run("Should fail if orgs don't exist in the database", func(t *testing.T) { @@ -38,12 +36,6 @@ func TestDashboardsAsConfig(t *testing.T) { orgFake.ExpectedError = nil }) - for i := 1; i <= 2; i++ { - orgCommand := models.CreateOrgCommand{Name: fmt.Sprintf("Main Org. %v", i)} - err := store.CreateOrg(context.Background(), &orgCommand) - require.NoError(t, err) - } - t.Run("default values should be applied", func(t *testing.T) { cfgProvider := configReader{path: appliedDefaults, log: logger, orgService: orgFake} cfg, err := cfgProvider.readConfig(context.Background()) diff --git a/pkg/services/provisioning/notifiers/config_reader_test.go b/pkg/services/provisioning/notifiers/config_reader_test.go index 00d885cb7b6..db4271b6aae 100644 --- a/pkg/services/provisioning/notifiers/config_reader_test.go +++ b/pkg/services/provisioning/notifiers/config_reader_test.go @@ -14,7 +14,8 @@ import ( encryptionservice "github.com/grafana/grafana/pkg/services/encryption/service" "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/services/org" - "github.com/grafana/grafana/pkg/services/org/orgtest" + "github.com/grafana/grafana/pkg/services/org/orgimpl" + "github.com/grafana/grafana/pkg/services/quota/quotatest" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/stretchr/testify/require" @@ -35,24 +36,22 @@ var ( func TestNotificationAsConfig(t *testing.T) { var sqlStore *sqlstore.SQLStore - var orgFake org.Service + var orgService org.Service var ns *alerting.AlertNotificationService logger := log.New("fake.log") - orgService := orgtest.NewOrgServiceFake() - orgService.ExpectedOrg = &org.Org{} encryptionService := encryptionservice.SetupTestService(t) t.Run("Testing notification as configuration", func(t *testing.T) { setup := func() { sqlStore = db.InitTestDB(t) - orgFake = orgtest.NewOrgServiceFake() + orgService, _ = orgimpl.ProvideService(sqlStore, sqlStore.Cfg, quotatest.New(false, nil)) nm := ¬ifications.NotificationService{} ns = alerting.ProvideService(sqlStore, encryptionService, nm) for i := 1; i < 5; i++ { - orgCommand := models.CreateOrgCommand{Name: fmt.Sprintf("Main Org. %v", i)} - err := sqlStore.CreateOrg(context.Background(), &orgCommand) + orgCommand := org.CreateOrgCommand{Name: fmt.Sprintf("Main Org. %v", i)} + _, err := orgService.CreateWithMember(context.Background(), &orgCommand) require.NoError(t, err) } @@ -73,7 +72,7 @@ func TestNotificationAsConfig(t *testing.T) { setup() _ = os.Setenv("TEST_VAR", "default") cfgProvider := &configReader{ - orgService: orgFake, + orgService: orgService, encryptionService: encryptionService, log: log.New("test logger"), } @@ -282,7 +281,7 @@ func TestNotificationAsConfig(t *testing.T) { t.Run("Broken yaml should return error", func(t *testing.T) { reader := &configReader{ - orgService: orgFake, + orgService: orgService, encryptionService: encryptionService, log: log.New("test logger"), } @@ -293,7 +292,7 @@ func TestNotificationAsConfig(t *testing.T) { t.Run("Skip invalid directory", func(t *testing.T) { cfgProvider := &configReader{ - orgService: orgFake, + orgService: orgService, encryptionService: encryptionService, log: log.New("test logger"), } @@ -307,7 +306,7 @@ func TestNotificationAsConfig(t *testing.T) { t.Run("Unknown notifier should return error", func(t *testing.T) { cfgProvider := &configReader{ - orgService: orgFake, + orgService: orgService, encryptionService: encryptionService, log: log.New("test logger"), } @@ -318,7 +317,7 @@ func TestNotificationAsConfig(t *testing.T) { t.Run("Read incorrect properties", func(t *testing.T) { cfgProvider := &configReader{ - orgService: orgFake, + orgService: orgService, encryptionService: encryptionService, log: log.New("test logger"), } diff --git a/pkg/services/serviceaccounts/api/api_test.go b/pkg/services/serviceaccounts/api/api_test.go index 49cc8a63329..92130e02611 100644 --- a/pkg/services/serviceaccounts/api/api_test.go +++ b/pkg/services/serviceaccounts/api/api_test.go @@ -61,8 +61,8 @@ func TestServiceAccountsAPI_CreateServiceAccount(t *testing.T) { store.Cfg.AutoAssignOrg = autoAssignOrg }() - orgCmd := &models.CreateOrgCommand{Name: "Some Test Org"} - err = store.CreateOrg(context.Background(), orgCmd) + orgCmd := &org.CreateOrgCommand{Name: "Some Test Org"} + _, err = orgService.CreateWithMember(context.Background(), orgCmd) require.Nil(t, err) type testCreateSATestCase struct { diff --git a/pkg/services/serviceaccounts/database/database_test.go b/pkg/services/serviceaccounts/database/database_test.go index be9011ed9bc..49be2c2c536 100644 --- a/pkg/services/serviceaccounts/database/database_test.go +++ b/pkg/services/serviceaccounts/database/database_test.go @@ -10,7 +10,6 @@ import ( "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/kvstore" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/apikey/apikeyimpl" "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/org/orgimpl" @@ -43,13 +42,13 @@ func TestStore_CreateServiceAccountOrgNonExistant(t *testing.T) { func TestStore_CreateServiceAccount(t *testing.T) { _, store := setupTestDatabase(t) - orgQuery := &models.CreateOrgCommand{Name: sqlstore.MainOrgName} - err := store.sqlStore.CreateOrg(context.Background(), orgQuery) + orgQuery := &org.CreateOrgCommand{Name: sqlstore.MainOrgName} + orgResult, err := store.orgService.CreateWithMember(context.Background(), orgQuery) require.NoError(t, err) t.Run("create service account", func(t *testing.T) { serviceAccountName := "new Service Account" - serviceAccountOrgId := orgQuery.Result.Id + serviceAccountOrgId := orgResult.ID serviceAccountRole := org.RoleAdmin isDisabled := true saForm := serviceaccounts.CreateServiceAccountForm{ @@ -175,7 +174,7 @@ func TestStore_MigrateApiKeys(t *testing.T) { store.sqlStore.Cfg.AutoAssignOrg = true store.sqlStore.Cfg.AutoAssignOrgId = 1 store.sqlStore.Cfg.AutoAssignOrgRole = "Viewer" - err := store.sqlStore.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "main"}) + _, err := store.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "main"}) require.NoError(t, err) key := tests.SetupApiKey(t, db, c.key) err = store.MigrateApiKey(context.Background(), key.OrgId, key.Id) @@ -252,7 +251,7 @@ func TestStore_MigrateAllApiKeys(t *testing.T) { store.sqlStore.Cfg.AutoAssignOrg = true store.sqlStore.Cfg.AutoAssignOrgId = 1 store.sqlStore.Cfg.AutoAssignOrgRole = "Viewer" - err := store.sqlStore.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "main"}) + _, err := store.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "main"}) require.NoError(t, err) for _, key := range c.keys { @@ -314,7 +313,7 @@ func TestStore_RevertApiKey(t *testing.T) { store.sqlStore.Cfg.AutoAssignOrg = true store.sqlStore.Cfg.AutoAssignOrgId = 1 store.sqlStore.Cfg.AutoAssignOrgRole = "Viewer" - err := store.sqlStore.CreateOrg(context.Background(), &models.CreateOrgCommand{Name: "main"}) + _, err := store.orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "main"}) require.NoError(t, err) key := tests.SetupApiKey(t, db, c.key) From 254577ba568d8010dcc96446e788866d618c96c8 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 28 Nov 2022 12:39:12 +0100 Subject: [PATCH 020/131] CloudWatch: Cross-account querying support (#59362) * Lattice: Point to private prerelease of aws-sdk-go (#515) * point to private prerelease of aws-sdk-go * fix build issue * Lattice: Adding a feature toggle (#549) * Adding a feature toggle for lattice * Change name of feature toggle * Lattice: List accounts (#543) * Separate layers * Introduce testify/mock library Co-authored-by: Shirley Leu <4163034+fridgepoet@users.noreply.github.com> * point to version that includes metric api changes (#574) * add accounts component (#575) * Test refactor: remove unneeded clientFactoryMock (#581) * Lattice: Add monitoring badge (#576) * add monitoring badge * fix tests * solve conflict * Lattice: Add dynamic label for account display name (#579) * Build: Automatically sync lattice-main with OSS * Lattice: Point to private prerelease of aws-sdk-go (#515) * point to private prerelease of aws-sdk-go * fix build issue * Lattice: Adding a feature toggle (#549) * Adding a feature toggle for lattice * Change name of feature toggle * Lattice: List accounts (#543) * Separate layers * Introduce testify/mock library Co-authored-by: Shirley Leu <4163034+fridgepoet@users.noreply.github.com> * point to version that includes metric api changes (#574) * add accounts component (#575) * Test refactor: remove unneeded clientFactoryMock (#581) * Lattice: Add monitoring badge (#576) * add monitoring badge * fix tests * solve conflict * add account label Co-authored-by: Shirley Leu <4163034+fridgepoet@users.noreply.github.com> Co-authored-by: Sarah Zinger * fix import * solve merge related problem * add account info (#608) * add back namespaces handler * Lattice: Parse account id and return it to frontend (#609) * parse account id and return to frontend * fix route test * only show badge when feature toggle is enabled (#615) * Lattice: Refactor resource response type and return account (#613) * refactor resource response type * remove not used file. * go lint * fix tests * remove commented code * Lattice: Use account as input when listing metric names and dimensions (#611) * use account in resource requests * add account to response * revert accountInfo to accountId * PR feedback * unit test account in list metrics response * remove not used asserts * don't assert on response that is not relevant to the test * removed dupe test * pr feedback * rename request package (#626) * Lattice: Move account component and add tooltip (#630) * move accounts component to the top of metric stat editor * add tooltip * CloudWatch: add account to GetMetricData queries (#627) * Add AccountId to metric stat query * Lattice: Account variable support (#625) * add variable support in accounts component * add account variable query type * update variables * interpolate variable before its sent to backend * handle variable change in hooks * remove not used import * Update public/app/plugins/datasource/cloudwatch/components/Account.tsx Co-authored-by: Sarah Zinger * Update public/app/plugins/datasource/cloudwatch/hooks.ts Co-authored-by: Sarah Zinger * add one more unit test Co-authored-by: Sarah Zinger * cleanup (#629) * Set account Id according to crossAccountQuerying feature flag in backend (#632) * CloudWatch: Change spelling of feature-toggle (#634) * Lattice Logs (#631) * Lattice Logs * Fixes after CR * Lattice: Bug: fix dimension keys request (#644) * fix dimension keys * fix lint * more lint * CloudWatch: Add tests for QueryData with AccountId (#637) * Update from breaking change (#645) * Update from breaking change * Remove extra interface and methods Co-authored-by: Shirley Leu <4163034+fridgepoet@users.noreply.github.com> * CloudWatch: Add business logic layer for getting log groups (#642) Co-authored-by: Sarah Zinger * Lattice: Fix - unset account id in region change handler (#646) * move reset of account to region change handler * fix broken test * Lattice: Add account id to metric stat query deep link (#656) add account id to metric stat link * CloudWatch: Add new log groups handler for cross-account querying (#643) * Lattice: Add feature tracking (#660) * add tracking for account id prescense in metrics query * also check feature toggle * fix broken test * CloudWatch: Add route for DescribeLogGroups for cross-account querying (#647) Co-authored-by: Erik Sundell * Lattice: Handle account id default value (#662) * make sure right type is returned * set right default values * Suggestions to lattice changes (#663) * Change ListMetricsWithPageLimit response to slice of non-pointers * Change GetAccountsForCurrentUserOrRole response to be not pointer * Clean test Cleanup calls in test * Remove CloudWatchAPI as part of mock * Resolve conflicts * Add Latest SDK (#672) * add tooltip (#674) * Docs: Add documentation for CloudWatch cross account querying (#676) * wip docs * change wordings * add sections about metrics and logs * change from monitoring to observability * Update docs/sources/datasources/aws-cloudwatch/_index.md Co-authored-by: Sarah Zinger * Update docs/sources/datasources/aws-cloudwatch/query-editor/index.md Co-authored-by: Fiona Artiaga <89225282+GrafanaWriter@users.noreply.github.com> * Update docs/sources/datasources/aws-cloudwatch/query-editor/index.md Co-authored-by: Fiona Artiaga <89225282+GrafanaWriter@users.noreply.github.com> * Update docs/sources/datasources/aws-cloudwatch/query-editor/index.md Co-authored-by: Sarah Zinger * Update docs/sources/datasources/aws-cloudwatch/query-editor/index.md Co-authored-by: Fiona Artiaga <89225282+GrafanaWriter@users.noreply.github.com> * apply pr feedback * fix file name * more pr feedback * pr feedback Co-authored-by: Sarah Zinger Co-authored-by: Fiona Artiaga <89225282+GrafanaWriter@users.noreply.github.com> * use latest version of the aws-sdk-go * Fix tests' mock response type * Remove change in Azure Monitor Co-authored-by: Sarah Zinger Co-authored-by: Shirley Leu <4163034+fridgepoet@users.noreply.github.com> Co-authored-by: Fiona Artiaga <89225282+GrafanaWriter@users.noreply.github.com> --- .betterer.results | 3 +- .../datasources/aws-cloudwatch/_index.md | 15 ++ .../aws-cloudwatch/query-editor/index.md | 27 ++ go.mod | 6 +- go.sum | 13 +- .../src/types/featureToggles.gen.ts | 1 + pkg/services/featuremgmt/registry.go | 5 + pkg/services/featuremgmt/toggles_gen.go | 4 + pkg/tsdb/cloudwatch/clients/metrics.go | 13 +- pkg/tsdb/cloudwatch/clients/metrics_test.go | 30 +++ pkg/tsdb/cloudwatch/cloudwatch.go | 29 ++- pkg/tsdb/cloudwatch/cloudwatch_test.go | 96 ++++++- pkg/tsdb/cloudwatch/log_actions.go | 25 +- pkg/tsdb/cloudwatch/log_actions_test.go | 72 ++++++ .../cloudwatch/metric_data_query_builder.go | 14 +- .../metric_data_query_builder_test.go | 99 +++++++- pkg/tsdb/cloudwatch/metric_find_query.go | 2 - pkg/tsdb/cloudwatch/mocks/accounts_service.go | 16 ++ .../cloudwatch/mocks/cloudwatch_metric_api.go | 6 +- .../cloudwatch/mocks/list_metrics_service.go | 20 +- pkg/tsdb/cloudwatch/mocks/logs.go | 37 +++ pkg/tsdb/cloudwatch/mocks/metrics_client.go | 5 +- pkg/tsdb/cloudwatch/mocks/oam_client.go | 20 ++ pkg/tsdb/cloudwatch/models/api.go | 44 +++- .../cloudwatch/models/cloudwatch_query.go | 21 +- .../models/cloudwatch_query_test.go | 164 +++++++++--- ...st.go => dimension_values_request_test.go} | 0 .../resources/log_groups_resource_request.go | 53 ++++ .../resources/metrics_resource_request.go | 6 +- .../models/resources/resource_request.go | 24 +- pkg/tsdb/cloudwatch/models/resources/types.go | 24 ++ pkg/tsdb/cloudwatch/models/types.go | 22 +- pkg/tsdb/cloudwatch/resource_handler.go | 2 + pkg/tsdb/cloudwatch/routes/accounts.go | 55 ++++ pkg/tsdb/cloudwatch/routes/accounts_test.go | 96 +++++++ pkg/tsdb/cloudwatch/routes/dimension_keys.go | 2 +- .../cloudwatch/routes/dimension_keys_test.go | 8 +- .../routes/dimension_values_test.go | 4 +- pkg/tsdb/cloudwatch/routes/log_groups.go | 49 ++++ pkg/tsdb/cloudwatch/routes/log_groups_test.go | 240 ++++++++++++++++++ pkg/tsdb/cloudwatch/routes/metrics.go | 10 +- pkg/tsdb/cloudwatch/routes/metrics_test.go | 27 +- pkg/tsdb/cloudwatch/routes/namespaces.go | 14 +- pkg/tsdb/cloudwatch/routes/namespaces_test.go | 17 +- pkg/tsdb/cloudwatch/services/accounts.go | 89 +++++++ pkg/tsdb/cloudwatch/services/accounts_test.go | 165 ++++++++++++ .../cloudwatch/services/hardcoded_metrics.go | 24 +- .../services/hardcoded_metrics_test.go | 4 +- pkg/tsdb/cloudwatch/services/list_metrics.go | 63 ++--- .../cloudwatch/services/list_metrics_test.go | 170 ++++++++++--- pkg/tsdb/cloudwatch/services/logs.go | 53 ++++ pkg/tsdb/cloudwatch/services/logs_test.go | 200 +++++++++++++++ pkg/tsdb/cloudwatch/services/utils.go | 27 ++ pkg/tsdb/cloudwatch/time_series_query.go | 4 +- pkg/tsdb/cloudwatch/time_series_query_test.go | 183 +++++++++++++ pkg/tsdb/cloudwatch/utils/utils.go | 3 + pkg/tsdb/cloudwatch/utils_test.go | 3 - .../plugins/datasource/cloudwatch/Search.tsx | 39 +++ .../__mocks__/CloudWatchDataSource.ts | 16 +- .../__mocks__/dashboardOnLoadedEvent.ts | 25 ++ .../cloudwatch/__mocks__/queries.ts | 68 ++++- .../plugins/datasource/cloudwatch/api.test.ts | 24 +- .../app/plugins/datasource/cloudwatch/api.ts | 68 +++-- .../cloudwatch/components/Account.test.tsx | 65 +++++ .../cloudwatch/components/Account.tsx | 54 ++++ .../components/AnnotationQueryEditor.test.tsx | 7 + .../CrossAccountLogsQueryField.test.tsx | 190 ++++++++++++++ .../components/CrossAccountLogsQueryField.tsx | 179 +++++++++++++ .../components/Dimensions/FilterItem.tsx | 12 +- .../components/LogGroupSelection.test.tsx | 86 +++++++ .../components/LogGroupSelection.tsx | 64 +++++ .../components/LogGroupSelector.tsx | 6 + .../cloudwatch/components/LogsQueryField.tsx | 31 +-- .../MetricStatEditor.test.tsx | 56 ++++ .../MetricStatEditor/MetricStatEditor.tsx | 40 ++- .../MetricsQueryEditor.test.tsx | 1 + .../MetricsQueryHeader.test.tsx | 4 + .../MetricsQueryEditor/MetricsQueryHeader.tsx | 18 +- .../components/PanelQueryEditor.test.tsx | 86 +++++++ .../components/QueryHeader.test.tsx | 106 ++++++++ .../cloudwatch/components/QueryHeader.tsx | 39 ++- .../SQLBuilderEditor.test.tsx | 2 +- .../SQLBuilderEditor/SQLBuilderSelectRow.tsx | 2 +- .../cloudwatch/components/Search.test.tsx | 42 +++ .../VariableQueryEditor.tsx | 7 +- .../cloudwatch/components/styles.ts | 80 ++++++ .../datasource/cloudwatch/datasource.test.ts | 16 +- .../datasource/cloudwatch/datasource.ts | 4 +- .../cloudwatch/dynamic-labels/language.ts | 3 + .../datasource/cloudwatch/hooks.test.ts | 141 ++++++++++ .../plugins/datasource/cloudwatch/hooks.ts | 110 +++++++- .../query-runner/CloudWatchLogsQueryRunner.ts | 17 +- .../CloudWatchMetricsQueryRunner.test.ts | 20 ++ .../CloudWatchMetricsQueryRunner.ts | 42 ++- .../query-runner/CloudWatchRequest.ts | 4 + .../datasource/cloudwatch/tracking.test.ts | 14 +- .../plugins/datasource/cloudwatch/tracking.ts | 9 +- .../plugins/datasource/cloudwatch/types.ts | 56 ++-- .../datasource/cloudwatch/variables.test.ts | 24 ++ .../datasource/cloudwatch/variables.ts | 115 ++++----- 100 files changed, 3945 insertions(+), 475 deletions(-) create mode 100644 pkg/tsdb/cloudwatch/mocks/accounts_service.go create mode 100644 pkg/tsdb/cloudwatch/mocks/logs.go create mode 100644 pkg/tsdb/cloudwatch/mocks/oam_client.go rename pkg/tsdb/cloudwatch/models/resources/{dimension_values_rquest_test.go => dimension_values_request_test.go} (100%) create mode 100644 pkg/tsdb/cloudwatch/models/resources/log_groups_resource_request.go create mode 100644 pkg/tsdb/cloudwatch/routes/accounts.go create mode 100644 pkg/tsdb/cloudwatch/routes/accounts_test.go create mode 100644 pkg/tsdb/cloudwatch/routes/log_groups.go create mode 100644 pkg/tsdb/cloudwatch/routes/log_groups_test.go create mode 100644 pkg/tsdb/cloudwatch/services/accounts.go create mode 100644 pkg/tsdb/cloudwatch/services/accounts_test.go create mode 100644 pkg/tsdb/cloudwatch/services/logs.go create mode 100644 pkg/tsdb/cloudwatch/services/logs_test.go create mode 100644 pkg/tsdb/cloudwatch/services/utils.go create mode 100644 pkg/tsdb/cloudwatch/utils/utils.go create mode 100644 public/app/plugins/datasource/cloudwatch/Search.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/Account.test.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/Account.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/CrossAccountLogsQueryField.test.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/CrossAccountLogsQueryField.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/LogGroupSelection.test.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/LogGroupSelection.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/Search.test.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/styles.ts create mode 100644 public/app/plugins/datasource/cloudwatch/hooks.test.ts diff --git a/.betterer.results b/.betterer.results index fd31bfb5837..c79fb67538c 100644 --- a/.betterer.results +++ b/.betterer.results @@ -5373,8 +5373,7 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "2"] ], "public/app/plugins/datasource/cloudwatch/components/QueryHeader.tsx:5381": [ - [0, 0, 0, "Do not use any type assertions.", "0"], - [0, 0, 0, "Do not use any type assertions.", "1"] + [0, 0, 0, "Do not use any type assertions.", "0"] ], "public/app/plugins/datasource/cloudwatch/datasource.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"] diff --git a/docs/sources/datasources/aws-cloudwatch/_index.md b/docs/sources/datasources/aws-cloudwatch/_index.md index 2c2ce5f34bd..6a384f95ea6 100644 --- a/docs/sources/datasources/aws-cloudwatch/_index.md +++ b/docs/sources/datasources/aws-cloudwatch/_index.md @@ -168,6 +168,21 @@ You can attach these permissions to the IAM role or IAM user you configured in [ } ``` +**Cross-account observability:** + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": ["oam:ListSinks", "oam:ListAttachedLinks"], + "Effect": "Allow", + "Resource": "*" + } + ] +} +``` + ### Configure CloudWatch settings #### Namespaces of Custom Metrics diff --git a/docs/sources/datasources/aws-cloudwatch/query-editor/index.md b/docs/sources/datasources/aws-cloudwatch/query-editor/index.md index d58c0402c6e..5110aeab486 100644 --- a/docs/sources/datasources/aws-cloudwatch/query-editor/index.md +++ b/docs/sources/datasources/aws-cloudwatch/query-editor/index.md @@ -214,6 +214,33 @@ When making `stats` queries in [Explore]({{< relref "../../../explore/" >}}), ma {{< figure src="/static/img/docs/v70/explore-mode-switcher.png" max-width="500px" class="docs-image--right" caption="Explore mode switcher" >}} +## Cross-account observability + +The CloudWatch plugin provides the ability to monitor and troubleshoot applications that span across multiple accounts within a region. Using cross-account observability, you can seamlessly search, visualize and analyze metrics and logs, without having to worry about account boundaries. + +> **Note:** This feature is currently behind the `cloudWatchCrossAccountQuerying` feature toggle. + +> You can enable feature toggles through configuration file or environment variables. See configuration [docs]({{< relref "../setup-grafana/configure-grafana/#feature_toggles" >}}) for details. +> Grafana Cloud users can access this feature by [opening a support ticket in the Cloud Portal](https://grafana.com/profile/org#support). + +### Getting started + +To enable cross-account observability, first enable it in CloudWatch using the official [CloudWatch docs](http://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account.html), then add [two new API actions]({{< relref "../#cross-account-observability" >}}) to the IAM policy attached to the role/user running the plugin. + +Cross-account querying is available in the plugin through the `Logs` mode and the `Metric search` mode. Once you have it configured correctly, you'll see a "Monitoring account" badge displayed in the query editor header. + +{{< figure src="/static/img/docs/cloudwatch/cloudwatch-monitoring-badge-9.3.0.png" max-width="1200px" caption="Monitoring account badge" >}} + +### Metrics editor + +When you select the `Builder` mode within the Metric search editor, a new Account field displays. Use the Account field to specify which of the linked accounts to target for the given query. By default, the `All` option is specified, which will target all linked accounts. + +While in `Code` mode, you can specify any math expression. If the Monitoring account badge displays in the query editor header, all `SEARCH` expressions entered in this field will be cross-account by default. You can limit the search to one or a set of accounts, as documented in the [AWS documentation](http://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account.html). + +### Logs editor + +The Log group selector allows you to specify what log groups to target in the logs query. If the Monitoring account badge is displayed in the query editor header, it is possible to search and select log groups across multiple accounts. You can use the Account field in the Log Group Selector to filter Log Groups by Account. If you have many log groups and do not see the log group you'd like to select in the selector, use the prefix search to narrow down the possible log groups. + ### Deep-link Grafana panels to the CloudWatch console {{< figure src="/static/img/docs/v70/cloudwatch-logs-deep-linking.png" max-width="500px" class="docs-image--right" caption="CloudWatch Logs deep linking" >}} diff --git a/go.mod b/go.mod index c04c7037488..15c6a52fe6e 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/BurntSushi/toml v1.1.0 github.com/Masterminds/semver v1.5.0 github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f - github.com/aws/aws-sdk-go v1.44.109 + github.com/aws/aws-sdk-go v1.44.146 github.com/beevik/etree v1.1.0 github.com/benbjohnson/clock v1.3.0 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b @@ -108,7 +108,7 @@ require ( go.opentelemetry.io/otel/trace v1.7.0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d - golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect + golang.org/x/net v0.1.0 // indirect golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 golang.org/x/sync v0.1.0 golang.org/x/time v0.0.0-20220609170525-579cf78fd858 @@ -230,7 +230,7 @@ require ( go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 go.uber.org/goleak v1.1.12 // indirect - golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect + golang.org/x/sys v0.1.0 // indirect golang.org/x/text v0.4.0 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index c55c5495966..cb755504a11 100644 --- a/go.sum +++ b/go.sum @@ -373,8 +373,9 @@ github.com/aws/aws-sdk-go v1.38.60/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.37/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.109 h1:+Na5JPeS0kiEHoBp5Umcuuf+IDqXqD0lXnM920E31YI= github.com/aws/aws-sdk-go v1.44.109/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.146 h1:7YdGgPxDPRJu/yYffzZp/H7yHzQ6AqmuNFZPYraaN8I= +github.com/aws/aws-sdk-go v1.44.146/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4= github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA= @@ -2865,8 +2866,8 @@ golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -3087,14 +3088,16 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 5d9debb1609..c80c834dff4 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -68,6 +68,7 @@ export interface FeatureToggles { objectStore?: boolean; traceqlEditor?: boolean; flameGraph?: boolean; + cloudWatchCrossAccountQuerying?: boolean; redshiftAsyncQueryDataSupport?: boolean; athenaAsyncQueryDataSupport?: boolean; increaseInMemDatabaseQueryCache?: boolean; diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index b08a0bc4a34..f015cf5afd5 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -295,6 +295,11 @@ var ( Description: "Show the flame graph", State: FeatureStateAlpha, }, + { + Name: "cloudWatchCrossAccountQuerying", + Description: "Use cross-account querying in CloudWatch datasource", + State: FeatureStateAlpha, + }, { Name: "redshiftAsyncQueryDataSupport", Description: "Enable async query data support for Redshift", diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 1c503d4867f..1e8aaa72ce4 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -215,6 +215,10 @@ const ( // Show the flame graph FlagFlameGraph = "flameGraph" + // FlagCloudWatchCrossAccountQuerying + // Use cross-account querying in CloudWatch datasource + FlagCloudWatchCrossAccountQuerying = "cloudWatchCrossAccountQuerying" + // FlagRedshiftAsyncQueryDataSupport // Enable async query data support for Redshift FlagRedshiftAsyncQueryDataSupport = "redshiftAsyncQueryDataSupport" diff --git a/pkg/tsdb/cloudwatch/clients/metrics.go b/pkg/tsdb/cloudwatch/clients/metrics.go index d205c968cee..f49743d49ea 100644 --- a/pkg/tsdb/cloudwatch/clients/metrics.go +++ b/pkg/tsdb/cloudwatch/clients/metrics.go @@ -6,6 +6,7 @@ import ( "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" ) type metricsClient struct { @@ -17,16 +18,20 @@ func NewMetricsClient(api models.CloudWatchMetricsAPIProvider, config *setting.C return &metricsClient{CloudWatchMetricsAPIProvider: api, config: config} } -func (l *metricsClient) ListMetricsWithPageLimit(params *cloudwatch.ListMetricsInput) ([]*cloudwatch.Metric, error) { - var cloudWatchMetrics []*cloudwatch.Metric +func (l *metricsClient) ListMetricsWithPageLimit(params *cloudwatch.ListMetricsInput) ([]resources.MetricResponse, error) { + var cloudWatchMetrics []resources.MetricResponse pageNum := 0 err := l.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { pageNum++ metrics.MAwsCloudWatchListMetrics.Inc() metrics, err := awsutil.ValuesAtPath(page, "Metrics") if err == nil { - for _, metric := range metrics { - cloudWatchMetrics = append(cloudWatchMetrics, metric.(*cloudwatch.Metric)) + for idx, metric := range metrics { + metric := resources.MetricResponse{Metric: metric.(*cloudwatch.Metric)} + if len(page.OwningAccounts) >= idx && params.IncludeLinkedAccounts != nil && *params.IncludeLinkedAccounts { + metric.AccountId = page.OwningAccounts[idx] + } + cloudWatchMetrics = append(cloudWatchMetrics, metric) } } return !lastPage && pageNum < l.config.AWSListMetricsPageLimit diff --git a/pkg/tsdb/cloudwatch/clients/metrics_test.go b/pkg/tsdb/cloudwatch/clients/metrics_test.go index 91636775ba5..0a28e34c5a2 100644 --- a/pkg/tsdb/cloudwatch/clients/metrics_test.go +++ b/pkg/tsdb/cloudwatch/clients/metrics_test.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -46,4 +47,33 @@ func TestMetricsClient(t *testing.T) { assert.Equal(t, len(metrics), len(response)) }) + + t.Run("Should return account id in case IncludeLinkedAccounts is set to true", func(t *testing.T) { + fakeApi := &mocks.FakeMetricsAPI{Metrics: []*cloudwatch.Metric{ + {MetricName: aws.String("Test_MetricName1")}, + {MetricName: aws.String("Test_MetricName2")}, + {MetricName: aws.String("Test_MetricName3")}, + }, OwningAccounts: []*string{aws.String("1234567890"), aws.String("1234567890"), aws.String("1234567895")}} + client := NewMetricsClient(fakeApi, &setting.Cfg{AWSListMetricsPageLimit: 100}) + + response, err := client.ListMetricsWithPageLimit(&cloudwatch.ListMetricsInput{IncludeLinkedAccounts: aws.Bool(true)}) + require.NoError(t, err) + expected := []resources.MetricResponse{ + {Metric: &cloudwatch.Metric{MetricName: aws.String("Test_MetricName1")}, AccountId: stringPtr("1234567890")}, + {Metric: &cloudwatch.Metric{MetricName: aws.String("Test_MetricName2")}, AccountId: stringPtr("1234567890")}, + {Metric: &cloudwatch.Metric{MetricName: aws.String("Test_MetricName3")}, AccountId: stringPtr("1234567895")}, + } + assert.Equal(t, expected, response) + }) + + t.Run("Should not return account id in case IncludeLinkedAccounts is set to false", func(t *testing.T) { + fakeApi := &mocks.FakeMetricsAPI{Metrics: []*cloudwatch.Metric{{MetricName: aws.String("Test_MetricName1")}}, OwningAccounts: []*string{aws.String("1234567890")}} + client := NewMetricsClient(fakeApi, &setting.Cfg{AWSListMetricsPageLimit: 100}) + + response, err := client.ListMetricsWithPageLimit(&cloudwatch.ListMetricsInput{IncludeLinkedAccounts: aws.Bool(false)}) + require.NoError(t, err) + assert.Nil(t, response[0].AccountId) + }) } + +func stringPtr(s string) *string { return &s } diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index e78dfb8318f..e0e5843815a 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "net/http" - "net/url" "regexp" "time" @@ -18,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/oam" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface" "github.com/grafana/grafana-aws-sdk/pkg/awsds" @@ -26,7 +26,6 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt" "github.com/grafana/grafana-plugin-sdk-go/backend/resource/httpadapter" "github.com/grafana/grafana-plugin-sdk-go/data" - "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/featuremgmt" @@ -122,8 +121,11 @@ func (e *cloudWatchExecutor) getRequestContext(pluginCtx backend.PluginContext, return models.RequestContext{}, err } return models.RequestContext{ + OAMClientProvider: NewOAMAPI(sess), MetricsClientProvider: clients.NewMetricsClient(NewMetricsAPI(sess), e.cfg), + LogsAPIProvider: NewLogsAPI(sess), Settings: instance.Settings, + Features: e.features, }, nil } @@ -178,11 +180,12 @@ func (e *cloudWatchExecutor) checkHealthMetrics(pluginCtx backend.PluginContext) } func (e *cloudWatchExecutor) checkHealthLogs(pluginCtx backend.PluginContext) error { - parameters := url.Values{ - "limit": []string{"1"}, + session, err := e.newSession(pluginCtx, defaultRegion) + if err != nil { + return err } - - _, err := e.handleGetLogGroups(pluginCtx, parameters) + logsClient := NewLogsAPI(session) + _, err = logsClient.DescribeLogGroups(&cloudwatchlogs.DescribeLogGroupsInput{Limit: aws.Int64(1)}) return err } @@ -423,6 +426,20 @@ var NewMetricsAPI = func(sess *session.Session) models.CloudWatchMetricsAPIProvi return cloudwatch.New(sess) } +// NewLogsAPI is a CloudWatch logs api factory. +// +// Stubbable by tests. +var NewLogsAPI = func(sess *session.Session) models.CloudWatchLogsAPIProvider { + return cloudwatchlogs.New(sess) +} + +// NewOAMAPI is a CloudWatch OAM api factory. +// +// Stubbable by tests. +var NewOAMAPI = func(sess *session.Session) models.OAMClientProvider { + return oam.New(sess) +} + // NewCWClient is a CloudWatch client factory. // // Stubbable by tests. diff --git a/pkg/tsdb/cloudwatch/cloudwatch_test.go b/pkg/tsdb/cloudwatch/cloudwatch_test.go index bca9bbe4e80..a11c15889fe 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch_test.go +++ b/pkg/tsdb/cloudwatch/cloudwatch_test.go @@ -24,7 +24,9 @@ import ( "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/utils" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -97,16 +99,18 @@ func TestNewInstanceSettings(t *testing.T) { func Test_CheckHealth(t *testing.T) { origNewMetricsAPI := NewMetricsAPI origNewCWLogsClient := NewCWLogsClient + origNewLogsAPI := NewLogsAPI t.Cleanup(func() { NewMetricsAPI = origNewMetricsAPI NewCWLogsClient = origNewCWLogsClient + NewLogsAPI = origNewLogsAPI }) var client fakeCheckHealthClient NewMetricsAPI = func(sess *session.Session) models.CloudWatchMetricsAPIProvider { return client } - NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI { + NewLogsAPI = func(sess *session.Session) models.CloudWatchLogsAPIProvider { return client } @@ -536,16 +540,86 @@ func TestQuery_ResourceRequest_DescribeLogGroups(t *testing.T) { }) } +func TestQuery_ResourceRequest_DescribeLogGroups_with_CrossAccountQuerying(t *testing.T) { + sender := &mockedCallResourceResponseSenderForOauth{} + origNewMetricsAPI := NewMetricsAPI + origNewOAMAPI := NewOAMAPI + origNewLogsAPI := NewLogsAPI + NewMetricsAPI = func(sess *session.Session) models.CloudWatchMetricsAPIProvider { return nil } + NewOAMAPI = func(sess *session.Session) models.OAMClientProvider { return nil } + t.Cleanup(func() { + NewOAMAPI = origNewOAMAPI + NewMetricsAPI = origNewMetricsAPI + NewLogsAPI = origNewLogsAPI + }) + + var logsApi mocks.LogsAPI + NewLogsAPI = func(sess *session.Session) models.CloudWatchLogsAPIProvider { + return &logsApi + } + + im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { + return DataSource{Settings: models.CloudWatchSettings{}}, nil + }) + + t.Run("maps log group api response to resource response of describe-log-groups", func(t *testing.T) { + logsApi = mocks.LogsAPI{} + logsApi.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []*cloudwatchlogs.LogGroup{ + {Arn: aws.String("arn:aws:logs:us-east-1:111:log-group:group_a"), LogGroupName: aws.String("group_a")}, + }, + }, nil) + req := &backend.CallResourceRequest{ + Method: "GET", + Path: `/describe-log-groups?logGroupPattern=some-pattern&accountId=some-account-id`, + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ID: 0}, + PluginID: "cloudwatch", + }, + } + + executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudWatchCrossAccountQuerying)) + err := executor.CallResource(context.Background(), req, sender) + assert.NoError(t, err) + + assert.JSONEq(t, `[ + { + "accountId":"111", + "value":{ + "arn":"arn:aws:logs:us-east-1:111:log-group:group_a", + "name":"group_a" + } + } + ]`, string(sender.Response.Body)) + + logsApi.AssertCalled(t, "DescribeLogGroups", + &cloudwatchlogs.DescribeLogGroupsInput{ + AccountIdentifiers: []*string{utils.Pointer("some-account-id")}, + IncludeLinkedAccounts: utils.Pointer(true), + Limit: utils.Pointer(int64(50)), + LogGroupNamePrefix: utils.Pointer("some-pattern"), + }) + }) +} + func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) { sender := &mockedCallResourceResponseSenderForOauth{} origNewMetricsAPI := NewMetricsAPI + origNewOAMAPI := NewOAMAPI + origNewLogsAPI := NewLogsAPI + NewOAMAPI = func(sess *session.Session) models.OAMClientProvider { return nil } + NewLogsAPI = func(sess *session.Session) models.CloudWatchLogsAPIProvider { return nil } t.Cleanup(func() { + NewOAMAPI = origNewOAMAPI NewMetricsAPI = origNewMetricsAPI + NewLogsAPI = origNewLogsAPI }) + var api mocks.FakeMetricsAPI NewMetricsAPI = func(sess *session.Session) models.CloudWatchMetricsAPIProvider { return &api } + im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { return DataSource{Settings: models.CloudWatchSettings{}}, nil }) @@ -580,10 +654,10 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) { sent := sender.Response require.NotNil(t, sent) require.Equal(t, http.StatusOK, sent.Status) - res := []string{} + res := []resources.ResourceResponse[string]{} err = json.Unmarshal(sent.Body, &res) require.Nil(t, err) - assert.Equal(t, []string{"Value1", "Value2", "Value7"}, res) + assert.Equal(t, []resources.ResourceResponse[string]{{Value: "Value1"}, {Value: "Value2"}, {Value: "Value7"}}, res) }) t.Run("Should handle dimension key filter query and return keys from the api", func(t *testing.T) { @@ -616,10 +690,10 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) { sent := sender.Response require.NotNil(t, sent) require.Equal(t, http.StatusOK, sent.Status) - res := []string{} + res := []resources.ResourceResponse[string]{} err = json.Unmarshal(sent.Body, &res) require.Nil(t, err) - assert.Equal(t, []string{"Test_DimensionName1", "Test_DimensionName2", "Test_DimensionName4", "Test_DimensionName5"}, res) + assert.Equal(t, []resources.ResourceResponse[string]{{Value: "Test_DimensionName1"}, {Value: "Test_DimensionName2"}, {Value: "Test_DimensionName4"}, {Value: "Test_DimensionName5"}}, res) }) t.Run("Should handle standard dimension key query and return hard coded keys", func(t *testing.T) { @@ -640,10 +714,10 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) { sent := sender.Response require.NotNil(t, sent) require.Equal(t, http.StatusOK, sent.Status) - res := []string{} + res := []resources.ResourceResponse[string]{} err = json.Unmarshal(sent.Body, &res) require.Nil(t, err) - assert.Equal(t, []string{"ClientId", "DomainName"}, res) + assert.Equal(t, []resources.ResourceResponse[string]{{Value: "ClientId"}, {Value: "DomainName"}}, res) }) t.Run("Should handle custom namespace dimension key query and return hard coded keys", func(t *testing.T) { @@ -664,10 +738,10 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) { sent := sender.Response require.NotNil(t, sent) require.Equal(t, http.StatusOK, sent.Status) - res := []string{} + res := []resources.ResourceResponse[string]{} err = json.Unmarshal(sent.Body, &res) require.Nil(t, err) - assert.Equal(t, []string{"ClientId", "DomainName"}, res) + assert.Equal(t, []resources.ResourceResponse[string]{{Value: "ClientId"}, {Value: "DomainName"}}, res) }) t.Run("Should handle custom namespace metrics query and return metrics from api", func(t *testing.T) { @@ -700,10 +774,10 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) { sent := sender.Response require.NotNil(t, sent) require.Equal(t, http.StatusOK, sent.Status) - res := []resources.Metric{} + res := []resources.ResourceResponse[resources.Metric]{} err = json.Unmarshal(sent.Body, &res) require.Nil(t, err) - assert.Equal(t, []resources.Metric{{Name: "Test_MetricName1", Namespace: "AWS/EC2"}, {Name: "Test_MetricName2", Namespace: "AWS/EC2"}, {Name: "Test_MetricName3", Namespace: "AWS/ECS"}, {Name: "Test_MetricName10", Namespace: "AWS/ECS"}, {Name: "Test_MetricName4", Namespace: "AWS/ECS"}, {Name: "Test_MetricName5", Namespace: "AWS/Redshift"}}, res) + assert.Equal(t, []resources.ResourceResponse[resources.Metric]{{Value: resources.Metric{Name: "Test_MetricName1", Namespace: "AWS/EC2"}}, {Value: resources.Metric{Name: "Test_MetricName2", Namespace: "AWS/EC2"}}, {Value: resources.Metric{Name: "Test_MetricName3", Namespace: "AWS/ECS"}}, {Value: resources.Metric{Name: "Test_MetricName10", Namespace: "AWS/ECS"}}, {Value: resources.Metric{Name: "Test_MetricName4", Namespace: "AWS/ECS"}}, {Value: resources.Metric{Name: "Test_MetricName5", Namespace: "AWS/Redshift"}}}, res) }) } diff --git a/pkg/tsdb/cloudwatch/log_actions.go b/pkg/tsdb/cloudwatch/log_actions.go index f2f52d9cd64..33344572afe 100644 --- a/pkg/tsdb/cloudwatch/log_actions.go +++ b/pkg/tsdb/cloudwatch/log_actions.go @@ -7,6 +7,7 @@ import ( "fmt" "math" "sort" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -17,6 +18,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/featuremgmt" ) const ( @@ -40,6 +42,7 @@ type LogQueryJson struct { EndTime *int64 LogGroupName string LogGroupNames []string + LogGroups []suggestData LogGroupNamePrefix string LogStreamName string StartFromHead bool @@ -224,15 +227,31 @@ func (e *cloudWatchExecutor) executeStartQuery(ctx context.Context, logsClient c // StartTime is effectively floored while here EndTime is ceiled and so we should get the logs user wants // and also a little bit more but as CW logs accept only seconds as integers there is not much to do about // that. - EndTime: aws.Int64(int64(math.Ceil(float64(endTime.UnixNano()) / 1e9))), - LogGroupNames: aws.StringSlice(parameters.LogGroupNames), - QueryString: aws.String(modifiedQueryString), + EndTime: aws.Int64(int64(math.Ceil(float64(endTime.UnixNano()) / 1e9))), + QueryString: aws.String(modifiedQueryString), + } + + if e.features.IsEnabled(featuremgmt.FlagCloudWatchCrossAccountQuerying) { + if parameters.LogGroups != nil && len(parameters.LogGroups) > 0 { + var logGroupIdentifiers []string + for _, lg := range parameters.LogGroups { + arn := lg.Value + // due to a bug in the startQuery api, we remove * from the arn, otherwise it throws an error + logGroupIdentifiers = append(logGroupIdentifiers, strings.TrimSuffix(arn, "*")) + } + startQueryInput.LogGroupIdentifiers = aws.StringSlice(logGroupIdentifiers) + } + } + + if startQueryInput.LogGroupIdentifiers == nil { + startQueryInput.LogGroupNames = aws.StringSlice(parameters.LogGroupNames) } if parameters.Limit != nil { startQueryInput.Limit = aws.Int64(*parameters.Limit) } + logger.Debug("calling startquery with context with input", "input", startQueryInput) return logsClient.StartQueryWithContext(ctx, startQueryInput) } diff --git a/pkg/tsdb/cloudwatch/log_actions_test.go b/pkg/tsdb/cloudwatch/log_actions_test.go index 00985a930f8..919f3d5b0c1 100644 --- a/pkg/tsdb/cloudwatch/log_actions_test.go +++ b/pkg/tsdb/cloudwatch/log_actions_test.go @@ -391,6 +391,78 @@ func Test_executeStartQuery(t *testing.T) { require.Len(t, cli.calls.startQueryWithContext, 1) assert.Nil(t, cli.calls.startQueryWithContext[0].Limit) }) + + t.Run("attaches logGroupIdentifiers if the crossAccount feature is enabled", func(t *testing.T) { + cli = fakeCWLogsClient{} + im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { + return DataSource{Settings: models.CloudWatchSettings{}}, nil + }) + executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudWatchCrossAccountQuerying)) + + _, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}}, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Unix(0, 0), To: time.Unix(1, 0)}, + JSON: json.RawMessage(`{ + "type": "logAction", + "subtype": "StartQuery", + "limit": 12, + "queryString":"fields @message", + "logGroups":[{"value": "fakeARN"}] + }`), + }, + }, + }) + + assert.NoError(t, err) + assert.Equal(t, []*cloudwatchlogs.StartQueryInput{ + { + StartTime: aws.Int64(0), + EndTime: aws.Int64(1), + Limit: aws.Int64(12), + QueryString: aws.String("fields @timestamp,ltrim(@log) as __log__grafana_internal__,ltrim(@logStream) as __logstream__grafana_internal__|fields @message"), + LogGroupIdentifiers: []*string{aws.String("fakeARN")}, + }, + }, cli.calls.startQueryWithContext) + }) + + t.Run("attaches logGroupIdentifiers if the crossAccount feature is enabled and strips out trailing *", func(t *testing.T) { + cli = fakeCWLogsClient{} + im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { + return DataSource{Settings: models.CloudWatchSettings{}}, nil + }) + executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudWatchCrossAccountQuerying)) + + _, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}}, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Unix(0, 0), To: time.Unix(1, 0)}, + JSON: json.RawMessage(`{ + "type": "logAction", + "subtype": "StartQuery", + "limit": 12, + "queryString":"fields @message", + "logGroups":[{"value": "*fake**ARN*"}] + }`), + }, + }, + }) + + assert.NoError(t, err) + assert.Equal(t, []*cloudwatchlogs.StartQueryInput{ + { + StartTime: aws.Int64(0), + EndTime: aws.Int64(1), + Limit: aws.Int64(12), + QueryString: aws.String("fields @timestamp,ltrim(@log) as __log__grafana_internal__,ltrim(@logStream) as __logstream__grafana_internal__|fields @message"), + LogGroupIdentifiers: []*string{aws.String("*fake**ARN")}, + }, + }, cli.calls.startQueryWithContext) + }) } func TestQuery_StopQuery(t *testing.T) { diff --git a/pkg/tsdb/cloudwatch/metric_data_query_builder.go b/pkg/tsdb/cloudwatch/metric_data_query_builder.go index 5baf0bb872b..b921a88739a 100644 --- a/pkg/tsdb/cloudwatch/metric_data_query_builder.go +++ b/pkg/tsdb/cloudwatch/metric_data_query_builder.go @@ -50,6 +50,7 @@ func (e *cloudWatchExecutor) buildMetricDataQuery(logger log.Logger, query *mode }) } mdq.MetricStat.Stat = aws.String(query.Statistic) + mdq.AccountId = query.AccountId } if mdq.Expression != nil { @@ -98,18 +99,27 @@ func buildSearchExpression(query *models.CloudWatchQuery, stat string) string { searchTerm = appendSearch(searchTerm, keyFilter) } + var account string + if query.AccountId != nil && *query.AccountId != "all" { + account = fmt.Sprintf(":aws.AccountId=%q", *query.AccountId) + } + if query.MatchExact { schema := fmt.Sprintf("%q", query.Namespace) if len(dimensionNames) > 0 { sort.Strings(dimensionNames) schema += fmt.Sprintf(",%s", join(dimensionNames, ",", `"`, `"`)) } - return fmt.Sprintf("REMOVE_EMPTY(SEARCH('{%s} %s', '%s', %s))", schema, searchTerm, stat, strconv.Itoa(query.Period)) + schema = fmt.Sprintf("{%s}", schema) + schemaSearchTermAndAccount := strings.TrimSpace(strings.Join([]string{schema, searchTerm, account}, " ")) + return fmt.Sprintf("REMOVE_EMPTY(SEARCH('%s', '%s', %s))", schemaSearchTermAndAccount, stat, strconv.Itoa(query.Period)) } sort.Strings(dimensionNamesWithoutKnownValues) searchTerm = appendSearch(searchTerm, join(dimensionNamesWithoutKnownValues, " ", `"`, `"`)) - return fmt.Sprintf(`REMOVE_EMPTY(SEARCH('Namespace="%s" %s', '%s', %s))`, query.Namespace, searchTerm, stat, strconv.Itoa(query.Period)) + namespace := fmt.Sprintf("Namespace=%q", query.Namespace) + namespaceSearchTermAndAccount := strings.TrimSpace(strings.Join([]string{namespace, searchTerm, account}, " ")) + return fmt.Sprintf(`REMOVE_EMPTY(SEARCH('%s', '%s', %s))`, namespaceSearchTermAndAccount, stat, strconv.Itoa(query.Period)) } func escapeDoubleQuotes(arr []string) []string { diff --git a/pkg/tsdb/cloudwatch/metric_data_query_builder_test.go b/pkg/tsdb/cloudwatch/metric_data_query_builder_test.go index b13c389504c..84b827ec925 100644 --- a/pkg/tsdb/cloudwatch/metric_data_query_builder_test.go +++ b/pkg/tsdb/cloudwatch/metric_data_query_builder_test.go @@ -3,11 +3,11 @@ package cloudwatch import ( "testing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - + "github.com/aws/aws-sdk-go/aws" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMetricDataQueryBuilder(t *testing.T) { @@ -24,6 +24,27 @@ func TestMetricDataQueryBuilder(t *testing.T) { assert.Equal(t, query.Namespace, *mdq.MetricStat.Metric.Namespace) }) + t.Run("should pass AccountId in metric stat query", func(t *testing.T) { + executor := newExecutor(nil, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) + query := getBaseQuery() + query.MetricEditorMode = models.MetricEditorModeBuilder + query.MetricQueryType = models.MetricQueryTypeSearch + query.AccountId = aws.String("some account id") + mdq, err := executor.buildMetricDataQuery(logger, query) + require.NoError(t, err) + assert.Equal(t, "some account id", *mdq.AccountId) + }) + + t.Run("should leave AccountId in metric stat query", func(t *testing.T) { + executor := newExecutor(nil, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) + query := getBaseQuery() + query.MetricEditorMode = models.MetricEditorModeBuilder + query.MetricQueryType = models.MetricQueryTypeSearch + mdq, err := executor.buildMetricDataQuery(logger, query) + require.NoError(t, err) + assert.Nil(t, mdq.AccountId) + }) + t.Run("should use custom built expression", func(t *testing.T) { executor := newExecutor(nil, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) query := getBaseQuery() @@ -112,6 +133,42 @@ func TestMetricDataQueryBuilder(t *testing.T) { assert.Nil(t, mdq.Label) }) } + + t.Run(`should not specify accountId when it is "all"`, func(t *testing.T) { + executor := newExecutor(nil, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudWatchDynamicLabels)) + query := &models.CloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Statistic: "Average", + Period: 60, + MatchExact: false, + AccountId: aws.String("all"), + } + + mdq, err := executor.buildMetricDataQuery(logger, query) + + assert.NoError(t, err) + require.Nil(t, mdq.MetricStat) + assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization"', 'Average', 60))`, *mdq.Expression) + }) + + t.Run("should set accountId when it is specified", func(t *testing.T) { + executor := newExecutor(nil, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudWatchDynamicLabels)) + query := &models.CloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Statistic: "Average", + Period: 60, + MatchExact: false, + AccountId: aws.String("12345"), + } + + mdq, err := executor.buildMetricDataQuery(logger, query) + + assert.NoError(t, err) + require.Nil(t, mdq.MetricStat) + assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" :aws.AccountId="12345"', 'Average', 60))`, *mdq.Expression) + }) }) t.Run("Query should be matched exact", func(t *testing.T) { @@ -199,6 +256,24 @@ func TestMetricDataQueryBuilder(t *testing.T) { assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","InstanceId","LoadBalancer"} MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, res) }) + t.Run("Query has multiple dimensions and an account Id", func(t *testing.T) { + query := &models.CloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "LoadBalancer": {"lb1", "lb2", "lb3"}, + "InstanceId": {"i-123", "*", "i-789"}, + }, + Period: 300, + Expression: "", + MatchExact: matchExact, + AccountId: aws.String("some account id"), + } + + res := buildSearchExpression(query, "Average") + assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","InstanceId","LoadBalancer"} MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3") :aws.AccountId="some account id"', 'Average', 300))`, res) + }) + t.Run("Query has a dimension key with a space", func(t *testing.T) { query := &models.CloudWatchQuery{ Namespace: "AWS/Kafka", @@ -301,6 +376,24 @@ func TestMetricDataQueryBuilder(t *testing.T) { res := buildSearchExpression(query, "Average") assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3") "InstanceId"', 'Average', 300))`, res) }) + + t.Run("query has multiple dimensions and an account Id", func(t *testing.T) { + query := &models.CloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "LoadBalancer": {"lb1", "lb2", "lb3"}, + "InstanceId": {"i-123", "*", "i-789"}, + }, + Period: 300, + Expression: "", + MatchExact: matchExact, + AccountId: aws.String("some account id"), + } + + res := buildSearchExpression(query, "Average") + assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3") "InstanceId" :aws.AccountId="some account id"', 'Average', 300))`, res) + }) }) t.Run("Query has invalid characters in dimension values", func(t *testing.T) { diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index cb7c658c483..cc4e1f3904f 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -17,7 +17,6 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" "github.com/grafana/grafana-plugin-sdk-go/backend" - "github.com/grafana/grafana/pkg/tsdb/cloudwatch/constants" ) @@ -316,7 +315,6 @@ func (e *cloudWatchExecutor) handleGetLogGroups(pluginCtx backend.PluginContext, if err != nil || response == nil { return nil, err } - result := make([]suggestData, 0) for _, logGroup := range response.LogGroups { logGroupName := *logGroup.LogGroupName diff --git a/pkg/tsdb/cloudwatch/mocks/accounts_service.go b/pkg/tsdb/cloudwatch/mocks/accounts_service.go new file mode 100644 index 00000000000..52f3aff63b5 --- /dev/null +++ b/pkg/tsdb/cloudwatch/mocks/accounts_service.go @@ -0,0 +1,16 @@ +package mocks + +import ( + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" + "github.com/stretchr/testify/mock" +) + +type AccountsServiceMock struct { + mock.Mock +} + +func (a *AccountsServiceMock) GetAccountsForCurrentUserOrRole() ([]resources.ResourceResponse[resources.Account], error) { + args := a.Called() + + return args.Get(0).([]resources.ResourceResponse[resources.Account]), args.Error(1) +} diff --git a/pkg/tsdb/cloudwatch/mocks/cloudwatch_metric_api.go b/pkg/tsdb/cloudwatch/mocks/cloudwatch_metric_api.go index 8508f6e5697..b36077229a5 100644 --- a/pkg/tsdb/cloudwatch/mocks/cloudwatch_metric_api.go +++ b/pkg/tsdb/cloudwatch/mocks/cloudwatch_metric_api.go @@ -9,9 +9,8 @@ import ( ) type FakeMetricsAPI struct { - cloudwatchiface.CloudWatchAPI - Metrics []*cloudwatch.Metric + OwningAccounts []*string MetricsPerPage int } @@ -23,7 +22,8 @@ func (c *FakeMetricsAPI) ListMetricsPages(input *cloudwatch.ListMetricsInput, fn for i, metrics := range chunks { response := fn(&cloudwatch.ListMetricsOutput{ - Metrics: metrics, + Metrics: metrics, + OwningAccounts: c.OwningAccounts, }, i+1 == len(chunks)) if !response { break diff --git a/pkg/tsdb/cloudwatch/mocks/list_metrics_service.go b/pkg/tsdb/cloudwatch/mocks/list_metrics_service.go index e809bc8518c..8c9fb73f697 100644 --- a/pkg/tsdb/cloudwatch/mocks/list_metrics_service.go +++ b/pkg/tsdb/cloudwatch/mocks/list_metrics_service.go @@ -9,26 +9,20 @@ type ListMetricsServiceMock struct { mock.Mock } -func (a *ListMetricsServiceMock) GetDimensionKeysByDimensionFilter(r resources.DimensionKeysRequest) ([]string, error) { +func (a *ListMetricsServiceMock) GetDimensionKeysByDimensionFilter(r resources.DimensionKeysRequest) ([]resources.ResourceResponse[string], error) { args := a.Called(r) - return args.Get(0).([]string), args.Error(1) + return args.Get(0).([]resources.ResourceResponse[string]), args.Error(1) } -func (a *ListMetricsServiceMock) GetDimensionValuesByDimensionFilter(r resources.DimensionValuesRequest) ([]string, error) { +func (a *ListMetricsServiceMock) GetDimensionValuesByDimensionFilter(r resources.DimensionValuesRequest) ([]resources.ResourceResponse[string], error) { args := a.Called(r) - return args.Get(0).([]string), args.Error(1) + return args.Get(0).([]resources.ResourceResponse[string]), args.Error(1) } -func (a *ListMetricsServiceMock) GetDimensionKeysByNamespace(namespace string) ([]string, error) { - args := a.Called(namespace) +func (a *ListMetricsServiceMock) GetMetricsByNamespace(r resources.MetricsRequest) ([]resources.ResourceResponse[resources.Metric], error) { + args := a.Called(r) - return args.Get(0).([]string), args.Error(1) -} - -func (a *ListMetricsServiceMock) GetMetricsByNamespace(namespace string) ([]resources.Metric, error) { - args := a.Called(namespace) - - return args.Get(0).([]resources.Metric), args.Error(1) + return args.Get(0).([]resources.ResourceResponse[resources.Metric]), args.Error(1) } diff --git a/pkg/tsdb/cloudwatch/mocks/logs.go b/pkg/tsdb/cloudwatch/mocks/logs.go new file mode 100644 index 00000000000..c6210eeceac --- /dev/null +++ b/pkg/tsdb/cloudwatch/mocks/logs.go @@ -0,0 +1,37 @@ +package mocks + +import ( + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" + "github.com/stretchr/testify/mock" +) + +type LogsAPI struct { + mock.Mock +} + +func (l *LogsAPI) DescribeLogGroups(input *cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error) { + args := l.Called(input) + + return args.Get(0).(*cloudwatchlogs.DescribeLogGroupsOutput), args.Error(1) +} + +type LogsService struct { + mock.Mock +} + +func (l *LogsService) GetLogGroups(request resources.LogGroupsRequest) ([]resources.ResourceResponse[resources.LogGroup], error) { + args := l.Called(request) + + return args.Get(0).([]resources.ResourceResponse[resources.LogGroup]), args.Error(1) +} + +type MockFeatures struct { + mock.Mock +} + +func (f *MockFeatures) IsEnabled(feature string) bool { + args := f.Called(feature) + + return args.Bool(0) +} diff --git a/pkg/tsdb/cloudwatch/mocks/metrics_client.go b/pkg/tsdb/cloudwatch/mocks/metrics_client.go index 3b0d818f7f4..057389ba71e 100644 --- a/pkg/tsdb/cloudwatch/mocks/metrics_client.go +++ b/pkg/tsdb/cloudwatch/mocks/metrics_client.go @@ -2,6 +2,7 @@ package mocks import ( "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" "github.com/stretchr/testify/mock" ) @@ -9,7 +10,7 @@ type FakeMetricsClient struct { mock.Mock } -func (m *FakeMetricsClient) ListMetricsWithPageLimit(params *cloudwatch.ListMetricsInput) ([]*cloudwatch.Metric, error) { +func (m *FakeMetricsClient) ListMetricsWithPageLimit(params *cloudwatch.ListMetricsInput) ([]resources.MetricResponse, error) { args := m.Called(params) - return args.Get(0).([]*cloudwatch.Metric), args.Error(1) + return args.Get(0).([]resources.MetricResponse), args.Error(1) } diff --git a/pkg/tsdb/cloudwatch/mocks/oam_client.go b/pkg/tsdb/cloudwatch/mocks/oam_client.go new file mode 100644 index 00000000000..2d617cc143a --- /dev/null +++ b/pkg/tsdb/cloudwatch/mocks/oam_client.go @@ -0,0 +1,20 @@ +package mocks + +import ( + "github.com/aws/aws-sdk-go/service/oam" + "github.com/stretchr/testify/mock" +) + +type FakeOAMClient struct { + mock.Mock +} + +func (o *FakeOAMClient) ListSinks(input *oam.ListSinksInput) (*oam.ListSinksOutput, error) { + args := o.Called(input) + return args.Get(0).(*oam.ListSinksOutput), args.Error(1) +} + +func (o *FakeOAMClient) ListAttachedLinks(input *oam.ListAttachedLinksInput) (*oam.ListAttachedLinksOutput, error) { + args := o.Called(input) + return args.Get(0).(*oam.ListAttachedLinksOutput), args.Error(1) +} diff --git a/pkg/tsdb/cloudwatch/models/api.go b/pkg/tsdb/cloudwatch/models/api.go index b86860e5efd..41455687f5f 100644 --- a/pkg/tsdb/cloudwatch/models/api.go +++ b/pkg/tsdb/cloudwatch/models/api.go @@ -1,21 +1,55 @@ package models import ( + "net/url" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/oam" + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" ) +type RequestContextFactoryFunc func(pluginCtx backend.PluginContext, region string) (reqCtx RequestContext, err error) + +type RouteHandlerFunc func(pluginCtx backend.PluginContext, reqContextFactory RequestContextFactoryFunc, parameters url.Values) ([]byte, *HttpError) + +type RequestContext struct { + MetricsClientProvider MetricsClientProvider + LogsAPIProvider CloudWatchLogsAPIProvider + OAMClientProvider OAMClientProvider + Settings CloudWatchSettings + Features featuremgmt.FeatureToggles +} + type ListMetricsProvider interface { - GetDimensionKeysByDimensionFilter(resources.DimensionKeysRequest) ([]string, error) - GetDimensionKeysByNamespace(string) ([]string, error) - GetDimensionValuesByDimensionFilter(resources.DimensionValuesRequest) ([]string, error) - GetMetricsByNamespace(namespace string) ([]resources.Metric, error) + GetDimensionKeysByDimensionFilter(resources.DimensionKeysRequest) ([]resources.ResourceResponse[string], error) + GetDimensionValuesByDimensionFilter(resources.DimensionValuesRequest) ([]resources.ResourceResponse[string], error) + GetMetricsByNamespace(r resources.MetricsRequest) ([]resources.ResourceResponse[resources.Metric], error) } type MetricsClientProvider interface { - ListMetricsWithPageLimit(params *cloudwatch.ListMetricsInput) ([]*cloudwatch.Metric, error) + ListMetricsWithPageLimit(params *cloudwatch.ListMetricsInput) ([]resources.MetricResponse, error) } type CloudWatchMetricsAPIProvider interface { ListMetricsPages(*cloudwatch.ListMetricsInput, func(*cloudwatch.ListMetricsOutput, bool) bool) error } + +type CloudWatchLogsAPIProvider interface { + DescribeLogGroups(*cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error) +} + +type OAMClientProvider interface { + ListSinks(*oam.ListSinksInput) (*oam.ListSinksOutput, error) + ListAttachedLinks(*oam.ListAttachedLinksInput) (*oam.ListAttachedLinksOutput, error) +} + +type LogGroupsProvider interface { + GetLogGroups(request resources.LogGroupsRequest) ([]resources.ResourceResponse[resources.LogGroup], error) +} + +type AccountsProvider interface { + GetAccountsForCurrentUserOrRole() ([]resources.ResourceResponse[resources.Account], error) +} diff --git a/pkg/tsdb/cloudwatch/models/cloudwatch_query.go b/pkg/tsdb/cloudwatch/models/cloudwatch_query.go index 248796e35b5..a6cf683152e 100644 --- a/pkg/tsdb/cloudwatch/models/cloudwatch_query.go +++ b/pkg/tsdb/cloudwatch/models/cloudwatch_query.go @@ -60,6 +60,7 @@ type CloudWatchQuery struct { TimezoneUTCOffset string MetricQueryType MetricQueryType MetricEditorMode MetricEditorMode + AccountId *string } func (q *CloudWatchQuery) GetGMDAPIMode(logger log.Logger) GMDApiMode { @@ -95,6 +96,10 @@ func (q *CloudWatchQuery) IsInferredSearchExpression() bool { return false } + if q.AccountId != nil && *q.AccountId == "all" { + return true + } + if len(q.Dimensions) == 0 { return !q.MatchExact } @@ -167,6 +172,9 @@ func (q *CloudWatchQuery) BuildDeepLink(startTime time.Time, endTime time.Time, if dynamicLabelEnabled { metricStatMeta.Label = q.Label } + if q.AccountId != nil { + metricStatMeta.AccountId = *q.AccountId + } metricStat = append(metricStat, metricStatMeta) link.Metrics = []interface{}{metricStat} } @@ -214,11 +222,13 @@ type metricsDataQuery struct { QueryType string `json:"type"` Hide *bool `json:"hide"` Alias string `json:"alias"` + AccountId *string `json:"accountId"` } // ParseMetricDataQueries decodes the metric data queries json, validates, sets default values and returns an array of CloudWatchQueries. // The CloudWatchQuery has a 1 to 1 mapping to a query editor row -func ParseMetricDataQueries(dataQueries []backend.DataQuery, startTime time.Time, endTime time.Time, dynamicLabelsEnabled bool) ([]*CloudWatchQuery, error) { +func ParseMetricDataQueries(dataQueries []backend.DataQuery, startTime time.Time, endTime time.Time, dynamicLabelsEnabled, + crossAccountQueryingEnabled bool) ([]*CloudWatchQuery, error) { var metricDataQueries = make(map[string]metricsDataQuery) for _, query := range dataQueries { var metricsDataQuery metricsDataQuery @@ -250,7 +260,7 @@ func ParseMetricDataQueries(dataQueries []backend.DataQuery, startTime time.Time Expression: mdq.Expression, } - if err := cwQuery.validateAndSetDefaults(refId, mdq, startTime, endTime); err != nil { + if err := cwQuery.validateAndSetDefaults(refId, mdq, startTime, endTime, crossAccountQueryingEnabled); err != nil { return nil, &QueryError{Err: err, RefID: refId} } @@ -267,7 +277,8 @@ func (q *CloudWatchQuery) migrateLegacyQuery(query metricsDataQuery, dynamicLabe q.Label = getLabel(query, dynamicLabelsEnabled) } -func (q *CloudWatchQuery) validateAndSetDefaults(refId string, metricsDataQuery metricsDataQuery, startTime, endTime time.Time) error { +func (q *CloudWatchQuery) validateAndSetDefaults(refId string, metricsDataQuery metricsDataQuery, startTime, endTime time.Time, + crossAccountQueryingEnabled bool) error { if metricsDataQuery.Statistic == nil && metricsDataQuery.Statistics == nil { return fmt.Errorf("query must have either statistic or statistics field") } @@ -283,6 +294,10 @@ func (q *CloudWatchQuery) validateAndSetDefaults(refId string, metricsDataQuery return fmt.Errorf("failed to parse dimensions: %v", err) } + if crossAccountQueryingEnabled { + q.AccountId = metricsDataQuery.AccountId + } + if metricsDataQuery.Id == "" { // Why not just use refId if id is not specified in the frontend? When specifying an id in the editor, // and alphabetical must be used. The id must be unique, so if an id like for example a, b or c would be used, diff --git a/pkg/tsdb/cloudwatch/models/cloudwatch_query_test.go b/pkg/tsdb/cloudwatch/models/cloudwatch_query_test.go index d2467b2ed46..89459f66bea 100644 --- a/pkg/tsdb/cloudwatch/models/cloudwatch_query_test.go +++ b/pkg/tsdb/cloudwatch/models/cloudwatch_query_test.go @@ -107,6 +107,53 @@ func TestCloudWatchQuery(t *testing.T) { require.NoError(t, err) assert.NotContains(t, deepLink, "label") }) + + t.Run("includes account id in case its a metric stat query and an account id is set", func(t *testing.T) { + startTime := time.Now() + endTime := startTime.Add(2 * time.Hour) + query := &CloudWatchQuery{ + RefId: "A", + Region: "us-east-1", + Expression: "", + Statistic: "Average", + Period: 300, + Id: "id1", + MatchExact: true, + AccountId: pointer("123456789"), + Label: "${PROP('Namespace')}", + Dimensions: map[string][]string{ + "InstanceId": {"i-12345678"}, + }, + MetricQueryType: MetricQueryTypeSearch, + MetricEditorMode: MetricEditorModeBuilder, + } + + deepLink, err := query.BuildDeepLink(startTime, endTime, false) + require.NoError(t, err) + assert.Contains(t, deepLink, "accountId%22%3A%22123456789") + }) + + t.Run("does not include account id in case its not a metric stat query", func(t *testing.T) { + startTime := time.Now() + endTime := startTime.Add(2 * time.Hour) + query := &CloudWatchQuery{ + RefId: "A", + Region: "us-east-1", + Statistic: "Average", + Expression: "SEARCH(someexpression)", + AccountId: pointer("123456789"), + Period: 300, + Id: "id1", + MatchExact: true, + Label: "${PROP('Namespace')}", + MetricQueryType: MetricQueryTypeSearch, + MetricEditorMode: MetricEditorModeRaw, + } + + deepLink, err := query.BuildDeepLink(startTime, endTime, false) + require.NoError(t, err) + assert.NotContains(t, deepLink, "accountId%22%3A%22123456789") + }) }) t.Run("SEARCH(someexpression) was specified in the query editor", func(t *testing.T) { @@ -269,7 +316,7 @@ func TestRequestParser(t *testing.T) { }, } - migratedQueries, err := ParseMetricDataQueries(oldQuery, time.Now(), time.Now(), false) + migratedQueries, err := ParseMetricDataQueries(oldQuery, time.Now(), time.Now(), false, false) assert.NoError(t, err) require.Len(t, migratedQueries, 1) require.NotNil(t, migratedQueries[0]) @@ -300,7 +347,7 @@ func TestRequestParser(t *testing.T) { }, } - results, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + results, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) require.NoError(t, err) require.Len(t, results, 1) res := results[0] @@ -343,7 +390,7 @@ func TestRequestParser(t *testing.T) { }, } - results, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + results, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) assert.NoError(t, err) require.Len(t, results, 1) res := results[0] @@ -376,7 +423,7 @@ func TestRequestParser(t *testing.T) { }, } - _, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + _, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) require.Error(t, err) assert.Equal(t, `error parsing query "", failed to parse dimensions: unknown type as dimension value`, err.Error()) @@ -405,7 +452,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { }, } - res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) assert.NoError(t, err) require.Len(t, res, 1) require.NotNil(t, res[0]) @@ -437,7 +484,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { to := time.Now() from := to.Local().Add(time.Minute * time.Duration(5)) - res, err := ParseMetricDataQueries(query, from, to, false) + res, err := ParseMetricDataQueries(query, from, to, false, false) require.NoError(t, err) require.Len(t, res, 1) assert.Equal(t, 60, res[0].Period) @@ -447,7 +494,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { to := time.Now() from := to.AddDate(0, 0, -1) - res, err := ParseMetricDataQueries(query, from, to, false) + res, err := ParseMetricDataQueries(query, from, to, false, false) require.NoError(t, err) require.Len(t, res, 1) assert.Equal(t, 60, res[0].Period) @@ -456,7 +503,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { t.Run("Time range is 2 days", func(t *testing.T) { to := time.Now() from := to.AddDate(0, 0, -2) - res, err := ParseMetricDataQueries(query, from, to, false) + res, err := ParseMetricDataQueries(query, from, to, false, false) require.NoError(t, err) require.Len(t, res, 1) assert.Equal(t, 300, res[0].Period) @@ -466,7 +513,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { to := time.Now() from := to.AddDate(0, 0, -7) - res, err := ParseMetricDataQueries(query, from, to, false) + res, err := ParseMetricDataQueries(query, from, to, false, false) require.NoError(t, err) require.Len(t, res, 1) assert.Equal(t, 900, res[0].Period) @@ -476,7 +523,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { to := time.Now() from := to.AddDate(0, 0, -30) - res, err := ParseMetricDataQueries(query, from, to, false) + res, err := ParseMetricDataQueries(query, from, to, false, false) require.NoError(t, err) require.Len(t, res, 1) assert.Equal(t, 3600, res[0].Period) @@ -486,7 +533,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { to := time.Now() from := to.AddDate(0, 0, -90) - res, err := ParseMetricDataQueries(query, from, to, false) + res, err := ParseMetricDataQueries(query, from, to, false, false) require.NoError(t, err) require.Len(t, res, 1) assert.Equal(t, 21600, res[0].Period) @@ -496,7 +543,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { to := time.Now() from := to.AddDate(-1, 0, 0) - res, err := ParseMetricDataQueries(query, from, to, false) + res, err := ParseMetricDataQueries(query, from, to, false, false) require.Nil(t, err) require.Len(t, res, 1) assert.Equal(t, 21600, res[0].Period) @@ -506,7 +553,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { to := time.Now() from := to.AddDate(-2, 0, 0) - res, err := ParseMetricDataQueries(query, from, to, false) + res, err := ParseMetricDataQueries(query, from, to, false, false) require.NoError(t, err) require.Len(t, res, 1) assert.Equal(t, 86400, res[0].Period) @@ -515,7 +562,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { t.Run("Time range is 2 days, but 16 days ago", func(t *testing.T) { to := time.Now().AddDate(0, 0, -14) from := to.AddDate(0, 0, -2) - res, err := ParseMetricDataQueries(query, from, to, false) + res, err := ParseMetricDataQueries(query, from, to, false, false) require.NoError(t, err) require.Len(t, res, 1) assert.Equal(t, 300, res[0].Period) @@ -524,7 +571,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { t.Run("Time range is 2 days, but 90 days ago", func(t *testing.T) { to := time.Now().AddDate(0, 0, -88) from := to.AddDate(0, 0, -2) - res, err := ParseMetricDataQueries(query, from, to, false) + res, err := ParseMetricDataQueries(query, from, to, false, false) require.NoError(t, err) require.Len(t, res, 1) assert.Equal(t, 3600, res[0].Period) @@ -533,7 +580,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { t.Run("Time range is 2 days, but 456 days ago", func(t *testing.T) { to := time.Now().AddDate(0, 0, -454) from := to.AddDate(0, 0, -2) - res, err := ParseMetricDataQueries(query, from, to, false) + res, err := ParseMetricDataQueries(query, from, to, false, false) require.NoError(t, err) require.Len(t, res, 1) assert.Equal(t, 21600, res[0].Period) @@ -548,7 +595,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { }`), }, } - _, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + _, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) require.Error(t, err) assert.Equal(t, `error parsing query "", failed to parse period as duration: time: invalid duration "invalid"`, err.Error()) }) @@ -563,7 +610,7 @@ func Test_ParseMetricDataQueries_periods(t *testing.T) { }, } - res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) assert.NoError(t, err) require.Len(t, res, 1) @@ -620,6 +667,18 @@ func Test_ParseMetricDataQueries_query_type_and_metric_editor_mode_and_GMD_query expectedMetricEditorMode: dummyTestEditorMode, expectedGMDApiMode: GMDApiModeMetricStat, }, + "no dimensions, matchExact is false": { + extraDataQueryJson: `"matchExact":false,`, + expectedMetricQueryType: MetricQueryTypeSearch, + expectedMetricEditorMode: MetricEditorModeBuilder, + expectedGMDApiMode: GMDApiModeInferredSearchExpression, + }, + "query metricQueryType": { + extraDataQueryJson: `"metricQueryType":1,`, + expectedMetricQueryType: MetricQueryTypeQuery, + expectedMetricEditorMode: MetricEditorModeBuilder, + expectedGMDApiMode: GMDApiModeSQLExpression, + }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { @@ -638,7 +697,7 @@ func Test_ParseMetricDataQueries_query_type_and_metric_editor_mode_and_GMD_query ), }, } - res, err := ParseMetricDataQueries(query, time.Now(), time.Now(), false) + res, err := ParseMetricDataQueries(query, time.Now(), time.Now(), false, false) require.NoError(t, err) require.Len(t, res, 1) require.NotNil(t, res[0]) @@ -664,7 +723,7 @@ func Test_ParseMetricDataQueries_hide_and_ReturnData(t *testing.T) { }`), }, } - res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) require.NoError(t, err) require.Len(t, res, 1) require.NotNil(t, res[0]) @@ -685,7 +744,7 @@ func Test_ParseMetricDataQueries_hide_and_ReturnData(t *testing.T) { }`), }, } - res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) require.NoError(t, err) require.Len(t, res, 1) require.NotNil(t, res[0]) @@ -706,7 +765,7 @@ func Test_ParseMetricDataQueries_hide_and_ReturnData(t *testing.T) { }`), }, } - res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) require.NoError(t, err) require.Len(t, res, 1) require.NotNil(t, res[0]) @@ -725,7 +784,7 @@ func Test_ParseMetricDataQueries_hide_and_ReturnData(t *testing.T) { }`), }, } - res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) require.NoError(t, err) require.Len(t, res, 1) require.NotNil(t, res[0]) @@ -746,7 +805,7 @@ func Test_ParseMetricDataQueries_hide_and_ReturnData(t *testing.T) { }`), }, } - res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) require.NoError(t, err) require.Len(t, res, 1) require.NotNil(t, res[0]) @@ -767,7 +826,7 @@ func Test_ParseMetricDataQueries_hide_and_ReturnData(t *testing.T) { }`), }, } - res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) require.NoError(t, err) require.Len(t, res, 1) require.NotNil(t, res[0]) @@ -790,7 +849,7 @@ func Test_ParseMetricDataQueries_ID(t *testing.T) { }`), }, } - res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) require.NoError(t, err) require.Len(t, res, 1) require.NotNil(t, res[0]) @@ -811,7 +870,7 @@ func Test_ParseMetricDataQueries_ID(t *testing.T) { }`), }, } - res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false) + res, err := ParseMetricDataQueries(query, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour), false, false) require.NoError(t, err) require.Len(t, res, 1) require.NotNil(t, res[0]) @@ -838,7 +897,7 @@ func Test_ParseMetricDataQueries_sets_label_when_label_is_present_in_json_query( }, } - res, err := ParseMetricDataQueries(query, time.Now(), time.Now(), true) + res, err := ParseMetricDataQueries(query, time.Now(), time.Now(), true, false) assert.NoError(t, err) require.Len(t, res, 1) require.NotNil(t, res[0]) @@ -902,7 +961,7 @@ func Test_ParseMetricDataQueries_migrate_alias_to_label(t *testing.T) { }, } - res, err := ParseMetricDataQueries(query, time.Now(), time.Now(), true) + res, err := ParseMetricDataQueries(query, time.Now(), time.Now(), true, false) assert.NoError(t, err) require.Len(t, res, 1) @@ -949,7 +1008,7 @@ func Test_ParseMetricDataQueries_migrate_alias_to_label(t *testing.T) { }, } - res, err := ParseMetricDataQueries(query, time.Now(), time.Now(), true) + res, err := ParseMetricDataQueries(query, time.Now(), time.Now(), true, false) assert.NoError(t, err) require.Len(t, res, 2) @@ -1019,7 +1078,7 @@ func Test_ParseMetricDataQueries_migrate_alias_to_label(t *testing.T) { }`, tc.labelJson)), }, } - res, err := ParseMetricDataQueries(query, time.Now(), time.Now(), tc.dynamicLabelsFeatureToggleEnabled) + res, err := ParseMetricDataQueries(query, time.Now(), time.Now(), tc.dynamicLabelsFeatureToggleEnabled, false) assert.NoError(t, err) require.Len(t, res, 1) @@ -1046,7 +1105,7 @@ func Test_ParseMetricDataQueries_statistics_and_query_type_validation_and_MatchE { JSON: []byte("{}"), }, - }, time.Now(), time.Now(), false) + }, time.Now(), time.Now(), false, false) assert.Error(t, err) assert.Equal(t, `error parsing query "", query must have either statistic or statistics field`, err.Error()) @@ -1059,7 +1118,7 @@ func Test_ParseMetricDataQueries_statistics_and_query_type_validation_and_MatchE { JSON: []byte(`{"type":"some other type", "statistic":"Average", "matchExact":false}`), }, - }, time.Now(), time.Now(), false) + }, time.Now(), time.Now(), false, false) assert.NoError(t, err) assert.Empty(t, actual) @@ -1071,7 +1130,7 @@ func Test_ParseMetricDataQueries_statistics_and_query_type_validation_and_MatchE { JSON: []byte(`{"statistic":"Average"}`), }, - }, time.Now(), time.Now(), false) + }, time.Now(), time.Now(), false, false) assert.NoError(t, err) assert.NotEmpty(t, actual) @@ -1083,7 +1142,7 @@ func Test_ParseMetricDataQueries_statistics_and_query_type_validation_and_MatchE { JSON: []byte(`{"statistic":"Average"}`), }, - }, time.Now(), time.Now(), false) + }, time.Now(), time.Now(), false, false) assert.NoError(t, err) assert.Len(t, actual, 1) @@ -1097,7 +1156,7 @@ func Test_ParseMetricDataQueries_statistics_and_query_type_validation_and_MatchE { JSON: []byte(`{"statistic":"Average","matchExact":false}`), }, - }, time.Now(), time.Now(), false) + }, time.Now(), time.Now(), false, false) assert.NoError(t, err) assert.Len(t, actual, 1) @@ -1105,3 +1164,36 @@ func Test_ParseMetricDataQueries_statistics_and_query_type_validation_and_MatchE assert.False(t, actual[0].MatchExact) }) } + +func Test_ParseMetricDataQueries_account_Id(t *testing.T) { + t.Run("account is set when cross account querying enabled", func(t *testing.T) { + actual, err := ParseMetricDataQueries( + []backend.DataQuery{ + { + JSON: []byte(`{"accountId":"some account id", "statistic":"Average"}`), + }, + }, time.Now(), time.Now(), false, true) + assert.NoError(t, err) + + require.Len(t, actual, 1) + require.NotNil(t, actual[0]) + require.NotNil(t, actual[0].AccountId) + assert.Equal(t, "some account id", *actual[0].AccountId) + }) + + t.Run("account is not set when cross account querying disabled", func(t *testing.T) { + actual, err := ParseMetricDataQueries( + []backend.DataQuery{ + { + JSON: []byte(`{"accountId":"some account id", "statistic":"Average"}`), + }, + }, time.Now(), time.Now(), false, false) + assert.NoError(t, err) + + require.Len(t, actual, 1) + require.NotNil(t, actual[0]) + assert.Nil(t, actual[0].AccountId) + }) +} + +func pointer[T any](arg T) *T { return &arg } diff --git a/pkg/tsdb/cloudwatch/models/resources/dimension_values_rquest_test.go b/pkg/tsdb/cloudwatch/models/resources/dimension_values_request_test.go similarity index 100% rename from pkg/tsdb/cloudwatch/models/resources/dimension_values_rquest_test.go rename to pkg/tsdb/cloudwatch/models/resources/dimension_values_request_test.go diff --git a/pkg/tsdb/cloudwatch/models/resources/log_groups_resource_request.go b/pkg/tsdb/cloudwatch/models/resources/log_groups_resource_request.go new file mode 100644 index 00000000000..bf1502c75d0 --- /dev/null +++ b/pkg/tsdb/cloudwatch/models/resources/log_groups_resource_request.go @@ -0,0 +1,53 @@ +package resources + +import ( + "fmt" + "net/url" + "strconv" +) + +const defaultLogGroupLimit = int64(50) + +type LogGroupsRequest struct { + ResourceRequest + Limit int64 + LogGroupNamePrefix, LogGroupNamePattern *string +} + +func (r LogGroupsRequest) IsTargetingAllAccounts() bool { + return *r.AccountId == "all" +} + +func ParseLogGroupsRequest(parameters url.Values) (LogGroupsRequest, error) { + logGroupNamePrefix := setIfNotEmptyString(parameters.Get("logGroupNamePrefix")) + logGroupPattern := setIfNotEmptyString(parameters.Get("logGroupPattern")) + if logGroupNamePrefix != nil && logGroupPattern != nil { + return LogGroupsRequest{}, fmt.Errorf("cannot set both log group name prefix and pattern") + } + + return LogGroupsRequest{ + Limit: getLimit(parameters.Get("limit")), + ResourceRequest: ResourceRequest{ + Region: parameters.Get("region"), + AccountId: setIfNotEmptyString(parameters.Get("accountId")), + }, + LogGroupNamePrefix: logGroupNamePrefix, + LogGroupNamePattern: logGroupPattern, + }, nil +} + +func setIfNotEmptyString(paramValue string) *string { + if paramValue == "" { + return nil + } + return ¶mValue +} + +func getLimit(limit string) int64 { + logGroupLimit := defaultLogGroupLimit + intLimit, err := strconv.ParseInt(limit, 10, 64) + if err == nil && intLimit > 0 { + logGroupLimit = intLimit + } + return logGroupLimit +} diff --git a/pkg/tsdb/cloudwatch/models/resources/metrics_resource_request.go b/pkg/tsdb/cloudwatch/models/resources/metrics_resource_request.go index 6bc749348cd..2aaf53931f4 100644 --- a/pkg/tsdb/cloudwatch/models/resources/metrics_resource_request.go +++ b/pkg/tsdb/cloudwatch/models/resources/metrics_resource_request.go @@ -17,13 +17,13 @@ type MetricsRequest struct { Namespace string } -func GetMetricsRequest(parameters url.Values) (*MetricsRequest, error) { +func GetMetricsRequest(parameters url.Values) (MetricsRequest, error) { resourceRequest, err := getResourceRequest(parameters) if err != nil { - return nil, err + return MetricsRequest{}, err } - return &MetricsRequest{ + return MetricsRequest{ ResourceRequest: resourceRequest, Namespace: parameters.Get("namespace"), }, nil diff --git a/pkg/tsdb/cloudwatch/models/resources/resource_request.go b/pkg/tsdb/cloudwatch/models/resources/resource_request.go index e13c11cb558..53b1525b62b 100644 --- a/pkg/tsdb/cloudwatch/models/resources/resource_request.go +++ b/pkg/tsdb/cloudwatch/models/resources/resource_request.go @@ -5,8 +5,15 @@ import ( "net/url" ) +const useLinkedAccountsId = "all" + type ResourceRequest struct { - Region string + Region string + AccountId *string +} + +func (r *ResourceRequest) ShouldTargetAllAccounts() bool { + return r.AccountId != nil && *r.AccountId == useLinkedAccountsId } func getResourceRequest(parameters url.Values) (*ResourceRequest, error) { @@ -14,9 +21,24 @@ func getResourceRequest(parameters url.Values) (*ResourceRequest, error) { Region: parameters.Get("region"), } + accountId := parameters.Get("accountId") + if accountId != "" { + request.AccountId = &accountId + } + if request.Region == "" { return nil, fmt.Errorf("region is required") } return request, nil } + +type LogsRequest struct { + Limit int64 + AccountId, LogGroupNamePrefix, LogGroupNamePattern *string + IsCrossAccountQueryingEnabled bool +} + +func (r LogsRequest) IsTargetingAllAccounts() bool { + return *r.AccountId == useLinkedAccountsId +} diff --git a/pkg/tsdb/cloudwatch/models/resources/types.go b/pkg/tsdb/cloudwatch/models/resources/types.go index a218740a438..728d74c5af6 100644 --- a/pkg/tsdb/cloudwatch/models/resources/types.go +++ b/pkg/tsdb/cloudwatch/models/resources/types.go @@ -1,11 +1,35 @@ package resources +import "github.com/aws/aws-sdk-go/service/cloudwatch" + type Dimension struct { Name string Value string } +type ResourceResponse[T any] struct { + AccountId *string `json:"accountId,omitempty"` + Value T `json:"value"` +} + +type MetricResponse struct { + *cloudwatch.Metric + AccountId *string `json:"accountId,omitempty"` +} + +type Account struct { + Id string `json:"id"` + Arn string `json:"arn"` + Label string `json:"label"` + IsMonitoringAccount bool `json:"isMonitoringAccount"` +} + type Metric struct { Name string `json:"name"` Namespace string `json:"namespace"` } + +type LogGroup struct { + Arn string `json:"arn"` + Name string `json:"name"` +} diff --git a/pkg/tsdb/cloudwatch/models/types.go b/pkg/tsdb/cloudwatch/models/types.go index df41487234d..70664869e3e 100644 --- a/pkg/tsdb/cloudwatch/models/types.go +++ b/pkg/tsdb/cloudwatch/models/types.go @@ -1,20 +1,5 @@ package models -import ( - "net/url" - - "github.com/grafana/grafana-plugin-sdk-go/backend" -) - -type RequestContext struct { - MetricsClientProvider MetricsClientProvider - Settings CloudWatchSettings -} - -type RequestContextFactoryFunc func(pluginCtx backend.PluginContext, region string) (reqCtx RequestContext, err error) - -type RouteHandlerFunc func(pluginCtx backend.PluginContext, reqContextFactory RequestContextFactoryFunc, parameters url.Values) ([]byte, *HttpError) - type cloudWatchLink struct { View string `json:"view"` Stacked bool `json:"stacked"` @@ -31,7 +16,8 @@ type metricExpression struct { } type metricStatMeta struct { - Stat string `json:"stat"` - Period int `json:"period"` - Label string `json:"label,omitempty"` + Stat string `json:"stat"` + Period int `json:"period"` + Label string `json:"label,omitempty"` + AccountId string `json:"accountId,omitempty"` } diff --git a/pkg/tsdb/cloudwatch/resource_handler.go b/pkg/tsdb/cloudwatch/resource_handler.go index d9c88baebd8..55d95b7caff 100644 --- a/pkg/tsdb/cloudwatch/resource_handler.go +++ b/pkg/tsdb/cloudwatch/resource_handler.go @@ -19,10 +19,12 @@ func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux { mux.HandleFunc("/ec2-instance-attribute", handleResourceReq(e.handleGetEc2InstanceAttribute)) mux.HandleFunc("/resource-arns", handleResourceReq(e.handleGetResourceArns)) mux.HandleFunc("/log-groups", handleResourceReq(e.handleGetLogGroups)) + mux.HandleFunc("/describe-log-groups", routes.ResourceRequestMiddleware(routes.LogGroupsHandler, logger, e.getRequestContext)) // supports CrossAccountQuerying mux.HandleFunc("/all-log-groups", handleResourceReq(e.handleGetAllLogGroups)) mux.HandleFunc("/metrics", routes.ResourceRequestMiddleware(routes.MetricsHandler, logger, e.getRequestContext)) mux.HandleFunc("/dimension-values", routes.ResourceRequestMiddleware(routes.DimensionValuesHandler, logger, e.getRequestContext)) mux.HandleFunc("/dimension-keys", routes.ResourceRequestMiddleware(routes.DimensionKeysHandler, logger, e.getRequestContext)) + mux.HandleFunc("/accounts", routes.ResourceRequestMiddleware(routes.AccountsHandler, logger, e.getRequestContext)) mux.HandleFunc("/namespaces", routes.ResourceRequestMiddleware(routes.NamespacesHandler, logger, e.getRequestContext)) return mux } diff --git a/pkg/tsdb/cloudwatch/routes/accounts.go b/pkg/tsdb/cloudwatch/routes/accounts.go new file mode 100644 index 00000000000..8d125c55ba4 --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/accounts.go @@ -0,0 +1,55 @@ +package routes + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/services" +) + +func AccountsHandler(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, parameters url.Values) ([]byte, *models.HttpError) { + region := parameters.Get("region") + if region == "" { + return nil, models.NewHttpError("error in AccountsHandler", http.StatusBadRequest, fmt.Errorf("region is required")) + } + + service, err := newAccountsService(pluginCtx, reqCtxFactory, region) + if err != nil { + return nil, models.NewHttpError("error in AccountsHandler", http.StatusInternalServerError, err) + } + + accounts, err := service.GetAccountsForCurrentUserOrRole() + if err != nil { + msg := "error getting accounts for current user or role" + switch { + case errors.Is(err, services.ErrAccessDeniedException): + return nil, models.NewHttpError(msg, http.StatusForbidden, err) + default: + return nil, models.NewHttpError(msg, http.StatusInternalServerError, err) + } + } + + accountsResponse, err := json.Marshal(accounts) + if err != nil { + return nil, models.NewHttpError("error in AccountsHandler", http.StatusInternalServerError, err) + } + + return accountsResponse, nil +} + +// newAccountService is an account service factory. +// +// Stubbable by tests. +var newAccountsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.AccountsProvider, error) { + oamClient, err := reqCtxFactory(pluginCtx, region) + if err != nil { + return nil, err + } + + return services.NewAccountsService(oamClient.OAMClientProvider), nil +} diff --git a/pkg/tsdb/cloudwatch/routes/accounts_test.go b/pkg/tsdb/cloudwatch/routes/accounts_test.go new file mode 100644 index 00000000000..6562f46b360 --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/accounts_test.go @@ -0,0 +1,96 @@ +package routes + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/services" + "github.com/stretchr/testify/assert" +) + +func Test_accounts_route(t *testing.T) { + origNewAccountsService := newAccountsService + t.Cleanup(func() { + newAccountsService = origNewAccountsService + }) + + t.Run("successfully returns array of accounts json", func(t *testing.T) { + mockAccountsService := mocks.AccountsServiceMock{} + mockAccountsService.On("GetAccountsForCurrentUserOrRole").Return([]resources.ResourceResponse[resources.Account]{{ + Value: resources.Account{ + Id: "123456789012", + Arn: "some arn", + Label: "some label", + IsMonitoringAccount: true, + }, + }}, nil) + newAccountsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.AccountsProvider, error) { + return &mockAccountsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/accounts?region=us-east-1", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(AccountsHandler, logger, nil)) + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) + assert.JSONEq(t, `[{"value":{"id":"123456789012", "arn":"some arn", "isMonitoringAccount":true, "label":"some label"}}]`, rr.Body.String()) + }) + + t.Run("rejects POST method", func(t *testing.T) { + rr := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/accounts?region=us-east-1", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(AccountsHandler, logger, nil)) + handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusMethodNotAllowed, rr.Code) + }) + + t.Run("requires region query value", func(t *testing.T) { + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/accounts", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(AccountsHandler, logger, nil)) + handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + }) + + t.Run("returns 403 when accounts service returns ErrAccessDeniedException", func(t *testing.T) { + mockAccountsService := mocks.AccountsServiceMock{} + mockAccountsService.On("GetAccountsForCurrentUserOrRole").Return([]resources.ResourceResponse[resources.Account](nil), + fmt.Errorf("%w: %s", services.ErrAccessDeniedException, "some AWS message")) + newAccountsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.AccountsProvider, error) { + return &mockAccountsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/accounts?region=us-east-1", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(AccountsHandler, logger, nil)) + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusForbidden, rr.Code) + assert.JSONEq(t, + `{"Message":"error getting accounts for current user or role: access denied. please check your IAM policy: some AWS message", + "Error":"access denied. please check your IAM policy: some AWS message","StatusCode":403}`, rr.Body.String()) + }) + + t.Run("returns 500 when accounts service returns unknown error", func(t *testing.T) { + mockAccountsService := mocks.AccountsServiceMock{} + mockAccountsService.On("GetAccountsForCurrentUserOrRole").Return([]resources.ResourceResponse[resources.Account](nil), fmt.Errorf("some error")) + newAccountsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.AccountsProvider, error) { + return &mockAccountsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/accounts?region=us-east-1", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(AccountsHandler, logger, nil)) + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusInternalServerError, rr.Code) + assert.Equal(t, `{"Message":"error getting accounts for current user or role: some error","Error":"some error","StatusCode":500}`, rr.Body.String()) + }) +} diff --git a/pkg/tsdb/cloudwatch/routes/dimension_keys.go b/pkg/tsdb/cloudwatch/routes/dimension_keys.go index 302225dbed3..906f8229170 100644 --- a/pkg/tsdb/cloudwatch/routes/dimension_keys.go +++ b/pkg/tsdb/cloudwatch/routes/dimension_keys.go @@ -22,7 +22,7 @@ func DimensionKeysHandler(pluginCtx backend.PluginContext, reqCtxFactory models. return nil, models.NewHttpError("error in DimensionKeyHandler", http.StatusInternalServerError, err) } - var response []string + var response []resources.ResourceResponse[string] switch dimensionKeysRequest.Type() { case resources.FilterDimensionKeysRequest: response, err = service.GetDimensionKeysByDimensionFilter(dimensionKeysRequest) diff --git a/pkg/tsdb/cloudwatch/routes/dimension_keys_test.go b/pkg/tsdb/cloudwatch/routes/dimension_keys_test.go index 068e0fd4bca..4b02727ffd4 100644 --- a/pkg/tsdb/cloudwatch/routes/dimension_keys_test.go +++ b/pkg/tsdb/cloudwatch/routes/dimension_keys_test.go @@ -31,7 +31,7 @@ func Test_DimensionKeys_Route(t *testing.T) { len(r.DimensionFilter) == 2 && assert.Contains(t, r.DimensionFilter, &resources.Dimension{Name: "NodeID", Value: "Shared"}) && assert.Contains(t, r.DimensionFilter, &resources.Dimension{Name: "stage", Value: "QueryCommit"}) - })).Return([]string{}, nil).Once() + })).Return([]resources.ResourceResponse[string]{}, nil).Once() newListMetricsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.ListMetricsProvider, error) { return &mockListMetricsService, nil } @@ -48,10 +48,10 @@ func Test_DimensionKeys_Route(t *testing.T) { }) haveBeenCalled := false usedNamespace := "" - services.GetHardCodedDimensionKeysByNamespace = func(namespace string) ([]string, error) { + services.GetHardCodedDimensionKeysByNamespace = func(namespace string) ([]resources.ResourceResponse[string], error) { haveBeenCalled = true usedNamespace = namespace - return []string{}, nil + return []resources.ResourceResponse[string]{}, nil } rr := httptest.NewRecorder() req := httptest.NewRequest("GET", "/dimension-keys?region=us-east-2&namespace=AWS/EC2&metricName=CPUUtilization", nil) @@ -66,7 +66,7 @@ func Test_DimensionKeys_Route(t *testing.T) { t.Run("return 500 if GetDimensionKeysByDimensionFilter returns an error", func(t *testing.T) { mockListMetricsService := mocks.ListMetricsServiceMock{} - mockListMetricsService.On("GetDimensionKeysByDimensionFilter", mock.Anything).Return([]string{}, fmt.Errorf("some error")) + mockListMetricsService.On("GetDimensionKeysByDimensionFilter", mock.Anything).Return([]resources.ResourceResponse[string]{}, fmt.Errorf("some error")) newListMetricsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.ListMetricsProvider, error) { return &mockListMetricsService, nil } diff --git a/pkg/tsdb/cloudwatch/routes/dimension_values_test.go b/pkg/tsdb/cloudwatch/routes/dimension_values_test.go index cf365c1a090..8ecdf240666 100644 --- a/pkg/tsdb/cloudwatch/routes/dimension_values_test.go +++ b/pkg/tsdb/cloudwatch/routes/dimension_values_test.go @@ -26,7 +26,7 @@ func Test_DimensionValues_Route(t *testing.T) { len(r.DimensionFilter) == 2 && assert.Contains(t, r.DimensionFilter, &resources.Dimension{Name: "NodeID", Value: "Shared"}) && assert.Contains(t, r.DimensionFilter, &resources.Dimension{Name: "stage", Value: "QueryCommit"}) - })).Return([]string{}, nil).Once() + })).Return([]resources.ResourceResponse[string]{}, nil).Once() newListMetricsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.ListMetricsProvider, error) { return &mockListMetricsService, nil } @@ -38,7 +38,7 @@ func Test_DimensionValues_Route(t *testing.T) { t.Run("returns 500 if GetDimensionValuesByDimensionFilter returns an error", func(t *testing.T) { mockListMetricsService := mocks.ListMetricsServiceMock{} - mockListMetricsService.On("GetDimensionValuesByDimensionFilter", mock.Anything).Return([]string{}, fmt.Errorf("some error")) + mockListMetricsService.On("GetDimensionValuesByDimensionFilter", mock.Anything).Return([]resources.ResourceResponse[string]{}, fmt.Errorf("some error")) newListMetricsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.ListMetricsProvider, error) { return &mockListMetricsService, nil } diff --git a/pkg/tsdb/cloudwatch/routes/log_groups.go b/pkg/tsdb/cloudwatch/routes/log_groups.go new file mode 100644 index 00000000000..d5a803ec01d --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/log_groups.go @@ -0,0 +1,49 @@ +package routes + +import ( + "encoding/json" + "net/http" + "net/url" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/services" +) + +func LogGroupsHandler(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, parameters url.Values) ([]byte, *models.HttpError) { + request, err := resources.ParseLogGroupsRequest(parameters) + if err != nil { + return nil, models.NewHttpError("cannot set both log group name prefix and pattern", http.StatusBadRequest, err) + } + + service, err := newLogGroupsService(pluginCtx, reqCtxFactory, request.Region) + if err != nil { + return nil, models.NewHttpError("newLogGroupsService error", http.StatusInternalServerError, err) + } + + logGroups, err := service.GetLogGroups(request) + if err != nil { + return nil, models.NewHttpError("GetLogGroups error", http.StatusInternalServerError, err) + } + + logGroupsResponse, err := json.Marshal(logGroups) + if err != nil { + return nil, models.NewHttpError("LogGroupsHandler json error", http.StatusInternalServerError, err) + } + + return logGroupsResponse, nil +} + +// newLogGroupsService is a describe log groups service factory. +// +// Stubbable by tests. +var newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + reqCtx, err := reqCtxFactory(pluginCtx, region) + if err != nil { + return nil, err + } + + return services.NewLogGroupsService(reqCtx.LogsAPIProvider, reqCtx.Features.IsEnabled(featuremgmt.FlagCloudWatchCrossAccountQuerying)), nil +} diff --git a/pkg/tsdb/cloudwatch/routes/log_groups_test.go b/pkg/tsdb/cloudwatch/routes/log_groups_test.go new file mode 100644 index 00000000000..4902679c93a --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/log_groups_test.go @@ -0,0 +1,240 @@ +package routes + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func Test_log_groups_route(t *testing.T) { + origLogGroupsService := newLogGroupsService + t.Cleanup(func() { + newLogGroupsService = origLogGroupsService + }) + + mockFeatures := mocks.MockFeatures{} + mockFeatures.On("IsEnabled", featuremgmt.FlagCloudWatchCrossAccountQuerying).Return(false) + reqCtxFunc := func(pluginCtx backend.PluginContext, region string) (reqCtx models.RequestContext, err error) { + return models.RequestContext{Features: &mockFeatures}, err + } + + t.Run("successfully returns 1 log group with account id", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroups", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroup]{{ + Value: resources.LogGroup{ + Arn: "some arn", + Name: "some name", + }, + AccountId: utils.Pointer("111"), + }}, nil) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-groups", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) + assert.JSONEq(t, `[{"value":{"name":"some name", "arn":"some arn"},"accountId":"111"}]`, rr.Body.String()) + }) + + t.Run("successfully returns multiple log groups with account id", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroups", mock.Anything).Return( + []resources.ResourceResponse[resources.LogGroup]{ + { + Value: resources.LogGroup{ + Arn: "arn 1", + Name: "name 1", + }, + AccountId: utils.Pointer("111"), + }, { + Value: resources.LogGroup{ + Arn: "arn 2", + Name: "name 2", + }, + AccountId: utils.Pointer("222"), + }, + }, nil) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-groups", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) + assert.JSONEq(t, `[ + { + "value":{ + "name":"name 1", + "arn":"arn 1" + }, + "accountId":"111" + }, + { + "value":{ + "name":"name 2", + "arn":"arn 2" + }, + "accountId":"222" + } + ]`, rr.Body.String()) + }) + + t.Run("returns error when both logGroupPrefix and logGroup Pattern are provided", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroups", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroup]{}, nil) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-groups?logGroupNamePrefix=some-prefix&logGroupPattern=some-pattern", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.JSONEq(t, `{"Error":"cannot set both log group name prefix and pattern", "Message":"cannot set both log group name prefix and pattern: cannot set both log group name prefix and pattern", "StatusCode":400}`, rr.Body.String()) + }) + + t.Run("passes default log group limit and nil for logGroupNamePrefix, accountId, and logGroupPattern", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroups", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroup]{}, nil) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-groups", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + mockLogsService.AssertCalled(t, "GetLogGroups", resources.LogGroupsRequest{ + Limit: 50, + ResourceRequest: resources.ResourceRequest{}, + LogGroupNamePrefix: nil, + LogGroupNamePattern: nil, + }) + }) + + t.Run("passes default log group limit and nil for logGroupNamePrefix when both are absent", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroups", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroup]{}, nil) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-groups", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + mockLogsService.AssertCalled(t, "GetLogGroups", resources.LogGroupsRequest{ + Limit: 50, + LogGroupNamePrefix: nil, + }) + }) + + t.Run("passes log group limit from query parameter", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroups", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroup]{}, nil) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-groups?limit=2", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + mockLogsService.AssertCalled(t, "GetLogGroups", resources.LogGroupsRequest{ + Limit: 2, + }) + }) + + t.Run("passes logGroupPrefix from query parameter", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroups", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroup]{}, nil) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-groups?logGroupNamePrefix=some-prefix", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + mockLogsService.AssertCalled(t, "GetLogGroups", resources.LogGroupsRequest{ + Limit: 50, + LogGroupNamePrefix: utils.Pointer("some-prefix"), + }) + }) + + t.Run("passes logGroupPattern from query parameter", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroups", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroup]{}, nil) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-groups?logGroupPattern=some-pattern", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + mockLogsService.AssertCalled(t, "GetLogGroups", resources.LogGroupsRequest{ + Limit: 50, + LogGroupNamePattern: utils.Pointer("some-pattern"), + }) + }) + + t.Run("passes logGroupPattern from query parameter", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroups", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroup]{}, nil) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-groups?accountId=some-account-id", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + mockLogsService.AssertCalled(t, "GetLogGroups", resources.LogGroupsRequest{ + Limit: 50, + ResourceRequest: resources.ResourceRequest{AccountId: utils.Pointer("some-account-id")}, + }) + }) + + t.Run("returns error if service returns error", func(t *testing.T) { + mockLogsService := mocks.LogsService{} + mockLogsService.On("GetLogGroups", mock.Anything). + Return([]resources.ResourceResponse[resources.LogGroup]{}, fmt.Errorf("some error")) + newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) { + return &mockLogsService, nil + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/log-groups", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupsHandler, logger, reqCtxFunc)) + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusInternalServerError, rr.Code) + assert.JSONEq(t, `{"Error":"some error","Message":"GetLogGroups error: some error","StatusCode":500}`, rr.Body.String()) + }) +} diff --git a/pkg/tsdb/cloudwatch/routes/metrics.go b/pkg/tsdb/cloudwatch/routes/metrics.go index db90f7b2f2c..54aadfc78c0 100644 --- a/pkg/tsdb/cloudwatch/routes/metrics.go +++ b/pkg/tsdb/cloudwatch/routes/metrics.go @@ -22,20 +22,20 @@ func MetricsHandler(pluginCtx backend.PluginContext, reqCtxFactory models.Reques return nil, models.NewHttpError("error in MetricsHandler", http.StatusInternalServerError, err) } - var metrics []resources.Metric + var response []resources.ResourceResponse[resources.Metric] switch metricsRequest.Type() { case resources.AllMetricsRequestType: - metrics = services.GetAllHardCodedMetrics() + response = services.GetAllHardCodedMetrics() case resources.MetricsByNamespaceRequestType: - metrics, err = services.GetHardCodedMetricsByNamespace(metricsRequest.Namespace) + response, err = services.GetHardCodedMetricsByNamespace(metricsRequest.Namespace) case resources.CustomNamespaceRequestType: - metrics, err = service.GetMetricsByNamespace(metricsRequest.Namespace) + response, err = service.GetMetricsByNamespace(metricsRequest) } if err != nil { return nil, models.NewHttpError("error in MetricsHandler", http.StatusInternalServerError, err) } - metricsResponse, err := json.Marshal(metrics) + metricsResponse, err := json.Marshal(response) if err != nil { return nil, models.NewHttpError("error in MetricsHandler", http.StatusInternalServerError, err) } diff --git a/pkg/tsdb/cloudwatch/routes/metrics_test.go b/pkg/tsdb/cloudwatch/routes/metrics_test.go index 061a846012c..b11da17af18 100644 --- a/pkg/tsdb/cloudwatch/routes/metrics_test.go +++ b/pkg/tsdb/cloudwatch/routes/metrics_test.go @@ -1,27 +1,24 @@ package routes import ( - "encoding/json" "fmt" "net/http" "net/http/httptest" "testing" "github.com/grafana/grafana-plugin-sdk-go/backend" - "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/services" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) func Test_Metrics_Route(t *testing.T) { t.Run("calls GetMetricsByNamespace when a CustomNamespaceRequestType is passed", func(t *testing.T) { mockListMetricsService := mocks.ListMetricsServiceMock{} - mockListMetricsService.On("GetMetricsByNamespace", mock.Anything).Return([]resources.Metric{}, nil) + mockListMetricsService.On("GetMetricsByNamespace", mock.Anything).Return([]resources.ResourceResponse[resources.Metric]{}, nil) newListMetricsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.ListMetricsProvider, error) { return &mockListMetricsService, nil } @@ -38,17 +35,14 @@ func Test_Metrics_Route(t *testing.T) { services.GetAllHardCodedMetrics = origGetAllHardCodedMetrics }) haveBeenCalled := false - services.GetAllHardCodedMetrics = func() []resources.Metric { + services.GetAllHardCodedMetrics = func() []resources.ResourceResponse[resources.Metric] { haveBeenCalled = true - return []resources.Metric{} + return []resources.ResourceResponse[resources.Metric]{} } rr := httptest.NewRecorder() req := httptest.NewRequest("GET", "/metrics?region=us-east-2", nil) handler := http.HandlerFunc(ResourceRequestMiddleware(MetricsHandler, logger, nil)) handler.ServeHTTP(rr, req) - res := []resources.Metric{} - err := json.Unmarshal(rr.Body.Bytes(), &res) - require.Nil(t, err) assert.True(t, haveBeenCalled) }) @@ -59,25 +53,22 @@ func Test_Metrics_Route(t *testing.T) { }) haveBeenCalled := false usedNamespace := "" - services.GetHardCodedMetricsByNamespace = func(namespace string) ([]resources.Metric, error) { + services.GetHardCodedMetricsByNamespace = func(namespace string) ([]resources.ResourceResponse[resources.Metric], error) { haveBeenCalled = true usedNamespace = namespace - return []resources.Metric{}, nil + return []resources.ResourceResponse[resources.Metric]{}, nil } rr := httptest.NewRecorder() req := httptest.NewRequest("GET", "/metrics?region=us-east-2&namespace=AWS/DMS", nil) handler := http.HandlerFunc(ResourceRequestMiddleware(MetricsHandler, logger, nil)) handler.ServeHTTP(rr, req) - res := []resources.Metric{} - err := json.Unmarshal(rr.Body.Bytes(), &res) - require.Nil(t, err) assert.True(t, haveBeenCalled) assert.Equal(t, "AWS/DMS", usedNamespace) }) t.Run("returns 500 if GetMetricsByNamespace returns an error", func(t *testing.T) { mockListMetricsService := mocks.ListMetricsServiceMock{} - mockListMetricsService.On("GetMetricsByNamespace", mock.Anything).Return([]resources.Metric{}, fmt.Errorf("some error")) + mockListMetricsService.On("GetMetricsByNamespace", mock.Anything).Return([]resources.ResourceResponse[resources.Metric]{}, fmt.Errorf("some error")) newListMetricsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.ListMetricsProvider, error) { return &mockListMetricsService, nil } diff --git a/pkg/tsdb/cloudwatch/routes/namespaces.go b/pkg/tsdb/cloudwatch/routes/namespaces.go index 040748c5407..a79f7bb0def 100644 --- a/pkg/tsdb/cloudwatch/routes/namespaces.go +++ b/pkg/tsdb/cloudwatch/routes/namespaces.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/services" ) @@ -18,14 +19,19 @@ func NamespacesHandler(pluginCtx backend.PluginContext, reqCtxFactory models.Req return nil, models.NewHttpError("error in NamespacesHandler", http.StatusInternalServerError, err) } - result := services.GetHardCodedNamespaces() + response := services.GetHardCodedNamespaces() customNamespace := reqCtx.Settings.Namespace if customNamespace != "" { - result = append(result, strings.Split(customNamespace, ",")...) + customNamespaces := strings.Split(customNamespace, ",") + for _, customNamespace := range customNamespaces { + response = append(response, resources.ResourceResponse[string]{Value: customNamespace}) + } } - sort.Strings(result) + sort.Slice(response, func(i, j int) bool { + return response[i].Value < response[j].Value + }) - namespacesResponse, err := json.Marshal(result) + namespacesResponse, err := json.Marshal(response) if err != nil { return nil, models.NewHttpError("error in NamespacesHandler", http.StatusInternalServerError, err) } diff --git a/pkg/tsdb/cloudwatch/routes/namespaces_test.go b/pkg/tsdb/cloudwatch/routes/namespaces_test.go index 9eeec6f0a94..f2006677b19 100644 --- a/pkg/tsdb/cloudwatch/routes/namespaces_test.go +++ b/pkg/tsdb/cloudwatch/routes/namespaces_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/services" ) @@ -28,9 +29,9 @@ func Test_Namespaces_Route(t *testing.T) { services.GetHardCodedNamespaces = origGetHardCodedNamespaces }) haveBeenCalled := false - services.GetHardCodedNamespaces = func() []string { + services.GetHardCodedNamespaces = func() []resources.ResourceResponse[string] { haveBeenCalled = true - return []string{} + return []resources.ResourceResponse[string]{} } rr := httptest.NewRecorder() req := httptest.NewRequest("GET", "/namespaces", nil) @@ -44,15 +45,15 @@ func Test_Namespaces_Route(t *testing.T) { t.Cleanup(func() { services.GetHardCodedNamespaces = origGetHardCodedNamespaces }) - services.GetHardCodedNamespaces = func() []string { - return []string{"AWS/EC2", "AWS/ELB"} + services.GetHardCodedNamespaces = func() []resources.ResourceResponse[string] { + return []resources.ResourceResponse[string]{{Value: "AWS/EC2"}, {Value: "AWS/ELB"}} } rr := httptest.NewRecorder() req := httptest.NewRequest("GET", "/namespaces", nil) customNamespaces = "customNamespace1,customNamespace2" handler := http.HandlerFunc(ResourceRequestMiddleware(NamespacesHandler, logger, factoryFunc)) handler.ServeHTTP(rr, req) - assert.JSONEq(t, `["AWS/EC2", "AWS/ELB", "customNamespace1", "customNamespace2"]`, rr.Body.String()) + assert.JSONEq(t, `[{"value":"AWS/EC2"}, {"value":"AWS/ELB"}, {"value":"customNamespace1"}, {"value":"customNamespace2"}]`, rr.Body.String()) }) t.Run("sorts result", func(t *testing.T) { @@ -60,14 +61,14 @@ func Test_Namespaces_Route(t *testing.T) { t.Cleanup(func() { services.GetHardCodedNamespaces = origGetHardCodedNamespaces }) - services.GetHardCodedNamespaces = func() []string { - return []string{"AWS/XYZ", "AWS/ELB"} + services.GetHardCodedNamespaces = func() []resources.ResourceResponse[string] { + return []resources.ResourceResponse[string]{{Value: "AWS/XYZ"}, {Value: "AWS/ELB"}} } rr := httptest.NewRecorder() req := httptest.NewRequest("GET", "/namespaces", nil) customNamespaces = "DCustomNamespace1,ACustomNamespace2" handler := http.HandlerFunc(ResourceRequestMiddleware(NamespacesHandler, logger, factoryFunc)) handler.ServeHTTP(rr, req) - assert.JSONEq(t, `["ACustomNamespace2", "AWS/ELB", "AWS/XYZ", "DCustomNamespace1"]`, rr.Body.String()) + assert.JSONEq(t, `[{"value":"ACustomNamespace2"}, {"value":"AWS/ELB"}, {"value":"AWS/XYZ"}, {"value":"DCustomNamespace1"}]`, rr.Body.String()) }) } diff --git a/pkg/tsdb/cloudwatch/services/accounts.go b/pkg/tsdb/cloudwatch/services/accounts.go new file mode 100644 index 00000000000..ab8db64147e --- /dev/null +++ b/pkg/tsdb/cloudwatch/services/accounts.go @@ -0,0 +1,89 @@ +package services + +import ( + "errors" + "fmt" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/oam" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" +) + +var ErrAccessDeniedException = errors.New("access denied. please check your IAM policy") + +type AccountsService struct { + models.OAMClientProvider +} + +func NewAccountsService(oamClient models.OAMClientProvider) models.AccountsProvider { + return &AccountsService{oamClient} +} + +func (a *AccountsService) GetAccountsForCurrentUserOrRole() ([]resources.ResourceResponse[resources.Account], error) { + var nextToken *string + sinks := []*oam.ListSinksItem{} + for { + response, err := a.ListSinks(&oam.ListSinksInput{NextToken: nextToken}) + if err != nil { + var aerr awserr.Error + if errors.As(err, &aerr) { + switch aerr.Code() { + // unlike many other services, OAM doesn't define this error code. however, it's returned in case calling role/user has insufficient permissions + case "AccessDeniedException": + return nil, fmt.Errorf("%w: %s", ErrAccessDeniedException, aerr.Message()) + } + } + } + if err != nil { + return nil, fmt.Errorf("ListSinks error: %w", err) + } + + sinks = append(sinks, response.Items...) + + if response.NextToken == nil { + break + } + nextToken = response.NextToken + } + + if len(sinks) == 0 { + return nil, nil + } + + sinkIdentifier := sinks[0].Arn + response := []resources.Account{{ + Id: getAccountId(*sinkIdentifier), + Label: *sinks[0].Name, + Arn: *sinkIdentifier, + IsMonitoringAccount: true, + }} + + nextToken = nil + for { + links, err := a.ListAttachedLinks(&oam.ListAttachedLinksInput{ + SinkIdentifier: sinkIdentifier, + NextToken: nextToken, + }) + if err != nil { + return nil, fmt.Errorf("ListAttachedLinks error: %w", err) + } + + for _, link := range links.Items { + arn := *link.LinkArn + response = append(response, resources.Account{ + Id: getAccountId(arn), + Label: *link.Label, + Arn: arn, + IsMonitoringAccount: false, + }) + } + + if links.NextToken == nil { + break + } + nextToken = links.NextToken + } + + return valuesToListMetricRespone(response), nil +} diff --git a/pkg/tsdb/cloudwatch/services/accounts_test.go b/pkg/tsdb/cloudwatch/services/accounts_test.go new file mode 100644 index 00000000000..7e74e6fbec9 --- /dev/null +++ b/pkg/tsdb/cloudwatch/services/accounts_test.go @@ -0,0 +1,165 @@ +package services + +import ( + "fmt" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/oam" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestHandleGetAccounts(t *testing.T) { + t.Run("Should return an error in case of insufficient permissions from ListSinks", func(t *testing.T) { + fakeOAMClient := &mocks.FakeOAMClient{} + fakeOAMClient.On("ListSinks", mock.Anything).Return(&oam.ListSinksOutput{}, awserr.New("AccessDeniedException", + "AWS message", nil)) + accounts := NewAccountsService(fakeOAMClient) + + resp, err := accounts.GetAccountsForCurrentUserOrRole() + + assert.Error(t, err) + assert.Nil(t, resp) + assert.Equal(t, err.Error(), "access denied. please check your IAM policy: AWS message") + assert.ErrorIs(t, err, ErrAccessDeniedException) + }) + + t.Run("Should return an error in case of any error from ListSinks", func(t *testing.T) { + fakeOAMClient := &mocks.FakeOAMClient{} + fakeOAMClient.On("ListSinks", mock.Anything).Return(&oam.ListSinksOutput{}, fmt.Errorf("some error")) + accounts := NewAccountsService(fakeOAMClient) + + resp, err := accounts.GetAccountsForCurrentUserOrRole() + + assert.Error(t, err) + assert.Nil(t, resp) + assert.Equal(t, err.Error(), "ListSinks error: some error") + }) + + t.Run("Should return empty array in case no monitoring account exists", func(t *testing.T) { + fakeOAMClient := &mocks.FakeOAMClient{} + fakeOAMClient.On("ListSinks", mock.Anything).Return(&oam.ListSinksOutput{}, nil) + accounts := NewAccountsService(fakeOAMClient) + + resp, err := accounts.GetAccountsForCurrentUserOrRole() + + assert.NoError(t, err) + assert.Empty(t, resp) + }) + + t.Run("Should return one monitoring account (the first) even though ListSinks returns multiple sinks", func(t *testing.T) { + fakeOAMClient := &mocks.FakeOAMClient{} + fakeOAMClient.On("ListSinks", mock.Anything).Return(&oam.ListSinksOutput{ + Items: []*oam.ListSinksItem{ + {Name: aws.String("Account 1"), Arn: aws.String("arn:aws:logs:us-east-1:123456789012:log-group:my-log-group1")}, + {Name: aws.String("Account 2"), Arn: aws.String("arn:aws:logs:us-east-1:123456789012:log-group:my-log-group2")}, + }, + NextToken: new(string), + }, nil).Once() + fakeOAMClient.On("ListSinks", mock.Anything).Return(&oam.ListSinksOutput{ + Items: []*oam.ListSinksItem{ + {Name: aws.String("Account 3"), Arn: aws.String("arn:aws:logs:us-east-1:123456789012:log-group:my-log-group3")}, + }, + NextToken: nil, + }, nil) + fakeOAMClient.On("ListAttachedLinks", mock.Anything).Return(&oam.ListAttachedLinksOutput{}, nil) + accounts := NewAccountsService(fakeOAMClient) + + resp, err := accounts.GetAccountsForCurrentUserOrRole() + + assert.NoError(t, err) + fakeOAMClient.AssertNumberOfCalls(t, "ListSinks", 2) + require.Len(t, resp, 1) + assert.True(t, resp[0].Value.IsMonitoringAccount) + assert.Equal(t, "Account 1", resp[0].Value.Label) + assert.Equal(t, "arn:aws:logs:us-east-1:123456789012:log-group:my-log-group1", resp[0].Value.Arn) + }) + + t.Run("Should merge the first sink with attached links", func(t *testing.T) { + fakeOAMClient := &mocks.FakeOAMClient{} + fakeOAMClient.On("ListSinks", mock.Anything).Return(&oam.ListSinksOutput{ + Items: []*oam.ListSinksItem{ + {Name: aws.String("Account 1"), Arn: aws.String("arn:aws:logs:us-east-1:123456789012:log-group:my-log-group1")}, + {Name: aws.String("Account 2"), Arn: aws.String("arn:aws:logs:us-east-1:123456789012:log-group:my-log-group2")}, + }, + NextToken: new(string), + }, nil).Once() + fakeOAMClient.On("ListSinks", mock.Anything).Return(&oam.ListSinksOutput{ + Items: []*oam.ListSinksItem{ + {Name: aws.String("Account 3"), Arn: aws.String("arn:aws:logs:us-east-1:123456789012:log-group:my-log-group3")}, + }, + NextToken: nil, + }, nil) + fakeOAMClient.On("ListAttachedLinks", mock.Anything).Return(&oam.ListAttachedLinksOutput{ + Items: []*oam.ListAttachedLinksItem{ + {Label: aws.String("Account 10"), LinkArn: aws.String("arn:aws:logs:us-east-1:123456789013:log-group:my-log-group10")}, + {Label: aws.String("Account 11"), LinkArn: aws.String("arn:aws:logs:us-east-1:123456789014:log-group:my-log-group11")}, + }, + NextToken: new(string), + }, nil).Once() + fakeOAMClient.On("ListAttachedLinks", mock.Anything).Return(&oam.ListAttachedLinksOutput{ + Items: []*oam.ListAttachedLinksItem{ + {Label: aws.String("Account 12"), LinkArn: aws.String("arn:aws:logs:us-east-1:123456789012:log-group:my-log-group12")}, + }, + NextToken: nil, + }, nil) + accounts := NewAccountsService(fakeOAMClient) + + resp, err := accounts.GetAccountsForCurrentUserOrRole() + + assert.NoError(t, err) + fakeOAMClient.AssertNumberOfCalls(t, "ListSinks", 2) + fakeOAMClient.AssertNumberOfCalls(t, "ListAttachedLinks", 2) + expectedAccounts := []resources.ResourceResponse[resources.Account]{ + {Value: resources.Account{Id: "123456789012", Label: "Account 1", Arn: "arn:aws:logs:us-east-1:123456789012:log-group:my-log-group1", IsMonitoringAccount: true}}, + {Value: resources.Account{Id: "123456789013", Label: "Account 10", Arn: "arn:aws:logs:us-east-1:123456789013:log-group:my-log-group10", IsMonitoringAccount: false}}, + {Value: resources.Account{Id: "123456789014", Label: "Account 11", Arn: "arn:aws:logs:us-east-1:123456789014:log-group:my-log-group11", IsMonitoringAccount: false}}, + {Value: resources.Account{Id: "123456789012", Label: "Account 12", Arn: "arn:aws:logs:us-east-1:123456789012:log-group:my-log-group12", IsMonitoringAccount: false}}, + } + assert.Equal(t, expectedAccounts, resp) + }) + + t.Run("Should call ListAttachedLinks with arn of first sink", func(t *testing.T) { + fakeOAMClient := &mocks.FakeOAMClient{} + fakeOAMClient.On("ListSinks", mock.Anything).Return(&oam.ListSinksOutput{ + Items: []*oam.ListSinksItem{ + {Name: aws.String("Account 1"), Arn: aws.String("arn:aws:logs:us-east-1:123456789012:log-group:my-log-group1")}, + }, + NextToken: new(string), + }, nil).Once() + fakeOAMClient.On("ListSinks", mock.Anything).Return(&oam.ListSinksOutput{ + Items: []*oam.ListSinksItem{ + {Name: aws.String("Account 3"), Arn: aws.String("arn:aws:logs:us-east-1:123456789012:log-group:my-log-group3")}, + }, + NextToken: nil, + }, nil).Once() + fakeOAMClient.On("ListAttachedLinks", mock.Anything).Return(&oam.ListAttachedLinksOutput{}, nil) + accounts := NewAccountsService(fakeOAMClient) + + _, _ = accounts.GetAccountsForCurrentUserOrRole() + + fakeOAMClient.AssertCalled(t, "ListAttachedLinks", &oam.ListAttachedLinksInput{ + SinkIdentifier: aws.String("arn:aws:logs:us-east-1:123456789012:log-group:my-log-group1"), + }) + }) + + t.Run("Should return an error in case of any error from ListAttachedLinks", func(t *testing.T) { + fakeOAMClient := &mocks.FakeOAMClient{} + fakeOAMClient.On("ListSinks", mock.Anything).Return(&oam.ListSinksOutput{ + Items: []*oam.ListSinksItem{{Name: aws.String("Account 1"), Arn: aws.String("arn:aws:logs:us-east-1:123456789012:log-group:my-log-group1")}}, + }, nil) + fakeOAMClient.On("ListAttachedLinks", mock.Anything).Return(&oam.ListAttachedLinksOutput{}, fmt.Errorf("some error")).Once() + accounts := NewAccountsService(fakeOAMClient) + + resp, err := accounts.GetAccountsForCurrentUserOrRole() + + assert.Error(t, err) + assert.Nil(t, resp) + assert.Equal(t, err.Error(), "ListAttachedLinks error: some error") + }) +} diff --git a/pkg/tsdb/cloudwatch/services/hardcoded_metrics.go b/pkg/tsdb/cloudwatch/services/hardcoded_metrics.go index 2268089b566..8680fdf7184 100644 --- a/pkg/tsdb/cloudwatch/services/hardcoded_metrics.go +++ b/pkg/tsdb/cloudwatch/services/hardcoded_metrics.go @@ -7,16 +7,16 @@ import ( "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" ) -var GetHardCodedDimensionKeysByNamespace = func(namespace string) ([]string, error) { - var dimensionKeys []string +var GetHardCodedDimensionKeysByNamespace = func(namespace string) ([]resources.ResourceResponse[string], error) { + var response []string exists := false - if dimensionKeys, exists = constants.NamespaceDimensionKeysMap[namespace]; !exists { + if response, exists = constants.NamespaceDimensionKeysMap[namespace]; !exists { return nil, fmt.Errorf("unable to find dimensions for namespace '%q'", namespace) } - return dimensionKeys, nil + return valuesToListMetricRespone(response), nil } -var GetHardCodedMetricsByNamespace = func(namespace string) ([]resources.Metric, error) { +var GetHardCodedMetricsByNamespace = func(namespace string) ([]resources.ResourceResponse[resources.Metric], error) { response := []resources.Metric{} exists := false var metrics []string @@ -28,10 +28,10 @@ var GetHardCodedMetricsByNamespace = func(namespace string) ([]resources.Metric, response = append(response, resources.Metric{Namespace: namespace, Name: metric}) } - return response, nil + return valuesToListMetricRespone(response), nil } -var GetAllHardCodedMetrics = func() []resources.Metric { +var GetAllHardCodedMetrics = func() []resources.ResourceResponse[resources.Metric] { response := []resources.Metric{} for namespace, metrics := range constants.NamespaceMetricsMap { for _, metric := range metrics { @@ -39,14 +39,14 @@ var GetAllHardCodedMetrics = func() []resources.Metric { } } - return response + return valuesToListMetricRespone(response) } -var GetHardCodedNamespaces = func() []string { - var namespaces []string +var GetHardCodedNamespaces = func() []resources.ResourceResponse[string] { + response := []string{} for key := range constants.NamespaceMetricsMap { - namespaces = append(namespaces, key) + response = append(response, key) } - return namespaces + return valuesToListMetricRespone(response) } diff --git a/pkg/tsdb/cloudwatch/services/hardcoded_metrics_test.go b/pkg/tsdb/cloudwatch/services/hardcoded_metrics_test.go index 4895ef9dc61..e5c9d4f6ade 100644 --- a/pkg/tsdb/cloudwatch/services/hardcoded_metrics_test.go +++ b/pkg/tsdb/cloudwatch/services/hardcoded_metrics_test.go @@ -19,7 +19,7 @@ func TestHardcodedMetrics_GetHardCodedDimensionKeysByNamespace(t *testing.T) { t.Run("Should return keys if namespace exist", func(t *testing.T) { resp, err := GetHardCodedDimensionKeysByNamespace("AWS/EC2") require.NoError(t, err) - assert.Equal(t, []string{"AutoScalingGroupName", "ImageId", "InstanceId", "InstanceType"}, resp) + assert.Equal(t, []resources.ResourceResponse[string]{{Value: "AutoScalingGroupName"}, {Value: "ImageId"}, {Value: "InstanceId"}, {Value: "InstanceType"}}, resp) }) } @@ -34,6 +34,6 @@ func TestHardcodedMetrics_GetHardCodedMetricsByNamespace(t *testing.T) { t.Run("Should return metrics if namespace exist", func(t *testing.T) { resp, err := GetHardCodedMetricsByNamespace("AWS/IoTAnalytics") require.NoError(t, err) - assert.Equal(t, []resources.Metric{{Name: "ActionExecution", Namespace: "AWS/IoTAnalytics"}, {Name: "ActivityExecutionError", Namespace: "AWS/IoTAnalytics"}, {Name: "IncomingMessages", Namespace: "AWS/IoTAnalytics"}}, resp) + assert.Equal(t, []resources.ResourceResponse[resources.Metric]{{Value: resources.Metric{Name: "ActionExecution", Namespace: "AWS/IoTAnalytics"}}, {Value: resources.Metric{Name: "ActivityExecutionError", Namespace: "AWS/IoTAnalytics"}}, {Value: resources.Metric{Name: "IncomingMessages", Namespace: "AWS/IoTAnalytics"}}}, resp) }) } diff --git a/pkg/tsdb/cloudwatch/services/list_metrics.go b/pkg/tsdb/cloudwatch/services/list_metrics.go index 8ab2d2bebad..f4f66914dfe 100644 --- a/pkg/tsdb/cloudwatch/services/list_metrics.go +++ b/pkg/tsdb/cloudwatch/services/list_metrics.go @@ -18,7 +18,7 @@ func NewListMetricsService(metricsClient models.MetricsClientProvider) models.Li return &ListMetricsService{metricsClient} } -func (l *ListMetricsService) GetDimensionKeysByDimensionFilter(r resources.DimensionKeysRequest) ([]string, error) { +func (l *ListMetricsService) GetDimensionKeysByDimensionFilter(r resources.DimensionKeysRequest) ([]resources.ResourceResponse[string], error) { input := &cloudwatch.ListMetricsInput{} if r.Namespace != "" { input.Namespace = aws.String(r.Namespace) @@ -27,13 +27,14 @@ func (l *ListMetricsService) GetDimensionKeysByDimensionFilter(r resources.Dimen input.MetricName = aws.String(r.MetricName) } setDimensionFilter(input, r.DimensionFilter) + setAccount(input, r.ResourceRequest) metrics, err := l.ListMetricsWithPageLimit(input) if err != nil { return nil, fmt.Errorf("%v: %w", "unable to call AWS API", err) } - var dimensionKeys []string + response := []resources.ResourceResponse[string]{} // remove duplicates dupCheck := make(map[string]struct{}) for _, metric := range metrics { @@ -56,26 +57,27 @@ func (l *ListMetricsService) GetDimensionKeysByDimensionFilter(r resources.Dimen } dupCheck[*dim.Name] = struct{}{} - dimensionKeys = append(dimensionKeys, *dim.Name) + response = append(response, resources.ResourceResponse[string]{AccountId: metric.AccountId, Value: *dim.Name}) } } - return dimensionKeys, nil + return response, nil } -func (l *ListMetricsService) GetDimensionValuesByDimensionFilter(r resources.DimensionValuesRequest) ([]string, error) { +func (l *ListMetricsService) GetDimensionValuesByDimensionFilter(r resources.DimensionValuesRequest) ([]resources.ResourceResponse[string], error) { input := &cloudwatch.ListMetricsInput{ Namespace: aws.String(r.Namespace), MetricName: aws.String(r.MetricName), } setDimensionFilter(input, r.DimensionFilter) + setAccount(input, r.ResourceRequest) metrics, err := l.ListMetricsWithPageLimit(input) if err != nil { return nil, fmt.Errorf("%v: %w", "unable to call AWS API", err) } - var dimensionValues []string + response := []resources.ResourceResponse[string]{} dupCheck := make(map[string]bool) for _, metric := range metrics { for _, dim := range metric.Dimensions { @@ -85,51 +87,33 @@ func (l *ListMetricsService) GetDimensionValuesByDimensionFilter(r resources.Dim } dupCheck[*dim.Value] = true - dimensionValues = append(dimensionValues, *dim.Value) + response = append(response, resources.ResourceResponse[string]{AccountId: metric.AccountId, Value: *dim.Value}) } } } - sort.Strings(dimensionValues) - return dimensionValues, nil + sort.Slice(response, func(i, j int) bool { + return response[i].Value < response[j].Value + }) + return response, nil } -func (l *ListMetricsService) GetDimensionKeysByNamespace(namespace string) ([]string, error) { - metrics, err := l.ListMetricsWithPageLimit(&cloudwatch.ListMetricsInput{Namespace: aws.String(namespace)}) - if err != nil { - return []string{}, err - } - - var dimensionKeys []string - dupCheck := make(map[string]struct{}) - for _, metric := range metrics { - for _, dim := range metric.Dimensions { - if _, exists := dupCheck[*dim.Name]; exists { - continue - } - - dupCheck[*dim.Name] = struct{}{} - dimensionKeys = append(dimensionKeys, *dim.Name) - } - } - - return dimensionKeys, nil -} - -func (l *ListMetricsService) GetMetricsByNamespace(namespace string) ([]resources.Metric, error) { - metrics, err := l.ListMetricsWithPageLimit(&cloudwatch.ListMetricsInput{Namespace: aws.String(namespace)}) +func (l *ListMetricsService) GetMetricsByNamespace(r resources.MetricsRequest) ([]resources.ResourceResponse[resources.Metric], error) { + input := &cloudwatch.ListMetricsInput{Namespace: aws.String(r.Namespace)} + setAccount(input, r.ResourceRequest) + metrics, err := l.ListMetricsWithPageLimit(input) if err != nil { return nil, err } - response := []resources.Metric{} + response := []resources.ResourceResponse[resources.Metric]{} dupCheck := make(map[string]struct{}) for _, metric := range metrics { if _, exists := dupCheck[*metric.MetricName]; exists { continue } dupCheck[*metric.MetricName] = struct{}{} - response = append(response, resources.Metric{Name: *metric.MetricName, Namespace: *metric.Namespace}) + response = append(response, resources.ResourceResponse[resources.Metric]{AccountId: metric.AccountId, Value: resources.Metric{Name: *metric.MetricName, Namespace: *metric.Namespace}}) } return response, nil @@ -146,3 +130,12 @@ func setDimensionFilter(input *cloudwatch.ListMetricsInput, dimensionFilter []*r input.Dimensions = append(input.Dimensions, df) } } + +func setAccount(input *cloudwatch.ListMetricsInput, r *resources.ResourceRequest) { + if r != nil && r.AccountId != nil { + input.IncludeLinkedAccounts = aws.Bool(true) + if !r.ShouldTargetAllAccounts() { + input.OwningAccount = r.AccountId + } + } +} diff --git a/pkg/tsdb/cloudwatch/services/list_metrics_test.go b/pkg/tsdb/cloudwatch/services/list_metrics_test.go index 34548ffaa98..8d63baabcbf 100644 --- a/pkg/tsdb/cloudwatch/services/list_metrics_test.go +++ b/pkg/tsdb/cloudwatch/services/list_metrics_test.go @@ -7,40 +7,55 @@ import ( "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) -var metricResponse = []*cloudwatch.Metric{ +const useLinkedAccountsId = "all" + +var metricResponse = []resources.MetricResponse{ { - MetricName: aws.String("CPUUtilization"), - Namespace: aws.String("AWS/EC2"), - Dimensions: []*cloudwatch.Dimension{ - {Name: aws.String("InstanceId"), Value: aws.String("i-1234567890abcdef0")}, - {Name: aws.String("InstanceType"), Value: aws.String("t2.micro")}, + Metric: &cloudwatch.Metric{ + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []*cloudwatch.Dimension{ + {Name: aws.String("InstanceId"), Value: aws.String("i-1234567890abcdef0")}, + {Name: aws.String("InstanceType"), Value: aws.String("t2.micro")}, + }, }, }, { - MetricName: aws.String("CPUUtilization"), - Namespace: aws.String("AWS/EC2"), - Dimensions: []*cloudwatch.Dimension{ - {Name: aws.String("InstanceId"), Value: aws.String("i-5234567890abcdef0")}, - {Name: aws.String("InstanceType"), Value: aws.String("t2.micro")}, - {Name: aws.String("AutoScalingGroupName"), Value: aws.String("my-asg")}, + Metric: &cloudwatch.Metric{ + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []*cloudwatch.Dimension{ + {Name: aws.String("InstanceId"), Value: aws.String("i-5234567890abcdef0")}, + {Name: aws.String("InstanceType"), Value: aws.String("t2.micro")}, + {Name: aws.String("AutoScalingGroupName"), Value: aws.String("my-asg")}, + }, }, }, { - MetricName: aws.String("CPUUtilization"), - Namespace: aws.String("AWS/EC2"), - Dimensions: []*cloudwatch.Dimension{ - {Name: aws.String("InstanceId"), Value: aws.String("i-64234567890abcdef0")}, - {Name: aws.String("InstanceType"), Value: aws.String("t3.micro")}, - {Name: aws.String("AutoScalingGroupName"), Value: aws.String("my-asg2")}, + Metric: &cloudwatch.Metric{ + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []*cloudwatch.Dimension{ + {Name: aws.String("InstanceId"), Value: aws.String("i-64234567890abcdef0")}, + {Name: aws.String("InstanceType"), Value: aws.String("t3.micro")}, + {Name: aws.String("AutoScalingGroupName"), Value: aws.String("my-asg2")}, + }, }, }, } +type validateInputTestCase[T resources.DimensionKeysRequest | resources.DimensionValuesRequest] struct { + name string + input T + listMetricsWithPageLimitInput *cloudwatch.ListMetricsInput +} + func TestListMetricsService_GetDimensionKeysByDimensionFilter(t *testing.T) { t.Run("Should filter out duplicates and keys matching dimension filter keys", func(t *testing.T) { fakeMetricsClient := &mocks.FakeMetricsClient{} @@ -51,27 +66,68 @@ func TestListMetricsService_GetDimensionKeysByDimensionFilter(t *testing.T) { ResourceRequest: &resources.ResourceRequest{Region: "us-east-1"}, Namespace: "AWS/EC2", MetricName: "CPUUtilization", - DimensionFilter: []*resources.Dimension{ - {Name: "InstanceId", Value: ""}, - }, + DimensionFilter: []*resources.Dimension{{Name: "InstanceId", Value: ""}}, }) require.NoError(t, err) - assert.Equal(t, []string{"InstanceType", "AutoScalingGroupName"}, resp) + assert.Equal(t, []resources.ResourceResponse[string]{{Value: "InstanceType"}, {Value: "AutoScalingGroupName"}}, resp) }) -} -func TestListMetricsService_GetDimensionKeysByNamespace(t *testing.T) { - t.Run("Should filter out duplicates and keys matching dimension filter keys", func(t *testing.T) { - fakeMetricsClient := &mocks.FakeMetricsClient{} - fakeMetricsClient.On("ListMetricsWithPageLimit", mock.Anything).Return(metricResponse, nil) - listMetricsService := NewListMetricsService(fakeMetricsClient) + testCases := []validateInputTestCase[resources.DimensionKeysRequest]{ + { + name: "Should set account correctly on list metric input if it cross account is defined on the request", + input: resources.DimensionKeysRequest{ + ResourceRequest: &resources.ResourceRequest{Region: "us-east-1", AccountId: utils.Pointer(useLinkedAccountsId)}, + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + DimensionFilter: []*resources.Dimension{{Name: "InstanceId", Value: ""}}, + }, + listMetricsWithPageLimitInput: &cloudwatch.ListMetricsInput{ + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []*cloudwatch.DimensionFilter{{Name: aws.String("InstanceId")}}, + IncludeLinkedAccounts: aws.Bool(true), + }, + }, + { + name: "Should set account correctly on list metric input if single account is defined on the request", + input: resources.DimensionKeysRequest{ + ResourceRequest: &resources.ResourceRequest{Region: "us-east-1", AccountId: utils.Pointer("1234567890")}, + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + DimensionFilter: []*resources.Dimension{{Name: "InstanceId", Value: ""}}, + }, + listMetricsWithPageLimitInput: &cloudwatch.ListMetricsInput{ + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []*cloudwatch.DimensionFilter{{Name: aws.String("InstanceId")}}, + IncludeLinkedAccounts: aws.Bool(true), + OwningAccount: aws.String("1234567890"), + }, + }, + { + name: "Should not set namespace and metricName on list metric input if empty strings are set for these in the request", + input: resources.DimensionKeysRequest{ + ResourceRequest: &resources.ResourceRequest{Region: "us-east-1"}, + Namespace: "", + MetricName: "", + DimensionFilter: []*resources.Dimension{{Name: "InstanceId", Value: ""}}, + }, + listMetricsWithPageLimitInput: &cloudwatch.ListMetricsInput{Dimensions: []*cloudwatch.DimensionFilter{{Name: aws.String("InstanceId")}}}, + }, + } - resp, err := listMetricsService.GetDimensionKeysByNamespace("AWS/EC2") - - require.NoError(t, err) - assert.Equal(t, []string{"InstanceId", "InstanceType", "AutoScalingGroupName"}, resp) - }) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeMetricsClient := &mocks.FakeMetricsClient{} + fakeMetricsClient.On("ListMetricsWithPageLimit", mock.Anything).Return(metricResponse, nil) + listMetricsService := NewListMetricsService(fakeMetricsClient) + res, err := listMetricsService.GetDimensionKeysByDimensionFilter(tc.input) + require.NoError(t, err) + require.NotEmpty(t, res) + fakeMetricsClient.AssertCalled(t, "ListMetricsWithPageLimit", tc.listMetricsWithPageLimitInput) + }) + } } func TestListMetricsService_GetDimensionValuesByDimensionFilter(t *testing.T) { @@ -91,6 +147,52 @@ func TestListMetricsService_GetDimensionValuesByDimensionFilter(t *testing.T) { }) require.NoError(t, err) - assert.Equal(t, []string{"i-1234567890abcdef0", "i-5234567890abcdef0", "i-64234567890abcdef0"}, resp) + assert.Equal(t, []resources.ResourceResponse[string]{{Value: "i-1234567890abcdef0"}, {Value: "i-5234567890abcdef0"}, {Value: "i-64234567890abcdef0"}}, resp) }) + + testCases := []validateInputTestCase[resources.DimensionValuesRequest]{ + { + name: "Should set account correctly on list metric input if it cross account is defined on the request", + input: resources.DimensionValuesRequest{ + ResourceRequest: &resources.ResourceRequest{Region: "us-east-1", AccountId: utils.Pointer(useLinkedAccountsId)}, + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + DimensionFilter: []*resources.Dimension{{Name: "InstanceId", Value: ""}}, + }, + listMetricsWithPageLimitInput: &cloudwatch.ListMetricsInput{ + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []*cloudwatch.DimensionFilter{{Name: aws.String("InstanceId")}}, + IncludeLinkedAccounts: aws.Bool(true), + }, + }, + { + name: "Should set account correctly on list metric input if single account is defined on the request", + input: resources.DimensionValuesRequest{ + ResourceRequest: &resources.ResourceRequest{Region: "us-east-1", AccountId: utils.Pointer("1234567890")}, + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + DimensionFilter: []*resources.Dimension{{Name: "InstanceId", Value: ""}}, + }, + listMetricsWithPageLimitInput: &cloudwatch.ListMetricsInput{ + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []*cloudwatch.DimensionFilter{{Name: aws.String("InstanceId")}}, + IncludeLinkedAccounts: aws.Bool(true), + OwningAccount: aws.String("1234567890"), + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeMetricsClient := &mocks.FakeMetricsClient{} + fakeMetricsClient.On("ListMetricsWithPageLimit", mock.Anything).Return(metricResponse, nil) + listMetricsService := NewListMetricsService(fakeMetricsClient) + res, err := listMetricsService.GetDimensionValuesByDimensionFilter(tc.input) + require.NoError(t, err) + require.Empty(t, res) + fakeMetricsClient.AssertCalled(t, "ListMetricsWithPageLimit", tc.listMetricsWithPageLimitInput) + }) + } } diff --git a/pkg/tsdb/cloudwatch/services/logs.go b/pkg/tsdb/cloudwatch/services/logs.go new file mode 100644 index 00000000000..64658bc51df --- /dev/null +++ b/pkg/tsdb/cloudwatch/services/logs.go @@ -0,0 +1,53 @@ +package services + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/utils" +) + +type LogGroupsService struct { + logGroupsAPI models.CloudWatchLogsAPIProvider + isCrossAccountEnabled bool +} + +func NewLogGroupsService(logsClient models.CloudWatchLogsAPIProvider, isCrossAccountEnabled bool) models.LogGroupsProvider { + return &LogGroupsService{logGroupsAPI: logsClient, isCrossAccountEnabled: isCrossAccountEnabled} +} + +func (s *LogGroupsService) GetLogGroups(req resources.LogGroupsRequest) ([]resources.ResourceResponse[resources.LogGroup], error) { + input := &cloudwatchlogs.DescribeLogGroupsInput{ + Limit: aws.Int64(req.Limit), + LogGroupNamePrefix: req.LogGroupNamePrefix, + } + + if s.isCrossAccountEnabled && req.AccountId != nil { + input.IncludeLinkedAccounts = aws.Bool(true) + if req.LogGroupNamePattern != nil { + input.LogGroupNamePrefix = req.LogGroupNamePattern + } + if !req.IsTargetingAllAccounts() { + // TODO: accept more than one account id in search + input.AccountIdentifiers = []*string{req.AccountId} + } + } + response, err := s.logGroupsAPI.DescribeLogGroups(input) + if err != nil || response == nil { + return nil, err + } + + var result []resources.ResourceResponse[resources.LogGroup] + for _, logGroup := range response.LogGroups { + result = append(result, resources.ResourceResponse[resources.LogGroup]{ + Value: resources.LogGroup{ + Arn: *logGroup.Arn, + Name: *logGroup.LogGroupName, + }, + AccountId: utils.Pointer(getAccountId(*logGroup.Arn)), + }) + } + + return result, nil +} diff --git a/pkg/tsdb/cloudwatch/services/logs_test.go b/pkg/tsdb/cloudwatch/services/logs_test.go new file mode 100644 index 00000000000..a1b81a8d094 --- /dev/null +++ b/pkg/tsdb/cloudwatch/services/logs_test.go @@ -0,0 +1,200 @@ +package services + +import ( + "fmt" + "testing" + + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func Test_GetLogGroups(t *testing.T) { + t.Run("Should map log groups response", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return( + &cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []*cloudwatchlogs.LogGroup{ + {Arn: utils.Pointer("arn:aws:logs:us-east-1:111:log-group:group_a"), LogGroupName: utils.Pointer("group_a")}, + {Arn: utils.Pointer("arn:aws:logs:us-east-1:222:log-group:group_b"), LogGroupName: utils.Pointer("group_b")}, + {Arn: utils.Pointer("arn:aws:logs:us-east-1:333:log-group:group_c"), LogGroupName: utils.Pointer("group_c")}, + }, + }, nil) + service := NewLogGroupsService(mockLogsAPI, false) + + resp, err := service.GetLogGroups(resources.LogGroupsRequest{}) + + assert.NoError(t, err) + assert.Equal(t, []resources.ResourceResponse[resources.LogGroup]{ + { + AccountId: utils.Pointer("111"), + Value: resources.LogGroup{Arn: "arn:aws:logs:us-east-1:111:log-group:group_a", Name: "group_a"}, + }, + { + AccountId: utils.Pointer("222"), + Value: resources.LogGroup{Arn: "arn:aws:logs:us-east-1:222:log-group:group_b", Name: "group_b"}, + }, + { + AccountId: utils.Pointer("333"), + Value: resources.LogGroup{Arn: "arn:aws:logs:us-east-1:333:log-group:group_c", Name: "group_c"}, + }, + }, resp) + }) + + t.Run("Should only use LogGroupNamePrefix even if LogGroupNamePattern passed in resource call", func(t *testing.T) { + // TODO: use LogGroupNamePattern when we have accounted for its behavior, still a little unexpected at the moment + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, nil) + service := NewLogGroupsService(mockLogsAPI, false) + + _, err := service.GetLogGroups(resources.LogGroupsRequest{ + Limit: 0, + LogGroupNamePrefix: utils.Pointer("test"), + }) + + assert.NoError(t, err) + mockLogsAPI.AssertCalled(t, "DescribeLogGroups", &cloudwatchlogs.DescribeLogGroupsInput{ + Limit: utils.Pointer(int64(0)), + LogGroupNamePrefix: utils.Pointer("test"), + }) + }) + + t.Run("Should call api without LogGroupNamePrefix nor LogGroupNamePattern if not passed in resource call", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, nil) + service := NewLogGroupsService(mockLogsAPI, false) + + _, err := service.GetLogGroups(resources.LogGroupsRequest{}) + + assert.NoError(t, err) + mockLogsAPI.AssertCalled(t, "DescribeLogGroups", &cloudwatchlogs.DescribeLogGroupsInput{ + Limit: utils.Pointer(int64(0)), + }) + }) + + t.Run("Should return an error when API returns error", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, + fmt.Errorf("some error")) + service := NewLogGroupsService(mockLogsAPI, false) + + _, err := service.GetLogGroups(resources.LogGroupsRequest{}) + + assert.Error(t, err) + assert.Equal(t, "some error", err.Error()) + }) +} + +func Test_GetLogGroups_crossAccountQuerying(t *testing.T) { + t.Run("Should not includeLinkedAccounts or accountId if isCrossAccountEnabled is set to false", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, nil) + service := NewLogGroupsService(mockLogsAPI, false) + + _, err := service.GetLogGroups(resources.LogGroupsRequest{ + ResourceRequest: resources.ResourceRequest{AccountId: utils.Pointer("accountId")}, + LogGroupNamePrefix: utils.Pointer("prefix"), + }) + + assert.NoError(t, err) + mockLogsAPI.AssertCalled(t, "DescribeLogGroups", &cloudwatchlogs.DescribeLogGroupsInput{ + Limit: utils.Pointer(int64(0)), + LogGroupNamePrefix: utils.Pointer("prefix"), + }) + }) + + t.Run("Should replace LogGroupNamePrefix if LogGroupNamePattern passed in resource call", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, nil) + service := NewLogGroupsService(mockLogsAPI, true) + + _, err := service.GetLogGroups(resources.LogGroupsRequest{ + ResourceRequest: resources.ResourceRequest{AccountId: utils.Pointer("accountId")}, + LogGroupNamePrefix: utils.Pointer("prefix"), + LogGroupNamePattern: utils.Pointer("pattern"), + }) + + assert.NoError(t, err) + mockLogsAPI.AssertCalled(t, "DescribeLogGroups", &cloudwatchlogs.DescribeLogGroupsInput{ + AccountIdentifiers: []*string{utils.Pointer("accountId")}, + Limit: utils.Pointer(int64(0)), + LogGroupNamePrefix: utils.Pointer("pattern"), + IncludeLinkedAccounts: utils.Pointer(true), + }) + }) + + t.Run("Should includeLinkedAccounts,and accountId if isCrossAccountEnabled is set to true", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, nil) + service := NewLogGroupsService(mockLogsAPI, true) + + _, err := service.GetLogGroups(resources.LogGroupsRequest{ + ResourceRequest: resources.ResourceRequest{AccountId: utils.Pointer("accountId")}, + }) + + assert.NoError(t, err) + mockLogsAPI.AssertCalled(t, "DescribeLogGroups", &cloudwatchlogs.DescribeLogGroupsInput{ + Limit: utils.Pointer(int64(0)), + IncludeLinkedAccounts: utils.Pointer(true), + AccountIdentifiers: []*string{utils.Pointer("accountId")}, + }) + }) + + t.Run("Should should not override prefix is there is no logGroupNamePattern", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, nil) + service := NewLogGroupsService(mockLogsAPI, true) + + _, err := service.GetLogGroups(resources.LogGroupsRequest{ + ResourceRequest: resources.ResourceRequest{AccountId: utils.Pointer("accountId")}, + LogGroupNamePrefix: utils.Pointer("prefix"), + }) + assert.NoError(t, err) + mockLogsAPI.AssertCalled(t, "DescribeLogGroups", &cloudwatchlogs.DescribeLogGroupsInput{ + AccountIdentifiers: []*string{utils.Pointer("accountId")}, + Limit: utils.Pointer(int64(0)), + LogGroupNamePrefix: utils.Pointer("prefix"), + IncludeLinkedAccounts: utils.Pointer(true), + }) + }) + + t.Run("Should not includeLinkedAccounts, or accountId if accountId is nil", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, nil) + service := NewLogGroupsService(mockLogsAPI, true) + + _, err := service.GetLogGroups(resources.LogGroupsRequest{ + LogGroupNamePrefix: utils.Pointer("prefix"), + }) + + assert.NoError(t, err) + mockLogsAPI.AssertCalled(t, "DescribeLogGroups", &cloudwatchlogs.DescribeLogGroupsInput{ + Limit: utils.Pointer(int64(0)), + LogGroupNamePrefix: utils.Pointer("prefix"), + }) + }) + + t.Run("Should should not override prefix is there is no logGroupNamePattern", func(t *testing.T) { + mockLogsAPI := &mocks.LogsAPI{} + mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, nil) + service := NewLogGroupsService(mockLogsAPI, true) + + _, err := service.GetLogGroups(resources.LogGroupsRequest{ + ResourceRequest: resources.ResourceRequest{ + AccountId: utils.Pointer("accountId"), + }, + LogGroupNamePrefix: utils.Pointer("prefix"), + }) + + assert.NoError(t, err) + mockLogsAPI.AssertCalled(t, "DescribeLogGroups", &cloudwatchlogs.DescribeLogGroupsInput{ + AccountIdentifiers: []*string{utils.Pointer("accountId")}, + IncludeLinkedAccounts: utils.Pointer(true), + Limit: utils.Pointer(int64(0)), + LogGroupNamePrefix: utils.Pointer("prefix"), + }) + }) +} diff --git a/pkg/tsdb/cloudwatch/services/utils.go b/pkg/tsdb/cloudwatch/services/utils.go new file mode 100644 index 00000000000..4684aa2acad --- /dev/null +++ b/pkg/tsdb/cloudwatch/services/utils.go @@ -0,0 +1,27 @@ +package services + +import ( + "strings" + + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources" +) + +func valuesToListMetricRespone[T any](values []T) []resources.ResourceResponse[T] { + var response []resources.ResourceResponse[T] + for _, value := range values { + response = append(response, resources.ResourceResponse[T]{Value: value}) + } + + return response +} + +func getAccountId(arn string) string { + // format: arn:partition:service:region:account-id:resource-id + parts := strings.Split(arn, ":") + + if len(parts) >= 4 { + return parts[4] + } + + return "" +} diff --git a/pkg/tsdb/cloudwatch/time_series_query.go b/pkg/tsdb/cloudwatch/time_series_query.go index b3d2fc4d8b3..fbec70a42b1 100644 --- a/pkg/tsdb/cloudwatch/time_series_query.go +++ b/pkg/tsdb/cloudwatch/time_series_query.go @@ -31,7 +31,9 @@ func (e *cloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, logger return nil, fmt.Errorf("invalid time range: start time must be before end time") } - requestQueries, err := models.ParseMetricDataQueries(req.Queries, startTime, endTime, e.features.IsEnabled(featuremgmt.FlagCloudWatchDynamicLabels)) + requestQueries, err := models.ParseMetricDataQueries(req.Queries, startTime, endTime, + e.features.IsEnabled(featuremgmt.FlagCloudWatchDynamicLabels), + e.features.IsEnabled(featuremgmt.FlagCloudWatchCrossAccountQuerying)) if err != nil { return nil, err } diff --git a/pkg/tsdb/cloudwatch/time_series_query_test.go b/pkg/tsdb/cloudwatch/time_series_query_test.go index 1dd7042ada6..29728c729fc 100644 --- a/pkg/tsdb/cloudwatch/time_series_query_test.go +++ b/pkg/tsdb/cloudwatch/time_series_query_test.go @@ -588,3 +588,186 @@ func Test_QueryData_response_data_frame_names(t *testing.T) { }) } } + +func TestTimeSeriesQuery_CrossAccountQuerying(t *testing.T) { + origNewCWClient := NewCWClient + t.Cleanup(func() { + NewCWClient = origNewCWClient + }) + var api mocks.MetricsAPI + + NewCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI { + return &api + } + im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { + return DataSource{Settings: models.CloudWatchSettings{}}, nil + }) + + t.Run("should call GetMetricDataInput with AccountId nil when no AccountId is provided", func(t *testing.T) { + api = mocks.MetricsAPI{} + api.On("GetMetricDataWithContext", mock.Anything, mock.Anything, mock.Anything).Return(&cloudwatch.GetMetricDataOutput{}, nil) + executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudWatchCrossAccountQuerying)) + + _, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, + }, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Now().Add(time.Hour * -2), To: time.Now().Add(time.Hour * -1)}, + JSON: json.RawMessage(`{ + "type": "timeSeriesQuery", + "subtype": "metrics", + "namespace": "AWS/EC2", + "metricName": "NetworkOut", + "dimensions": { + "InstanceId": "i-00645d91ed77d87ac" + }, + "region": "us-east-2", + "id": "a", + "alias": "NetworkOut", + "statistic": "Maximum", + "period": "300", + "hide": false, + "matchExact": true, + "refId": "A" + }`), + }, + }, + }) + require.NoError(t, err) + actualInput, ok := api.Calls[0].Arguments[1].(*cloudwatch.GetMetricDataInput) + require.True(t, ok) + require.Len(t, actualInput.MetricDataQueries, 1) + + assert.Nil(t, actualInput.MetricDataQueries[0].Expression) + assert.Nil(t, actualInput.MetricDataQueries[0].AccountId) + }) + + t.Run("should call GetMetricDataInput with AccountId nil when feature flag is false", func(t *testing.T) { + api = mocks.MetricsAPI{} + api.On("GetMetricDataWithContext", mock.Anything, mock.Anything, mock.Anything).Return(&cloudwatch.GetMetricDataOutput{}, nil) + executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) + _, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, + }, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Now().Add(time.Hour * -2), To: time.Now().Add(time.Hour * -1)}, + JSON: json.RawMessage(`{ + "type": "timeSeriesQuery", + "subtype": "metrics", + "namespace": "AWS/EC2", + "metricName": "NetworkOut", + "dimensions": { + "InstanceId": "i-00645d91ed77d87ac" + }, + "region": "us-east-2", + "id": "a", + "alias": "NetworkOut", + "statistic": "Maximum", + "period": "300", + "hide": false, + "matchExact": true, + "refId": "A", + "accountId":"some account Id" + }`), + }, + }, + }) + require.NoError(t, err) + actualInput, ok := api.Calls[0].Arguments[1].(*cloudwatch.GetMetricDataInput) + require.True(t, ok) + require.Len(t, actualInput.MetricDataQueries, 1) + + assert.Nil(t, actualInput.MetricDataQueries[0].Expression) + assert.Nil(t, actualInput.MetricDataQueries[0].AccountId) + }) + + t.Run("should call GetMetricDataInput with AccountId in a MetricStat query", func(t *testing.T) { + api = mocks.MetricsAPI{} + api.On("GetMetricDataWithContext", mock.Anything, mock.Anything, mock.Anything).Return(&cloudwatch.GetMetricDataOutput{}, nil) + executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudWatchCrossAccountQuerying)) + _, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, + }, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Now().Add(time.Hour * -2), To: time.Now().Add(time.Hour * -1)}, + JSON: json.RawMessage(`{ + "type": "timeSeriesQuery", + "subtype": "metrics", + "namespace": "AWS/EC2", + "metricName": "NetworkOut", + "dimensions": { + "InstanceId": "i-00645d91ed77d87ac" + }, + "region": "us-east-2", + "id": "a", + "alias": "NetworkOut", + "statistic": "Maximum", + "period": "300", + "hide": false, + "matchExact": true, + "refId": "A", + "accountId":"some account Id" + }`), + }, + }, + }) + require.NoError(t, err) + actualInput, ok := api.Calls[0].Arguments[1].(*cloudwatch.GetMetricDataInput) + require.True(t, ok) + require.Len(t, actualInput.MetricDataQueries, 1) + + require.NotNil(t, actualInput.MetricDataQueries[0].AccountId) + assert.Equal(t, "some account Id", *actualInput.MetricDataQueries[0].AccountId) + }) + + t.Run("should GetMetricDataInput with AccountId in an inferred search expression query", func(t *testing.T) { + api = mocks.MetricsAPI{} + api.On("GetMetricDataWithContext", mock.Anything, mock.Anything, mock.Anything).Return(&cloudwatch.GetMetricDataOutput{}, nil) + executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudWatchCrossAccountQuerying)) + _, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, + }, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Now().Add(time.Hour * -2), To: time.Now().Add(time.Hour * -1)}, + JSON: json.RawMessage(`{ + "type": "timeSeriesQuery", + "subtype": "metrics", + "namespace": "AWS/EC2", + "metricName": "NetworkOut", + "dimensions": { + "InstanceId": "*" + }, + "region": "us-east-2", + "id": "a", + "alias": "NetworkOut", + "statistic": "Maximum", + "period": "300", + "hide": false, + "matchExact": true, + "refId": "A", + "accountId":"some account Id" + }`), + }, + }, + }) + require.NoError(t, err) + actualInput, ok := api.Calls[0].Arguments[1].(*cloudwatch.GetMetricDataInput) + require.True(t, ok) + require.Len(t, actualInput.MetricDataQueries, 1) + + require.NotNil(t, actualInput.MetricDataQueries[0].Expression) + assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","InstanceId"} MetricName="NetworkOut" :aws.AccountId="some account Id"', 'Maximum', 300))`, *actualInput.MetricDataQueries[0].Expression) + }) +} diff --git a/pkg/tsdb/cloudwatch/utils/utils.go b/pkg/tsdb/cloudwatch/utils/utils.go new file mode 100644 index 00000000000..65a4cc8ba4d --- /dev/null +++ b/pkg/tsdb/cloudwatch/utils/utils.go @@ -0,0 +1,3 @@ +package utils + +func Pointer[T any](arg T) *T { return &arg } diff --git a/pkg/tsdb/cloudwatch/utils_test.go b/pkg/tsdb/cloudwatch/utils_test.go index 90577a090dc..0b3a07039c0 100644 --- a/pkg/tsdb/cloudwatch/utils_test.go +++ b/pkg/tsdb/cloudwatch/utils_test.go @@ -166,9 +166,6 @@ func (c fakeRGTAClient) GetResourcesPages(in *resourcegroupstaggingapi.GetResour } type fakeCheckHealthClient struct { - cloudwatchiface.CloudWatchAPI - cloudwatchlogsiface.CloudWatchLogsAPI - listMetricsPages func(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error describeLogGroups func(input *cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error) } diff --git a/public/app/plugins/datasource/cloudwatch/Search.tsx b/public/app/plugins/datasource/cloudwatch/Search.tsx new file mode 100644 index 00000000000..e7af681ffe1 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/Search.tsx @@ -0,0 +1,39 @@ +import { debounce } from 'lodash'; +import React, { useEffect, useMemo, useState } from 'react'; + +import { Icon, Input, useStyles2 } from '@grafana/ui'; + +import getStyles from './components/styles'; + +// TODO: consider moving search into grafana/ui, this is mostly the same as that in azure monitor +const Search = ({ searchFn, searchPhrase }: { searchPhrase: string; searchFn: (searchPhrase: string) => void }) => { + const [searchFilter, setSearchFilter] = useState(searchPhrase); + const styles = useStyles2(getStyles); + + const debouncedSearch = useMemo(() => debounce(searchFn, 600), [searchFn]); + + useEffect(() => { + return () => { + // Stop the invocation of the debounced function after unmounting + debouncedSearch?.cancel(); + }; + }, [debouncedSearch]); + + return ( + } + value={searchFilter} + onChange={(event) => { + const searchPhrase = event.currentTarget.value; + setSearchFilter(searchPhrase); + debouncedSearch(searchPhrase); + }} + placeholder="search by log group name prefix" + /> + ); +}; + +export default Search; diff --git a/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts b/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts index dcc5c5364ef..192561b8e1b 100644 --- a/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts +++ b/public/app/plugins/datasource/cloudwatch/__mocks__/CloudWatchDataSource.ts @@ -71,7 +71,7 @@ export function setupMockedDataSource({ if (variables) { templateService = setupMockedTemplateService(variables); if (mockGetVariableName) { - templateService.getVariableName = (name: string) => name; + templateService.getVariableName = (name: string) => name.replace('$', ''); } } @@ -83,6 +83,7 @@ export function setupMockedDataSource({ datasource.api.getRegions = jest.fn().mockResolvedValue([]); datasource.api.getDimensionKeys = jest.fn().mockResolvedValue([]); datasource.api.getMetrics = jest.fn().mockResolvedValue([]); + datasource.api.getAccounts = jest.fn().mockResolvedValue([]); datasource.logsQueryRunner.defaultLogGroups = []; const fetchMock = jest.fn().mockReturnValue(of({})); setBackendSrv({ @@ -241,3 +242,16 @@ export const periodIntervalVariable: CustomVariableModel = { hide: VariableHide.dontHide, type: 'custom', }; + +export const accountIdVariable: CustomVariableModel = { + ...initialCustomVariableModelState, + id: 'accountId', + name: 'accountId', + current: { + value: 'templatedaccountId', + text: 'templatedaccountId', + selected: true, + }, + options: [{ value: 'templatedRegion', text: 'templatedRegion', selected: true }], + multi: false, +}; diff --git a/public/app/plugins/datasource/cloudwatch/__mocks__/dashboardOnLoadedEvent.ts b/public/app/plugins/datasource/cloudwatch/__mocks__/dashboardOnLoadedEvent.ts index 1161f75b6e5..8ba9e8149c6 100644 --- a/public/app/plugins/datasource/cloudwatch/__mocks__/dashboardOnLoadedEvent.ts +++ b/public/app/plugins/datasource/cloudwatch/__mocks__/dashboardOnLoadedEvent.ts @@ -9,6 +9,31 @@ export const CloudWatchDashboardLoadedEvent = new DashboardLoadedEvent({ grafanaVersion: 'v9.0.0', queries: { cloudwatch: [ + { + accountId: '123456789', + datasource: { + type: 'cloudwatch', + uid: '123', + }, + dimensions: { + InstanceId: 'i-123', + }, + expression: '', + hide: false, + id: '', + label: '', + matchExact: true, + metricEditorMode: 0, + metricName: 'CPUUtilization', + metricQueryType: 0, + namespace: 'AWS/EC2', + period: '', + queryMode: 'Metrics', + refId: 'A', + region: 'us-east-1', + sqlExpression: '', + statistic: 'Average', + }, { alias: '', datasource: { diff --git a/public/app/plugins/datasource/cloudwatch/__mocks__/queries.ts b/public/app/plugins/datasource/cloudwatch/__mocks__/queries.ts index ebd054e6610..1280b72be72 100644 --- a/public/app/plugins/datasource/cloudwatch/__mocks__/queries.ts +++ b/public/app/plugins/datasource/cloudwatch/__mocks__/queries.ts @@ -1,6 +1,25 @@ +import { QueryEditorExpressionType } from '../expressions'; import { CloudWatchMetricsQuery, MetricQueryType, MetricEditorMode, CloudWatchLogsQuery } from '../types'; -export const validMetricsQuery: CloudWatchMetricsQuery = { +export const validMetricSearchCodeQuery: CloudWatchMetricsQuery = { + id: '', + queryMode: 'Metrics', + region: 'us-east-2', + namespace: 'AWS/EC2', + period: '3000', + alias: '', + metricName: 'CPUUtilization', + dimensions: { InstanceId: 'i-123' }, + matchExact: true, + statistic: 'Average', + expression: 'SEARCH()', + refId: 'A', + metricQueryType: MetricQueryType.Search, + metricEditorMode: MetricEditorMode.Code, + hide: false, +}; + +export const validMetricSearchBuilderQuery: CloudWatchMetricsQuery = { id: '', queryMode: 'Metrics', region: 'us-east-2', @@ -14,6 +33,53 @@ export const validMetricsQuery: CloudWatchMetricsQuery = { expression: '', refId: 'A', metricQueryType: MetricQueryType.Search, + metricEditorMode: MetricEditorMode.Builder, + hide: false, +}; + +export const validMetricQueryBuilderQuery: CloudWatchMetricsQuery = { + id: '', + queryMode: 'Metrics', + region: 'us-east-2', + namespace: 'AWS/EC2', + period: '3000', + alias: '', + metricName: 'CPUUtilization', + dimensions: { InstanceId: 'i-123' }, + matchExact: true, + statistic: 'Average', + sql: { + select: { + type: QueryEditorExpressionType.Function, + name: 'AVERAGE', + parameters: [ + { + type: QueryEditorExpressionType.FunctionParameter, + name: 'CPUUtilization', + }, + ], + }, + }, + refId: 'A', + metricQueryType: MetricQueryType.Query, + metricEditorMode: MetricEditorMode.Builder, + hide: false, +}; + +export const validMetricQueryCodeQuery: CloudWatchMetricsQuery = { + id: '', + queryMode: 'Metrics', + region: 'us-east-2', + namespace: 'AWS/EC2', + period: '3000', + alias: '', + metricName: 'CPUUtilization', + dimensions: { InstanceId: 'i-123' }, + matchExact: true, + statistic: 'Average', + sqlExpression: 'SELECT * FROM "AWS/EC2" WHERE "InstanceId" = \'i-123\'', + refId: 'A', + metricQueryType: MetricQueryType.Query, metricEditorMode: MetricEditorMode.Code, hide: false, }; diff --git a/public/app/plugins/datasource/cloudwatch/api.test.ts b/public/app/plugins/datasource/cloudwatch/api.test.ts index 6199876b77e..99fbbcc32f2 100644 --- a/public/app/plugins/datasource/cloudwatch/api.test.ts +++ b/public/app/plugins/datasource/cloudwatch/api.test.ts @@ -75,12 +75,16 @@ describe('api', () => { it('when getAllMetrics is called', async () => { const getMock = jest.fn().mockResolvedValue([ { - namespace: 'AWS/EC2', - name: 'CPUUtilization', + value: { + namespace: 'AWS/EC2', + name: 'CPUUtilization', + }, }, { - namespace: 'AWS/Redshift', - name: 'CPUPercentage', + value: { + namespace: 'AWS/Redshift', + name: 'CPUPercentage', + }, }, ]); const { api } = setupMockedAPI({ getMock }); @@ -94,12 +98,16 @@ describe('api', () => { it('when getMetrics', async () => { const getMock = jest.fn().mockResolvedValue([ { - namespace: 'AWS/EC2', - name: 'CPUUtilization', + value: { + namespace: 'AWS/EC2', + name: 'CPUUtilization', + }, }, { - namespace: 'AWS/EC2', - name: 'CPUPercentage', + value: { + namespace: 'AWS/EC2', + name: 'CPUPercentage', + }, }, ]); const { api } = setupMockedAPI({ getMock }); diff --git a/public/app/plugins/datasource/cloudwatch/api.ts b/public/app/plugins/datasource/cloudwatch/api.ts index a6342b11499..1edf5b49bad 100644 --- a/public/app/plugins/datasource/cloudwatch/api.ts +++ b/public/app/plugins/datasource/cloudwatch/api.ts @@ -1,6 +1,6 @@ import { memoize } from 'lodash'; -import { DataSourceInstanceSettings, SelectableValue, toOption } from '@grafana/data'; +import { DataSourceInstanceSettings, SelectableValue } from '@grafana/data'; import { getBackendSrv } from '@grafana/runtime'; import { TemplateSrv } from 'app/features/templating/template_srv'; @@ -11,8 +11,12 @@ import { GetDimensionKeysRequest, GetDimensionValuesRequest, GetMetricsRequest, + LogGroupResponse, MetricResponse, MultiFilters, + Account, + ResourceRequest, + ResourceResponse, } from './types'; export interface SelectableResourceValue extends SelectableValue { @@ -33,6 +37,18 @@ export class CloudWatchAPI extends CloudWatchRequest { return getBackendSrv().get(`/api/datasources/${this.instanceSettings.id}/resources/${subtype}`, parameters); } + getAccounts({ region }: ResourceRequest): Promise { + return this.memoizedGetRequest>>('accounts', { + region: this.templateSrv.replace(region), + }).then((accounts) => accounts.map((a) => a.value)); + } + + isMonitoringAccount(region: string): Promise { + return this.getAccounts({ region }) + .then((accounts) => accounts.some((account) => account.isMonitoringAccount)) + .catch(() => false); + } + getRegions() { return this.memoizedGetRequest('regions').then((regions) => [ { label: 'default', value: 'default', text: 'default' }, @@ -41,8 +57,8 @@ export class CloudWatchAPI extends CloudWatchRequest { } getNamespaces() { - return this.memoizedGetRequest('namespaces').then((namespaces) => - namespaces.map((n) => ({ label: n, value: n })) + return this.memoizedGetRequest>>('namespaces').then((namespaces) => + namespaces.map((n) => ({ label: n.value, value: n.value })) ); } @@ -53,6 +69,20 @@ export class CloudWatchAPI extends CloudWatchRequest { }); } + async describeCrossAccountLogGroups(params: DescribeLogGroupsRequest): Promise { + return this.memoizedGetRequest>>('describe-log-groups', { + ...params, + region: this.templateSrv.replace(this.getActualRegion(params.region)), + accountId: this.templateSrv.replace(params.accountId), + }).then((resourceResponse) => + resourceResponse.map((resource) => ({ + label: resource.value.name, + value: resource.value.arn, + text: resource.accountId || '', + })) + ); + } + async describeAllLogGroups(params: DescribeLogGroupsRequest) { return this.memoizedGetRequest('all-log-groups', { ...params, @@ -60,21 +90,26 @@ export class CloudWatchAPI extends CloudWatchRequest { }); } - async getMetrics({ region, namespace }: GetMetricsRequest): Promise>> { + async getMetrics({ region, namespace, accountId }: GetMetricsRequest): Promise>> { if (!namespace) { return []; } - return this.memoizedGetRequest('metrics', { + return this.memoizedGetRequest>>('metrics', { region: this.templateSrv.replace(this.getActualRegion(region)), namespace: this.templateSrv.replace(namespace), - }).then((metrics) => metrics.map((m) => ({ label: m.name, value: m.name }))); + accountId: this.templateSrv.replace(accountId), + }).then((metrics) => metrics.map((m) => ({ label: m.value.name, value: m.value.name }))); } - async getAllMetrics({ region }: GetMetricsRequest): Promise> { - return this.memoizedGetRequest('metrics', { + async getAllMetrics({ + region, + accountId, + }: GetMetricsRequest): Promise> { + return this.memoizedGetRequest>>('metrics', { region: this.templateSrv.replace(this.getActualRegion(region)), - }).then((metrics) => metrics.map((m) => ({ metricName: m.name, namespace: m.namespace }))); + accountId: this.templateSrv.replace(accountId), + }).then((metrics) => metrics.map((m) => ({ metricName: m.value.name, namespace: m.value.namespace }))); } async getDimensionKeys({ @@ -82,13 +117,15 @@ export class CloudWatchAPI extends CloudWatchRequest { namespace = '', dimensionFilters = {}, metricName = '', + accountId, }: GetDimensionKeysRequest): Promise>> { - return this.memoizedGetRequest('dimension-keys', { + return this.memoizedGetRequest>>('dimension-keys', { region: this.templateSrv.replace(this.getActualRegion(region)), namespace: this.templateSrv.replace(namespace), + accountId: this.templateSrv.replace(accountId), + metricName: this.templateSrv.replace(metricName), dimensionFilters: JSON.stringify(this.convertDimensionFormat(dimensionFilters, {})), - metricName, - }).then((dimensionKeys) => dimensionKeys.map(toOption)); + }).then((r) => r.map((r) => ({ label: r.value, value: r.value }))); } async getDimensionValues({ @@ -97,19 +134,20 @@ export class CloudWatchAPI extends CloudWatchRequest { namespace, dimensionFilters = {}, metricName = '', + accountId, }: GetDimensionValuesRequest) { if (!namespace || !metricName) { return []; } - const values = await this.memoizedGetRequest('dimension-values', { + const values = await this.memoizedGetRequest>>('dimension-values', { region: this.templateSrv.replace(this.getActualRegion(region)), namespace: this.templateSrv.replace(namespace), metricName: this.templateSrv.replace(metricName.trim()), dimensionKey: this.templateSrv.replace(dimensionKey), dimensionFilters: JSON.stringify(this.convertDimensionFormat(dimensionFilters, {})), - }).then((dimensionValues) => dimensionValues.map(toOption)); - + accountId: this.templateSrv.replace(accountId), + }).then((r) => r.map((r) => ({ label: r.value, value: r.value }))); return values; } diff --git a/public/app/plugins/datasource/cloudwatch/components/Account.test.tsx b/public/app/plugins/datasource/cloudwatch/components/Account.test.tsx new file mode 100644 index 00000000000..e2803524fd1 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/Account.test.tsx @@ -0,0 +1,65 @@ +import { render, screen } from '@testing-library/react'; +import React from 'react'; +import selectEvent from 'react-select-event'; + +import { Account } from './Account'; + +export const AccountOptions = [ + { + value: '123456789', + label: 'test-account1', + description: '123456789', + }, + { + value: '432156789013', + label: 'test-account2', + description: '432156789013', + }, + { + value: '999999999999', + label: 'test-account3', + description: '999999999999', + }, + { + label: 'Template Variables', + options: [ + { + value: '$fakeVar', + label: '$fakeVar', + }, + ], + }, +]; +describe('Account', () => { + const props = { + accountOptions: AccountOptions, + region: 'us-east-2', + onChange: jest.fn(), + accountId: '123456789012', + }; + + it('should not render if there are no accounts', async () => { + render(); + expect(screen.queryByLabelText('Account Selection')).not.toBeInTheDocument(); + }); + + it('should render a selectable field of accounts if there are accounts', async () => { + const onChange = jest.fn(); + render(); + expect(screen.getByLabelText('Account Selection')).toBeInTheDocument(); + await selectEvent.select(screen.getByLabelText('Account Selection'), 'test-account3', { container: document.body }); + expect(onChange).toBeCalledWith('999999999999'); + }); + + it("should default to 'all' if there is no selection", () => { + render(); + expect(screen.getByLabelText('Account Selection')).toBeInTheDocument(); + expect(screen.getByText('All')).toBeInTheDocument(); + }); + + it('should select an uninterpolated template variable if it has been selected', () => { + render(); + expect(screen.getByLabelText('Account Selection')).toBeInTheDocument(); + expect(screen.getByText('$fakeVar')).toBeInTheDocument(); + }); +}); diff --git a/public/app/plugins/datasource/cloudwatch/components/Account.tsx b/public/app/plugins/datasource/cloudwatch/components/Account.tsx new file mode 100644 index 00000000000..57cde6ac83a --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/Account.tsx @@ -0,0 +1,54 @@ +import React, { useMemo } from 'react'; + +import { SelectableValue } from '@grafana/data'; +import { EditorField } from '@grafana/experimental'; +import { Select } from '@grafana/ui'; + +export interface Props { + onChange: (accountId?: string) => void; + accountOptions: Array>; + accountId?: string; +} + +export const ALL_ACCOUNTS_OPTION = { + label: 'All', + value: 'all', + description: 'Target all linked accounts', +}; + +export function Account({ accountId, onChange, accountOptions }: Props) { + const selectedAccountExistsInOptions = useMemo( + () => + accountOptions.find((a) => { + if (a.options) { + const matchingTemplateVar = a.options.find((tempVar: SelectableValue) => { + return tempVar.value === accountId; + }); + return matchingTemplateVar; + } + return a.value === accountId; + }), + [accountOptions, accountId] + ); + + if (accountOptions.length === 0) { + return null; + } + + return ( + +
+ + + + + + + + + {isLoading && ( + + + + )} + {!isLoading && selectableLogGroups.length === 0 && ( + + + + )} + {!isLoading && + selectableLogGroups.map((row) => ( + + + + + + ))} + +
Log GroupAccount nameAccount ID
+ +
No log groups found
+ handleSelectCheckbox(row, ev.currentTarget.checked)} + value={!!(row.value && selectedLogGroups.some((lg) => lg.value === row.value))} + /> + + {accountNameById[row.text]}{row.text}
+ + +
+ + +
+ + +
+ +
+ +
+ {props.selectedLogGroups.map((lg) => ( +
+ {lg.label} + props.onChange(props.selectedLogGroups.filter((slg) => slg.value !== lg.value))} + /> +
+ ))} +
+ + ); +}; diff --git a/public/app/plugins/datasource/cloudwatch/components/Dimensions/FilterItem.tsx b/public/app/plugins/datasource/cloudwatch/components/Dimensions/FilterItem.tsx index 6fa2a829cd3..4dea46fead0 100644 --- a/public/app/plugins/datasource/cloudwatch/components/Dimensions/FilterItem.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/Dimensions/FilterItem.tsx @@ -34,7 +34,7 @@ const excludeCurrentKey = (dimensions: Dimensions, currentKey: string | undefine export const FilterItem: FunctionComponent = ({ filter, - metricStat: { region, namespace, metricName, dimensions }, + metricStat: { region, namespace, metricName, dimensions, accountId }, datasource, dimensionKeys, disableExpressions, @@ -58,6 +58,7 @@ export const FilterItem: FunctionComponent = ({ region, namespace, metricName, + accountId, }) .then((result: Array>) => { if (result.length && !disableExpressions && !result.some((o) => o.value === wildcardOption.value)) { @@ -67,7 +68,14 @@ export const FilterItem: FunctionComponent = ({ }); }; - const [state, loadOptions] = useAsyncFn(loadDimensionValues, [filter.key, dimensions]); + const [state, loadOptions] = useAsyncFn(loadDimensionValues, [ + filter.key, + dimensions, + region, + namespace, + metricName, + accountId, + ]); const theme = useTheme2(); const styles = getOperatorStyles(theme); diff --git a/public/app/plugins/datasource/cloudwatch/components/LogGroupSelection.test.tsx b/public/app/plugins/datasource/cloudwatch/components/LogGroupSelection.test.tsx new file mode 100644 index 00000000000..63c4c904bbc --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/LogGroupSelection.test.tsx @@ -0,0 +1,86 @@ +import { render, screen, waitFor } from '@testing-library/react'; +// eslint-disable-next-line lodash/import-scope +import lodash from 'lodash'; +import React from 'react'; + +import { config } from '@grafana/runtime'; + +import { setupMockedDataSource } from '../__mocks__/CloudWatchDataSource'; +import { CloudWatchLogsQuery } from '../types'; + +import { LogGroupSelection } from './LogGroupSelection'; + +const originalFeatureToggleValue = config.featureToggles.cloudWatchCrossAccountQuerying; +const originalDebounce = lodash.debounce; + +const defaultProps = { + datasource: setupMockedDataSource().datasource, + query: { + queryMode: 'Logs', + id: '', + region: '', + refId: '', + } as CloudWatchLogsQuery, + onChange: jest.fn(), + onRunQuery: jest.fn(), +}; +describe('LogGroupSelection', () => { + beforeEach(() => { + lodash.debounce = jest.fn().mockImplementation((fn) => { + fn.cancel = () => {}; + return fn; + }); + }); + afterEach(() => { + config.featureToggles.cloudWatchCrossAccountQuerying = originalFeatureToggleValue; + lodash.debounce = originalDebounce; + }); + it('renders the old logGroupSelector when the feature toggle is disabled and there are no linked accounts', async () => { + config.featureToggles.cloudWatchCrossAccountQuerying = false; + render(); + await waitFor(() => screen.getByText('Choose Log Groups')); + expect(screen.queryByText('Select Log Groups')).not.toBeInTheDocument(); + }); + it('renders the old logGroupSelector when the feature toggle is disabled but there are linked accounts', async () => { + config.featureToggles.cloudWatchCrossAccountQuerying = false; + const ds = setupMockedDataSource().datasource; + ds.api.getAccounts = () => + Promise.resolve([ + { + arn: 'arn', + id: 'accountId', + label: 'label', + isMonitoringAccount: true, + }, + ]); + + render(); + await waitFor(() => screen.getByText('Choose Log Groups')); + expect(screen.queryByText('Select Log Groups')).not.toBeInTheDocument(); + }); + + it('renders the old logGroupSelector when the feature toggle is enabled but there are no linked accounts', async () => { + config.featureToggles.cloudWatchCrossAccountQuerying = true; + render(); + await waitFor(() => screen.getByText('Choose Log Groups')); + expect(screen.queryByText('Select Log Groups')).not.toBeInTheDocument(); + }); + + it('renders the new logGroupSelector when the feature toggle is enabled and there are linked accounts', async () => { + config.featureToggles.cloudWatchCrossAccountQuerying = true; + const ds = setupMockedDataSource().datasource; + ds.api.getAccounts = () => + Promise.resolve([ + { + arn: 'arn', + id: 'accountId', + label: 'label', + isMonitoringAccount: true, + }, + ]); + + render(); + await waitFor(() => screen.getByText('Select Log Groups')); + expect(screen.queryByText('Choose Log Groups')).not.toBeInTheDocument(); + }); +}); diff --git a/public/app/plugins/datasource/cloudwatch/components/LogGroupSelection.tsx b/public/app/plugins/datasource/cloudwatch/components/LogGroupSelection.tsx new file mode 100644 index 00000000000..36b0cc7ef88 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/LogGroupSelection.tsx @@ -0,0 +1,64 @@ +import { css } from '@emotion/css'; +import React from 'react'; + +import { config } from '@grafana/runtime'; +import { LegacyForms } from '@grafana/ui'; + +import { SelectableResourceValue } from '../api'; +import { CloudWatchDatasource } from '../datasource'; +import { useAccountOptions } from '../hooks'; +import { CloudWatchLogsQuery, CloudWatchQuery, DescribeLogGroupsRequest } from '../types'; + +import { CrossAccountLogsQueryField } from './CrossAccountLogsQueryField'; +import { LogGroupSelector } from './LogGroupSelector'; + +type Props = { + datasource: CloudWatchDatasource; + query: CloudWatchLogsQuery; + onChange: (value: CloudWatchQuery) => void; + onRunQuery: () => void; +}; + +const rowGap = css` + gap: 3px; +`; + +export const LogGroupSelection = ({ datasource, query, onChange, onRunQuery }: Props) => { + const accountState = useAccountOptions(datasource.api, query.region); + + return ( +
+ {config.featureToggles.cloudWatchCrossAccountQuerying && accountState?.value?.length ? ( + ) => + datasource.api.describeCrossAccountLogGroups({ region: query.region, ...params }) + } + onChange={(selectedLogGroups: SelectableResourceValue[]) => { + onChange({ ...query, logGroups: selectedLogGroups, logGroupNames: [] }); + }} + accountOptions={accountState.value} + onRunQuery={onRunQuery} + selectedLogGroups={query.logGroups ?? []} /* todo handle defaults */ + /> + ) : ( + + } + /> + )} +
+ ); +}; diff --git a/public/app/plugins/datasource/cloudwatch/components/LogGroupSelector.tsx b/public/app/plugins/datasource/cloudwatch/components/LogGroupSelector.tsx index 7e5a3e09c67..b3aa44e9c85 100644 --- a/public/app/plugins/datasource/cloudwatch/components/LogGroupSelector.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/LogGroupSelector.tsx @@ -109,6 +109,12 @@ export const LogGroupSelector: React.FC = ({ // Config editor does not fetch new log group options unless changes have been saved saved && getAvailableLogGroupOptions(); + + // if component unmounts in the middle of setting state, we reset state and unsubscribe from fetchLogGroupOptions + return () => { + setAvailableLogGroups([]); + setLoadingLogGroups(false); + }; // this hook shouldn't get called every time selectedLogGroups or onChange updates // eslint-disable-next-line react-hooks/exhaustive-deps }, [datasource, region, saved]); diff --git a/public/app/plugins/datasource/cloudwatch/components/LogsQueryField.tsx b/public/app/plugins/datasource/cloudwatch/components/LogsQueryField.tsx index cdbb29a4cc4..487345359e2 100644 --- a/public/app/plugins/datasource/cloudwatch/components/LogsQueryField.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/LogsQueryField.tsx @@ -7,7 +7,6 @@ import { Editor } from 'slate-react'; import { AbsoluteTimeRange, QueryEditorProps } from '@grafana/data'; import { BracesPlugin, - LegacyForms, QueryField, SlatePrism, TypeaheadInput, @@ -26,7 +25,7 @@ import syntax from '../syntax'; import { CloudWatchJsonData, CloudWatchLogsQuery, CloudWatchQuery } from '../types'; import { getStatsGroups } from '../utils/query/getStatsGroups'; -import { LogGroupSelector } from './LogGroupSelector'; +import { LogGroupSelection } from './LogGroupSelection'; import QueryHeader from './QueryHeader'; export interface CloudWatchLogsQueryFieldProps @@ -39,14 +38,9 @@ export interface CloudWatchLogsQueryFieldProps query: CloudWatchLogsQuery; } -const rowGap = css` - gap: 3px; -`; - const addPaddingToButton = css` padding: 1px 4px; `; - interface State { hint: | { @@ -128,7 +122,7 @@ class CloudWatchLogsQueryField extends React.PureComponent -
- - } - /> -
+
{ExtraFieldElement} diff --git a/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.test.tsx b/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.test.tsx index 7b10802d6a9..e97032161c8 100644 --- a/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.test.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.test.tsx @@ -3,10 +3,14 @@ import userEvent from '@testing-library/user-event'; import React from 'react'; import selectEvent from 'react-select-event'; +import { config } from '@grafana/runtime'; + import { MetricStatEditor } from '..'; import { setupMockedDataSource } from '../../__mocks__/CloudWatchDataSource'; +import { validMetricSearchBuilderQuery } from '../../__mocks__/queries'; import { MetricStat } from '../../types'; +const originalFeatureToggleValue = config.featureToggles.cloudWatchCrossAccountQuerying; const ds = setupMockedDataSource({ variables: [], }); @@ -33,6 +37,9 @@ const props = { }; describe('MetricStatEditor', () => { + afterEach(() => { + config.featureToggles.cloudWatchCrossAccountQuerying = originalFeatureToggleValue; + }); describe('statistics field', () => { test.each([['Average', 'p23.23', 'p34', '$statistic']])('should accept valid values', async (statistic) => { const onChange = jest.fn(); @@ -199,4 +206,53 @@ describe('MetricStatEditor', () => { expect(await screen.findByText(expected)).toBeInTheDocument(); }); }); + + describe('account id', () => { + it('should set value to "all" when its a monitoring account and no account id is defined in the query', async () => { + config.featureToggles.cloudWatchCrossAccountQuerying = true; + const onChange = jest.fn(); + props.datasource.api.isMonitoringAccount = jest.fn().mockResolvedValue(true); + props.datasource.api.getAccounts = jest.fn().mockResolvedValue([ + { + value: '123456789', + label: 'test-account1', + description: '123456789', + }, + { + value: '432156789013', + label: 'test-account2', + description: '432156789013', + }, + ]); + await act(async () => { + render( + + ); + }); + expect(onChange).toHaveBeenCalledWith({ ...validMetricSearchBuilderQuery, accountId: 'all' }); + expect(await screen.findByText('Account')).toBeInTheDocument(); + }); + + it('should unset value when no accounts were found and an account id is defined in the query', async () => { + config.featureToggles.cloudWatchCrossAccountQuerying = true; + const onChange = jest.fn(); + props.datasource.api.isMonitoringAccount = jest.fn().mockResolvedValue(false); + props.datasource.api.getAccounts = jest.fn().mockResolvedValue([]); + await act(async () => { + render( + + ); + }); + expect(onChange).toHaveBeenCalledWith({ ...validMetricSearchBuilderQuery, accountId: undefined }); + expect(await screen.queryByText('Account')).not.toBeInTheDocument(); + }); + }); }); diff --git a/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx b/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx index 332f62649d1..72fc3b17817 100644 --- a/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx +++ b/public/app/plugins/datasource/cloudwatch/components/MetricStatEditor/MetricStatEditor.tsx @@ -1,15 +1,17 @@ -import React from 'react'; +import React, { useEffect } from 'react'; import { SelectableValue } from '@grafana/data'; import { EditorField, EditorFieldGroup, EditorRow, EditorRows, EditorSwitch } from '@grafana/experimental'; +import { config } from '@grafana/runtime'; import { Select } from '@grafana/ui'; import { Dimensions } from '..'; import { CloudWatchDatasource } from '../../datasource'; -import { useDimensionKeys, useMetrics, useNamespaces } from '../../hooks'; +import { useAccountOptions, useDimensionKeys, useMetrics, useNamespaces } from '../../hooks'; import { standardStatistics } from '../../standardStatistics'; import { MetricStat } from '../../types'; import { appendTemplateVariables, toOption } from '../../utils/utils'; +import { Account } from '../Account'; export type Props = { refId: string; @@ -28,10 +30,22 @@ export function MetricStatEditor({ onChange, onRunQuery, }: React.PropsWithChildren) { - const { region, namespace } = metricStat; const namespaces = useNamespaces(datasource); - const metrics = useMetrics(datasource, region, namespace); + const metrics = useMetrics(datasource, metricStat); const dimensionKeys = useDimensionKeys(datasource, { ...metricStat, dimensionFilters: metricStat.dimensions }); + const accountState = useAccountOptions(datasource.api, metricStat.region); + + useEffect(() => { + datasource.api.isMonitoringAccount(metricStat.region).then((isMonitoringAccount) => { + if (isMonitoringAccount && !accountState.loading && accountState.value?.length && !metricStat.accountId) { + onChange({ ...metricStat, accountId: 'all' }); + } + + if (!accountState.loading && accountState.value && !accountState.value.length && metricStat.accountId) { + onChange({ ...metricStat, accountId: undefined }); + } + }); + }, [accountState, metricStat, onChange, datasource.api]); const onMetricStatChange = (metricStat: MetricStat) => { onChange(metricStat); @@ -59,6 +73,16 @@ export function MetricStatEditor({ return ( + {!disableExpressions && config.featureToggles.cloudWatchCrossAccountQuerying && ( + { + onChange({ ...metricStat, accountId }); + onRunQuery(); + }} + accountOptions={accountState?.value || []} + > + )} = ['random', 'response']; +const options: Array = ['random', 'response', 'random edges']; diff --git a/public/app/plugins/datasource/testdata/datasource.ts b/public/app/plugins/datasource/testdata/datasource.ts index fd08daf5273..fd9e42ea665 100644 --- a/public/app/plugins/datasource/testdata/datasource.ts +++ b/public/app/plugins/datasource/testdata/datasource.ts @@ -19,7 +19,7 @@ import { DataSourceWithBackend, getBackendSrv, getGrafanaLiveSrv, getTemplateSrv import { getSearchFilterScopedVar } from 'app/features/variables/utils'; import { queryMetricTree } from './metricTree'; -import { generateRandomNodes, savedNodesResponse } from './nodeGraphUtils'; +import { generateRandomEdges, generateRandomNodes, savedNodesResponse } from './nodeGraphUtils'; import { runStream } from './runStreams'; import { flameGraphData } from './testData/flameGraphResponse'; import { Scenario, TestDataQuery } from './types'; @@ -210,6 +210,9 @@ export class TestDataDataSource extends DataSourceWithBackend { case 'response': frames = savedNodesResponse(); break; + case 'random edges': + frames = [generateRandomEdges(target.nodes?.count)]; + break; default: throw new Error(`Unknown node_graph sub type ${type}`); } diff --git a/public/app/plugins/datasource/testdata/nodeGraphUtils.ts b/public/app/plugins/datasource/testdata/nodeGraphUtils.ts index 894e5652401..c4b5b86448f 100644 --- a/public/app/plugins/datasource/testdata/nodeGraphUtils.ts +++ b/public/app/plugins/datasource/testdata/nodeGraphUtils.ts @@ -13,7 +13,7 @@ export function generateRandomNodes(count = 10) { const nodes = []; const root = { - id: '0', + id: 'root', title: 'root', subTitle: 'client', success: 1, @@ -44,11 +44,11 @@ export function generateRandomNodes(count = 10) { for (let i = 0; i <= additionalEdges; i++) { const sourceIndex = Math.floor(Math.random() * Math.floor(nodes.length - 1)); const targetIndex = Math.floor(Math.random() * Math.floor(nodes.length - 1)); - if (sourceIndex === targetIndex || nodes[sourceIndex].id === '0' || nodes[sourceIndex].id === '0') { + if (sourceIndex === targetIndex || nodes[sourceIndex].id === '0' || nodes[targetIndex].id === '0') { continue; } - nodes[sourceIndex].edges.push(nodes[sourceIndex].id); + nodes[sourceIndex].edges.push(nodes[targetIndex].id); } const nodeFields: Record & { values: ArrayVector }> = { @@ -108,27 +108,14 @@ export function generateRandomNodes(count = 10) { meta: { preferredVisualisationType: 'nodeGraph' }, }); - const edgeFields: any = { - [NodeGraphDataFrameFieldNames.id]: { - values: new ArrayVector(), - type: FieldType.string, - }, - [NodeGraphDataFrameFieldNames.source]: { - values: new ArrayVector(), - type: FieldType.string, - }, - [NodeGraphDataFrameFieldNames.target]: { - values: new ArrayVector(), - type: FieldType.string, - }, - }; - const edgesFrame = new MutableDataFrame({ name: 'edges', - fields: Object.keys(edgeFields).map((key) => ({ - ...edgeFields[key], - name: key, - })), + fields: [ + { name: NodeGraphDataFrameFieldNames.id, values: new ArrayVector(), type: FieldType.string }, + { name: NodeGraphDataFrameFieldNames.source, values: new ArrayVector(), type: FieldType.string }, + { name: NodeGraphDataFrameFieldNames.target, values: new ArrayVector(), type: FieldType.string }, + { name: NodeGraphDataFrameFieldNames.mainStat, values: new ArrayVector(), type: FieldType.number }, + ], meta: { preferredVisualisationType: 'nodeGraph' }, }); @@ -148,9 +135,10 @@ export function generateRandomNodes(count = 10) { continue; } edgesSet.add(id); - edgeFields.id.values.add(`${node.id}--${edge}`); - edgeFields.source.values.add(node.id); - edgeFields.target.values.add(edge); + edgesFrame.fields[0].values.add(`${node.id}--${edge}`); + edgesFrame.fields[1].values.add(node.id); + edgesFrame.fields[2].values.add(edge); + edgesFrame.fields[3].values.add(Math.random() * 100); } } @@ -161,7 +149,7 @@ function makeRandomNode(index: number) { const success = Math.random(); const error = 1 - success; return { - id: index.toString(), + id: `service:${index}`, title: `service:${index}`, subTitle: 'service', success, @@ -175,3 +163,8 @@ function makeRandomNode(index: number) { export function savedNodesResponse(): any { return [new MutableDataFrame(nodes), new MutableDataFrame(edges)]; } + +// Generates node graph data but only returns the edges +export function generateRandomEdges(count = 10) { + return generateRandomNodes(count)[1]; +} diff --git a/public/app/plugins/datasource/testdata/types.ts b/public/app/plugins/datasource/testdata/types.ts index 3e1d97ca488..82495788dcc 100644 --- a/public/app/plugins/datasource/testdata/types.ts +++ b/public/app/plugins/datasource/testdata/types.ts @@ -30,7 +30,7 @@ export interface TestDataQuery extends DataQuery { } export interface NodesQuery { - type?: 'random' | 'response'; + type?: 'random' | 'response' | 'random edges'; count?: number; } diff --git a/public/app/plugins/panel/nodeGraph/Node.tsx b/public/app/plugins/panel/nodeGraph/Node.tsx index 75b2b1f5f6d..498c843f462 100644 --- a/public/app/plugins/panel/nodeGraph/Node.tsx +++ b/public/app/plugins/panel/nodeGraph/Node.tsx @@ -96,9 +96,14 @@ export const Node = memo(function Node(props: {
- {node.mainStat && statToString(node.mainStat, node.dataFrameRowIndex)} + + {node.mainStat && statToString(node.mainStat.config, node.mainStat.values.get(node.dataFrameRowIndex))} +
- {node.secondaryStat && statToString(node.secondaryStat, node.dataFrameRowIndex)} + + {node.secondaryStat && + statToString(node.secondaryStat.config, node.secondaryStat.values.get(node.dataFrameRowIndex))} +
{ }); it('can zoom in and out', async () => { - render( []} />); + render( + []} + /> + ); const zoomIn = await screen.findByTitle(/Zoom in/); const zoomOut = await screen.findByTitle(/Zoom out/); @@ -44,8 +49,8 @@ describe('NodeGraph', () => { dataFrames={[ makeNodesDataFrame(3), makeEdgesDataFrame([ - [0, 1], - [1, 2], + { source: '0', target: '1' }, + { source: '1', target: '2' }, ]), ]} getLinks={() => []} @@ -70,7 +75,7 @@ describe('NodeGraph', () => { it('shows context menu when clicking on node or edge', async () => { render( { return [ { @@ -98,8 +103,8 @@ describe('NodeGraph', () => { dataFrames={[ makeNodesDataFrame(3), makeEdgesDataFrame([ - [0, 1], - [1, 2], + { source: '0', target: '1' }, + { source: '1', target: '2' }, ]), ]} getLinks={() => []} @@ -117,8 +122,8 @@ describe('NodeGraph', () => { dataFrames={[ makeNodesDataFrame(3), makeEdgesDataFrame([ - [0, 1], - [0, 2], + { source: '0', target: '1' }, + { source: '0', target: '2' }, ]), ]} getLinks={() => []} @@ -137,10 +142,10 @@ describe('NodeGraph', () => { dataFrames={[ makeNodesDataFrame(5), makeEdgesDataFrame([ - [0, 1], - [0, 2], - [2, 3], - [3, 4], + { source: '0', target: '1' }, + { source: '0', target: '2' }, + { source: '2', target: '3' }, + { source: '3', target: '4' }, ]), ]} getLinks={() => []} @@ -162,10 +167,10 @@ describe('NodeGraph', () => { dataFrames={[ makeNodesDataFrame(5), makeEdgesDataFrame([ - [0, 1], - [1, 2], - [2, 3], - [3, 4], + { source: '0', target: '1' }, + { source: '1', target: '2' }, + { source: '2', target: '3' }, + { source: '3', target: '4' }, ]), ]} getLinks={() => []} @@ -192,8 +197,8 @@ describe('NodeGraph', () => { dataFrames={[ makeNodesDataFrame(3), makeEdgesDataFrame([ - [0, 1], - [1, 2], + { source: '0', target: '1' }, + { source: '1', target: '2' }, ]), ]} getLinks={() => []} diff --git a/public/app/plugins/panel/nodeGraph/NodeGraph.tsx b/public/app/plugins/panel/nodeGraph/NodeGraph.tsx index d4ea4365bf3..a42d64eab54 100644 --- a/public/app/plugins/panel/nodeGraph/NodeGraph.tsx +++ b/public/app/plugins/panel/nodeGraph/NodeGraph.tsx @@ -4,7 +4,7 @@ import React, { memo, MouseEvent, useCallback, useEffect, useMemo, useState } fr import useMeasure from 'react-use/lib/useMeasure'; import { DataFrame, GrafanaTheme2, LinkModel } from '@grafana/data'; -import { Icon, Spinner, useStyles2, useTheme2 } from '@grafana/ui'; +import { Icon, Spinner, useStyles2 } from '@grafana/ui'; import { Edge } from './Edge'; import { EdgeArrowMarker } from './EdgeArrowMarker'; @@ -123,13 +123,11 @@ export function NodeGraph({ getLinks, dataFrames, nodeLimit }: Props) { const firstNodesDataFrame = nodesDataFrames[0]; const firstEdgesDataFrame = edgesDataFrames[0]; - const theme = useTheme2(); - // TODO we should be able to allow multiple dataframes for both edges and nodes, could be issue with node ids which in // that case should be unique or figure a way to link edges and nodes dataframes together. const processed = useMemo( - () => processNodes(firstNodesDataFrame, firstEdgesDataFrame, theme), - [firstEdgesDataFrame, firstNodesDataFrame, theme] + () => processNodes(firstNodesDataFrame, firstEdgesDataFrame), + [firstEdgesDataFrame, firstNodesDataFrame] ); // We need hover state here because for nodes we also highlight edges and for edges have labels separate to make @@ -162,7 +160,7 @@ export function NodeGraph({ getLinks, dataFrames, nodeLimit }: Props) { focusedNodeId ); - // If we move from grid to graph layout and we have focused node lets get its position to center there. We want do + // If we move from grid to graph layout, and we have focused node lets get its position to center there. We want to // do it specifically only in that case. const focusPosition = useFocusPositionOnLayout(config, nodes, focusedNodeId); const { panRef, zoomRef, onStepUp, onStepDown, isPanning, position, scale, isMaxZoom, isMinZoom } = usePanAndZoom( @@ -180,7 +178,7 @@ export function NodeGraph({ getLinks, dataFrames, nodeLimit }: Props) { ); const styles = useStyles2(getStyles); - // This cannot be inline func or it will create infinite render cycle. + // This cannot be inline func, or it will create infinite render cycle. const topLevelRef = useCallback( (r: HTMLDivElement) => { measureRef(r); diff --git a/public/app/plugins/panel/nodeGraph/layout.ts b/public/app/plugins/panel/nodeGraph/layout.ts index 30e4b57a6d7..36c3e6beabd 100644 --- a/public/app/plugins/panel/nodeGraph/layout.ts +++ b/public/app/plugins/panel/nodeGraph/layout.ts @@ -199,7 +199,7 @@ function gridLayout( const val1 = sort!.field.values.get(node1.dataFrameRowIndex); const val2 = sort!.field.values.get(node2.dataFrameRowIndex); - // Lets pretend we don't care about type of the stats for a while (they can be strings) + // Let's pretend we don't care about type of the stats for a while (they can be strings) return sort!.ascending ? val1 - val2 : val2 - val1; }); } diff --git a/public/app/plugins/panel/nodeGraph/types.ts b/public/app/plugins/panel/nodeGraph/types.ts index 130a02923d6..b9f08b60913 100644 --- a/public/app/plugins/panel/nodeGraph/types.ts +++ b/public/app/plugins/panel/nodeGraph/types.ts @@ -35,6 +35,8 @@ export type NodeDatum = SimulationNodeDatum & { color?: Field; }; +export type NodeDatumFromEdge = NodeDatum & { mainStatNumeric?: number; secondaryStatNumeric?: number }; + // This is the data we have before the graph is laid out with source and target being string IDs. type LinkDatum = SimulationLinkDatum & { source: string; diff --git a/public/app/plugins/panel/nodeGraph/useContextMenu.tsx b/public/app/plugins/panel/nodeGraph/useContextMenu.tsx index a8ab98ad701..cdc3cda0615 100644 --- a/public/app/plugins/panel/nodeGraph/useContextMenu.tsx +++ b/public/app/plugins/panel/nodeGraph/useContextMenu.tsx @@ -1,7 +1,7 @@ import { css } from '@emotion/css'; import React, { MouseEvent, useCallback, useState } from 'react'; -import { DataFrame, Field, GrafanaTheme2, LinkModel } from '@grafana/data'; +import { DataFrame, GrafanaTheme2, LinkModel } from '@grafana/data'; import { ContextMenu, MenuGroup, MenuItem, useStyles2, useTheme2 } from '@grafana/ui'; import { Config } from './layout'; @@ -14,8 +14,10 @@ import { getEdgeFields, getNodeFields } from './utils'; */ export function useContextMenu( getLinks: (dataFrame: DataFrame, rowIndex: number) => LinkModel[], - nodes: DataFrame, - edges: DataFrame, + // This can be undefined if we only use edge dataframe + nodes: DataFrame | undefined, + // This can be undefined if we have only single node + edges: DataFrame | undefined, config: Config, setConfig: (config: Config) => void, setFocusedNodeId: (id: string) => void @@ -28,13 +30,9 @@ export function useContextMenu( const onNodeOpen = useCallback( (event: MouseEvent, node: NodeDatum) => { - let label = 'Show in Grid layout'; - let showGridLayout = true; - - if (config.gridLayout) { - label = 'Show in Graph layout'; - showGridLayout = false; - } + const [label, showGridLayout] = config.gridLayout + ? ['Show in Graph layout', false] + : ['Show in Grid layout', true]; const extraNodeItem = [ { @@ -47,18 +45,11 @@ export function useContextMenu( }, ]; - const renderer = getItemsRenderer(getLinks(nodes, node.dataFrameRowIndex), node, extraNodeItem); + const links = nodes ? getLinks(nodes, node.dataFrameRowIndex) : []; + const renderer = getItemsRenderer(links, node, extraNodeItem); if (renderer) { - setMenu( - } - renderMenuItems={renderer} - onClose={() => setMenu(undefined)} - x={event.pageX} - y={event.pageY} - /> - ); + setMenu(makeContextMenu(, renderer, event, setMenu)); } }, [config, nodes, getLinks, setMenu, setConfig, setFocusedNodeId] @@ -66,18 +57,16 @@ export function useContextMenu( const onEdgeOpen = useCallback( (event: MouseEvent, edge: EdgeDatum) => { - const renderer = getItemsRenderer(getLinks(edges, edge.dataFrameRowIndex), edge); + if (!edges) { + // This could happen if we have only one node and no edges, in which case this is not needed as there is no edge + // to click on. + return; + } + const links = getLinks(edges, edge.dataFrameRowIndex); + const renderer = getItemsRenderer(links, edge); if (renderer) { - setMenu( - } - renderMenuItems={renderer} - onClose={() => setMenu(undefined)} - x={event.pageX} - y={event.pageY} - /> - ); + setMenu(makeContextMenu(, renderer, event, setMenu)); } }, [edges, getLinks, setMenu] @@ -86,6 +75,23 @@ export function useContextMenu( return { onEdgeOpen, onNodeOpen, MenuComponent: menu }; } +function makeContextMenu( + header: JSX.Element, + renderer: () => React.ReactNode, + event: MouseEvent, + setMenu: (el: JSX.Element | undefined) => void +) { + return ( + header} + renderMenuItems={renderer} + onClose={() => setMenu(undefined)} + x={event.pageX} + y={event.pageY} + /> + ); +} + function getItemsRenderer( links: LinkModel[], item: T, @@ -173,24 +179,45 @@ function getItems(links: LinkModel[]) { }); } -function NodeHeader(props: { node: NodeDatum; nodes: DataFrame }) { - const index = props.node.dataFrameRowIndex; - const fields = getNodeFields(props.nodes); - return ( -
- {fields.title &&
- ); +function NodeHeader({ node, nodes }: { node: NodeDatum; nodes?: DataFrame }) { + const index = node.dataFrameRowIndex; + if (nodes) { + const fields = getNodeFields(nodes); + + return ( +
+ {fields.title && ( +
+ ); + } else { + // Fallback if we don't have nodes dataFrame. Can happen if we use just the edges frame to construct this. + return ( +
+ {node.title &&
+ ); + } } function EdgeHeader(props: { edge: EdgeDatum; edges: DataFrame }) { const index = props.edge.dataFrameRowIndex; - const fields = getEdgeFields(props.edges); const styles = getLabelStyles(useTheme2()); + const fields = getEdgeFields(props.edges); const valueSource = fields.source?.values.get(index) || ''; const valueTarget = fields.target?.values.get(index) || ''; @@ -205,20 +232,18 @@ function EdgeHeader(props: { edge: EdgeDatum; edges: DataFrame }) {
)} {fields.details.map((f) => ( -