From ea741dda6badd73ca3b3025d31f8c66167025990 Mon Sep 17 00:00:00 2001 From: Karl Persson Date: Tue, 10 Oct 2023 14:17:16 +0200 Subject: [PATCH 001/102] Signingkeys: Add local cache (#76234) * IDForwarding: change audience to be prefixed by org and remove JTI * IDForwarding: Construct new signer each time we want to sign a token. * SigningKeys: Simplify storage layer and move logic to service * SigningKeys: Add private key to local cache --- pkg/services/auth/idimpl/service.go | 14 +- pkg/services/auth/idimpl/signer.go | 58 +++-- pkg/services/authn/clients/ext_jwt_test.go | 6 +- .../oauthserver/oasimpl/service_test.go | 8 +- pkg/services/signingkeys/signingkeys.go | 9 + .../signingkeys/signingkeysimpl/service.go | 176 ++++++++++++- .../signingkeysimpl/service_test.go | 69 +++-- .../signingkeys/signingkeystest/fake.go | 33 +-- .../signingkeys/signingkeystore/fake.go | 56 ++-- .../signingkeys/signingkeystore/store.go | 184 +++----------- .../signingkeys/signingkeystore/store_test.go | 239 +++++------------- 11 files changed, 386 insertions(+), 466 deletions(-) diff --git a/pkg/services/auth/idimpl/service.go b/pkg/services/auth/idimpl/service.go index 19a33033a81..08b65b4df41 100644 --- a/pkg/services/auth/idimpl/service.go +++ b/pkg/services/auth/idimpl/service.go @@ -3,7 +3,6 @@ package idimpl import ( "context" "fmt" - "strconv" "time" "github.com/go-jose/go-jose/v3/jwt" @@ -67,10 +66,9 @@ func (s *Service) SignIdentity(ctx context.Context, id identity.Requester) (stri now := time.Now() token, err := s.signer.SignIDToken(ctx, &auth.IDClaims{ Claims: jwt.Claims{ - ID: identifier, Issuer: s.cfg.AppURL, - Audience: jwt.Audience{strconv.FormatInt(id.GetOrgID(), 10)}, - Subject: fmt.Sprintf("%s:%s", namespace, identifier), + Audience: getAudience(id.GetOrgID()), + Subject: getSubject(namespace, identifier), Expiry: jwt.NewNumericDate(now.Add(tokenTTL)), IssuedAt: jwt.NewNumericDate(now), }, @@ -102,6 +100,14 @@ func (s *Service) hook(ctx context.Context, identity *authn.Identity, _ *authn.R return nil } +func getAudience(orgID int64) jwt.Audience { + return jwt.Audience{fmt.Sprintf("org:%d", orgID)} +} + +func getSubject(namespace, identifier string) string { + return fmt.Sprintf("%s:%s", namespace, identifier) +} + func prefixCacheKey(key string) string { return fmt.Sprintf("%s-%s", cachePrefix, key) } diff --git a/pkg/services/auth/idimpl/signer.go b/pkg/services/auth/idimpl/signer.go index 15300c0979e..4cd8a090503 100644 --- a/pkg/services/auth/idimpl/signer.go +++ b/pkg/services/auth/idimpl/signer.go @@ -11,40 +11,20 @@ import ( "github.com/grafana/grafana/pkg/services/signingkeys" ) -const idSignerKeyPrefix = "id" +const ( + keyPrefix = "id" + headerKeyID = "kid" +) var _ auth.IDSigner = (*LocalSigner)(nil) func ProvideLocalSigner(keyService signingkeys.Service, features featuremgmt.FeatureToggles) (*LocalSigner, error) { - if features.IsEnabled(featuremgmt.FlagIdForwarding) { - id, key, err := keyService.GetOrCreatePrivateKey(context.Background(), idSignerKeyPrefix, jose.ES256) - if err != nil { - return nil, err - } - - // FIXME: Handle key rotation - signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.ES256, Key: key}, &jose.SignerOptions{ - ExtraHeaders: map[jose.HeaderKey]interface{}{ - "kid": id, - }, - }) - - if err != nil { - return nil, err - } - - return &LocalSigner{ - features: features, - signer: signer, - }, nil - } - - return &LocalSigner{features: features}, nil + return &LocalSigner{features, keyService}, nil } type LocalSigner struct { - signer jose.Signer - features featuremgmt.FeatureToggles + features featuremgmt.FeatureToggles + keyService signingkeys.Service } func (s *LocalSigner) SignIDToken(ctx context.Context, claims *auth.IDClaims) (string, error) { @@ -52,7 +32,12 @@ func (s *LocalSigner) SignIDToken(ctx context.Context, claims *auth.IDClaims) (s return "", nil } - builder := jwt.Signed(s.signer).Claims(claims.Claims) + signer, err := s.getSigner(ctx) + if err != nil { + return "", err + } + + builder := jwt.Signed(signer).Claims(claims.Claims) token, err := builder.CompactSerialize() if err != nil { @@ -61,3 +46,20 @@ func (s *LocalSigner) SignIDToken(ctx context.Context, claims *auth.IDClaims) (s return token, nil } + +func (s *LocalSigner) getSigner(ctx context.Context) (jose.Signer, error) { + id, key, err := s.keyService.GetOrCreatePrivateKey(ctx, keyPrefix, jose.ES256) + if err != nil { + return nil, err + } + + signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.ES256, Key: key}, &jose.SignerOptions{ + ExtraHeaders: map[jose.HeaderKey]any{headerKeyID: id}, + }) + + if err != nil { + return nil, err + } + + return signer, nil +} diff --git a/pkg/services/authn/clients/ext_jwt_test.go b/pkg/services/authn/clients/ext_jwt_test.go index c3a6a82ab0c..7f2f3e91f7d 100644 --- a/pkg/services/authn/clients/ext_jwt_test.go +++ b/pkg/services/authn/clients/ext_jwt_test.go @@ -2,7 +2,6 @@ package clients import ( "context" - "crypto" "crypto/rand" "crypto/rsa" "fmt" @@ -516,8 +515,9 @@ func setupTestCtx(t *testing.T, cfg *setting.Cfg) *testEnv { } } - signingKeysSvc := &signingkeystest.FakeSigningKeysService{ExpectedKeys: map[string]crypto.Signer{ - signingkeys.ServerPrivateKeyID: pk}, + signingKeysSvc := &signingkeystest.FakeSigningKeysService{ + ExpectedSinger: pk, + ExpectedKeyID: signingkeys.ServerPrivateKeyID, } userSvc := &usertest.FakeUserService{} diff --git a/pkg/services/extsvcauth/oauthserver/oasimpl/service_test.go b/pkg/services/extsvcauth/oauthserver/oasimpl/service_test.go index 1188a6f800d..c97c5cc2c78 100644 --- a/pkg/services/extsvcauth/oauthserver/oasimpl/service_test.go +++ b/pkg/services/extsvcauth/oauthserver/oasimpl/service_test.go @@ -2,7 +2,6 @@ package oasimpl import ( "context" - "crypto" "crypto/rand" "crypto/rsa" "encoding/base64" @@ -93,10 +92,9 @@ func setupTestEnv(t *testing.T) *TestEnv { } env.S.oauthProvider = newProvider(config, env.S, &signingkeystest.FakeSigningKeysService{ - ExpectedKeys: map[string]crypto.Signer{ - "default": pk, - }, - ExpectedError: nil, + ExpectedSinger: pk, + ExpectedKeyID: "default", + ExpectedError: nil, }) return env diff --git a/pkg/services/signingkeys/signingkeys.go b/pkg/services/signingkeys/signingkeys.go index bbb7ceba529..f5678829886 100644 --- a/pkg/services/signingkeys/signingkeys.go +++ b/pkg/services/signingkeys/signingkeys.go @@ -10,6 +10,7 @@ package signingkeys import ( "context" "crypto" + "time" "github.com/go-jose/go-jose/v3" ) @@ -26,3 +27,11 @@ type Service interface { GetJWKS(ctx context.Context) (jose.JSONWebKeySet, error) GetOrCreatePrivateKey(ctx context.Context, keyPrefix string, alg jose.SignatureAlgorithm) (string, crypto.Signer, error) } + +type SigningKey struct { + KeyID string `xorm:"key_id"` + PrivateKey []byte `xorm:"private_key"` + AddedAt time.Time `xorm:"added_at"` + ExpiresAt *time.Time `xorm:"expires_at"` + Alg jose.SignatureAlgorithm `xorm:"alg"` +} diff --git a/pkg/services/signingkeys/signingkeysimpl/service.go b/pkg/services/signingkeys/signingkeysimpl/service.go index 5e314d7edd0..500f0606dd9 100644 --- a/pkg/services/signingkeys/signingkeysimpl/service.go +++ b/pkg/services/signingkeys/signingkeysimpl/service.go @@ -6,7 +6,10 @@ import ( "crypto/ecdsa" "crypto/elliptic" "crypto/rand" + "crypto/x509" + "encoding/base64" "encoding/json" + "encoding/pem" "errors" "net/http" "strings" @@ -17,6 +20,7 @@ import ( "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/infra/localcache" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/remotecache" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" @@ -31,9 +35,11 @@ func ProvideEmbeddedSigningKeysService(dbStore db.DB, secretsService secrets.Ser remoteCache remotecache.CacheStorage, routerRegister routing.RouteRegister, ) (*Service, error) { s := &Service{ - log: log.New("auth.key_service"), - store: signingkeystore.NewSigningKeyStore(dbStore, secretsService), - remoteCache: remoteCache, + log: log.New("auth.key_service"), + store: signingkeystore.NewSigningKeyStore(dbStore), + secretsService: secretsService, + remoteCache: remoteCache, + localCache: localcache.New(1*time.Hour, 1*time.Hour), } s.registerAPIEndpoints(routerRegister) @@ -46,14 +52,17 @@ func ProvideEmbeddedSigningKeysService(dbStore db.DB, secretsService secrets.Ser // // The service is under active development and is not yet ready for production use. type Service struct { - log log.Logger - store signingkeystore.SigningStore - remoteCache remotecache.CacheStorage + log log.Logger + store signingkeystore.SigningStore + secretsService secrets.Service + remoteCache remotecache.CacheStorage + localCache *localcache.CacheService } const ( jwksCacheKey = "signingkeys-jwks" - defaultExpiry = 12 * time.Hour + jwksTTL = 12 * time.Hour + privateKeyTTL = 60 * time.Second ) // GetJWKS returns the JSON Web Key Set (JWKS) with all the keys that can be used to verify tokens (public keys) @@ -66,15 +75,20 @@ func (s *Service) GetJWKS(ctx context.Context) (jose.JSONWebKeySet, error) { } } - jwks, err := s.store.GetJWKS(ctx) + keys, err := s.store.List(ctx) if err != nil { return jose.JSONWebKeySet{}, err } + jwks, err := s.buildJWKS(ctx, keys) + if err != nil { + return jwks, err + } + // cache jwks jwksBytes, err := json.Marshal(jwks) if err == nil { - if err := s.remoteCache.Set(ctx, jwksCacheKey, jwksBytes, defaultExpiry); err != nil { + if err := s.remoteCache.Set(ctx, jwksCacheKey, jwksBytes, jwksTTL); err != nil { s.log.Warn("Failed to cache JWKS", "err", err) } } @@ -82,6 +96,24 @@ func (s *Service) GetJWKS(ctx context.Context) (jose.JSONWebKeySet, error) { return jwks, err } +func (s *Service) buildJWKS(ctx context.Context, keys []signingkeys.SigningKey) (jose.JSONWebKeySet, error) { + var jwks jose.JSONWebKeySet + for _, key := range keys { + assertedKey, err := s.decodePrivateKey(ctx, key.PrivateKey) + if err != nil { + return jwks, err + } + + jwks.Keys = append(jwks.Keys, jose.JSONWebKey{ + Key: assertedKey.Public(), + Algorithm: string(key.Alg), + KeyID: key.KeyID, + Use: "sig", + }) + } + return jwks, nil +} + // GetOrCreatePrivateKey returns the private key with the specified key ID. If the key does not exist, it will be // created with the specified algorithm. // The key will be automatically rotated at the beginning of each month. The previous key will be kept for 30 days. @@ -93,30 +125,146 @@ func (s *Service) GetOrCreatePrivateKey(ctx context.Context, } keyID := keyMonthScopedID(keyPrefix, alg) - signer, err := s.store.GetPrivateKey(ctx, keyID) + signer, err := s.getPrivateKey(ctx, keyID) if err == nil { return keyID, signer, nil } + + // we only want to create a new signing key if none exits for keyID + if !errors.Is(err, signingkeys.ErrSigningKeyNotFound) { + return "", nil, err + } + s.log.Debug("Private key not found, generating new key", "keyID", keyID, "err", err) + signer, err = s.addPrivateKey(ctx, keyID, alg, false) + if err != nil { + return "", nil, err + } + + return keyID, signer, nil +} + +func (s *Service) getPrivateKey(ctx context.Context, keyID string) (crypto.Signer, error) { + if key, ok := s.localCache.Get(keyID); ok { + return key.(crypto.Signer), nil + } + + key, err := s.store.Get(ctx, keyID) + if err != nil { + return nil, err + } + + singer, err := s.decodePrivateKey(ctx, key.PrivateKey) + if err != nil { + return nil, err + } + + s.localCache.Set(keyID, singer, privateKeyTTL) + return singer, nil +} + +func (s *Service) addPrivateKey(ctx context.Context, keyID string, alg jose.SignatureAlgorithm, force bool) (crypto.Signer, error) { privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { s.log.Error("Error generating private key", "err", err) - return "", nil, signingkeys.ErrKeyGenerationFailed.Errorf("Error generating private key: %v", err) + return nil, signingkeys.ErrKeyGenerationFailed.Errorf("Error generating private key: %v", err) + } + + encoded, err := s.encodePrivateKey(ctx, privateKey) + if err != nil { + s.log.Error("Error encoding private key", "err", err) + return nil, err } expiry := time.Now().Add(30 * 24 * time.Hour) - if signer, err = s.store.AddPrivateKey(ctx, keyID, alg, privateKey, &expiry, false); err != nil && !errors.Is(err, signingkeys.ErrSigningKeyAlreadyExists) { - return "", nil, err + key, err := s.store.Add(ctx, &signingkeys.SigningKey{ + KeyID: keyID, + PrivateKey: encoded, + ExpiresAt: &expiry, + Alg: alg, + }, force) + + if err != nil && !errors.Is(err, signingkeys.ErrSigningKeyAlreadyExists) { + return nil, err } + signer, err := s.decodePrivateKey(ctx, key.PrivateKey) + if err != nil { + return nil, err + } + + // invalidate local cache + s.localCache.Delete(keyID) + // invalidate cache if err := s.remoteCache.Delete(ctx, jwksCacheKey); err != nil { // not a critical error, key might not be in cache s.log.Debug("Failed to invalidate JWKS cache", "err", err) } - return keyID, signer, nil + return signer, nil +} + +func (s *Service) encodePrivateKey(ctx context.Context, privateKey crypto.Signer) ([]byte, error) { + // Encode private key to binary format + pKeyBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + return nil, err + } + + // Encode private key to PEM format + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: pKeyBytes, + }) + + encrypted, err := s.secretsService.Encrypt(ctx, privateKeyPEM, secrets.WithoutScope()) + if err != nil { + return nil, err + } + + encoded := make([]byte, base64.StdEncoding.EncodedLen(len(encrypted))) + base64.StdEncoding.Encode(encoded, encrypted) + return encoded, nil +} + +func (s *Service) decodePrivateKey(ctx context.Context, privateKey []byte) (crypto.Signer, error) { + // Bail out if empty string since it'll cause a segfault in Decrypt + if len(privateKey) == 0 { + return nil, errors.New("private key is empty") + } + + payload := make([]byte, base64.StdEncoding.DecodedLen(len(privateKey))) + _, err := base64.StdEncoding.Decode(payload, privateKey) + if err != nil { + return nil, err + } + + decrypted, err := s.secretsService.Decrypt(ctx, payload) + if err != nil { + return nil, err + } + + block, _ := pem.Decode(decrypted) + if block == nil { + return nil, errors.New("failed to decode private key PEM") + } + + if block.Type != "PRIVATE KEY" { + return nil, errors.New("invalid block type") + } + + parsedKey, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, err + } + + assertedKey, ok := parsedKey.(crypto.Signer) + if !ok { + return nil, errors.New("failed to assert private key as crypto.Signer") + } + return assertedKey, nil } func keyMonthScopedID(keyPrefix string, alg jose.SignatureAlgorithm) string { diff --git a/pkg/services/signingkeys/signingkeysimpl/service_test.go b/pkg/services/signingkeys/signingkeysimpl/service_test.go index aac3953bc82..239c255f622 100644 --- a/pkg/services/signingkeys/signingkeysimpl/service_test.go +++ b/pkg/services/signingkeys/signingkeysimpl/service_test.go @@ -13,13 +13,16 @@ import ( "time" "github.com/go-jose/go-jose/v3" + "github.com/grafana/grafana/pkg/services/signingkeys" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/infra/localcache" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/remotecache" - "github.com/grafana/grafana/pkg/services/signingkeys" + secretstest "github.com/grafana/grafana/pkg/services/secrets/fakes" "github.com/grafana/grafana/pkg/services/signingkeys/signingkeystore" "github.com/grafana/grafana/pkg/services/user" "github.com/grafana/grafana/pkg/web/webtest" @@ -32,28 +35,33 @@ ielIkb6/Ys51o7KjHxtANhPesw== -----END PRIVATE KEY-----` ) -func getPrivateKey(t *testing.T) *ecdsa.PrivateKey { +func getPrivateKey(t *testing.T, svc *Service) []byte { pemBlock, _ := pem.Decode([]byte(privateKeyPem)) privateKey, err := x509.ParsePKCS8PrivateKey(pemBlock.Bytes) require.NoError(t, err) - return privateKey.(*ecdsa.PrivateKey) + + bytes, err := svc.encodePrivateKey(context.Background(), privateKey.(*ecdsa.PrivateKey)) + require.NoError(t, err) + return bytes } -func TestEmbeddedKeyService_GetJWKS_OnlyPublicKeyShared(t *testing.T) { - mockStore := signingkeystore.NewFakeStore() - cacheStorage := remotecache.NewFakeCacheStorage() - - _, err := mockStore.AddPrivateKey(context.Background(), signingkeys.ServerPrivateKeyID, jose.ES256, getPrivateKey(t), nil, false) - require.NoError(t, err) - - _, err = mockStore.AddPrivateKey(context.Background(), "other", jose.ES256, getPrivateKey(t), nil, false) - require.NoError(t, err) +func TestIntegrationEmbeddedKeyService_GetJWKS_OnlyPublicKeyShared(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } svc := &Service{ - log: log.NewNopLogger(), - store: mockStore, - remoteCache: cacheStorage, + log: log.NewNopLogger(), + store: signingkeystore.NewSigningKeyStore(db.InitTestDB(t)), + secretsService: secretstest.NewFakeSecretsService(), + remoteCache: remotecache.NewFakeCacheStorage(), + localCache: localcache.New(privateKeyTTL, 10*time.Hour), } + + _, _, err := svc.GetOrCreatePrivateKey(context.Background(), "key-1", jose.ES256) + require.NoError(t, err) + _, _, err = svc.GetOrCreatePrivateKey(context.Background(), "key-2", jose.ES256) + require.NoError(t, err) jwks, err := svc.GetJWKS(context.Background()) require.NoError(t, err) @@ -79,14 +87,18 @@ func TestEmbeddedKeyService_GetJWKS_OnlyPublicKeyShared(t *testing.T) { } } -func TestEmbeddedKeyService_GetOrCreatePrivateKey(t *testing.T) { - mockStore := signingkeystore.NewFakeStore() +func TestIntegrationEmbeddedKeyService_GetOrCreatePrivateKey(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } cacheStorage := remotecache.NewFakeCacheStorage() svc := &Service{ - log: log.NewNopLogger(), - store: mockStore, - remoteCache: cacheStorage, + log: log.NewNopLogger(), + store: signingkeystore.NewSigningKeyStore(db.InitTestDB(t)), + secretsService: secretstest.NewFakeSecretsService(), + remoteCache: cacheStorage, + localCache: localcache.New(privateKeyTTL, 10*time.Hour), } wantedKeyID := keyMonthScopedID("test", jose.ES256) @@ -110,7 +122,6 @@ func TestEmbeddedKeyService_GetOrCreatePrivateKey(t *testing.T) { // new key is generated, so jwks cache should be voided require.Len(t, cacheStorage.Storage, 0) - assert.Contains(t, mockStore.PrivateKeys, wantedKeyID) err = cacheStorage.Set(context.Background(), jwksCacheKey, []byte("invalid"), 0) require.NoError(t, err) @@ -122,7 +133,6 @@ func TestEmbeddedKeyService_GetOrCreatePrivateKey(t *testing.T) { require.Equal(t, key, key2) require.Equal(t, wantedKeyID, id) - assert.Len(t, mockStore.PrivateKeys, 1) // no new key is generated, so jwks cache should not be voided require.Len(t, cacheStorage.Storage, 1) } @@ -132,9 +142,11 @@ func TestExposeJWKS(t *testing.T) { mockStore := signingkeystore.NewFakeStore() cacheStorage := remotecache.NewFakeCacheStorage() svc := &Service{ - log: log.NewNopLogger(), - store: mockStore, - remoteCache: cacheStorage, + log: log.NewNopLogger(), + store: mockStore, + remoteCache: cacheStorage, + secretsService: secretstest.NewFakeSecretsService(), + localCache: localcache.New(privateKeyTTL, 10*time.Hour), } routerRegister := routing.NewRouteRegister() @@ -142,8 +154,13 @@ func TestExposeJWKS(t *testing.T) { svc.registerAPIEndpoints(routerRegister) server := webtest.NewServer(t, routerRegister) + _, err := mockStore.Add(context.Background(), &signingkeys.SigningKey{ + KeyID: "test-key", + PrivateKey: getPrivateKey(t, svc), + AddedAt: time.Now(), + Alg: jose.ES256, + }, false) - _, err := mockStore.AddPrivateKey(context.Background(), "test-key", jose.ES256, getPrivateKey(t), nil, false) require.NoError(t, err) // create a new request context diff --git a/pkg/services/signingkeys/signingkeystest/fake.go b/pkg/services/signingkeys/signingkeystest/fake.go index 48620bfc0b1..3f7e23634a4 100644 --- a/pkg/services/signingkeys/signingkeystest/fake.go +++ b/pkg/services/signingkeys/signingkeystest/fake.go @@ -3,15 +3,14 @@ package signingkeystest import ( "context" "crypto" - "time" "github.com/go-jose/go-jose/v3" ) type FakeSigningKeysService struct { ExpectedJSONWebKeySet jose.JSONWebKeySet - ExpectedJSONWebKey jose.JSONWebKey - ExpectedKeys map[string]crypto.Signer + ExpectedKeyID string + ExpectedSinger crypto.Signer ExpectedError error } @@ -19,30 +18,6 @@ func (s *FakeSigningKeysService) GetJWKS(ctx context.Context) (jose.JSONWebKeySe return s.ExpectedJSONWebKeySet, nil } -// GetPublicKey returns the public key with the specified key ID -func (s *FakeSigningKeysService) GetPublicKey(ctx context.Context, keyID string) (crypto.PublicKey, error) { - return s.ExpectedKeys[keyID].Public(), s.ExpectedError -} - -// GetPrivateKey returns the private key with the specified key ID -func (s *FakeSigningKeysService) GetPrivateKey(ctx context.Context, keyID string) (crypto.PrivateKey, error) { - return s.ExpectedKeys[keyID], s.ExpectedError -} - -// AddPrivateKey adds a private key to the service -func (s *FakeSigningKeysService) AddPrivateKey(ctx context.Context, keyID string, - privateKey crypto.Signer, alg jose.SignatureAlgorithm, expiresAt *time.Time, force bool) error { - if s.ExpectedError != nil { - return s.ExpectedError - } - s.ExpectedKeys[keyID] = privateKey - return nil -} - -func (s *FakeSigningKeysService) GetOrCreatePrivateKey(ctx context.Context, - keyPrefix string, alg jose.SignatureAlgorithm) (string, crypto.Signer, error) { - if s.ExpectedError != nil { - return "", nil, s.ExpectedError - } - return keyPrefix, s.ExpectedKeys[keyPrefix], nil +func (s *FakeSigningKeysService) GetOrCreatePrivateKey(ctx context.Context, keyPrefix string, alg jose.SignatureAlgorithm) (string, crypto.Signer, error) { + return s.ExpectedKeyID, s.ExpectedSinger, s.ExpectedError } diff --git a/pkg/services/signingkeys/signingkeystore/fake.go b/pkg/services/signingkeys/signingkeystore/fake.go index 63fed1841df..b78721353e7 100644 --- a/pkg/services/signingkeys/signingkeystore/fake.go +++ b/pkg/services/signingkeys/signingkeystore/fake.go @@ -3,13 +3,15 @@ package signingkeystore import ( "context" "crypto" - "fmt" - "time" "github.com/go-jose/go-jose/v3" + "github.com/grafana/grafana/pkg/services/signingkeys" ) +var _ SigningStore = (*FakeStore)(nil) + type FakeStore struct { + Keys map[string]signingkeys.SigningKey PrivateKeys map[string]crypto.Signer jwks jose.JSONWebKeySet } @@ -17,46 +19,34 @@ type FakeStore struct { func NewFakeStore() *FakeStore { return &FakeStore{ PrivateKeys: make(map[string]crypto.Signer), + Keys: make(map[string]signingkeys.SigningKey), jwks: jose.JSONWebKeySet{}, } } -func (s *FakeStore) GetJWKS(ctx context.Context) (jose.JSONWebKeySet, error) { - return s.jwks, nil -} - -func (s *FakeStore) AddPrivateKey(ctx context.Context, keyID string, alg jose.SignatureAlgorithm, - privateKey crypto.Signer, expiresAt *time.Time, force bool) (crypto.Signer, error) { +func (s *FakeStore) Add(ctx context.Context, key *signingkeys.SigningKey, force bool) (*signingkeys.SigningKey, error) { if !force { - if key, ok := s.PrivateKeys[keyID]; ok { - if !hasExpired(key) { - return nil, fmt.Errorf("key already exists and has not expired") - } + if _, ok := s.Keys[key.KeyID]; ok { + return nil, signingkeys.ErrSigningKeyAlreadyExists } } - s.PrivateKeys[keyID] = privateKey + s.Keys[key.KeyID] = *key + return key, nil +} - jwk := jose.JSONWebKey{ - Key: privateKey.Public(), - Algorithm: string(alg), - KeyID: keyID, - Use: "sig", +func (s *FakeStore) List(ctx context.Context) ([]signingkeys.SigningKey, error) { + out := make([]signingkeys.SigningKey, 0, len(s.Keys)) + for _, key := range s.Keys { + out = append(out, key) + } + return out, nil +} + +func (s *FakeStore) Get(ctx context.Context, keyID string) (*signingkeys.SigningKey, error) { + if key, ok := s.Keys[keyID]; ok { + return &key, nil } - s.jwks.Keys = append(s.jwks.Keys, jwk) - - return privateKey, nil -} - -func (s *FakeStore) GetPrivateKey(ctx context.Context, keyID string) (crypto.Signer, error) { - if key, ok := s.PrivateKeys[keyID]; ok { - return key, nil - } - - return nil, fmt.Errorf("key not found") -} - -func hasExpired(key crypto.Signer) bool { - return false + return nil, signingkeys.ErrSigningKeyNotFound } diff --git a/pkg/services/signingkeys/signingkeystore/store.go b/pkg/services/signingkeys/signingkeystore/store.go index 0f6f23b3d4f..0c01a2850a1 100644 --- a/pkg/services/signingkeys/signingkeystore/store.go +++ b/pkg/services/signingkeys/signingkeystore/store.go @@ -2,11 +2,7 @@ package signingkeystore import ( "context" - "crypto" - "crypto/x509" "database/sql" - "encoding/base64" - "encoding/pem" "errors" "time" @@ -15,21 +11,19 @@ import ( "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/localcache" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/signingkeys" "github.com/grafana/grafana/pkg/services/sqlstore" ) type SigningStore interface { - // GetJWKS returns the JSON Web Key Set for the service - GetJWKS(ctx context.Context) (jose.JSONWebKeySet, error) - // AddPrivateKey adds a private key to the service. If the key already exists, it will be updated if force is true. + // List returns all non expired keys + List(ctx context.Context) ([]signingkeys.SigningKey, error) + // Add adds a signing key to the database. If the key already exists, it will be updated if force is true. // If force is false, the key will only be updated if it has expired. If the key does not exist, it will be added. - // If expiresAt is nil, the key will not expire. Retrieve the result key with GetPrivateKey. - AddPrivateKey(ctx context.Context, keyID string, alg jose.SignatureAlgorithm, - privateKey crypto.Signer, expiresAt *time.Time, force bool) (crypto.Signer, error) - // GetPrivateKey returns the private key with the specified key ID - GetPrivateKey(ctx context.Context, keyID string) (crypto.Signer, error) + // If expiresAt is nil, the key will not expire. Retrieve the result key with Get. + Add(ctx context.Context, key *signingkeys.SigningKey, force bool) (*signingkeys.SigningKey, error) + // Get returns the signing key with the specified key ID + Get(ctx context.Context, keyID string) (*signingkeys.SigningKey, error) } var _ SigningStore = (*Store)(nil) @@ -37,10 +31,9 @@ var _ SigningStore = (*Store)(nil) const cleanupRateLimitKey = "signingkeys-cleanup" type Store struct { - dbStore db.DB - secretsService secrets.Service - log log.Logger - localCache *localcache.CacheService + dbStore db.DB + log log.Logger + localCache *localcache.CacheService } type SigningKey struct { @@ -52,90 +45,57 @@ type SigningKey struct { Alg jose.SignatureAlgorithm `json:"alg" xorm:"alg" db:"alg"` } -func NewSigningKeyStore(dbStore db.DB, secretsService secrets.Service) *Store { +func NewSigningKeyStore(dbStore db.DB) *Store { return &Store{ - dbStore: dbStore, - secretsService: secretsService, - log: log.New("signing.key_service"), - localCache: localcache.New(12*time.Hour, 4*time.Hour), + dbStore: dbStore, + log: log.New("signing.key_service"), + localCache: localcache.New(12*time.Hour, 4*time.Hour), } } -// GetJWKS returns the JSON Web Key Set (JWKS) for the service. Expired keys will not be returned. -func (s *Store) GetJWKS(ctx context.Context) (jose.JSONWebKeySet, error) { - keySet := jose.JSONWebKeySet{} +func (s *Store) List(ctx context.Context) ([]signingkeys.SigningKey, error) { + var keys []signingkeys.SigningKey - keys := []*SigningKey{} err := s.dbStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error { return dbSession.SQL("SELECT * FROM signing_key WHERE expires_at IS NULL OR expires_at > ?", time.Now()).Find(&keys) }) - if err != nil { - return keySet, err - } - - for _, key := range keys { - assertedKey, err := s.decodePrivateKey(ctx, key) - if err != nil { - return keySet, err - } - - keySet.Keys = append(keySet.Keys, jose.JSONWebKey{ - Key: assertedKey.Public(), - Algorithm: string(key.Alg), - KeyID: key.KeyID, - Use: "sig", - }) - } - - return keySet, nil -} - -// AddPrivateKey adds a private key to the service. -func (s *Store) AddPrivateKey(ctx context.Context, - keyID string, alg jose.SignatureAlgorithm, privateKey crypto.Signer, expiresAt *time.Time, force bool) (crypto.Signer, error) { - privateKeyPEM, err := s.encodePrivateKey(ctx, privateKey) if err != nil { return nil, err } - key := &SigningKey{ - KeyID: keyID, - PrivateKey: privateKeyPEM, - AddedAt: time.Now(), - Alg: alg, - ExpiresAt: expiresAt, - } + return keys, nil +} - var signer crypto.Signer - err = s.dbStore.WithTransactionalDbSession(ctx, func(tx *sqlstore.DBSession) error { - existingKey := SigningKey{} - _, err := tx.SQL("SELECT * FROM signing_key WHERE key_id = ?", keyID).Get(&existingKey) +// Add adds a private key to the service. +func (s *Store) Add(ctx context.Context, key *signingkeys.SigningKey, force bool) (*signingkeys.SigningKey, error) { + var result *signingkeys.SigningKey + + err := s.dbStore.WithTransactionalDbSession(ctx, func(tx *sqlstore.DBSession) error { + existingKey := &signingkeys.SigningKey{} + exists, err := tx.SQL("SELECT * FROM signing_key WHERE key_id = ?", key.KeyID).Get(existingKey) if err != nil && !errors.Is(err, sql.ErrNoRows) { return err } - if len(existingKey.PrivateKey) == 0 { + if !exists { _, err = tx.Exec("INSERT INTO signing_key (key_id, private_key, added_at, alg, expires_at) VALUES (?, ?, ?, ?, ?)", key.KeyID, key.PrivateKey, key.AddedAt, key.Alg, key.ExpiresAt, ) - signer = privateKey + result = key return err } if force || (existingKey.ExpiresAt != nil && existingKey.ExpiresAt.Before(time.Now())) { _, err = tx.Exec("UPDATE signing_key SET private_key = ?, added_at = ?, alg = ?, expires_at = ? WHERE key_id = ?", key.PrivateKey, key.AddedAt, key.Alg, key.ExpiresAt, key.KeyID) - signer = privateKey + + result = key return err } - signer, err = s.decodePrivateKey(ctx, &existingKey) - if err != nil { - return err - } - - return signingkeys.ErrSigningKeyAlreadyExists.Errorf("The specified key already exists: %s", keyID) + result = existingKey + return signingkeys.ErrSigningKeyAlreadyExists.Errorf("The specified key already exists: %s", existingKey.KeyID) }) if _, ok := s.localCache.Get(cleanupRateLimitKey); !ok { @@ -156,14 +116,18 @@ func (s *Store) AddPrivateKey(ctx context.Context, s.localCache.Set(cleanupRateLimitKey, true, 1*time.Hour) } - return signer, err + return result, err } -// GetPrivateKey returns the private key with the specified key ID. Expired keys will not be returned. -func (s *Store) GetPrivateKey(ctx context.Context, keyID string) (crypto.Signer, error) { - key := SigningKey{} +// Get implements SigningStore. +func (s *Store) Get(ctx context.Context, keyID string) (*signingkeys.SigningKey, error) { + key := signingkeys.SigningKey{} err := s.dbStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error { - _, err := dbSession.SQL("SELECT * FROM signing_key WHERE key_id = ?", keyID).Get(&key) + exists, err := dbSession.SQL("SELECT * FROM signing_key WHERE key_id = ?", keyID).Get(&key) + if !exists { + return signingkeys.ErrSigningKeyNotFound.Errorf("The specified key was not found: %s", keyID) + } + return err }) @@ -176,73 +140,7 @@ func (s *Store) GetPrivateKey(ctx context.Context, keyID string) (crypto.Signer, return nil, signingkeys.ErrSigningKeyNotFound.Errorf("The specified key was not found: %s", keyID) } - signKey, err := s.decodePrivateKey(ctx, &key) - if err != nil { - return nil, err - } - - return signKey, nil -} - -func (s *Store) encodePrivateKey(ctx context.Context, privateKey crypto.Signer) ([]byte, error) { - // Encode private key to binary format - pKeyBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) - if err != nil { - return nil, err - } - - // Encode private key to PEM format - privateKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "PRIVATE KEY", - Bytes: pKeyBytes, - }) - - encrypted, err := s.secretsService.Encrypt(ctx, privateKeyPEM, secrets.WithoutScope()) - if err != nil { - return nil, err - } - - encoded := make([]byte, base64.StdEncoding.EncodedLen(len(encrypted))) - base64.StdEncoding.Encode(encoded, encrypted) - return encoded, nil -} - -func (s *Store) decodePrivateKey(ctx context.Context, signingKey *SigningKey) (crypto.Signer, error) { - // Bail out if empty string since it'll cause a segfault in Decrypt - if len(signingKey.PrivateKey) == 0 { - return nil, errors.New("private key is empty") - } - - payload := make([]byte, base64.StdEncoding.DecodedLen(len(signingKey.PrivateKey))) - _, err := base64.StdEncoding.Decode(payload, signingKey.PrivateKey) - if err != nil { - return nil, err - } - - decrypted, err := s.secretsService.Decrypt(ctx, payload) - if err != nil { - return nil, err - } - - block, _ := pem.Decode(decrypted) - if block == nil { - return nil, errors.New("failed to decode private key PEM") - } - - if block.Type != "PRIVATE KEY" { - return nil, errors.New("invalid block type") - } - - parsedKey, err := x509.ParsePKCS8PrivateKey(block.Bytes) - if err != nil { - return nil, err - } - - assertedKey, ok := parsedKey.(crypto.Signer) - if !ok { - return nil, errors.New("failed to assert private key as crypto.Signer") - } - return assertedKey, nil + return &key, nil } // cleanupExpiredKeys removes expired keys from the database that have expired more than 61 days ago diff --git a/pkg/services/signingkeys/signingkeystore/store_test.go b/pkg/services/signingkeys/signingkeystore/store_test.go index 01375fad4c7..54e587dcd40 100644 --- a/pkg/services/signingkeys/signingkeystore/store_test.go +++ b/pkg/services/signingkeys/signingkeystore/store_test.go @@ -2,204 +2,81 @@ package signingkeystore import ( "context" - "crypto" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/rsa" "testing" "time" - "github.com/go-jose/go-jose/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/infra/db" - "github.com/grafana/grafana/pkg/services/secrets/fakes" "github.com/grafana/grafana/pkg/services/signingkeys" ) func TestIntegrationSigningKeyStore(t *testing.T) { - ctx := context.Background() - - testCases := []struct { - name string - keyFunc func() (crypto.Signer, error) - keyID string - alg jose.SignatureAlgorithm - expected jose.JSONWebKey - }{ - { - name: "RSA key", - keyFunc: func() (crypto.Signer, error) { - return rsa.GenerateKey(rand.Reader, 2048) - }, - keyID: "test-rsa-key", - alg: jose.RS256, - expected: jose.JSONWebKey{ - Key: &rsa.PublicKey{}, - Algorithm: "RS256", - KeyID: "test-rsa-key", - Use: "sig", - }, - }, - { - name: "Elliptic Curve key", - keyFunc: func() (crypto.Signer, error) { - return ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - }, - keyID: "test-ec-key", - alg: jose.ES256, - expected: jose.JSONWebKey{ - Key: &ecdsa.PublicKey{}, - Algorithm: "ES256", - KeyID: "test-ec-key", - Use: "sig", - }, - }, + if testing.Short() { + t.Skip("skipping integration test") } - for _, tc := range testCases { - dbStore := db.InitTestDB(t) - secretSvc := fakes.NewFakeSecretsService() - store := NewSigningKeyStore(dbStore, secretSvc) - - t.Run(tc.name, func(t *testing.T) { - key, err := tc.keyFunc() - assert.NoError(t, err) - - _, err = store.AddPrivateKey(ctx, tc.keyID, tc.alg, key, nil, true) - assert.NoError(t, err) - - retrievedKey, err := store.GetPrivateKey(ctx, tc.keyID) - require.NoError(t, err) - - assert.Equal(t, key.Public(), retrievedKey.Public()) - - jwks, err := store.GetJWKS(ctx) - assert.NoError(t, err) - - require.Len(t, jwks.Keys, 1) - assert.Equal(t, key.Public(), jwks.Keys[0].Key) - assert.Equal(t, tc.expected.Algorithm, jwks.Keys[0].Algorithm) - assert.Equal(t, tc.expected.KeyID, jwks.Keys[0].KeyID) - assert.Equal(t, tc.expected.Use, jwks.Keys[0].Use) - }) - } -} - -func TestIntegrationAddPrivateKey(t *testing.T) { - ctx := context.Background() - - dbStore := db.InitTestDB(t) - secretSvc := fakes.NewFakeSecretsService() - store := NewSigningKeyStore(dbStore, secretSvc) - - key1 := generateRSAKey(t) - key2 := generateECKey(t) - key3 := generateECKey(t) - - testCases := []struct { - name string - keyID string - alg jose.SignatureAlgorithm - privateKey crypto.Signer - expiresAt *time.Time - force bool - expectedErr error - expectedKey crypto.Signer - expectedGot crypto.Signer - }{ - { - name: "Add new private key", - keyID: "test-key-1", - alg: jose.RS256, - privateKey: key1, - force: false, - expectedKey: key1, - expectedGot: key1, - }, - { - name: "Add new private key with expiration", - keyID: "test-key-2", - alg: jose.ES256, - privateKey: key2, - expiresAt: &[]time.Time{time.Now().Add(24 * time.Hour)}[0], - force: false, - expectedKey: key2, - expectedGot: key2, - }, - { - name: "Fail to replace unexpired key", - keyID: "test-key-1", - alg: jose.RS256, - privateKey: key3, - expiresAt: &[]time.Time{time.Now().Add(-24 * time.Hour)}[0], - force: false, - expectedErr: signingkeys.ErrSigningKeyAlreadyExists, - expectedKey: key1, - expectedGot: key1, - }, - { - name: "Replace key1 private key with force, already expired", - keyID: "test-key-1", - alg: jose.ES256, - privateKey: key3, - expiresAt: &[]time.Time{time.Now().Add(-24 * time.Hour)}[0], - force: true, - expectedKey: nil, - expectedGot: key3, - }, - { - name: "Replace key1 private key with no force, is expired", - keyID: "test-key-1", - alg: jose.ES256, - privateKey: key1, - expiresAt: &[]time.Time{time.Now().Add(24 * time.Hour)}[0], - force: false, - expectedKey: nil, - expectedGot: key1, - }, + setup := func() (context.Context, *Store) { + return context.Background(), NewSigningKeyStore(db.InitTestDB(t)) } - _, exists := store.localCache.Get(cleanupRateLimitKey) - require.False(t, exists) + t.Run("Should successfully add new singing key", func(_ *testing.T) { + ctx, store := setup() + key, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", AddedAt: time.Now().UTC(), PrivateKey: []byte{}}, false) + require.NoError(t, err) + assert.Equal(t, "1", key.KeyID) + }) - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - got, err := store.AddPrivateKey(ctx, tc.keyID, tc.alg, tc.privateKey, tc.expiresAt, tc.force) - if tc.expectedErr != nil { - assert.ErrorIs(t, err, tc.expectedErr) - } else { - assert.NoError(t, err) - } + t.Run("Should return old key if already exists", func(_ *testing.T) { + ctx, store := setup() + key, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().UTC()}, false) + require.NoError(t, err) + assert.Equal(t, "1", key.KeyID) - if tc.expectedGot != nil { - assert.Equal(t, tc.expectedGot.Public(), got.Public()) - } else { - assert.Nil(t, got) - } + // try to add the same key again with a different AddedAt + key2, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().Add(10 * time.Minute).UTC()}, false) + require.ErrorIs(t, err, signingkeys.ErrSigningKeyAlreadyExists) + assert.Equal(t, "1", key2.KeyID) + }) - if tc.expectedKey != nil { - retrievedKey, err := store.GetPrivateKey(ctx, tc.keyID) - assert.NoError(t, err) - assert.Equal(t, tc.expectedKey.Public(), retrievedKey.Public()) - } - }) - } + t.Run("Should update old key when force is true", func(t *testing.T) { + ctx, store := setup() + key, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().UTC()}, false) + require.NoError(t, err) + assert.Equal(t, "1", key.KeyID) - _, exists = store.localCache.Get(cleanupRateLimitKey) - require.True(t, exists) -} - -func generateRSAKey(t *testing.T) *rsa.PrivateKey { - key, err := rsa.GenerateKey(rand.Reader, 2048) - require.NoError(t, err) - return key -} - -func generateECKey(t *testing.T) *ecdsa.PrivateKey { - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - require.NoError(t, err) - return key + // try to add the same key again with a different AddedAt and force is true + key2, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().Add(10 * time.Minute).UTC()}, true) + require.NoError(t, err) + assert.Equal(t, "1", key2.KeyID) + assert.NotEqual(t, key.AddedAt, key2.AddedAt) + }) + + t.Run("Should update old key when expired", func(t *testing.T) { + ctx, store := setup() + key, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().UTC(), ExpiresAt: &time.Time{}}, false) + require.NoError(t, err) + assert.Equal(t, "1", key.KeyID) + + // try to add the same key again with a different AddedAt and force is false + key2, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().Add(10 * time.Minute).UTC()}, false) + require.NoError(t, err) + assert.Equal(t, "1", key2.KeyID) + assert.NotEqual(t, key.AddedAt, key2.AddedAt) + }) + + t.Run("List should return all keys that are not expired", func(t *testing.T) { + ctx, store := setup() + _, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().UTC()}, false) + require.NoError(t, err) + _, err = store.Add(ctx, &signingkeys.SigningKey{KeyID: "2", PrivateKey: []byte{}, AddedAt: time.Now().UTC(), ExpiresAt: &time.Time{}}, false) + require.NoError(t, err) + _, err = store.Add(ctx, &signingkeys.SigningKey{KeyID: "3", PrivateKey: []byte{}, AddedAt: time.Now().UTC()}, false) + require.NoError(t, err) + + keys, err := store.List(ctx) + require.NoError(t, err) + require.Len(t, keys, 2) + }) } From 42218fbdbb0a77e3f55e3ba243270b183fa2cbfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Tue, 10 Oct 2023 14:33:58 +0200 Subject: [PATCH 002/102] DashboardScene: Support auto migration for angular panels (#76100) * DashboardScene: Support auto migration for angular panels * minor tweak * Update scenes * Review fix * Update --- .betterer.results | 23 +++++++----- package.json | 2 +- .../serialization/angularMigration.test.ts | 35 +++++++++++++++++++ .../serialization/angularMigration.ts | 22 ++++++++++++ .../sceneVariablesSetToVariables.test.ts | 1 - .../transformSaveModelToScene.ts | 3 ++ .../features/dashboard/state/PanelModel.ts | 2 +- public/app/features/scenes/apps/utils.ts | 2 +- .../scenes/scenes/queryVariableDemo.tsx | 6 ++-- .../scenes/scenes/repeatingPanels.tsx | 2 +- yarn.lock | 10 +++--- 11 files changed, 86 insertions(+), 22 deletions(-) create mode 100644 public/app/features/dashboard-scene/serialization/angularMigration.test.ts create mode 100644 public/app/features/dashboard-scene/serialization/angularMigration.ts diff --git a/.betterer.results b/.betterer.results index 3fb02eb3288..8f8c6ee10c8 100644 --- a/.betterer.results +++ b/.betterer.results @@ -3007,6 +3007,10 @@ exports[`better eslint`] = { "public/app/features/dashboard-scene/inspect/InspectJsonTab.tsx:5381": [ [0, 0, 0, "Use data-testid for E2E selectors instead of aria-label", "0"] ], + "public/app/features/dashboard-scene/serialization/angularMigration.test.ts:5381": [ + [0, 0, 0, "Unexpected any. Specify a different type.", "0"], + [0, 0, 0, "Unexpected any. Specify a different type.", "1"] + ], "public/app/features/dashboard-scene/serialization/transformSaveModelToScene.test.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], [0, 0, 0, "Unexpected any. Specify a different type.", "1"] @@ -3579,18 +3583,19 @@ exports[`better eslint`] = { [0, 0, 0, "Do not use any type assertions.", "9"], [0, 0, 0, "Unexpected any. Specify a different type.", "10"], [0, 0, 0, "Unexpected any. Specify a different type.", "11"], - [0, 0, 0, "Do not use any type assertions.", "12"], - [0, 0, 0, "Unexpected any. Specify a different type.", "13"], - [0, 0, 0, "Do not use any type assertions.", "14"], - [0, 0, 0, "Unexpected any. Specify a different type.", "15"], + [0, 0, 0, "Unexpected any. Specify a different type.", "12"], + [0, 0, 0, "Do not use any type assertions.", "13"], + [0, 0, 0, "Unexpected any. Specify a different type.", "14"], + [0, 0, 0, "Do not use any type assertions.", "15"], [0, 0, 0, "Unexpected any. Specify a different type.", "16"], [0, 0, 0, "Unexpected any. Specify a different type.", "17"], - [0, 0, 0, "Do not use any type assertions.", "18"], - [0, 0, 0, "Unexpected any. Specify a different type.", "19"], - [0, 0, 0, "Do not use any type assertions.", "20"], - [0, 0, 0, "Unexpected any. Specify a different type.", "21"], + [0, 0, 0, "Unexpected any. Specify a different type.", "18"], + [0, 0, 0, "Do not use any type assertions.", "19"], + [0, 0, 0, "Unexpected any. Specify a different type.", "20"], + [0, 0, 0, "Do not use any type assertions.", "21"], [0, 0, 0, "Unexpected any. Specify a different type.", "22"], - [0, 0, 0, "Unexpected any. Specify a different type.", "23"] + [0, 0, 0, "Unexpected any. Specify a different type.", "23"], + [0, 0, 0, "Unexpected any. Specify a different type.", "24"] ], "public/app/features/dashboard/state/TimeModel.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], diff --git a/package.json b/package.json index 82e096ab989..07bd5cf092d 100644 --- a/package.json +++ b/package.json @@ -250,7 +250,7 @@ "@grafana/lezer-traceql": "0.0.6", "@grafana/monaco-logql": "^0.0.7", "@grafana/runtime": "workspace:*", - "@grafana/scenes": "^1.13.0", + "@grafana/scenes": "^1.15.0", "@grafana/schema": "workspace:*", "@grafana/ui": "workspace:*", "@kusto/monaco-kusto": "^7.4.0", diff --git a/public/app/features/dashboard-scene/serialization/angularMigration.test.ts b/public/app/features/dashboard-scene/serialization/angularMigration.test.ts new file mode 100644 index 00000000000..c8a8d890889 --- /dev/null +++ b/public/app/features/dashboard-scene/serialization/angularMigration.test.ts @@ -0,0 +1,35 @@ +import { getPanelPlugin } from '@grafana/data/test/__mocks__/pluginMocks'; +import { PanelModel } from 'app/features/dashboard/state'; + +import { getAngularPanelMigrationHandler } from './angularMigration'; + +describe('getAngularPanelMigrationHandler', () => { + describe('Given an old angular panel', () => { + it('Should call migration handler', () => { + const onPanelTypeChanged = (panel: PanelModel, prevPluginId: string, prevOptions: Record) => { + panel.fieldConfig = { defaults: { unit: 'bytes' }, overrides: [] }; + return { name: prevOptions.angular.oldOptionProp }; + }; + + const reactPlugin = getPanelPlugin({ id: 'timeseries' }).setPanelChangeHandler(onPanelTypeChanged as any); + + const oldModel = new PanelModel({ + autoMigrateFrom: 'graph', + oldOptionProp: 'old name', + type: 'timeseries', + }); + + const mutatedModel = { + id: 1, + type: 'timeseries', + options: {}, + fieldConfig: { defaults: {}, overrides: [] }, + }; + + getAngularPanelMigrationHandler(oldModel)(mutatedModel, reactPlugin); + + expect(mutatedModel.options).toEqual({ name: 'old name' }); + expect(mutatedModel.fieldConfig).toEqual({ defaults: { unit: 'bytes' }, overrides: [] }); + }); + }); +}); diff --git a/public/app/features/dashboard-scene/serialization/angularMigration.ts b/public/app/features/dashboard-scene/serialization/angularMigration.ts new file mode 100644 index 00000000000..0eb88a32886 --- /dev/null +++ b/public/app/features/dashboard-scene/serialization/angularMigration.ts @@ -0,0 +1,22 @@ +import { PanelModel as PanelModelFromData, PanelPlugin } from '@grafana/data'; +import { autoMigrateAngular, PanelModel } from 'app/features/dashboard/state/PanelModel'; + +export function getAngularPanelMigrationHandler(oldModel: PanelModel) { + return function handleAngularPanelMigrations(panel: PanelModelFromData, plugin: PanelPlugin) { + if (plugin.angularPanelCtrl) { + panel.options = { angularOptions: oldModel.getOptionsToRemember() }; + return; + } + + if (oldModel.autoMigrateFrom) { + const wasAngular = autoMigrateAngular[oldModel.autoMigrateFrom] != null; + const oldOptions = oldModel.getOptionsToRemember(); + const prevPluginId = oldModel.autoMigrateFrom; + + if (plugin.onPanelTypeChanged) { + const prevOptions = wasAngular ? { angular: oldOptions } : oldOptions.options; + Object.assign(panel.options, plugin.onPanelTypeChanged(panel, prevPluginId, prevOptions, panel.fieldConfig)); + } + } + }; +} diff --git a/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.test.ts b/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.test.ts index 12af487dc7d..fb528d04bca 100644 --- a/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.test.ts +++ b/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.test.ts @@ -122,7 +122,6 @@ describe('sceneVariablesSetToVariables', () => { "query": "query", "refresh": 1, "regex": "", - "sort": 1, } `); }); diff --git a/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts b/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts index 3ee6e03618e..690a7bc735d 100644 --- a/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts +++ b/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts @@ -46,6 +46,8 @@ import { RowRepeaterBehavior } from '../scene/RowRepeaterBehavior'; import { createPanelDataProvider } from '../utils/createPanelDataProvider'; import { getVizPanelKeyForPanelId } from '../utils/utils'; +import { getAngularPanelMigrationHandler } from './angularMigration'; + export interface DashboardLoaderState { dashboard?: DashboardScene; isLoading?: boolean; @@ -338,6 +340,7 @@ export function buildGridItemForPanel(panel: PanelModel): SceneGridItemLike { menu: new VizPanelMenu({ $behaviors: [panelMenuBehavior], }), + _UNSAFE_customMigrationHandler: getAngularPanelMigrationHandler(panel), }; if (panel.timeFrom || panel.timeShift) { diff --git a/public/app/features/dashboard/state/PanelModel.ts b/public/app/features/dashboard/state/PanelModel.ts index ab336f1eafb..d053d2c756d 100644 --- a/public/app/features/dashboard/state/PanelModel.ts +++ b/public/app/features/dashboard/state/PanelModel.ts @@ -394,7 +394,7 @@ export class PanelModel implements DataConfigSource, IPanelModel { } } - private getOptionsToRemember() { + public getOptionsToRemember(): any { return Object.keys(this).reduce((acc, property) => { if (notPersistedProperties[property] || mustKeepProps[property]) { return acc; diff --git a/public/app/features/scenes/apps/utils.ts b/public/app/features/scenes/apps/utils.ts index eb37d5dc16e..48afdf1efee 100644 --- a/public/app/features/scenes/apps/utils.ts +++ b/public/app/features/scenes/apps/utils.ts @@ -50,7 +50,7 @@ export function getVariablesDefinitions() { new QueryVariable({ name: 'instance', datasource: { uid: 'gdev-prometheus' }, - query: { query: 'label_values(grafana_http_request_duration_seconds_sum, instance)' }, + query: { query: 'label_values(grafana_http_request_duration_seconds_sum, instance)', refId: 'A' }, }), ], }); diff --git a/public/app/features/scenes/scenes/queryVariableDemo.tsx b/public/app/features/scenes/scenes/queryVariableDemo.tsx index d51143cf783..d1f1e025426 100644 --- a/public/app/features/scenes/scenes/queryVariableDemo.tsx +++ b/public/app/features/scenes/scenes/queryVariableDemo.tsx @@ -31,19 +31,19 @@ export function getQueryVariableDemo(): DashboardScene { new QueryVariable({ name: 'instance (using datasource variable)', refresh: VariableRefresh.onTimeRangeChanged, - query: { query: 'label_values(go_gc_duration_seconds, ${metric})' }, + query: { query: 'label_values(go_gc_duration_seconds, ${metric})', refId: 'A' }, datasource: { uid: '${datasource}' }, }), new QueryVariable({ name: 'label values (on time range refresh)', refresh: VariableRefresh.onTimeRangeChanged, - query: { query: 'label_values(go_gc_duration_seconds, ${metric})' }, + query: { query: 'label_values(go_gc_duration_seconds, ${metric})', refId: 'B' }, datasource: { uid: 'gdev-prometheus', type: 'prometheus' }, }), new QueryVariable({ name: 'legacy (graphite)', refresh: VariableRefresh.onTimeRangeChanged, - query: { queryType: 'Default', target: 'stats.response.*' }, + query: { queryType: 'Default', target: 'stats.response.*', refId: 'C' }, datasource: { uid: 'gdev-graphite', type: 'graphite' }, }), ], diff --git a/public/app/features/scenes/scenes/repeatingPanels.tsx b/public/app/features/scenes/scenes/repeatingPanels.tsx index b2440e6f3bd..8286f8cfc71 100644 --- a/public/app/features/scenes/scenes/repeatingPanels.tsx +++ b/public/app/features/scenes/scenes/repeatingPanels.tsx @@ -79,7 +79,7 @@ export function getRepeatingPanelsDemo(): DashboardScene { function changeVariable(variable: TestVariable) { const sub = variable.subscribeToState((state, old) => { if (!state.loading && old.loading) { - if (variable.state.optionsToReturn.length === 2) { + if (variable.state.optionsToReturn?.length === 2) { variable.setState({ query: 'ABC', optionsToReturn: [ diff --git a/yarn.lock b/yarn.lock index eaa1fe156ec..2d0ae3143a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3343,9 +3343,9 @@ __metadata: languageName: unknown linkType: soft -"@grafana/scenes@npm:^1.13.0": - version: 1.13.0 - resolution: "@grafana/scenes@npm:1.13.0" +"@grafana/scenes@npm:^1.15.0": + version: 1.15.0 + resolution: "@grafana/scenes@npm:1.15.0" dependencies: "@grafana/e2e-selectors": 10.0.2 react-grid-layout: 1.3.4 @@ -3357,7 +3357,7 @@ __metadata: "@grafana/runtime": 10.0.3 "@grafana/schema": 10.0.3 "@grafana/ui": 10.0.3 - checksum: 3a6c4419ee324d49f41b9569dc86ca7ffa926438eedd762aa79c3327c39cd16b9671047ad02f5de3c52190a81b4139cdb75653a4b3741753f851ca0c9155b9bf + checksum: 8416327ad9a0e3be35d5a106788248d5886f6aa220777cf0772ef73a3f64ecac889f58a8e804beea0de339acd571d6e49a10278180cf1c50a2a51f44e03bf93e languageName: node linkType: hard @@ -17654,7 +17654,7 @@ __metadata: "@grafana/lezer-traceql": 0.0.6 "@grafana/monaco-logql": ^0.0.7 "@grafana/runtime": "workspace:*" - "@grafana/scenes": ^1.13.0 + "@grafana/scenes": ^1.15.0 "@grafana/schema": "workspace:*" "@grafana/tsconfig": ^1.3.0-rc1 "@grafana/ui": "workspace:*" From f40b695267edce16085ba2324be5a22fdbdd69c4 Mon Sep 17 00:00:00 2001 From: Fabrizio <135109076+fabrizio-grafana@users.noreply.github.com> Date: Tue, 10 Oct 2023 14:48:37 +0200 Subject: [PATCH 003/102] Docs: Add entry for TraceQL query editor (#76119) --- docs/sources/whatsnew/whats-new-next/index.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/sources/whatsnew/whats-new-next/index.md b/docs/sources/whatsnew/whats-new-next/index.md index ed3d0086270..724719b637c 100644 --- a/docs/sources/whatsnew/whats-new-next/index.md +++ b/docs/sources/whatsnew/whats-new-next/index.md @@ -83,6 +83,17 @@ To learn more, refer to the following video demo, as well as the [Grafana Variab {{< video-embed src="/media/docs/tempo/screen-recording-grafana-10.2-tempo-query-type-template-variables.mp4" >}} +## Improved TraceQL query editor + + + + +_Generally available in Grafana Cloud_ + +The [TraceQL query editor](https://grafana.com/docs/tempo/latest/traceql/#traceql-query-editor) has been improved to facilitate the creation of TraceQL queries. In particular, it now features improved autocompletion, syntax highlighting, and error reporting. + +{{< video-embed src="/media/docs/tempo/screen-recording-grafana-10.2-traceql-query-editor-improvements.mp4" >}} + ## Grafana OnCall integration for Alerting From 1f9bd7466d0ae1de67c1fc05b4ca647a64107372 Mon Sep 17 00:00:00 2001 From: Andre Pereira Date: Tue, 10 Oct 2023 13:53:43 +0100 Subject: [PATCH 004/102] Tempo: Multi-line support for error markers in TraceQL editor (#75904) * Don't show error when editor is empty * Added multi-line support for error markers * Fixed bug where new lines weren't being counted as characters * Take startLine and endLine into account in column calculations * Fix --- .../tempo/traceql/TraceQLEditor.tsx | 2 +- .../tempo/traceql/errorHighlighting.ts | 25 ++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.tsx b/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.tsx index c8f2dd7f4f5..42d83347505 100644 --- a/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.tsx +++ b/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.tsx @@ -70,7 +70,7 @@ export function TraceQLEditor(props: Props) { // Register callback for query changes editor.onDidChangeModelContent((changeEvent) => { const model = editor.getModel(); - if (!model) { + if (!model || model.getValue() === '') { return; } diff --git a/public/app/plugins/datasource/tempo/traceql/errorHighlighting.ts b/public/app/plugins/datasource/tempo/traceql/errorHighlighting.ts index b9b50641500..dff1c7f9715 100644 --- a/public/app/plugins/datasource/tempo/traceql/errorHighlighting.ts +++ b/public/app/plugins/datasource/tempo/traceql/errorHighlighting.ts @@ -106,17 +106,30 @@ export const setErrorMarkers = ( model, 'owner', // default value errorNodes.map((errorNode) => { + let startLine = 0; + let endLine = 0; + let start = errorNode.from; + let end = errorNode.to; + + while (start > 0) { + startLine++; + start -= model.getLineLength(startLine) + 1; // new lines don't count for getLineLength() but they still count as a character for the parser + } + while (end > 0) { + endLine++; + end -= model.getLineLength(endLine) + 1; + } + return { message: computeErrorMessage(errorNode), severity: monaco.MarkerSeverity.Error, - // As of now, we support only single-line queries - startLineNumber: 0, - endLineNumber: 0, + startLineNumber: startLine, + endLineNumber: endLine, - // `+ 1` because squiggles seem shifted by one - startColumn: errorNode.from + 1, - endColumn: errorNode.to + 1, + // `+ 2` because of the above computations + startColumn: start + model.getLineLength(startLine) + 2, + endColumn: end + model.getLineLength(endLine) + 2, }; }) ); From ac27fea5bc3f2bef0a87e6cbeb4b8c7f31bc5c7e Mon Sep 17 00:00:00 2001 From: Esteban Beltran Date: Tue, 10 Oct 2023 15:41:20 +0200 Subject: [PATCH 005/102] Sandbox: Add basic e2e tests for datasources inside sandbox (#76226) * Sandbox: initial dummy datasource plugin for e2e * WIP: tests * Add metrics to plugin.json so it shows up in explore * Fix false positives in frontend-sandbox-datasource.spec.ts * Change typed name to be static * Add code to delete the datasource after tests are complete * Add fail on status code * Update tests for more config * Replace visit with existing page * Delete cleanup code --- .../img/logo.svg | 1 + .../module.js | 115 ++++++++++++++ .../plugin.json | 26 ++++ .../frontend-sandbox-datasource.spec.ts | 145 ++++++++++++++++++ 4 files changed, 287 insertions(+) create mode 100644 e2e/custom-plugins/frontend-sandbox-datasource-test/img/logo.svg create mode 100644 e2e/custom-plugins/frontend-sandbox-datasource-test/module.js create mode 100644 e2e/custom-plugins/frontend-sandbox-datasource-test/plugin.json create mode 100644 e2e/various-suite/frontend-sandbox-datasource.spec.ts diff --git a/e2e/custom-plugins/frontend-sandbox-datasource-test/img/logo.svg b/e2e/custom-plugins/frontend-sandbox-datasource-test/img/logo.svg new file mode 100644 index 00000000000..08ec9068d45 --- /dev/null +++ b/e2e/custom-plugins/frontend-sandbox-datasource-test/img/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/e2e/custom-plugins/frontend-sandbox-datasource-test/module.js b/e2e/custom-plugins/frontend-sandbox-datasource-test/module.js new file mode 100644 index 00000000000..39a1058a505 --- /dev/null +++ b/e2e/custom-plugins/frontend-sandbox-datasource-test/module.js @@ -0,0 +1,115 @@ +/* + * This is a dummy plugin to test the frontend sandbox + * It is not meant to be used in any other way + * This file doesn't require any compilation + */ +define(['react', '@grafana/data'], function (React, grafanaData) { + const { DataSourcePlugin, DataSourceApi, MutableDataFrame, FieldType } = grafanaData; + const { useState } = React; + + const QueryEditor = (props) => { + const [value, setValue] = useState(''); + + const handleChange = (event) => { + setValue(event.target.value); + props.onChange({ + ...props.query, + testValue: event.target.value, + }); + props.onRunQuery(); + }; + + return React.createElement( + 'div', + null, + React.createElement('label', { htmlFor: 'inputField' }, 'Dummy input field'), + React.createElement('input', { + type: 'text', + id: 'inputField', + 'data-testid': 'sandbox-query-editor-query-input', + value: value, + onChange: handleChange, + }) + ); + }; + + const ConfigEditor = (props) => { + const { onOptionsChange, options } = props; + const value = options.jsonData.input || ''; + + const handleChange = (event) => { + onOptionsChange({ + ...options, + jsonData: { + input: event.target.value, + }, + }); + }; + + return React.createElement( + 'div', + null, + React.createElement('label', { htmlFor: 'inputField' }, 'Test Config field'), + React.createElement('input', { + type: 'text', + id: 'inputField', + 'data-testid': 'sandbox-config-editor-query-input', + value: value, + onChange: handleChange, + }) + ); + }; + + class BasicDataSource extends DataSourceApi { + constructor(instanceSettings) { + super(instanceSettings); + } + + // this is a test query it'll generate a logarithmic series starting now-1h + async query(options) { + const promises = options.targets.map(async (target) => { + const query = target; + + const timestamps = []; + const values = []; + + const endTime = Date.now(); + const startTime = endTime - 3600000; // 1 hour in milliseconds + + // Define the logarithmic base and increment factor + const base = 2; + const increment = 0.1; + + // Generate the data points + for (let i = 0; i < 50; i++) { + // Calculate the timestamp + const timestamp = startTime + ((endTime - startTime) / 50) * i; + timestamps.push(timestamp); + + // Calculate the value using logarithmic increase + const value = Math.pow(base, increment * i); + values.push(value); + } + + return new MutableDataFrame({ + refId: query.refId, + fields: [ + { name: 'Time', type: FieldType.time, values: timestamps }, + { name: 'Value', type: FieldType.number, values: values }, + ], + }); + }); + return Promise.all(promises).then((data) => ({ data })); + } + async testDatasource() { + return { + status: 'success', + message: 'Sandbox Success', + }; + } + } + + const plugin = new DataSourcePlugin(BasicDataSource).setConfigEditor(ConfigEditor).setQueryEditor(QueryEditor); + + return { plugin }; +}); diff --git a/e2e/custom-plugins/frontend-sandbox-datasource-test/plugin.json b/e2e/custom-plugins/frontend-sandbox-datasource-test/plugin.json new file mode 100644 index 00000000000..c1b84c78d82 --- /dev/null +++ b/e2e/custom-plugins/frontend-sandbox-datasource-test/plugin.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://raw.githubusercontent.com/grafana/grafana/master/docs/sources/developers/plugins/plugin.schema.json", + "type": "datasource", + "name": "Sandbox datasource test plugin", + "id": "sandbox-test-datasource", + "metrics": true, + "info": { + "keywords": ["explore", "datasource"], + "description": "", + "author": { + "name": "Grafana" + }, + "logos": { + "small": "img/logo.svg", + "large": "img/logo.svg" + }, + "links": [], + "screenshots": [], + "version": "1.0.0", + "updated": "2023-06-27" + }, + "dependencies": { + "grafanaDependency": ">=10.0", + "plugins": [] + } +} diff --git a/e2e/various-suite/frontend-sandbox-datasource.spec.ts b/e2e/various-suite/frontend-sandbox-datasource.spec.ts new file mode 100644 index 00000000000..2881e7dfe5d --- /dev/null +++ b/e2e/various-suite/frontend-sandbox-datasource.spec.ts @@ -0,0 +1,145 @@ +import { random } from 'lodash'; + +import { e2e } from '../utils'; + +const DATASOURCE_ID = 'sandbox-test-datasource'; +let DATASOURCE_CONNECTION_ID = ''; +const DATASOURCE_TYPED_NAME = 'SandboxDatasourceInstance'; + +describe('Datasource sandbox', () => { + before(() => { + e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'), true); + + e2e.pages.AddDataSource.visit(); + e2e.pages.AddDataSource.dataSourcePluginsV2('Sandbox datasource test plugin') + .scrollIntoView() + .should('be.visible') // prevents flakiness + .click(); + e2e.pages.DataSource.name().clear(); + e2e.pages.DataSource.name().type(DATASOURCE_TYPED_NAME); + e2e.pages.DataSource.saveAndTest().click(); + cy.url().then((url) => { + const split = url.split('/'); + DATASOURCE_CONNECTION_ID = split[split.length - 1]; + }); + }); + beforeEach(() => { + e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'), true); + }); + + describe('Config Editor', () => { + describe('Sandbox disabled', () => { + beforeEach(() => { + cy.window().then((win) => { + win.localStorage.setItem('grafana.featureToggles', 'pluginsFrontendSandbox=0'); + }); + }); + it('Should not render a sandbox wrapper around the datasource config editor', () => { + e2e.pages.EditDataSource.visit(DATASOURCE_CONNECTION_ID); + cy.wait(300); // wait to prevent false positives because cypress checks too fast + cy.get(`div[data-plugin-sandbox="${DATASOURCE_ID}"]`).should('not.exist'); + }); + }); + + describe('Sandbox enabled', () => { + beforeEach(() => { + cy.window().then((win) => { + win.localStorage.setItem('grafana.featureToggles', 'pluginsFrontendSandbox=1'); + }); + }); + + it('Should render a sandbox wrapper around the datasource config editor', () => { + e2e.pages.EditDataSource.visit(DATASOURCE_CONNECTION_ID); + cy.get(`div[data-plugin-sandbox="${DATASOURCE_ID}"]`).should('exist'); + }); + + it('Should store values in jsonData and secureJsonData correctly', () => { + e2e.pages.EditDataSource.visit(DATASOURCE_CONNECTION_ID); + + const valueToStore = 'test' + random(100); + + cy.get('[data-testid="sandbox-config-editor-query-input"]').should('not.be.disabled'); + cy.get('[data-testid="sandbox-config-editor-query-input"]').type(valueToStore); + cy.get('[data-testid="sandbox-config-editor-query-input"]').should('have.value', valueToStore); + + e2e.pages.DataSource.saveAndTest().click(); + e2e.pages.DataSource.alert().should('exist').contains('Sandbox Success', {}); + + // validate the value was stored + e2e.pages.EditDataSource.visit(DATASOURCE_CONNECTION_ID); + cy.get('[data-testid="sandbox-config-editor-query-input"]').should('not.be.disabled'); + cy.get('[data-testid="sandbox-config-editor-query-input"]').should('have.value', valueToStore); + }); + }); + }); + + describe('Explore Page', () => { + describe('Sandbox disabled', () => { + beforeEach(() => { + cy.window().then((win) => { + win.localStorage.setItem('grafana.featureToggles', 'pluginsFrontendSandbox=0'); + }); + }); + + it('Should not wrap the query editor in a sandbox wrapper', () => { + e2e.pages.Explore.visit(); + e2e.components.DataSourcePicker.container().should('be.visible').click(); + cy.contains(DATASOURCE_TYPED_NAME).scrollIntoView().should('be.visible').click(); + + cy.wait(300); // wait to prevent false positives because cypress checks too fast + cy.get(`div[data-plugin-sandbox="${DATASOURCE_ID}"]`).should('not.exist'); + }); + + it('Should accept values when typed', () => { + e2e.pages.Explore.visit(); + e2e.components.DataSourcePicker.container().should('be.visible').click(); + cy.contains(DATASOURCE_TYPED_NAME).scrollIntoView().should('be.visible').click(); + + const valueToType = 'test' + random(100); + + cy.get('[data-testid="sandbox-query-editor-query-input"]').should('not.be.disabled'); + cy.get('[data-testid="sandbox-query-editor-query-input"]').type(valueToType); + cy.get('[data-testid="sandbox-query-editor-query-input"]').should('have.value', valueToType); + }); + }); + + describe('Sandbox enabled', () => { + beforeEach(() => { + cy.window().then((win) => { + win.localStorage.setItem('grafana.featureToggles', 'pluginsFrontendSandbox=1'); + }); + }); + + it('Should wrap the query editor in a sandbox wrapper', () => { + e2e.pages.Explore.visit(); + e2e.components.DataSourcePicker.container().should('be.visible').click(); + cy.contains(DATASOURCE_TYPED_NAME).scrollIntoView().should('be.visible').click(); + + cy.get(`div[data-plugin-sandbox="${DATASOURCE_ID}"]`).should('exist'); + }); + + it('Should accept values when typed', () => { + e2e.pages.Explore.visit(); + e2e.components.DataSourcePicker.container().should('be.visible').click(); + cy.contains(DATASOURCE_TYPED_NAME).scrollIntoView().should('be.visible').click(); + + const valueToType = 'test' + random(100); + + cy.get('[data-testid="sandbox-query-editor-query-input"]').should('not.be.disabled'); + cy.get('[data-testid="sandbox-query-editor-query-input"]').type(valueToType); + cy.get('[data-testid="sandbox-query-editor-query-input"]').should('have.value', valueToType); + + // typing the query editor should reflect in the url + cy.url().should('include', valueToType); + }); + }); + }); + + afterEach(() => { + e2e.flows.revertAllChanges(); + }); + + after(() => { + cy.clearCookies(); + }); +}); From de2d8f50e86bd348f0c0ab22bc8859878f82b331 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 10 Oct 2023 14:49:22 +0100 Subject: [PATCH 006/102] Update dependency @react-types/shared to v3.21.0 (#76251) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index 07bd5cf092d..b3a0b4c3615 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,7 @@ "@react-types/button": "3.9.0", "@react-types/menu": "3.9.2", "@react-types/overlays": "3.8.0", - "@react-types/shared": "3.18.1", + "@react-types/shared": "3.21.0", "@rtsao/plugin-proposal-class-properties": "7.0.1-patch.1", "@swc/core": "1.3.38", "@swc/helpers": "0.4.14", diff --git a/yarn.lock b/yarn.lock index 2d0ae3143a9..ce7dfd83649 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6824,16 +6824,7 @@ __metadata: languageName: node linkType: hard -"@react-types/shared@npm:3.18.1": - version: 3.18.1 - resolution: "@react-types/shared@npm:3.18.1" - peerDependencies: - react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 - checksum: 298822d1cca26b5da1bdf53a57d26d5619f3af816db3fdecdc8b99a3a305ca8ee352347ca43f426a5d79045b08cf32c00c6b083ad4a1002abf673de9dd4a9bb5 - languageName: node - linkType: hard - -"@react-types/shared@npm:^3.18.1, @react-types/shared@npm:^3.21.0": +"@react-types/shared@npm:3.21.0, @react-types/shared@npm:^3.18.1, @react-types/shared@npm:^3.21.0": version: 3.21.0 resolution: "@react-types/shared@npm:3.21.0" peerDependencies: @@ -17687,7 +17678,7 @@ __metadata: "@react-types/button": 3.9.0 "@react-types/menu": 3.9.2 "@react-types/overlays": 3.8.0 - "@react-types/shared": 3.18.1 + "@react-types/shared": 3.21.0 "@reduxjs/toolkit": 1.9.5 "@remix-run/router": ^1.5.0 "@rtsao/plugin-proposal-class-properties": 7.0.1-patch.1 From 930c7533409ab366d8d6d874a26901ff768f9a12 Mon Sep 17 00:00:00 2001 From: Ashley Harrison Date: Tue, 10 Oct 2023 14:55:52 +0100 Subject: [PATCH 007/102] Navigation: Implement logic for docking nav menu (#76188) * Create a state for dockedMegaMenu and the function to manage it * Add the dockedMenu icon and handle the status when clicking it * Add Megamenu to section nav area when it is docked * get logic working * fix mobile * refactor state + persist in localStorage * adjust icon and don't use position absolute * restore old rudderstack tracking * use Flex instead * adjust feature toggle to be experimental * extract out localStorage handling into utils * don't need separate file * use store.set/get instead --------- Co-authored-by: eledobleefe --- .../feature-toggles/index.md | 2 +- packages/grafana-data/src/types/icon.ts | 1 + pkg/services/featuremgmt/registry.go | 2 +- pkg/services/featuremgmt/toggles_gen.csv | 2 +- .../core/components/AppChrome/AppChrome.tsx | 14 +++++- .../components/AppChrome/AppChromeMenu.tsx | 4 +- .../components/AppChrome/AppChromeService.tsx | 24 +++++---- .../DockedMegaMenu/MegaMenu.test.tsx | 2 +- .../AppChrome/DockedMegaMenu/MegaMenu.tsx | 49 ++++++++++++++++--- .../AppChrome/DockedMegaMenu/MegaMenuItem.tsx | 13 +++-- .../AppChrome/MegaMenu/MegaMenu.test.tsx | 2 +- .../AppChrome/MegaMenu/NavBarMenu.tsx | 4 +- public/app/core/components/Page/Page.tsx | 3 +- public/locales/de-DE/grafana.json | 5 ++ public/locales/en-US/grafana.json | 5 ++ public/locales/es-ES/grafana.json | 5 ++ public/locales/fr-FR/grafana.json | 5 ++ public/locales/pseudo-LOCALE/grafana.json | 5 ++ public/locales/zh-Hans/grafana.json | 5 ++ 19 files changed, 117 insertions(+), 35 deletions(-) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 342e22368fb..06727f9e537 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -64,7 +64,6 @@ Some features are enabled by default. You can disable these feature by setting t | `newDBLibrary` | Use jmoiron/sqlx rather than xorm for a few backend services | | `autoMigrateOldPanels` | Migrate old angular panels to supported versions (graph, table-old, worldmap, etc) | | `disableAngular` | Dynamic flag to disable angular at runtime. The preferred method is to set `angular_support_enabled` to `false` in the [security] settings, which allows you to change the state at runtime. | -| `dockedMegaMenu` | Enable support for a persistent (docked) navigation menu | | `grpcServer` | Run the GRPC server | | `accessControlOnCall` | Access control primitives for OnCall | | `nestedFolders` | Enable folder nesting | @@ -97,6 +96,7 @@ Experimental features might be changed or removed without prior notice. | `scenes` | Experimental framework to build interactive dashboards | | `disableSecretsCompatibility` | Disable duplicated secret storage in legacy tables | | `logRequestsInstrumentedAsUnknown` | Logs the path for requests that are instrumented as unknown | +| `dockedMegaMenu` | Enable support for a persistent (docked) navigation menu | | `showDashboardValidationWarnings` | Show warnings when dashboards do not validate against the schema | | `mysqlAnsiQuotes` | Use double quotes to escape keyword in a MySQL query | | `alertingBacktesting` | Rule backtesting API for alerting | diff --git a/packages/grafana-data/src/types/icon.ts b/packages/grafana-data/src/types/icon.ts index 9d957baf273..d2e9d9303bf 100644 --- a/packages/grafana-data/src/types/icon.ts +++ b/packages/grafana-data/src/types/icon.ts @@ -225,6 +225,7 @@ export const availableIconsIndex = { 'vertical-align-bottom': true, 'vertical-align-center': true, 'vertical-align-top': true, + 'web-section-alt': true, 'wrap-text': true, rss: true, x: true, diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 37fa797b436..cde567f677d 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -164,7 +164,7 @@ var ( { Name: "dockedMegaMenu", Description: "Enable support for a persistent (docked) navigation menu", - Stage: FeatureStagePublicPreview, + Stage: FeatureStageExperimental, FrontendOnly: true, Owner: grafanaFrontendPlatformSquad, }, diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 0a3882cb846..539bf40ec1f 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -22,7 +22,7 @@ disableSecretsCompatibility,experimental,@grafana/hosted-grafana-team,false,fals logRequestsInstrumentedAsUnknown,experimental,@grafana/hosted-grafana-team,false,false,false,false dataConnectionsConsole,GA,@grafana/plugins-platform-backend,false,false,false,false topnav,deprecated,@grafana/grafana-frontend-platform,false,false,false,false -dockedMegaMenu,preview,@grafana/grafana-frontend-platform,false,false,false,true +dockedMegaMenu,experimental,@grafana/grafana-frontend-platform,false,false,false,true grpcServer,preview,@grafana/grafana-app-platform-squad,false,false,false,false entityStore,experimental,@grafana/grafana-app-platform-squad,true,false,false,false cloudWatchCrossAccountQuerying,GA,@grafana/aws-datasources,false,false,false,false diff --git a/public/app/core/components/AppChrome/AppChrome.tsx b/public/app/core/components/AppChrome/AppChrome.tsx index 5c2f458f9c1..ae02e558f86 100644 --- a/public/app/core/components/AppChrome/AppChrome.tsx +++ b/public/app/core/components/AppChrome/AppChrome.tsx @@ -10,6 +10,7 @@ import { CommandPalette } from 'app/features/commandPalette/CommandPalette'; import { KioskMode } from 'app/types'; import { AppChromeMenu } from './AppChromeMenu'; +import { MegaMenu as DockedMegaMenu } from './DockedMegaMenu/MegaMenu'; import { MegaMenu } from './MegaMenu/MegaMenu'; import { NavToolbar } from './NavToolbar/NavToolbar'; import { SectionNav } from './SectionNav/SectionNav'; @@ -53,7 +54,7 @@ export function AppChrome({ children }: Props) { pageNav={state.pageNav} actions={state.actions} onToggleSearchBar={chrome.onToggleSearchBar} - onToggleMegaMenu={chrome.onToggleMegaMenu} + onToggleMegaMenu={() => chrome.setMegaMenu(state.megaMenu === 'closed' ? 'open' : 'closed')} onToggleKioskMode={chrome.onToggleKioskMode} /> @@ -64,6 +65,9 @@ export function AppChrome({ children }: Props) { {state.layout === PageLayoutType.Standard && state.sectionNav && !config.featureToggles.dockedMegaMenu && ( )} + {config.featureToggles.dockedMegaMenu && !state.chromeless && state.megaMenu === 'docked' && ( + chrome.setMegaMenu('closed')} /> + )}
{children}
@@ -74,7 +78,7 @@ export function AppChrome({ children }: Props) { {config.featureToggles.dockedMegaMenu ? ( ) : ( - chrome.setMegaMenu(false)} /> + chrome.setMegaMenu('closed')} /> )} @@ -102,6 +106,12 @@ const getStyles = (theme: GrafanaTheme2) => { contentChromeless: css({ paddingTop: 0, }), + dockedMegaMenu: css({ + background: theme.colors.background.primary, + borderRight: `1px solid ${theme.colors.border.weak}`, + borderTop: `1px solid ${theme.colors.border.weak}`, + zIndex: theme.zIndex.navbarFixed, + }), topNav: css({ display: 'flex', position: 'fixed', diff --git a/public/app/core/components/AppChrome/AppChromeMenu.tsx b/public/app/core/components/AppChrome/AppChromeMenu.tsx index ab164b0639d..a204e7fc9b3 100644 --- a/public/app/core/components/AppChrome/AppChromeMenu.tsx +++ b/public/app/core/components/AppChrome/AppChromeMenu.tsx @@ -27,8 +27,8 @@ export function AppChromeMenu({}: Props) { const animationSpeed = theme.transitions.duration.shortest; const animationStyles = useStyles2(getAnimStyles, animationSpeed); - const isOpen = state.megaMenuOpen; - const onClose = () => chrome.setMegaMenu(false); + const isOpen = state.megaMenu === 'open'; + const onClose = () => chrome.setMegaMenu('closed'); const { overlayProps, underlayProps } = useOverlay( { diff --git a/public/app/core/components/AppChrome/AppChromeService.tsx b/public/app/core/components/AppChrome/AppChromeService.tsx index dafbbdcaf68..c39a25766d0 100644 --- a/public/app/core/components/AppChrome/AppChromeService.tsx +++ b/public/app/core/components/AppChrome/AppChromeService.tsx @@ -2,7 +2,7 @@ import { useObservable } from 'react-use'; import { BehaviorSubject } from 'rxjs'; import { AppEvents, NavModel, NavModelItem, PageLayoutType, UrlQueryValue } from '@grafana/data'; -import { locationService, reportInteraction } from '@grafana/runtime'; +import { config, locationService, reportInteraction } from '@grafana/runtime'; import appEvents from 'app/core/app_events'; import { t } from 'app/core/internationalization'; import store from 'app/core/store'; @@ -17,11 +17,13 @@ export interface AppChromeState { pageNav?: NavModelItem; actions?: React.ReactNode; searchBarHidden?: boolean; - megaMenuOpen?: boolean; + megaMenu: 'open' | 'closed' | 'docked'; kioskMode: KioskMode | null; layout: PageLayoutType; } +const DOCKED_LOCAL_STORAGE_KEY = 'grafana.navigation.docked'; + export class AppChromeService { searchBarStorageKey = 'SearchBar_Hidden'; private currentRoute?: RouteDescriptor; @@ -31,6 +33,8 @@ export class AppChromeService { chromeless: true, // start out hidden to not flash it on pages without chrome sectionNav: { node: { text: t('nav.home.title', 'Home') }, main: { text: '' } }, searchBarHidden: store.getBool(this.searchBarStorageKey, false), + megaMenu: + config.featureToggles.dockedMegaMenu && store.getBool(DOCKED_LOCAL_STORAGE_KEY, false) ? 'docked' : 'closed', kioskMode: null, layout: PageLayoutType.Canvas, }); @@ -93,14 +97,14 @@ export class AppChromeService { return useObservable(this.state, this.state.getValue()); } - public onToggleMegaMenu = () => { - const isOpen = !this.state.getValue().megaMenuOpen; - reportInteraction('grafana_toggle_menu_clicked', { action: isOpen ? 'open' : 'close' }); - this.update({ megaMenuOpen: isOpen }); - }; - - public setMegaMenu = (megaMenuOpen: boolean) => { - this.update({ megaMenuOpen }); + public setMegaMenu = (newMegaMenuState: AppChromeState['megaMenu']) => { + if (config.featureToggles.dockedMegaMenu) { + store.set(DOCKED_LOCAL_STORAGE_KEY, newMegaMenuState === 'docked'); + reportInteraction('grafana_mega_menu_state', { state: newMegaMenuState }); + } else { + reportInteraction('grafana_toggle_menu_clicked', { action: newMegaMenuState === 'open' ? 'open' : 'close' }); + } + this.update({ megaMenu: newMegaMenuState }); }; public onToggleSearchBar = () => { diff --git a/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.test.tsx b/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.test.tsx index ad5dd4e8bb0..a914fda4cac 100644 --- a/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.test.tsx +++ b/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.test.tsx @@ -35,7 +35,7 @@ const setup = () => { ]; const grafanaContext = getGrafanaContextMock(); - grafanaContext.chrome.onToggleMegaMenu(); + grafanaContext.chrome.setMegaMenu('open'); return render( diff --git a/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.tsx b/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.tsx index 6915a9a2270..d4459cb11ad 100644 --- a/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.tsx +++ b/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.tsx @@ -6,6 +6,9 @@ import { useLocation } from 'react-router-dom'; import { GrafanaTheme2 } from '@grafana/data'; import { CustomScrollbar, Icon, IconButton, useStyles2 } from '@grafana/ui'; +import { Flex } from '@grafana/ui/src/unstable'; +import { useGrafana } from 'app/core/context/GrafanaContext'; +import { t } from 'app/core/internationalization'; import { useSelector } from 'app/types'; import { MegaMenuItem } from './MegaMenuItem'; @@ -22,6 +25,8 @@ export const MegaMenu = React.memo( const navBarTree = useSelector((state) => state.navBarTree); const styles = useStyles2(getStyles); const location = useLocation(); + const { chrome } = useGrafana(); + const state = chrome.useState(); const navTree = cloneDeep(navBarTree); @@ -32,13 +37,16 @@ export const MegaMenu = React.memo( const activeItem = getActiveItem(navItems, location.pathname); + const handleDockedMenu = () => { + chrome.setMegaMenu(state.megaMenu === 'docked' ? 'closed' : 'docked'); + }; + return (
    - {navItems.map((link) => ( - + {navItems.map((link, index) => ( + + + {index === 0 && ( + + )} + ))}
@@ -65,8 +92,9 @@ const getStyles = (theme: GrafanaTheme2) => ({ content: css({ display: 'flex', flexDirection: 'column', - flexGrow: 1, + height: '100%', minHeight: 0, + position: 'relative', }), mobileHeader: css({ display: 'flex', @@ -79,10 +107,15 @@ const getStyles = (theme: GrafanaTheme2) => ({ }, }), itemList: css({ - display: 'grid', - gridAutoRows: `minmax(${theme.spacing(6)}, auto)`, - gridTemplateColumns: `minmax(${MENU_WIDTH}, auto)`, + display: 'flex', + flexDirection: 'column', listStyleType: 'none', minWidth: MENU_WIDTH, + [theme.breakpoints.up('md')]: { + width: MENU_WIDTH, + }, + }), + dockMenuButton: css({ + marginRight: theme.spacing(2), }), }); diff --git a/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenuItem.tsx b/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenuItem.tsx index c99000e5431..2518d035889 100644 --- a/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenuItem.tsx +++ b/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenuItem.tsx @@ -15,11 +15,11 @@ import { hasChildMatch } from './utils'; interface Props { link: NavModelItem; activeItem?: NavModelItem; - onClose?: () => void; + onClick?: () => void; level?: number; } -export function MegaMenuItem({ link, activeItem, level = 0, onClose }: Props) { +export function MegaMenuItem({ link, activeItem, level = 0, onClick }: Props) { const styles = useStyles2(getStyles); const FeatureHighlightWrapper = link.highlightText ? FeatureHighlight : React.Fragment; const isActive = link === activeItem; @@ -29,13 +29,13 @@ export function MegaMenuItem({ link, activeItem, level = 0, onClose }: Props) { const showExpandButton = linkHasChildren(link) || link.emptyMessage; return ( -
  • +
  • { link.onClick?.(); - onClose?.(); + onClick?.(); }} target={link.target} url={link.url} @@ -75,7 +75,7 @@ export function MegaMenuItem({ link, activeItem, level = 0, onClose }: Props) { key={`${link.text}-${childLink.text}`} link={childLink} activeItem={activeItem} - onClose={onClose} + onClick={onClick} level={level + 1} /> )) @@ -121,6 +121,9 @@ const getStyles = (theme: GrafanaTheme2) => ({ alignItems: 'center', fontWeight: theme.typography.fontWeightMedium, }), + listItem: css({ + flex: 1, + }), isActive: css({ color: theme.colors.text.primary, diff --git a/public/app/core/components/AppChrome/MegaMenu/MegaMenu.test.tsx b/public/app/core/components/AppChrome/MegaMenu/MegaMenu.test.tsx index a411c977ff9..59551594e72 100644 --- a/public/app/core/components/AppChrome/MegaMenu/MegaMenu.test.tsx +++ b/public/app/core/components/AppChrome/MegaMenu/MegaMenu.test.tsx @@ -29,7 +29,7 @@ const setup = () => { ]; const grafanaContext = getGrafanaContextMock(); - grafanaContext.chrome.onToggleMegaMenu(); + grafanaContext.chrome.setMegaMenu('open'); return render( diff --git a/public/app/core/components/AppChrome/MegaMenu/NavBarMenu.tsx b/public/app/core/components/AppChrome/MegaMenu/NavBarMenu.tsx index 6879e4f1cda..18e6411a900 100644 --- a/public/app/core/components/AppChrome/MegaMenu/NavBarMenu.tsx +++ b/public/app/core/components/AppChrome/MegaMenu/NavBarMenu.tsx @@ -46,10 +46,10 @@ export function NavBarMenu({ activeItem, navItems, searchBarHidden, onClose }: P ); useEffect(() => { - if (state.megaMenuOpen) { + if (state.megaMenu === 'open') { setIsOpen(true); } - }, [state.megaMenuOpen]); + }, [state.megaMenu]); return ( diff --git a/public/app/core/components/Page/Page.tsx b/public/app/core/components/Page/Page.tsx index 595cd466199..f2490a2d91f 100644 --- a/public/app/core/components/Page/Page.tsx +++ b/public/app/core/components/Page/Page.tsx @@ -3,6 +3,7 @@ import { css, cx } from '@emotion/css'; import React, { useLayoutEffect } from 'react'; import { GrafanaTheme2, PageLayoutType } from '@grafana/data'; +import { config } from '@grafana/runtime'; import { CustomScrollbar, useStyles2 } from '@grafana/ui'; import { useGrafana } from 'app/core/context/GrafanaContext'; @@ -108,7 +109,7 @@ const getStyles = (theme: GrafanaTheme2) => { margin: theme.spacing(0, 0, 0, 0), [theme.breakpoints.up('md')]: { - margin: theme.spacing(2, 2, 0, 1), + margin: theme.spacing(2, 2, 0, config.featureToggles.dockedMegaMenu ? 2 : 1), padding: theme.spacing(3), }, }), diff --git a/public/locales/de-DE/grafana.json b/public/locales/de-DE/grafana.json index 6b13a287858..d4a27b44448 100644 --- a/public/locales/de-DE/grafana.json +++ b/public/locales/de-DE/grafana.json @@ -722,6 +722,11 @@ "kiosk": { "tv-alert": "Drücke ESC, um den Kiosk-Modus zu verlassen" }, + "megamenu": { + "close": "", + "dock": "", + "undock": "" + }, "toolbar": { "enable-kiosk": "Kiosk-Modus aktivieren", "toggle-menu": "Menü umschalten", diff --git a/public/locales/en-US/grafana.json b/public/locales/en-US/grafana.json index 9d4ae262e9d..2680839a488 100644 --- a/public/locales/en-US/grafana.json +++ b/public/locales/en-US/grafana.json @@ -722,6 +722,11 @@ "kiosk": { "tv-alert": "Press ESC to exit kiosk mode" }, + "megamenu": { + "close": "Close menu", + "dock": "Dock menu", + "undock": "Undock menu" + }, "toolbar": { "enable-kiosk": "Enable kiosk mode", "toggle-menu": "Toggle menu", diff --git a/public/locales/es-ES/grafana.json b/public/locales/es-ES/grafana.json index 55fbec548ea..efdcacb0dad 100644 --- a/public/locales/es-ES/grafana.json +++ b/public/locales/es-ES/grafana.json @@ -728,6 +728,11 @@ "kiosk": { "tv-alert": "Pulse ESC para salir del modo de quiosco" }, + "megamenu": { + "close": "", + "dock": "", + "undock": "" + }, "toolbar": { "enable-kiosk": "Activar el modo de quiosco", "toggle-menu": "Activar o desactivar menú", diff --git a/public/locales/fr-FR/grafana.json b/public/locales/fr-FR/grafana.json index fa76398f0ba..dcdca2e1f2d 100644 --- a/public/locales/fr-FR/grafana.json +++ b/public/locales/fr-FR/grafana.json @@ -728,6 +728,11 @@ "kiosk": { "tv-alert": "Appuyez sur ESC pour quitter le mode kiosque" }, + "megamenu": { + "close": "", + "dock": "", + "undock": "" + }, "toolbar": { "enable-kiosk": "Activer le mode kiosque", "toggle-menu": "Afficher/Masquer le menu", diff --git a/public/locales/pseudo-LOCALE/grafana.json b/public/locales/pseudo-LOCALE/grafana.json index e54ce1ec676..d8e037af51e 100644 --- a/public/locales/pseudo-LOCALE/grafana.json +++ b/public/locales/pseudo-LOCALE/grafana.json @@ -722,6 +722,11 @@ "kiosk": { "tv-alert": "Přęşş ĒŜC ŧő ęχįŧ ĸįőşĸ mőđę" }, + "megamenu": { + "close": "Cľőşę męʼnū", + "dock": "Đőčĸ męʼnū", + "undock": "Ůʼnđőčĸ męʼnū" + }, "toolbar": { "enable-kiosk": "Ēʼnäþľę ĸįőşĸ mőđę", "toggle-menu": "Ŧőģģľę męʼnū", diff --git a/public/locales/zh-Hans/grafana.json b/public/locales/zh-Hans/grafana.json index f7bcbfbe56f..061da99ba8b 100644 --- a/public/locales/zh-Hans/grafana.json +++ b/public/locales/zh-Hans/grafana.json @@ -716,6 +716,11 @@ "kiosk": { "tv-alert": "按 ESC 退出 kiosk 模式" }, + "megamenu": { + "close": "", + "dock": "", + "undock": "" + }, "toolbar": { "enable-kiosk": "启用 kiosk 模式", "toggle-menu": "切换菜单", From 9665cecc2751f4c914ab3d328224b53cd4ff6470 Mon Sep 17 00:00:00 2001 From: Stefan Mitic Date: Tue, 10 Oct 2023 09:56:34 -0400 Subject: [PATCH 008/102] Chore: remove gf-forms and LegacyForms from Ldap Page (#76156) * Chore: remove LegacyForms from Ldap Page * remove references to gf-form * fix form control submit function --- .betterer.results | 3 - public/app/features/admin/ldap/LdapPage.tsx | 68 ++++++++++----------- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/.betterer.results b/.betterer.results index 8f8c6ee10c8..115753f74ad 100644 --- a/.betterer.results +++ b/.betterer.results @@ -1762,9 +1762,6 @@ exports[`better eslint`] = { [0, 0, 0, "Styles should be written using objects.", "1"], [0, 0, 0, "Styles should be written using objects.", "2"] ], - "public/app/features/admin/ldap/LdapPage.tsx:5381": [ - [0, 0, 0, "Unexpected any. Specify a different type.", "0"] - ], "public/app/features/alerting/AlertTab.tsx:5381": [ [0, 0, 0, "Do not use any type assertions.", "0"], [0, 0, 0, "Unexpected any. Specify a different type.", "1"], diff --git a/public/app/features/admin/ldap/LdapPage.tsx b/public/app/features/admin/ldap/LdapPage.tsx index 5a0e68fc1b7..e82df553ddc 100644 --- a/public/app/features/admin/ldap/LdapPage.tsx +++ b/public/app/features/admin/ldap/LdapPage.tsx @@ -3,8 +3,7 @@ import { connect, ConnectedProps } from 'react-redux'; import { NavModelItem } from '@grafana/data'; import { featureEnabled } from '@grafana/runtime'; -import { Alert, Button, LegacyForms } from '@grafana/ui'; -const { FormField } = LegacyForms; +import { Alert, Button, Field, Form, HorizontalGroup, Input } from '@grafana/ui'; import { Page } from 'app/core/components/Page/Page'; import { contextSrv } from 'app/core/core'; import { GrafanaRouteComponentProps } from 'app/core/navigation/types'; @@ -42,6 +41,10 @@ interface State { isLoading: boolean; } +interface FormModel { + username: string; +} + const pageNav: NavModelItem = { text: 'LDAP', subTitle: `Verify your LDAP and user mapping configuration.`, @@ -76,9 +79,7 @@ export class LdapPage extends PureComponent { return await loadUserMapping(username); } - search = (event: any) => { - event.preventDefault(); - const username = event.target.elements['username'].value; + search = (username: string) => { if (username) { this.fetchUserMapping(username); } @@ -98,11 +99,9 @@ export class LdapPage extends PureComponent { <> {ldapError && ldapError.title && ( -
    - - {ldapError.body} - -
    + + {ldapError.body} + )} @@ -111,31 +110,32 @@ export class LdapPage extends PureComponent { {canReadLDAPUser && ( <> -

    Test user mapping

    -
    -
    - - - -
    +

    Test user mapping

    +
    this.search(data.username)}> + {({ register }) => ( + + + + + + + )} +
    {userError && userError.title && ( -
    - - {userError.body} - -
    + + {userError.body} + )} {ldapUser && } From 473f8899c5d49cc8f93e5616fbf91a848d4b25c8 Mon Sep 17 00:00:00 2001 From: Tobias Skarhed <1438972+tskarhed@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:21:11 +0200 Subject: [PATCH 009/102] Chore: Add @deprecation to LegacyForms (#76055) * Add deprecations to LegacyForms * Fix whitespace * Update FormField.tsx * Update Input.tsx * Update IndicatorsContainer.tsx * Update NoOptionsMessage.tsx * Update Select.tsx * Update Switch.tsx * Update SecretFormField.tsx * Update index.ts * Update Select.tsx * Add deprecated Links * Update deprecation comments * Update deprecation message --- packages/grafana-ui/src/components/FormField/FormField.tsx | 4 ++++ .../grafana-ui/src/components/Forms/Legacy/Input/Input.tsx | 2 ++ .../components/Forms/Legacy/Select/IndicatorsContainer.tsx | 2 ++ .../src/components/Forms/Legacy/Select/NoOptionsMessage.tsx | 3 +++ .../grafana-ui/src/components/Forms/Legacy/Select/Select.tsx | 3 +++ .../grafana-ui/src/components/Forms/Legacy/Switch/Switch.tsx | 1 + .../src/components/SecretFormField/SecretFormField.tsx | 5 +++++ packages/grafana-ui/src/components/index.ts | 1 + 8 files changed, 21 insertions(+) diff --git a/packages/grafana-ui/src/components/FormField/FormField.tsx b/packages/grafana-ui/src/components/FormField/FormField.tsx index 0048f69a3c7..27e2e75ab23 100644 --- a/packages/grafana-ui/src/components/FormField/FormField.tsx +++ b/packages/grafana-ui/src/components/FormField/FormField.tsx @@ -2,6 +2,8 @@ import { css, cx } from '@emotion/css'; import React, { InputHTMLAttributes } from 'react'; import { InlineFormLabel } from '../FormLabel/FormLabel'; +import { Field } from '../Forms/Field'; +import { InlineField } from '../Forms/InlineField'; import { PopoverContent } from '../Tooltip'; export interface Props extends InputHTMLAttributes { @@ -23,6 +25,8 @@ const defaultProps = { /** * Default form field including label used in Grafana UI. Default input element is simple . You can also pass * custom inputEl if required in which case inputWidth and inputProps are ignored. + * @deprecated Please use the {@link Field} component, {@link https://developers.grafana.com/ui/latest/index.html?path=/story/forms-field--simple See Storybook}. + * For inline fields, use {@link InlineField}, {@link https://developers.grafana.com/ui/latest/index.html?path=/story/forms-inlinefield--basic See Storybook}. */ export const FormField = ({ label, diff --git a/packages/grafana-ui/src/components/Forms/Legacy/Input/Input.tsx b/packages/grafana-ui/src/components/Forms/Legacy/Input/Input.tsx index 76742d372c6..4561ae71c84 100644 --- a/packages/grafana-ui/src/components/Forms/Legacy/Input/Input.tsx +++ b/packages/grafana-ui/src/components/Forms/Legacy/Input/Input.tsx @@ -4,6 +4,7 @@ import React, { PureComponent, ChangeEvent } from 'react'; import { ValidationEvents, ValidationRule } from '../../../../types'; import { validate, EventsWithValidation, hasValidationEvent } from '../../../../utils'; +/** @deprecated Please use the `Input` component, which does not require this enum. */ export enum LegacyInputStatus { Invalid = 'invalid', Valid = 'valid', @@ -24,6 +25,7 @@ interface State { error: string | null; } +/** @deprecated Please use the `Input` component. {@link https://developers.grafana.com/ui/latest/index.html?path=/story/forms-input--simple See Storybook for example.} */ export class Input extends PureComponent { static defaultProps = { className: '', diff --git a/packages/grafana-ui/src/components/Forms/Legacy/Select/IndicatorsContainer.tsx b/packages/grafana-ui/src/components/Forms/Legacy/Select/IndicatorsContainer.tsx index e27792e4a00..0ec07bebd8d 100644 --- a/packages/grafana-ui/src/components/Forms/Legacy/Select/IndicatorsContainer.tsx +++ b/packages/grafana-ui/src/components/Forms/Legacy/Select/IndicatorsContainer.tsx @@ -2,7 +2,9 @@ import React from 'react'; import { components, IndicatorsContainerProps } from 'react-select'; import { Icon } from '../../../Icon/Icon'; +import { Select } from '../../../Select/Select'; +/** @deprecated Please use the {@link Select} component*/ export const IndicatorsContainer = (props: IndicatorsContainerProps) => { const isOpen = props.selectProps.menuIsOpen; return ( diff --git a/packages/grafana-ui/src/components/Forms/Legacy/Select/NoOptionsMessage.tsx b/packages/grafana-ui/src/components/Forms/Legacy/Select/NoOptionsMessage.tsx index f596a7f9c66..88693083998 100644 --- a/packages/grafana-ui/src/components/Forms/Legacy/Select/NoOptionsMessage.tsx +++ b/packages/grafana-ui/src/components/Forms/Legacy/Select/NoOptionsMessage.tsx @@ -3,8 +3,11 @@ import { components, NoticeProps, GroupBase } from 'react-select'; import { SelectableValue } from '@grafana/data'; +import { Select } from '../../../Select/Select'; + export type Props = NoticeProps, boolean, GroupBase>>; +/** @deprecated Please use the {@link Select} component*/ export const NoOptionsMessage = (props: Props) => { const { children } = props; return ( diff --git a/packages/grafana-ui/src/components/Forms/Legacy/Select/Select.tsx b/packages/grafana-ui/src/components/Forms/Legacy/Select/Select.tsx index 9258232c331..b8ee928a1ae 100644 --- a/packages/grafana-ui/src/components/Forms/Legacy/Select/Select.tsx +++ b/packages/grafana-ui/src/components/Forms/Legacy/Select/Select.tsx @@ -51,6 +51,8 @@ export const MenuList = (props: MenuListProps) => { ); }; + +/** @deprecated Please use the `Select` component, as seen {@link https://developers.grafana.com/ui/latest/index.html?path=/story/forms-select--basic in Storybook}. */ export class Select extends PureComponent> { declare context: React.ContextType; static contextType = ThemeContext; @@ -168,6 +170,7 @@ export class Select extends PureComponent> { } } +/** @deprecated Please use the `Select` component with async functionality, as seen {@link https://developers.grafana.com/ui/latest/index.html?path=/story/forms-select--basic-select-async in Storybook}. */ export class AsyncSelect extends PureComponent> { static contextType = ThemeContext; diff --git a/packages/grafana-ui/src/components/Forms/Legacy/Switch/Switch.tsx b/packages/grafana-ui/src/components/Forms/Legacy/Switch/Switch.tsx index 043a6b08891..e5d21915180 100644 --- a/packages/grafana-ui/src/components/Forms/Legacy/Switch/Switch.tsx +++ b/packages/grafana-ui/src/components/Forms/Legacy/Switch/Switch.tsx @@ -22,6 +22,7 @@ export interface State { id: string; } +/** @deprecated Please use the `Switch` component, {@link https://developers.grafana.com/ui/latest/index.html?path=/story/forms-switch--controlled as seen in Storybook} */ export class Switch extends PureComponent { state = { id: uniqueId(), diff --git a/packages/grafana-ui/src/components/SecretFormField/SecretFormField.tsx b/packages/grafana-ui/src/components/SecretFormField/SecretFormField.tsx index 6939a75cf98..94e4ee9808f 100644 --- a/packages/grafana-ui/src/components/SecretFormField/SecretFormField.tsx +++ b/packages/grafana-ui/src/components/SecretFormField/SecretFormField.tsx @@ -4,6 +4,8 @@ import React, { InputHTMLAttributes } from 'react'; import { Button } from '../Button/Button'; import { FormField } from '../FormField/FormField'; +import { Field } from '../Forms/Field'; +import { SecretInput } from '../SecretInput'; import { PopoverContent } from '../Tooltip'; export interface Props extends Omit, 'onReset'> { @@ -33,11 +35,14 @@ const getSecretFormFieldStyles = () => { }), }; }; + /** * Form field that has 2 states configured and not configured. If configured it will not show its contents and adds * a reset button that will clear the input and makes it accessible. In non configured state it behaves like normal * form field. This is used for passwords or anything that is encrypted on the server and is later returned encrypted * to the user (like datasource passwords). + * + * @deprecated Please use the {@link SecretInput} component with a {@link Field} instead, {@link https://developers.grafana.com/ui/latest/index.html?path=/story/forms-secretinput--basic as seen in Storybook} */ export const SecretFormField = ({ label = 'Password', diff --git a/packages/grafana-ui/src/components/index.ts b/packages/grafana-ui/src/components/index.ts index cee13eb356c..1a7f161adcf 100644 --- a/packages/grafana-ui/src/components/index.ts +++ b/packages/grafana-ui/src/components/index.ts @@ -265,6 +265,7 @@ export { type UserView } from './UsersIndicator/types'; export { InlineFormLabel } from './FormLabel/FormLabel'; export { Divider } from './Divider/Divider'; +/** @deprecated Please use non-legacy versions of these components */ const LegacyForms = { SecretFormField, FormField, From 2a2401e673a7828b937a66b03b928ca25ad52a00 Mon Sep 17 00:00:00 2001 From: Ivan Ortega Alba Date: Tue, 10 Oct 2023 16:23:08 +0200 Subject: [PATCH 010/102] useOpenAIStream: Improve error handling (#76212) * Simplify error message * Add timeout * Unsubscribe when unmount --------- Co-authored-by: Nathan Marrs --- .../components/GenAI/GenAIButton.test.tsx | 4 +- .../components/GenAI/GenAIButton.tsx | 8 ++- .../dashboard/components/GenAI/hooks.ts | 49 ++++++++++++++++--- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/public/app/features/dashboard/components/GenAI/GenAIButton.test.tsx b/public/app/features/dashboard/components/GenAI/GenAIButton.test.tsx index 44dc08ec314..dc94a9ae1e0 100644 --- a/public/app/features/dashboard/components/GenAI/GenAIButton.test.tsx +++ b/public/app/features/dashboard/components/GenAI/GenAIButton.test.tsx @@ -208,7 +208,9 @@ describe('GenAIButton', () => { // The tooltip keeps interactive to be able to click the link await userEvent.hover(tooltip); expect(tooltip).toBeVisible(); - expect(tooltip).toHaveTextContent('Something went wrong'); + expect(tooltip).toHaveTextContent( + 'Failed to generate content using OpenAI. Please try again or if the problem persist, contact your organization admin.' + ); }); it('should call the onClick callback', async () => { diff --git a/public/app/features/dashboard/components/GenAI/GenAIButton.tsx b/public/app/features/dashboard/components/GenAI/GenAIButton.tsx index 2a69bc7c0f9..12a76bf4bd5 100644 --- a/public/app/features/dashboard/components/GenAI/GenAIButton.tsx +++ b/public/app/features/dashboard/components/GenAI/GenAIButton.tsx @@ -173,7 +173,13 @@ export const GenAIButton = ({
    {isFirstHistoryEntry && } {!hasHistory && ( - + {button} )} diff --git a/public/app/features/dashboard/components/GenAI/hooks.ts b/public/app/features/dashboard/components/GenAI/hooks.ts index 02cc60b89e8..e3857500aef 100644 --- a/public/app/features/dashboard/components/GenAI/hooks.ts +++ b/public/app/features/dashboard/components/GenAI/hooks.ts @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useCallback, useEffect, useState } from 'react'; import { useAsync } from 'react-use'; import { Subscription } from 'rxjs'; @@ -18,6 +18,8 @@ export enum StreamStatus { COMPLETED = 'completed', } +export const TIMEOUT = 10000; + // TODO: Add tests export function useOpenAIStream( model = OPEN_AI_MODEL, @@ -46,6 +48,21 @@ export function useOpenAIStream( const [error, setError] = useState(); const { error: notifyError } = useAppNotification(); + const onError = useCallback( + (e: Error) => { + setStreamStatus(StreamStatus.IDLE); + setMessages([]); + setError(e); + notifyError( + 'Failed to generate content using OpenAI', + `Please try again or if the problem persists, contact your organization admin.` + ); + console.error(e); + logError(e, { messages: JSON.stringify(messages), model, temperature: String(temperature) }); + }, + [messages, model, temperature, notifyError] + ); + const { error: enabledError, value: enabled } = useAsync( async () => await isLLMPluginEnabled(), [isLLMPluginEnabled] @@ -80,13 +97,7 @@ export function useOpenAIStream( enabled, stream: stream.subscribe({ next: setReply, - error: (e: Error) => { - setStreamStatus(StreamStatus.IDLE); - setMessages([]); - setError(e); - notifyError('OpenAI Error', `${e.message}`); - logError(e, { messages: JSON.stringify(messages), model, temperature: String(temperature) }); - }, + error: onError, complete: () => { setStreamStatus(StreamStatus.COMPLETED); setTimeout(() => { @@ -99,6 +110,28 @@ export function useOpenAIStream( }; }, [messages, enabled]); + // Unsubscribe from the stream when the component unmounts. + useEffect(() => { + return () => { + if (value?.stream) { + value.stream.unsubscribe(); + } + }; + }, [value]); + + // If the stream is generating and we haven't received a reply, it times out. + useEffect(() => { + let timeout: NodeJS.Timeout | undefined; + if (streamStatus === StreamStatus.GENERATING && reply === '') { + timeout = setTimeout(() => { + onError(new Error(`OpenAI stream timed out after ${TIMEOUT}ms`)); + }, TIMEOUT); + } + return () => { + timeout && clearTimeout(timeout); + }; + }, [streamStatus, reply, onError]); + if (asyncError || enabledError) { setError(asyncError || enabledError); } From 9dd38de5c12ec95481cb789d3a9937f6a34c9005 Mon Sep 17 00:00:00 2001 From: Gabriel MABILLE Date: Tue, 10 Oct 2023 16:29:31 +0200 Subject: [PATCH 011/102] RBAC: Make fixed role UIDs deterministic (#76239) * Add fixed role UID Co-authored-by: Karl Persson * Use base64 url encoding --------- Co-authored-by: Karl Persson --- pkg/services/accesscontrol/models.go | 1 + pkg/services/accesscontrol/roles.go | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/pkg/services/accesscontrol/models.go b/pkg/services/accesscontrol/models.go index 87a95f89a89..f09ef0b4fec 100644 --- a/pkg/services/accesscontrol/models.go +++ b/pkg/services/accesscontrol/models.go @@ -321,6 +321,7 @@ func (cmd *SaveExternalServiceRoleCommand) Validate() error { const ( GlobalOrgID = 0 FixedRolePrefix = "fixed:" + FixedRoleUIDPrefix = "fixed_" ManagedRolePrefix = "managed:" BasicRolePrefix = "basic:" PluginRolePrefix = "plugins:" diff --git a/pkg/services/accesscontrol/roles.go b/pkg/services/accesscontrol/roles.go index 198b4583790..6f6a8822dde 100644 --- a/pkg/services/accesscontrol/roles.go +++ b/pkg/services/accesscontrol/roles.go @@ -1,6 +1,9 @@ package accesscontrol import ( + // #nosec G505 Used only for generating a 160 bit hash, it's not used for security purposes + "crypto/sha1" + "encoding/base64" "fmt" "strings" "sync" @@ -253,6 +256,15 @@ func ConcatPermissions(permissions ...[]Permission) []Permission { return perms } +// FixedRoleUID generates a UID of 34 bytes: "fixed_" + base64(sha1(roleName)) +func FixedRoleUID(roleName string) string { + // #nosec G505 Used only for generating a 160 bit hash, it's not used for security purposes + hasher := sha1.New() + hasher.Write([]byte(roleName)) + + return fmt.Sprintf("%s%s", FixedRoleUIDPrefix, base64.RawURLEncoding.EncodeToString(hasher.Sum(nil))) +} + // ValidateFixedRole errors when a fixed role does not match expected pattern func ValidateFixedRole(role RoleDTO) error { if !strings.HasPrefix(role.Name, FixedRolePrefix) { From ceb6f8b4092e42fc0e6465707dd3d84b6d94acca Mon Sep 17 00:00:00 2001 From: Karl Persson Date: Tue, 10 Oct 2023 16:30:20 +0200 Subject: [PATCH 012/102] Authn: error logs (#76264) * Reduce to debug for session need rotation error * try to extract log level from error and fallback to warning --- pkg/services/authn/authnimpl/service.go | 22 ++++++++++++++-------- pkg/services/authn/error.go | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/services/authn/authnimpl/service.go b/pkg/services/authn/authnimpl/service.go index 2a2e165eac9..4c50c97e9ff 100644 --- a/pkg/services/authn/authnimpl/service.go +++ b/pkg/services/authn/authnimpl/service.go @@ -226,17 +226,12 @@ func (s *Service) authenticate(ctx context.Context, c authn.Client, r *authn.Req r.OrgID = orgIDFromRequest(r) identity, err := c.Authenticate(ctx, r) if err != nil { - log := s.log.FromContext(ctx).Warn - if errors.Is(err, authn.ErrTokenNeedsRotation) { - log = s.log.FromContext(ctx).Debug - } - - log("Failed to authenticate request", "client", c.Name(), "error", err) + s.errorLogFunc(ctx, err)("Failed to authenticate request", "client", c.Name(), "error", err) return nil, err } if err := s.runPostAuthHooks(ctx, identity, r); err != nil { - s.log.FromContext(ctx).Warn("Failed to run post auth hook", "client", c.Name(), "id", identity.ID, "error", err) + s.errorLogFunc(ctx, err)("Failed to run post auth hook", "client", c.Name(), "id", identity.ID, "error", err) return nil, err } @@ -246,7 +241,7 @@ func (s *Service) authenticate(ctx context.Context, c authn.Client, r *authn.Req if hc, ok := c.(authn.HookClient); ok { if err := hc.Hook(ctx, identity, r); err != nil { - s.log.FromContext(ctx).Warn("Failed to run post client auth hook", "client", c.Name(), "id", identity.ID, "error", err) + s.errorLogFunc(ctx, err)("Failed to run post client auth hook", "client", c.Name(), "id", identity.ID, "error", err) return nil, err } } @@ -355,6 +350,17 @@ func (s *Service) SyncIdentity(ctx context.Context, identity *authn.Identity) er return s.runPostAuthHooks(ctx, identity, r) } +func (s *Service) errorLogFunc(ctx context.Context, err error) func(msg string, ctx ...any) { + l := s.log.FromContext(ctx) + + var grfErr errutil.Error + if errors.As(err, &grfErr) { + return grfErr.LogLevel.LogFunc(l) + } + + return l.Warn +} + func orgIDFromRequest(r *authn.Request) int64 { if r.HTTPRequest == nil { return 0 diff --git a/pkg/services/authn/error.go b/pkg/services/authn/error.go index 33fbdba1320..053ceeacfab 100644 --- a/pkg/services/authn/error.go +++ b/pkg/services/authn/error.go @@ -3,7 +3,7 @@ package authn import "github.com/grafana/grafana/pkg/util/errutil" var ( - ErrTokenNeedsRotation = errutil.Unauthorized("session.token.rotate") + ErrTokenNeedsRotation = errutil.Unauthorized("session.token.rotate", errutil.WithLogLevel(errutil.LevelDebug)) ErrUnsupportedClient = errutil.BadRequest("auth.client.unsupported") ErrClientNotConfigured = errutil.BadRequest("auth.client.notConfigured") ErrUnsupportedIdentity = errutil.NotImplemented("auth.identity.unsupported") From 810fbc3327841da6d21f945e75cad9daf040e625 Mon Sep 17 00:00:00 2001 From: Yuri Tseretyan Date: Tue, 10 Oct 2023 15:51:50 +0100 Subject: [PATCH 013/102] SSE: Support hysteresis threshold expression (#70998) * extend threshold command with second evaluator called `unloadEvaluator` * Introduce a new expression command Hysteresis and update Threshold unmarshaller to create the HysteresisCommand if the second eval * add feature flag `recoveryThreshold` * update unmarshal threshold command to not re-marshall because it breaks frame definition by shuffling the schema and data fields --- .../feature-toggles/index.md | 1 + .../src/types/featureToggles.gen.ts | 1 + pkg/expr/graph.go | 2 +- pkg/expr/hysteresis.go | 122 ++++++++++++ pkg/expr/hysteresis_test.go | 188 ++++++++++++++++++ pkg/expr/mathexp/types.go | 5 + pkg/expr/nodes.go | 4 +- pkg/expr/threshold.go | 95 +++++---- pkg/expr/threshold_test.go | 128 ++++++++---- pkg/services/featuremgmt/registry.go | 8 + pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + 12 files changed, 484 insertions(+), 75 deletions(-) create mode 100644 pkg/expr/hysteresis.go create mode 100644 pkg/expr/hysteresis_test.go diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 06727f9e537..8412aa950d1 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -148,6 +148,7 @@ Experimental features might be changed or removed without prior notice. | `transformationsVariableSupport` | Allows using variables in transformations | | `kubernetesPlaylists` | Use the kubernetes API in the frontend for playlists | | `navAdminSubsections` | Splits the administration section of the nav tree into subsections | +| `recoveryThreshold` | Enables feature recovery threshold (aka hysteresis) for threshold server-side expression | ## Development feature toggles diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 70e420bb05d..4acfe8c7aed 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -141,4 +141,5 @@ export interface FeatureToggles { transformationsVariableSupport?: boolean; kubernetesPlaylists?: boolean; navAdminSubsections?: boolean; + recoveryThreshold?: boolean; } diff --git a/pkg/expr/graph.go b/pkg/expr/graph.go index d45bae86b2a..86668d27579 100644 --- a/pkg/expr/graph.go +++ b/pkg/expr/graph.go @@ -225,7 +225,7 @@ func (s *Service) buildGraph(req *Request) (*simple.DirectedGraph, error) { case TypeDatasourceNode: node, err = s.buildDSNode(dp, rn, req) case TypeCMDNode: - node, err = buildCMDNode(dp, rn) + node, err = buildCMDNode(rn, s.features) case TypeMLNode: if s.features.IsEnabled(featuremgmt.FlagMlExpressions) { node, err = s.buildMLNode(dp, rn, req) diff --git a/pkg/expr/hysteresis.go b/pkg/expr/hysteresis.go new file mode 100644 index 00000000000..9526f835715 --- /dev/null +++ b/pkg/expr/hysteresis.go @@ -0,0 +1,122 @@ +package expr + +import ( + "context" + "fmt" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/data" + + "github.com/grafana/grafana/pkg/expr/mathexp" + "github.com/grafana/grafana/pkg/infra/tracing" +) + +type Fingerprints map[data.Fingerprint]struct{} + +// HysteresisCommand is a special case of ThresholdCommand that encapsulates two thresholds that are applied depending on the results of the previous evaluations: +// - first threshold - "loading", is used when the metric is determined as not loaded, i.e. it does not exist in the data provided by the reader. +// - second threshold - "unloading", is used when the metric is determined as loaded. +// To determine whether a metric is loaded, the command uses LoadedDimensions that is supposed to contain data.Fingerprint of +// the metrics that were loaded during the previous evaluation. +// The result of the execution of the command is the same as ThresholdCommand: 0 or 1 for each metric. +type HysteresisCommand struct { + RefID string + ReferenceVar string + LoadingThresholdFunc ThresholdCommand + UnloadingThresholdFunc ThresholdCommand + LoadedDimensions Fingerprints +} + +func (h *HysteresisCommand) NeedsVars() []string { + return []string{h.ReferenceVar} +} + +func (h *HysteresisCommand) Execute(ctx context.Context, now time.Time, vars mathexp.Vars, tracer tracing.Tracer) (mathexp.Results, error) { + results := vars[h.ReferenceVar] + + // shortcut for NoData + if results.IsNoData() { + return mathexp.Results{Values: mathexp.Values{mathexp.NewNoData()}}, nil + } + if h.LoadedDimensions == nil || len(h.LoadedDimensions) == 0 { + return h.LoadingThresholdFunc.Execute(ctx, now, vars, tracer) + } + var loadedVals, unloadedVals mathexp.Values + for _, value := range results.Values { + _, ok := h.LoadedDimensions[value.GetLabels().Fingerprint()] + if ok { + loadedVals = append(loadedVals, value) + } else { + unloadedVals = append(unloadedVals, value) + } + } + + if len(loadedVals) == 0 { // if all values are unloaded + return h.LoadingThresholdFunc.Execute(ctx, now, vars, tracer) + } + if len(unloadedVals) == 0 { // if all values are loaded + return h.UnloadingThresholdFunc.Execute(ctx, now, vars, tracer) + } + + defer func() { + // return back the old values + vars[h.ReferenceVar] = results + }() + + vars[h.ReferenceVar] = mathexp.Results{Values: unloadedVals} + loadingResults, err := h.LoadingThresholdFunc.Execute(ctx, now, vars, tracer) + if err != nil { + return mathexp.Results{}, fmt.Errorf("failed to execute loading threshold: %w", err) + } + vars[h.ReferenceVar] = mathexp.Results{Values: loadedVals} + unloadingResults, err := h.UnloadingThresholdFunc.Execute(ctx, now, vars, tracer) + if err != nil { + return mathexp.Results{}, fmt.Errorf("failed to execute unloading threshold: %w", err) + } + + return mathexp.Results{Values: append(loadingResults.Values, unloadingResults.Values...)}, nil +} + +func NewHysteresisCommand(refID string, referenceVar string, loadCondition ThresholdCommand, unloadCondition ThresholdCommand, l Fingerprints) (*HysteresisCommand, error) { + return &HysteresisCommand{ + RefID: refID, + LoadingThresholdFunc: loadCondition, + UnloadingThresholdFunc: unloadCondition, + ReferenceVar: referenceVar, + LoadedDimensions: l, + }, nil +} + +// FingerprintsFromFrame converts data.Frame to Fingerprints. +// The input data frame must have a single field of uint64 type. +// Returns error if the input data frame has invalid format +func FingerprintsFromFrame(frame *data.Frame) (Fingerprints, error) { + frameType, frameVersion := frame.TypeInfo("") + if frameType != "fingerprints" { + return nil, fmt.Errorf("invalid format of loaded dimensions frame: expected frame type 'fingerprints'") + } + if frameVersion.Greater(data.FrameTypeVersion{1, 0}) { + return nil, fmt.Errorf("invalid format of loaded dimensions frame: expected frame type 'fingerprints' of version 1.0 or lower") + } + if len(frame.Fields) != 1 { + return nil, fmt.Errorf("invalid format of loaded dimensions frame: expected a single field but got %d", len(frame.Fields)) + } + fld := frame.Fields[0] + if fld.Type() != data.FieldTypeUint64 { + return nil, fmt.Errorf("invalid format of loaded dimensions frame: the field type must be uint64 but got %s", fld.Type().String()) + } + result := make(Fingerprints, fld.Len()) + for i := 0; i < fld.Len(); i++ { + val, ok := fld.ConcreteAt(i) + if !ok { + continue + } + switch v := val.(type) { + case uint64: + result[data.Fingerprint(v)] = struct{}{} + default: + return nil, fmt.Errorf("cannot read the value at index [%d], expected uint64 but got '%T'", i, val) + } + } + return result, nil +} diff --git a/pkg/expr/hysteresis_test.go b/pkg/expr/hysteresis_test.go new file mode 100644 index 00000000000..957e045174c --- /dev/null +++ b/pkg/expr/hysteresis_test.go @@ -0,0 +1,188 @@ +package expr + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/expr/mathexp" + "github.com/grafana/grafana/pkg/infra/tracing" +) + +func TestHysteresisExecute(t *testing.T) { + number := func(label string, value float64) mathexp.Number { + n := mathexp.NewNumber("A", data.Labels{"label": label}) + n.SetValue(&value) + return n + } + fingerprint := func(label string) data.Fingerprint { + return data.Labels{"label": label}.Fingerprint() + } + + tracer := tracing.InitializeTracerForTest() + + var loadThreshold = 100.0 + var unloadThreshold = 30.0 + + testCases := []struct { + name string + loadedDimensions Fingerprints + input mathexp.Values + expected mathexp.Values + expectedError error + }{ + { + name: "return NoData when no data", + loadedDimensions: Fingerprints{0: struct{}{}}, + input: mathexp.Values{mathexp.NewNoData()}, + expected: mathexp.Values{mathexp.NewNoData()}, + }, + { + name: "use only loaded condition if no loaded metrics", + loadedDimensions: Fingerprints{}, + input: mathexp.Values{ + number("value1", loadThreshold+1), + number("value2", loadThreshold), + number("value3", loadThreshold-1), + number("value4", unloadThreshold+1), + number("value5", unloadThreshold), + number("value6", unloadThreshold-1), + }, + expected: mathexp.Values{ + number("value1", 1), + number("value2", 0), + number("value3", 0), + number("value4", 0), + number("value5", 0), + number("value6", 0), + }, + }, + { + name: "evaluate loaded metrics against unloaded threshold", + loadedDimensions: Fingerprints{ + fingerprint("value4"): {}, + fingerprint("value5"): {}, + fingerprint("value6"): {}, + }, + input: mathexp.Values{ + number("value1", loadThreshold+1), + number("value2", loadThreshold), + number("value3", loadThreshold-1), + number("value4", unloadThreshold+1), + number("value5", unloadThreshold), + number("value6", unloadThreshold-1), + }, + expected: mathexp.Values{ + number("value1", 1), + number("value2", 0), + number("value3", 0), + number("value4", 1), + number("value5", 0), + number("value6", 0), + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cmd := &HysteresisCommand{ + RefID: "B", + ReferenceVar: "A", + LoadingThresholdFunc: ThresholdCommand{ + ReferenceVar: "A", + RefID: "B", + ThresholdFunc: ThresholdIsAbove, + Conditions: []float64{loadThreshold}, + }, + UnloadingThresholdFunc: ThresholdCommand{ + ReferenceVar: "A", + RefID: "B", + ThresholdFunc: ThresholdIsAbove, + Conditions: []float64{unloadThreshold}, + }, + LoadedDimensions: tc.loadedDimensions, + } + + result, err := cmd.Execute(context.Background(), time.Now(), mathexp.Vars{ + "A": mathexp.Results{Values: tc.input}, + }, tracer) + if tc.expectedError != nil { + require.ErrorIs(t, err, tc.expectedError) + return + } + require.NoError(t, err) + require.EqualValues(t, result.Values, tc.expected) + }) + } +} + +func TestLoadedDimensionsFromFrame(t *testing.T) { + correctType := &data.FrameMeta{Type: "fingerprints", TypeVersion: data.FrameTypeVersion{1, 0}} + testCases := []struct { + name string + frame *data.Frame + expected Fingerprints + expectedError bool + }{ + { + name: "should fail if frame has wrong type", + frame: data.NewFrame("test").SetMeta(&data.FrameMeta{Type: "test"}), + expectedError: true, + }, + { + name: "should fail if frame has unsupported version", + frame: data.NewFrame("test").SetMeta(&data.FrameMeta{Type: "fingerprints", TypeVersion: data.FrameTypeVersion{1, 1}}), + expectedError: true, + }, + { + name: "should fail if frame has no fields", + frame: data.NewFrame("test").SetMeta(correctType), + expectedError: true, + }, + { + name: "should fail if frame has many fields", + frame: data.NewFrame("test", + data.NewField("fingerprints", nil, []uint64{}), + data.NewField("test", nil, []string{}), + ).SetMeta(correctType), + expectedError: true, + }, + { + name: "should fail if frame has field of a wrong type", + frame: data.NewFrame("test", + data.NewField("fingerprints", nil, []int64{}), + ).SetMeta(correctType), + expectedError: true, + }, + { + name: "should fail if frame has nullable uint64 field", + frame: data.NewFrame("test", + data.NewField("fingerprints", nil, []*uint64{}), + ).SetMeta(correctType), + expectedError: true, + }, + { + name: "should create LoadedMetrics", + frame: data.NewFrame("test", + data.NewField("fingerprints", nil, []uint64{1, 2, 3, 4, 5}), + ).SetMeta(correctType), + expected: Fingerprints{1: {}, 2: {}, 3: {}, 4: {}, 5: {}}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + result, err := FingerprintsFromFrame(testCase.frame) + if testCase.expectedError { + require.Error(t, err) + } else { + require.EqualValues(t, testCase.expected, result) + b, _ := json.Marshal(testCase.frame) + t.Log(string(b)) + } + }) + } +} diff --git a/pkg/expr/mathexp/types.go b/pkg/expr/mathexp/types.go index 38a975caed8..ca5069650b7 100644 --- a/pkg/expr/mathexp/types.go +++ b/pkg/expr/mathexp/types.go @@ -13,6 +13,11 @@ type Results struct { Error error } +// IsNoData checks whether the result contains NoData value +func (r Results) IsNoData() bool { + return len(r.Values) == 0 || len(r.Values) == 1 && r.Values[0].Type() == parse.TypeNoData +} + // Values is a slice of Value interfaces type Values []Value diff --git a/pkg/expr/nodes.go b/pkg/expr/nodes.go index 9991c8ea8d5..bb98c995f3b 100644 --- a/pkg/expr/nodes.go +++ b/pkg/expr/nodes.go @@ -96,7 +96,7 @@ func (gn *CMDNode) Execute(ctx context.Context, now time.Time, vars mathexp.Vars return gn.Command.Execute(ctx, now, vars, s.tracer) } -func buildCMDNode(dp *simple.DirectedGraph, rn *rawNode) (*CMDNode, error) { +func buildCMDNode(rn *rawNode, toggles featuremgmt.FeatureToggles) (*CMDNode, error) { commandType, err := rn.GetCommandType() if err != nil { return nil, fmt.Errorf("invalid command type in expression '%v': %w", rn.RefID, err) @@ -120,7 +120,7 @@ func buildCMDNode(dp *simple.DirectedGraph, rn *rawNode) (*CMDNode, error) { case TypeClassicConditions: node.Command, err = classic.UnmarshalConditionsCmd(rn.Query, rn.RefID) case TypeThreshold: - node.Command, err = UnmarshalThresholdCommand(rn) + node.Command, err = UnmarshalThresholdCommand(rn, toggles) default: return nil, fmt.Errorf("expression command type '%v' in expression '%v' not implemented", commandType, rn.RefID) } diff --git a/pkg/expr/threshold.go b/pkg/expr/threshold.go index 588cd65e30f..f49eea59b5f 100644 --- a/pkg/expr/threshold.go +++ b/pkg/expr/threshold.go @@ -7,8 +7,11 @@ import ( "strings" "time" + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/expr/mathexp" "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/grafana/grafana/pkg/services/featuremgmt" ) type ThresholdCommand struct { @@ -16,6 +19,7 @@ type ThresholdCommand struct { RefID string ThresholdFunc string Conditions []float64 + Invert bool } const ( @@ -39,6 +43,8 @@ func NewThresholdCommand(refID, referenceVar, thresholdFunc string, conditions [ if len(conditions) < 1 { return nil, fmt.Errorf("incorrect number of arguments: got %d but need 1", len(conditions)) } + default: + return nil, fmt.Errorf("expected threshold function to be one of [%s], got %s", strings.Join(supportedThresholdFuncs, ", "), thresholdFunc) } return &ThresholdCommand{ @@ -49,50 +55,48 @@ func NewThresholdCommand(refID, referenceVar, thresholdFunc string, conditions [ }, nil } -type ThresholdConditionJSON struct { - Evaluator ConditionEvalJSON `json:"evaluator"` -} - type ConditionEvalJSON struct { Params []float64 `json:"params"` Type string `json:"type"` // e.g. "gt" } // UnmarshalResampleCommand creates a ResampleCMD from Grafana's frontend query. -func UnmarshalThresholdCommand(rn *rawNode) (*ThresholdCommand, error) { - rawQuery := rn.Query - - rawExpression, ok := rawQuery["expression"] - if !ok { +func UnmarshalThresholdCommand(rn *rawNode, features featuremgmt.FeatureToggles) (Command, error) { + cmdConfig := ThresholdCommandConfig{} + if err := json.Unmarshal(rn.QueryRaw, &cmdConfig); err != nil { + return nil, fmt.Errorf("failed to parse the threshold command: %w", err) + } + if cmdConfig.Expression == "" { return nil, fmt.Errorf("no variable specified to reference for refId %v", rn.RefID) } - referenceVar, ok := rawExpression.(string) - if !ok { - return nil, fmt.Errorf("expected threshold variable to be a string, got %T for refId %v", rawExpression, rn.RefID) - } - - jsonFromM, err := json.Marshal(rawQuery["conditions"]) - if err != nil { - return nil, fmt.Errorf("failed to remarshal threshold expression body: %w", err) - } - var conditions []ThresholdConditionJSON - if err = json.Unmarshal(jsonFromM, &conditions); err != nil { - return nil, fmt.Errorf("failed to unmarshal remarshaled threshold expression body: %w", err) - } - - for _, condition := range conditions { - if !IsSupportedThresholdFunc(condition.Evaluator.Type) { - return nil, fmt.Errorf("expected threshold function to be one of %s, got %s", strings.Join(supportedThresholdFuncs, ", "), condition.Evaluator.Type) - } - } + referenceVar := cmdConfig.Expression // we only support one condition for now, we might want to turn this in to "OR" expressions later - if len(conditions) != 1 { + if len(cmdConfig.Conditions) != 1 { return nil, fmt.Errorf("threshold expression requires exactly one condition") } - firstCondition := conditions[0] + firstCondition := cmdConfig.Conditions[0] - return NewThresholdCommand(rn.RefID, referenceVar, firstCondition.Evaluator.Type, firstCondition.Evaluator.Params) + threshold, err := NewThresholdCommand(rn.RefID, referenceVar, firstCondition.Evaluator.Type, firstCondition.Evaluator.Params) + if err != nil { + return nil, fmt.Errorf("invalid condition: %w", err) + } + if firstCondition.UnloadEvaluator != nil && features.IsEnabled(featuremgmt.FlagRecoveryThreshold) { + unloading, err := NewThresholdCommand(rn.RefID, referenceVar, firstCondition.UnloadEvaluator.Type, firstCondition.UnloadEvaluator.Params) + unloading.Invert = true + if err != nil { + return nil, fmt.Errorf("invalid unloadCondition: %w", err) + } + var d Fingerprints + if firstCondition.LoadedDimensions != nil { + d, err = FingerprintsFromFrame(firstCondition.LoadedDimensions) + if err != nil { + return nil, fmt.Errorf("failed to parse loaded dimensions: %w", err) + } + } + return NewHysteresisCommand(rn.RefID, referenceVar, *threshold, *unloading, d) + } + return threshold, nil } // NeedsVars returns the variable names (refIds) that are dependencies @@ -102,7 +106,7 @@ func (tc *ThresholdCommand) NeedsVars() []string { } func (tc *ThresholdCommand) Execute(ctx context.Context, now time.Time, vars mathexp.Vars, tracer tracing.Tracer) (mathexp.Results, error) { - mathExpression, err := createMathExpression(tc.ReferenceVar, tc.ThresholdFunc, tc.Conditions) + mathExpression, err := createMathExpression(tc.ReferenceVar, tc.ThresholdFunc, tc.Conditions, tc.Invert) if err != nil { return mathexp.Results{}, err } @@ -116,19 +120,25 @@ func (tc *ThresholdCommand) Execute(ctx context.Context, now time.Time, vars mat } // createMathExpression converts all the info we have about a "threshold" expression in to a Math expression -func createMathExpression(referenceVar string, thresholdFunc string, args []float64) (string, error) { +func createMathExpression(referenceVar string, thresholdFunc string, args []float64, invert bool) (string, error) { + var exp string switch thresholdFunc { case ThresholdIsAbove: - return fmt.Sprintf("${%s} > %f", referenceVar, args[0]), nil + exp = fmt.Sprintf("${%s} > %f", referenceVar, args[0]) case ThresholdIsBelow: - return fmt.Sprintf("${%s} < %f", referenceVar, args[0]), nil + exp = fmt.Sprintf("${%s} < %f", referenceVar, args[0]) case ThresholdIsWithinRange: - return fmt.Sprintf("${%s} > %f && ${%s} < %f", referenceVar, args[0], referenceVar, args[1]), nil + exp = fmt.Sprintf("${%s} > %f && ${%s} < %f", referenceVar, args[0], referenceVar, args[1]) case ThresholdIsOutsideRange: - return fmt.Sprintf("${%s} < %f || ${%s} > %f", referenceVar, args[0], referenceVar, args[1]), nil + exp = fmt.Sprintf("${%s} < %f || ${%s} > %f", referenceVar, args[0], referenceVar, args[1]) default: return "", fmt.Errorf("failed to evaluate threshold expression: no such threshold function %s", thresholdFunc) } + + if invert { + return fmt.Sprintf("!(%s)", exp), nil + } + return exp, nil } func IsSupportedThresholdFunc(name string) bool { @@ -142,3 +152,14 @@ func IsSupportedThresholdFunc(name string) bool { return isSupported } + +type ThresholdCommandConfig struct { + Expression string `json:"expression"` + Conditions []ThresholdConditionJSON `json:"conditions"` +} + +type ThresholdConditionJSON struct { + Evaluator ConditionEvalJSON `json:"evaluator"` + UnloadEvaluator *ConditionEvalJSON `json:"unloadEvaluator"` + LoadedDimensions *data.Frame `json:"loadedDimensions"` +} diff --git a/pkg/expr/threshold_test.go b/pkg/expr/threshold_test.go index 6a99d60f5a2..5368ae33486 100644 --- a/pkg/expr/threshold_test.go +++ b/pkg/expr/threshold_test.go @@ -2,9 +2,13 @@ package expr import ( "encoding/json" + "fmt" + "sort" "testing" "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/services/featuremgmt" ) func TestNewThresholdCommand(t *testing.T) { @@ -82,6 +86,7 @@ func TestUnmarshalThresholdCommand(t *testing.T) { query string shouldError bool expectedError string + assert func(*testing.T, Command) } cases := []testCase{ @@ -97,7 +102,13 @@ func TestUnmarshalThresholdCommand(t *testing.T) { } }] }`, - shouldError: false, + assert: func(t *testing.T, command Command) { + require.IsType(t, &ThresholdCommand{}, command) + cmd := command.(*ThresholdCommand) + require.Equal(t, []string{"A"}, cmd.NeedsVars()) + require.Equal(t, "gt", cmd.ThresholdFunc) + require.Equal(t, []float64{20.0, 80.0}, cmd.Conditions) + }, }, { description: "unmarshal with missing conditions should error", @@ -107,17 +118,7 @@ func TestUnmarshalThresholdCommand(t *testing.T) { "conditions": [] }`, shouldError: true, - expectedError: "requires exactly one condition", - }, - { - description: "unmarshal with missing conditions should error", - query: `{ - "expression" : "A", - "type": "threshold", - "conditions": [] - }`, - shouldError: true, - expectedError: "requires exactly one condition", + expectedError: "threshold expression requires exactly one condition", }, { description: "unmarshal with unsupported threshold function", @@ -141,37 +142,86 @@ func TestUnmarshalThresholdCommand(t *testing.T) { "type": "threshold", "conditions": [] }`, - shouldError: true, - expectedError: "expected threshold variable to be a string", + shouldError: true, + }, + { + description: "unmarshal as hysteresis command if two evaluators", + query: `{ + "expression": "B", + "conditions": [ + { + "evaluator": { + "params": [ + 100 + ], + "type": "gt" + }, + "unloadEvaluator": { + "params": [ + 31 + ], + "type": "lt" + }, + "loadedDimensions": {"schema":{"name":"test","meta":{"type":"fingerprints","typeVersion":[1,0]},"fields":[{"name":"fingerprints","type":"number","typeInfo":{"frame":"uint64"}}]},"data":{"values":[[1,2,3,4,5]]}} + } + ] + }`, + assert: func(t *testing.T, c Command) { + require.IsType(t, &HysteresisCommand{}, c) + cmd := c.(*HysteresisCommand) + require.Equal(t, []string{"B"}, cmd.NeedsVars()) + require.Equal(t, []string{"B"}, cmd.LoadingThresholdFunc.NeedsVars()) + require.Equal(t, "gt", cmd.LoadingThresholdFunc.ThresholdFunc) + require.Equal(t, []float64{100.0}, cmd.LoadingThresholdFunc.Conditions) + require.Equal(t, []string{"B"}, cmd.UnloadingThresholdFunc.NeedsVars()) + require.Equal(t, "lt", cmd.UnloadingThresholdFunc.ThresholdFunc) + require.Equal(t, []float64{31.0}, cmd.UnloadingThresholdFunc.Conditions) + require.True(t, cmd.UnloadingThresholdFunc.Invert) + require.NotNil(t, cmd.LoadedDimensions) + actual := make([]uint64, 0, len(cmd.LoadedDimensions)) + for fingerprint := range cmd.LoadedDimensions { + actual = append(actual, uint64(fingerprint)) + } + sort.Slice(actual, func(i, j int) bool { + return actual[i] < actual[j] + }) + + require.EqualValues(t, []uint64{1, 2, 3, 4, 5}, actual) + }, }, } for _, tc := range cases { - q := []byte(tc.query) + t.Run(tc.description, func(t *testing.T) { + q := []byte(tc.query) + var qmap = make(map[string]any) + require.NoError(t, json.Unmarshal(q, &qmap)) - var qmap = make(map[string]any) - require.NoError(t, json.Unmarshal(q, &qmap)) + cmd, err := UnmarshalThresholdCommand(&rawNode{ + RefID: "", + Query: qmap, + QueryRaw: []byte(tc.query), + QueryType: "", + DataSource: nil, + }, featuremgmt.WithFeatures(featuremgmt.FlagRecoveryThreshold)) - cmd, err := UnmarshalThresholdCommand(&rawNode{ - RefID: "", - Query: qmap, - QueryType: "", - DataSource: nil, + if tc.shouldError { + require.Nil(t, cmd) + require.NotNil(t, err) + require.Contains(t, err.Error(), tc.expectedError) + } else { + require.Nil(t, err) + require.NotNil(t, cmd) + if tc.assert != nil { + tc.assert(t, cmd) + } + } }) - - if tc.shouldError { - require.Nil(t, cmd) - require.NotNil(t, err) - require.Contains(t, err.Error(), tc.expectedError) - } else { - require.Nil(t, err) - require.NotNil(t, cmd) - } } } func TestThresholdCommandVars(t *testing.T) { - cmd, err := NewThresholdCommand("B", "A", "is_above", []float64{}) + cmd, err := NewThresholdCommand("B", "A", "lt", []float64{1.0}) require.Nil(t, err) require.Equal(t, cmd.NeedsVars(), []string{"A"}) } @@ -219,17 +269,25 @@ func TestCreateMathExpression(t *testing.T) { for _, tc := range cases { t.Run(tc.description, func(t *testing.T) { - expr, err := createMathExpression(tc.ref, tc.function, tc.params) + expr, err := createMathExpression(tc.ref, tc.function, tc.params, false) require.Nil(t, err) require.NotNil(t, expr) - require.Equal(t, expr, tc.expected) + require.Equal(t, tc.expected, expr) + + t.Run("inverted", func(t *testing.T) { + expr, err := createMathExpression(tc.ref, tc.function, tc.params, true) + require.Nil(t, err) + require.NotNil(t, expr) + + require.Equal(t, fmt.Sprintf("!(%s)", tc.expected), expr) + }) }) } t.Run("should error if function is unsupported", func(t *testing.T) { - expr, err := createMathExpression("A", "foo", []float64{0}) + expr, err := createMathExpression("A", "foo", []float64{0}, false) require.Equal(t, expr, "") require.NotNil(t, err) require.Contains(t, err.Error(), "no such threshold function") diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index cde567f677d..c42da0ee7bc 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -859,5 +859,13 @@ var ( FrontendOnly: false, Owner: grafanaFrontendPlatformSquad, }, + { + Name: "recoveryThreshold", + Description: "Enables feature recovery threshold (aka hysteresis) for threshold server-side expression", + Stage: FeatureStageExperimental, + FrontendOnly: false, + Owner: grafanaAlertingSquad, + RequiresRestart: true, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 539bf40ec1f..0409ef6b370 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -122,3 +122,4 @@ enableNativeHTTPHistogram,experimental,@grafana/hosted-grafana-team,false,false, transformationsVariableSupport,experimental,@grafana/grafana-bi-squad,false,false,false,true kubernetesPlaylists,experimental,@grafana/grafana-app-platform-squad,false,false,false,true navAdminSubsections,experimental,@grafana/grafana-frontend-platform,false,false,false,false +recoveryThreshold,experimental,@grafana/alerting-squad,false,false,true,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 1c793fde7bc..95da20c5195 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -498,4 +498,8 @@ const ( // FlagNavAdminSubsections // Splits the administration section of the nav tree into subsections FlagNavAdminSubsections = "navAdminSubsections" + + // FlagRecoveryThreshold + // Enables feature recovery threshold (aka hysteresis) for threshold server-side expression + FlagRecoveryThreshold = "recoveryThreshold" ) From b75b747881f35271e387ae99372dad43af27c316 Mon Sep 17 00:00:00 2001 From: Domas Date: Tue, 10 Oct 2023 17:57:05 +0300 Subject: [PATCH 014/102] Tempo data source: Fix service map query error when a duration data frame has no data (#76269) * fix tempo service graph req when data frame is empty * also report error to Faro --- public/app/features/query/state/runRequest.ts | 6 +-- .../datasource/tempo/datasource.test.ts | 16 ++++++- .../plugins/datasource/tempo/datasource.ts | 47 ++++++++++--------- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/public/app/features/query/state/runRequest.ts b/public/app/features/query/state/runRequest.ts index cf389aff076..c9195b45475 100644 --- a/public/app/features/query/state/runRequest.ts +++ b/public/app/features/query/state/runRequest.ts @@ -18,7 +18,7 @@ import { PanelData, TimeRange, } from '@grafana/data'; -import { config, toDataQueryError } from '@grafana/runtime'; +import { config, toDataQueryError, logError } from '@grafana/runtime'; import { isExpressionReference } from '@grafana/runtime/src/utils/DataSourceWithBackend'; import { backendSrv } from 'app/core/services/backend_srv'; import { queryIsEmpty } from 'app/core/utils/query'; @@ -156,8 +156,8 @@ export function runRequest( }), // handle errors catchError((err) => { - const errLog = typeof err === 'string' ? err : JSON.stringify(err); - console.error('runRequest.catchError', errLog); + console.error('runRequest.catchError', err); + logError(err); return of({ ...state.panelData, state: LoadingState.Error, diff --git a/public/app/plugins/datasource/tempo/datasource.test.ts b/public/app/plugins/datasource/tempo/datasource.test.ts index 7a41119d45e..9091e9f61d3 100644 --- a/public/app/plugins/datasource/tempo/datasource.test.ts +++ b/public/app/plugins/datasource/tempo/datasource.test.ts @@ -987,7 +987,15 @@ const backendSrvWithPrometheus = { return { query() { return of({ - data: [rateMetric, errorRateMetric, durationMetric, totalsPromMetric, secondsPromMetric, failedPromMetric], + data: [ + rateMetric, + errorRateMetric, + durationMetric, + emptyDurationMetric, + totalsPromMetric, + secondsPromMetric, + failedPromMetric, + ], }); }, }; @@ -1079,6 +1087,12 @@ const durationMetric = createDataFrame({ ], }); +const emptyDurationMetric = createDataFrame({ + refId: + 'histogram_quantile(.9, sum(rate(traces_spanmetrics_latency_bucket{span_name=~"HTTP GET - root"}[$__range])) by (le))', + fields: [], +}); + const totalsPromMetric = createDataFrame({ refId: 'traces_service_graph_request_total', fields: [ diff --git a/public/app/plugins/datasource/tempo/datasource.ts b/public/app/plugins/datasource/tempo/datasource.ts index 47b6190227e..b9d40dbb218 100644 --- a/public/app/plugins/datasource/tempo/datasource.ts +++ b/public/app/plugins/datasource/tempo/datasource.ts @@ -1187,30 +1187,33 @@ function getServiceGraphView( }); } - if (duration.length > 0 && duration[0].fields?.length > 1) { + if (duration.length > 0) { let durationObj: any = {}; - duration.map((d) => { - const delimiter = d.refId?.includes('span_name=~"') ? 'span_name=~"' : 'span_name="'; - const name = d.refId?.split(delimiter)[1].split('"}')[0]; - durationObj[name] = { value: d.fields[1].values[0] }; - }); - - df.fields.push({ - ...duration[0].fields[1], - name: 'Duration (p90)', - values: getRateAlignedValues({ ...rate }, durationObj), - config: { - links: [ - makePromLink( - 'Duration', - buildLinkExpr(buildExpr(durationMetric, 'span_name="${__data.fields[0]}"', request)), - datasourceUid, - false - ), - ], - unit: 's', - }, + duration.forEach((d) => { + if (d.fields.length > 1) { + const delimiter = d.refId?.includes('span_name=~"') ? 'span_name=~"' : 'span_name="'; + const name = d.refId?.split(delimiter)[1].split('"}')[0]; + durationObj[name] = { value: d.fields[1].values[0] }; + } }); + if (Object.keys(durationObj).length > 0) { + df.fields.push({ + ...duration[0].fields[1], + name: 'Duration (p90)', + values: getRateAlignedValues({ ...rate }, durationObj), + config: { + links: [ + makePromLink( + 'Duration', + buildLinkExpr(buildExpr(durationMetric, 'span_name="${__data.fields[0]}"', request)), + datasourceUid, + false + ), + ], + unit: 's', + }, + }); + } } if (df.fields.length > 0 && df.fields[0].values) { From 186a9c2331579b5bd3225243ef656b2fd772b7bd Mon Sep 17 00:00:00 2001 From: Bilal Khan <64713734+ibilalkayy@users.noreply.github.com> Date: Tue, 10 Oct 2023 20:01:13 +0500 Subject: [PATCH 015/102] improved the sentence to make it clear for entering admin in both the username and password (#76141) * improved the sentence to make it clear for entering admin in both the username and password * Update docs/sources/getting-started/build-first-dashboard.md --------- Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com> --- docs/sources/getting-started/build-first-dashboard.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/getting-started/build-first-dashboard.md b/docs/sources/getting-started/build-first-dashboard.md index c88459425e7..47783916f7b 100644 --- a/docs/sources/getting-started/build-first-dashboard.md +++ b/docs/sources/getting-started/build-first-dashboard.md @@ -32,7 +32,7 @@ To sign in to Grafana for the first time: The default HTTP port that Grafana listens to is `3000` unless you have configured a different port. -1. On the sign-in page, enter `admin` for the username and password. +1. On the sign-in page, enter `admin` for both the username and password. 1. Click **Sign in**. If successful, you will see a prompt to change the password. From d25163a6458f30907baaf2f0cfe9e820318270dd Mon Sep 17 00:00:00 2001 From: guangwu Date: Tue, 10 Oct 2023 23:11:37 +0800 Subject: [PATCH 016/102] fix: Verification typo (#73137) * fix: Verification typo * fix: change to verification --- docs/sources/datasources/mysql/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/datasources/mysql/_index.md b/docs/sources/datasources/mysql/_index.md index ed016c1e70a..985653e5f9b 100644 --- a/docs/sources/datasources/mysql/_index.md +++ b/docs/sources/datasources/mysql/_index.md @@ -120,7 +120,7 @@ datasources: password: ${GRAFANA_MYSQL_PASSWORD} ``` -##### Using TLS Verificaiton +##### Using TLS verification ```yaml apiVersion: 1 From a9becedbbb2340f81afdf55117f1d40ee3d1fd67 Mon Sep 17 00:00:00 2001 From: Kyle Cunningham Date: Tue, 10 Oct 2023 10:31:03 -0500 Subject: [PATCH 017/102] Transformations: Add Numeric Index in Title (#76265) Add index to transformation title --- .../TransformationsEditor/TransformationOperationRow.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx b/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx index 3d9070acd4d..9a05c7ce589 100644 --- a/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx +++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx @@ -160,7 +160,7 @@ export const TransformationOperationRow = ({ Date: Tue, 10 Oct 2023 16:34:23 +0100 Subject: [PATCH 018/102] Update dependency @rushstack/rig-package to v0.5.1 (#76267) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b3a0b4c3615..799e2dcdf90 100644 --- a/package.json +++ b/package.json @@ -417,7 +417,7 @@ "resolutions": { "underscore": "1.13.6", "@types/slate": "0.47.11", - "@rushstack/rig-package": "0.4.0", + "@rushstack/rig-package": "0.5.1", "@rushstack/ts-command-line": "4.15.1", "ngtemplate-loader/loader-utils": "^2.0.0", "trim": "1.0.1", From 0d390382d3a7e3956059bdb994593d5fe92f9c7d Mon Sep 17 00:00:00 2001 From: Josh Hunt Date: Tue, 10 Oct 2023 16:00:33 +0000 Subject: [PATCH 019/102] I18n: Reset back to default plural seperators (#76284) * reset to standard plural seperators for crowdin compatibility * update docs --- contribute/internationalization.md | 4 +-- .../app/core/internationalization/index.tsx | 2 -- public/locales/de-DE/grafana.json | 24 ++++++------- public/locales/en-US/grafana.json | 24 ++++++------- public/locales/es-ES/grafana.json | 36 +++++++++---------- public/locales/fr-FR/grafana.json | 36 +++++++++---------- public/locales/i18next-parser.config.js | 2 -- public/locales/pseudo-LOCALE/grafana.json | 24 ++++++------- public/locales/zh-Hans/grafana.json | 12 +++---- 9 files changed, 80 insertions(+), 84 deletions(-) diff --git a/contribute/internationalization.md b/contribute/internationalization.md index 9c5503add03..726fb7b4b10 100644 --- a/contribute/internationalization.md +++ b/contribute/internationalization.md @@ -183,8 +183,8 @@ Once extracted with `yarn i18n:extract` you will need to manually edit the [Engl ```json { "inbox": { - "heading__one": "You got {{count}} message", - "heading__other": "You got {{count}} messages" + "heading_one": "You got {{count}} message", + "heading_other": "You got {{count}} messages" } } ``` diff --git a/public/app/core/internationalization/index.tsx b/public/app/core/internationalization/index.tsx index 6cf9aa881cc..05eca138eb7 100644 --- a/public/app/core/internationalization/index.tsx +++ b/public/app/core/internationalization/index.tsx @@ -35,8 +35,6 @@ export function initializeI18n(language: string) { // If translations are empty strings (no translation), fall back to the default value in source code returnEmptyString: false, - - pluralSeparator: '__', }; let init = i18n; if (language === 'detect') { diff --git a/public/locales/de-DE/grafana.json b/public/locales/de-DE/grafana.json index d4a27b44448..61e16dbea00 100644 --- a/public/locales/de-DE/grafana.json +++ b/public/locales/de-DE/grafana.json @@ -45,16 +45,16 @@ "new-folder-name-required-phrase": "" }, "counts": { - "alertRule__one": "", - "alertRule__other": "", - "dashboard__one": "", - "dashboard__other": "", - "folder__one": "", - "folder__other": "", - "libraryPanel__one": "", - "libraryPanel__other": "", - "total__one": "", - "total__other": "" + "alertRule_one": "", + "alertRule_other": "", + "dashboard_one": "", + "dashboard_other": "", + "folder_one": "", + "folder_other": "", + "libraryPanel_one": "", + "libraryPanel_other": "", + "total_one": "", + "total_other": "" }, "dashboards-tree": { "collapse-folder-button": "", @@ -429,8 +429,8 @@ }, "library-panels": { "modal": { - "body__one": "", - "body__other": "", + "body_one": "", + "body_other": "", "button-cancel": "", "button-view-panel1": "", "button-view-panel2": "", diff --git a/public/locales/en-US/grafana.json b/public/locales/en-US/grafana.json index 2680839a488..601a0ea8b87 100644 --- a/public/locales/en-US/grafana.json +++ b/public/locales/en-US/grafana.json @@ -45,16 +45,16 @@ "new-folder-name-required-phrase": "Folder name is required." }, "counts": { - "alertRule__one": "{{count}} alert rule", - "alertRule__other": "{{count}} alert rules", - "dashboard__one": "{{count}} dashboard", - "dashboard__other": "{{count}} dashboards", - "folder__one": "{{count}} folder", - "folder__other": "{{count}} folders", - "libraryPanel__one": "{{count}} library panel", - "libraryPanel__other": "{{count}} library panels", - "total__one": "{{count}} item", - "total__other": "{{count}} items" + "alertRule_one": "{{count}} alert rule", + "alertRule_other": "{{count}} alert rules", + "dashboard_one": "{{count}} dashboard", + "dashboard_other": "{{count}} dashboards", + "folder_one": "{{count}} folder", + "folder_other": "{{count}} folders", + "libraryPanel_one": "{{count}} library panel", + "libraryPanel_other": "{{count}} library panels", + "total_one": "{{count}} item", + "total_other": "{{count}} items" }, "dashboards-tree": { "collapse-folder-button": "Collapse folder {{title}}", @@ -429,8 +429,8 @@ }, "library-panels": { "modal": { - "body__one": "This panel is being used in {{count}} dashboard. Please choose which dashboard to view the panel in:", - "body__other": "This panel is being used in {{count}} dashboards. Please choose which dashboard to view the panel in:", + "body_one": "This panel is being used in {{count}} dashboard. Please choose which dashboard to view the panel in:", + "body_other": "This panel is being used in {{count}} dashboards. Please choose which dashboard to view the panel in:", "button-cancel": "<0>Cancel", "button-view-panel1": "View panel in {{label}}...", "button-view-panel2": "View panel in dashboard...", diff --git a/public/locales/es-ES/grafana.json b/public/locales/es-ES/grafana.json index efdcacb0dad..671d2059313 100644 --- a/public/locales/es-ES/grafana.json +++ b/public/locales/es-ES/grafana.json @@ -45,21 +45,21 @@ "new-folder-name-required-phrase": "" }, "counts": { - "alertRule__one": "", - "alertRule__many": "", - "alertRule__other": "", - "dashboard__one": "", - "dashboard__many": "", - "dashboard__other": "", - "folder__one": "", - "folder__many": "", - "folder__other": "", - "libraryPanel__one": "", - "libraryPanel__many": "", - "libraryPanel__other": "", - "total__one": "", - "total__many": "", - "total__other": "" + "alertRule_one": "", + "alertRule_many": "", + "alertRule_other": "", + "dashboard_one": "", + "dashboard_many": "", + "dashboard_other": "", + "folder_one": "", + "folder_many": "", + "folder_other": "", + "libraryPanel_one": "", + "libraryPanel_many": "", + "libraryPanel_other": "", + "total_one": "", + "total_many": "", + "total_other": "" }, "dashboards-tree": { "collapse-folder-button": "", @@ -434,9 +434,9 @@ }, "library-panels": { "modal": { - "body__one": "", - "body__many": "", - "body__other": "", + "body_one": "", + "body_many": "", + "body_other": "", "button-cancel": "", "button-view-panel1": "", "button-view-panel2": "", diff --git a/public/locales/fr-FR/grafana.json b/public/locales/fr-FR/grafana.json index dcdca2e1f2d..48997a11644 100644 --- a/public/locales/fr-FR/grafana.json +++ b/public/locales/fr-FR/grafana.json @@ -45,21 +45,21 @@ "new-folder-name-required-phrase": "" }, "counts": { - "alertRule__one": "", - "alertRule__many": "", - "alertRule__other": "", - "dashboard__one": "", - "dashboard__many": "", - "dashboard__other": "", - "folder__one": "", - "folder__many": "", - "folder__other": "", - "libraryPanel__one": "", - "libraryPanel__many": "", - "libraryPanel__other": "", - "total__one": "", - "total__many": "", - "total__other": "" + "alertRule_one": "", + "alertRule_many": "", + "alertRule_other": "", + "dashboard_one": "", + "dashboard_many": "", + "dashboard_other": "", + "folder_one": "", + "folder_many": "", + "folder_other": "", + "libraryPanel_one": "", + "libraryPanel_many": "", + "libraryPanel_other": "", + "total_one": "", + "total_many": "", + "total_other": "" }, "dashboards-tree": { "collapse-folder-button": "", @@ -434,9 +434,9 @@ }, "library-panels": { "modal": { - "body__one": "", - "body__many": "", - "body__other": "", + "body_one": "", + "body_many": "", + "body_other": "", "button-cancel": "", "button-view-panel1": "", "button-view-panel2": "", diff --git a/public/locales/i18next-parser.config.js b/public/locales/i18next-parser.config.js index 9c30a722ccc..0628220503a 100644 --- a/public/locales/i18next-parser.config.js +++ b/public/locales/i18next-parser.config.js @@ -6,8 +6,6 @@ module.exports = { output: './public/locales/$LOCALE/$NAMESPACE.json', - pluralSeparator: '__', - sort: true, createOldCatalogs: false, diff --git a/public/locales/pseudo-LOCALE/grafana.json b/public/locales/pseudo-LOCALE/grafana.json index d8e037af51e..5af63ced102 100644 --- a/public/locales/pseudo-LOCALE/grafana.json +++ b/public/locales/pseudo-LOCALE/grafana.json @@ -45,16 +45,16 @@ "new-folder-name-required-phrase": "Főľđęř ʼnämę įş řęqūįřęđ." }, "counts": { - "alertRule__one": "{{count}} äľęřŧ řūľę", - "alertRule__other": "{{count}} äľęřŧ řūľęş", - "dashboard__one": "{{count}} đäşĥþőäřđ", - "dashboard__other": "{{count}} đäşĥþőäřđş", - "folder__one": "{{count}} ƒőľđęř", - "folder__other": "{{count}} ƒőľđęřş", - "libraryPanel__one": "{{count}} ľįþřäřy päʼnęľ", - "libraryPanel__other": "{{count}} ľįþřäřy päʼnęľş", - "total__one": "{{count}} įŧęm", - "total__other": "{{count}} įŧęmş" + "alertRule_one": "{{count}} äľęřŧ řūľę", + "alertRule_other": "{{count}} äľęřŧ řūľęş", + "dashboard_one": "{{count}} đäşĥþőäřđ", + "dashboard_other": "{{count}} đäşĥþőäřđş", + "folder_one": "{{count}} ƒőľđęř", + "folder_other": "{{count}} ƒőľđęřş", + "libraryPanel_one": "{{count}} ľįþřäřy päʼnęľ", + "libraryPanel_other": "{{count}} ľįþřäřy päʼnęľş", + "total_one": "{{count}} įŧęm", + "total_other": "{{count}} įŧęmş" }, "dashboards-tree": { "collapse-folder-button": "Cőľľäpşę ƒőľđęř {{title}}", @@ -429,8 +429,8 @@ }, "library-panels": { "modal": { - "body__one": "Ŧĥįş päʼnęľ įş þęįʼnģ ūşęđ įʼn {{count}} đäşĥþőäřđ. Pľęäşę čĥőőşę ŵĥįčĥ đäşĥþőäřđ ŧő vįęŵ ŧĥę päʼnęľ įʼn:", - "body__other": "Ŧĥįş päʼnęľ įş þęįʼnģ ūşęđ įʼn {{count}} đäşĥþőäřđş. Pľęäşę čĥőőşę ŵĥįčĥ đäşĥþőäřđ ŧő vįęŵ ŧĥę päʼnęľ įʼn:", + "body_one": "Ŧĥįş päʼnęľ įş þęįʼnģ ūşęđ įʼn {{count}} đäşĥþőäřđ. Pľęäşę čĥőőşę ŵĥįčĥ đäşĥþőäřđ ŧő vįęŵ ŧĥę päʼnęľ įʼn:", + "body_other": "Ŧĥįş päʼnęľ įş þęįʼnģ ūşęđ įʼn {{count}} đäşĥþőäřđş. Pľęäşę čĥőőşę ŵĥįčĥ đäşĥþőäřđ ŧő vįęŵ ŧĥę päʼnęľ įʼn:", "button-cancel": "<0>Cäʼnčęľ", "button-view-panel1": "Vįęŵ päʼnęľ įʼn {{label}}...", "button-view-panel2": "Vįęŵ päʼnęľ įʼn đäşĥþőäřđ...", diff --git a/public/locales/zh-Hans/grafana.json b/public/locales/zh-Hans/grafana.json index 061da99ba8b..23bfbaee5cb 100644 --- a/public/locales/zh-Hans/grafana.json +++ b/public/locales/zh-Hans/grafana.json @@ -45,11 +45,11 @@ "new-folder-name-required-phrase": "" }, "counts": { - "alertRule__other": "", - "dashboard__other": "", - "folder__other": "", - "libraryPanel__other": "", - "total__other": "" + "alertRule_other": "", + "dashboard_other": "", + "folder_other": "", + "libraryPanel_other": "", + "total_other": "" }, "dashboards-tree": { "collapse-folder-button": "", @@ -424,7 +424,7 @@ }, "library-panels": { "modal": { - "body__other": "", + "body_other": "", "button-cancel": "", "button-view-panel1": "", "button-view-panel2": "", From cada1f040aeab4cadb6491aae2235b49f0bd81e6 Mon Sep 17 00:00:00 2001 From: Jo Date: Tue, 10 Oct 2023 18:07:23 +0200 Subject: [PATCH 020/102] Auth: Add support for role mapping and allowed groups in Google OIDC (#76266) * support google oauth allowed_groups. unify allowed groups logic * add role mapping for google oauth * add documentation * add addendums * remove extra isGroupMember * add to sample ini * Apply suggestions from code review Co-authored-by: Gabriel MABILLE --------- Co-authored-by: Gabriel MABILLE --- conf/defaults.ini | 10 +- conf/sample.ini | 6 +- .../configure-authentication/gitlab/index.md | 8 +- .../configure-authentication/google/index.md | 75 +++++++- pkg/login/social/azuread_oauth.go | 19 +- pkg/login/social/azuread_oauth_test.go | 10 +- pkg/login/social/gitlab_oauth.go | 17 -- pkg/login/social/google_oauth.go | 21 +++ pkg/login/social/google_oauth_test.go | 175 +++++++++++++++++- pkg/login/social/social.go | 24 ++- pkg/setting/setting.go | 4 +- 11 files changed, 297 insertions(+), 72 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index 1e3feb44292..6712b15c734 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -644,7 +644,11 @@ token_url = https://oauth2.googleapis.com/token api_url = https://openidconnect.googleapis.com/v1/userinfo allowed_domains = hosted_domain = -skip_org_role_sync = false +allowed_groups = +role_attribute_path = +role_attribute_strict = false +allow_assign_grafana_admin = false +skip_org_role_sync = true tls_skip_verify_insecure = false tls_client_cert = tls_client_key = @@ -828,7 +832,7 @@ assume_role_enabled = true list_metrics_page_limit = 500 # Experimental, for use in Grafana Cloud only. Please do not set. -external_id = +external_id = #################################### Azure ############################### [azure] @@ -1226,7 +1230,7 @@ url = # Tenant ID to use in requests to the Alertmanager. # It will also be used for the basic auth username. -tenant = +tenant = # Optional password for basic authentication. # If not present, the tenant ID will be set in the X-Scope-OrgID header. diff --git a/conf/sample.ini b/conf/sample.ini index e58ab27a662..6972722789c 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -626,6 +626,10 @@ ;api_url = https://openidconnect.googleapis.com/v1/userinfo ;allowed_domains = ;hosted_domain = +;allowed_groups = +;role_attribute_path = +;role_attribute_strict = false +;allow_assign_grafana_admin = false ;skip_org_role_sync = false ;use_pkce = true @@ -781,7 +785,7 @@ ; list_metrics_page_limit = 500 # Experimental, for use in Grafana Cloud only. Please do not set. -; external_id = +; external_id = #################################### Azure ############################### [azure] diff --git a/docs/sources/setup-grafana/configure-security/configure-authentication/gitlab/index.md b/docs/sources/setup-grafana/configure-security/configure-authentication/gitlab/index.md index 3c649fa07f9..3b0a69f3481 100644 --- a/docs/sources/setup-grafana/configure-security/configure-authentication/gitlab/index.md +++ b/docs/sources/setup-grafana/configure-security/configure-authentication/gitlab/index.md @@ -79,15 +79,15 @@ The table below describes all GitLab OAuth configuration options. Like any other | `api_url` | No | Grafana uses `/user` endpoint to obtain GitLab user information compatible with [OpenID UserInfo](https://connect2id.com/products/server/docs/api/userinfo). | `https://gitlab.com/api/v4` | | `name` | No | Name used to refer to the GitLab authentication in the Grafana user interface. | `GitLab` | | `icon` | No | Icon used for GitLab authentication in the Grafana user interface. | `gitlab` | -| `scopes` | No | List of comma- or space-separated GitLab OAuth scopes. | `openid email profile` | +| `scopes` | No | List of comma or space-separated GitLab OAuth scopes. | `openid email profile` | | `allow_sign_up` | No | Whether to allow new Grafana user creation through GitLab login. If set to `false`, then only existing Grafana users can log in with GitLab OAuth. | `true` | | `auto_login` | No | Set to `true` to enable users to bypass the login screen and automatically log in. This setting is ignored if you configure multiple auth providers to use auto-login. | `false` | | `role_attribute_path` | No | [JMESPath](http://jmespath.org/examples.html) expression to use for Grafana role lookup. Grafana will first evaluate the expression using the GitLab OAuth token. If no role is found, Grafana creates a JSON data with `groups` key that maps to groups obtained from GitLab's `/oauth/userinfo` endpoint, and evaluates the expression using this data. Finally, if a valid role is still not found, the expression is evaluated against the user information retrieved from `api_url/users` endpoint and groups retrieved from `api_url/groups` endpoint. The result of the evaluation should be a valid Grafana role (`Viewer`, `Editor`, `Admin` or `GrafanaAdmin`). For more information on user role mapping, refer to [Configure role mapping]({{< relref "#configure-role-mapping" >}}). | | | `role_attribute_strict` | No | Set to `true` to deny user login if the Grafana role cannot be extracted using `role_attribute_path`. For more information on user role mapping, refer to [Configure role mapping]({{< relref "#configure-role-mapping" >}}). | `false` | | `allow_assign_grafana_admin` | No | Set to `true` to enable automatic sync of the Grafana server administrator role. If this option is set to `true` and the result of evaluating `role_attribute_path` for a user is `GrafanaAdmin`, Grafana grants the user the server administrator privileges and organization administrator role. If this option is set to `false` and the result of evaluating `role_attribute_path` for a user is `GrafanaAdmin`, Grafana grants the user only organization administrator role. For more information on user role mapping, refer to [Configure role mapping]({{< relref "#configure-role-mapping" >}}). | `false` | | `skip_org_role_sync` | No | Set to `true` to stop automatically syncing user roles. | `false` | -| `allowed_domains` | No | List of comma- or space-separated domains. User must belong to at least one domain to log in. | | -| `allowed_groups` | No | List of comma- or space-separated groups. The user should be a member of at least one group to log in. | | +| `allowed_domains` | No | List of comma or space-separated domains. User must belong to at least one domain to log in. | | +| `allowed_groups` | No | List of comma or space-separated groups. The user should be a member of at least one group to log in. | | | `tls_skip_verify_insecure` | No | If set to `true`, the client accepts any certificate presented by the server and any host name in that certificate. _You should only use this for testing_, because this mode leaves SSL/TLS susceptible to man-in-the-middle attacks. | `false` | | `tls_client_cert` | No | The path to the certificate. | | | `tls_client_key` | No | The path to the key. | | @@ -115,7 +115,7 @@ Refresh token fetching and access token expiration check is enabled by default f To limit access to authenticated users that are members of one or more [GitLab groups](https://docs.gitlab.com/ce/user/group/index.html), set `allowed_groups` -to a comma- or space-separated list of groups. +to a comma or space-separated list of groups. GitLab's groups are referenced by the group name. For example, `developers`. To reference a subgroup `frontend`, use `developers/frontend`. Note that in GitLab, the group or subgroup name does not always match its display name, especially if the display name contains spaces or special characters. diff --git a/docs/sources/setup-grafana/configure-security/configure-authentication/google/index.md b/docs/sources/setup-grafana/configure-security/configure-authentication/google/index.md index 0a0e14234fa..2c0b586c523 100644 --- a/docs/sources/setup-grafana/configure-security/configure-authentication/google/index.md +++ b/docs/sources/setup-grafana/configure-security/configure-authentication/google/index.md @@ -100,16 +100,6 @@ This setting is ignored if multiple auth providers are configured to use auto lo auto_login = true ``` -## Skip organization role sync - -We do not currently sync roles from Google and instead set the AutoAssigned role to the user at first login. The default setting for `skip_org_role_sync` is `true`, which means that role modifications can still be made through the user interface. - -```ini -[auth.google] -# .. -skip_org_role_sync = true -``` - ### Configure team sync for Google OAuth > Available in Grafana v10.1.0 and later versions. @@ -132,3 +122,68 @@ With team sync, you can easily add users to teams by utilizing their Google grou The external group ID for a Google group is the group's email address, such as `dev@grafana.com`. To learn more about Team Sync, refer to [Configure Team Sync]({{< relref "../../configure-team-sync" >}}). + +### Configure allowed groups + +> Available in Grafana v10.2.0 and later versions. + +To limit access to authenticated users that are members of one or more groups, set `allowed_groups` +to a comma or space separated list of groups. + +Google groups are referenced by the group email key. For example, `developers@google.com`. + +> Note: Add the `https://www.googleapis.com/auth/cloud-identity.groups.readonly` scope to your Grafana `[auth.google]` scopes configuration to retrieve groups + +## Configure role mapping + +> Available in Grafana v10.2.0 and later versions. + +Unless `skip_org_role_sync` option is enabled, the user's role will be set to the role mapped from Google upon user login. If no mapping is set the default instance role is used. + +The user's role is retrieved using a [JMESPath](http://jmespath.org/examples.html) expression from the `role_attribute_path` configuration option. +To map the server administrator role, use the `allow_assign_grafana_admin` configuration option. + +If no valid role is found, the user is assigned the role specified by [the `auto_assign_org_role` option]({{< relref "../../../configure-grafana#auto_assign_org_role" >}}). +You can disable this default role assignment by setting `role_attribute_strict = true`. +This setting denies user access if no role or an invalid role is returned. + +To ease configuration of a proper JMESPath expression, go to [JMESPath](http://jmespath.org/) to test and evaluate expressions with custom payloads. + +> By default skip_org_role_sync is enabled. skip_org_role_sync will default to false in Grafana v10.3.0 and later versions. + +### Role mapping examples + +This section includes examples of JMESPath expressions used for role mapping. + +#### Map roles using user information from OAuth token + +In this example, the user with email `admin@company.com` has been granted the `Admin` role. +All other users are granted the `Viewer` role. + +```ini +role_attribute_path = email=='admin@company.com' && 'Admin' || 'Viewer' +skip_org_role_sync = false +``` + +#### Map roles using groups + +In this example, the user from Google group 'example-group@google.com' have been granted the `Editor` role. +All other users are granted the `Viewer` role. + +```ini +role_attribute_path = contains(groups[*], 'example-group@google.com') && 'Editor' || 'Viewer' +skip_org_role_sync = false +``` + +> Note: Add the `https://www.googleapis.com/auth/cloud-identity.groups.readonly` scope to your Grafana `[auth.google]` scopes configuration to retrieve groups + +#### Map server administrator role + +In this example, the user with email `admin@company.com` has been granted the `Admin` organization role as well as the Grafana server admin role. +All other users are granted the `Viewer` role. + +```ini +allow_assign_grafana_admin = true +skip_org_role_sync = false +role_attribute_path = email=='admin@company.com' && 'GrafanaAdmin' || 'Viewer' +``` diff --git a/pkg/login/social/azuread_oauth.go b/pkg/login/social/azuread_oauth.go index e7dc8b11b63..640552e9aae 100644 --- a/pkg/login/social/azuread_oauth.go +++ b/pkg/login/social/azuread_oauth.go @@ -23,7 +23,6 @@ type SocialAzureAD struct { *SocialBase cache remotecache.CacheStorage allowedOrganizations []string - allowedGroups []string forceUseGraphAPI bool skipOrgRoleSync bool } @@ -99,7 +98,7 @@ func (s *SocialAzureAD) UserInfo(ctx context.Context, client *http.Client, token return nil, fmt.Errorf("failed to extract groups: %w", err) } s.log.Debug("AzureAD OAuth: extracted groups", "email", email, "groups", fmt.Sprintf("%v", groups)) - if !s.IsGroupMember(groups) { + if !s.isGroupMember(groups) { return nil, errMissingGroupMembership } @@ -182,22 +181,6 @@ func (s *SocialAzureAD) validateIDTokenSignature(ctx context.Context, client *ht return nil, &Error{"AzureAD OAuth: signing key not found"} } -func (s *SocialAzureAD) IsGroupMember(groups []string) bool { - if len(s.allowedGroups) == 0 { - return true - } - - for _, allowedGroup := range s.allowedGroups { - for _, group := range groups { - if group == allowedGroup { - return true - } - } - } - - return false -} - func (claims *azureClaims) extractEmail() string { if claims.Email == "" { if claims.PreferredUsername != "" { diff --git a/pkg/login/social/azuread_oauth_test.go b/pkg/login/social/azuread_oauth_test.go index 5818293d200..d8819629c4b 100644 --- a/pkg/login/social/azuread_oauth_test.go +++ b/pkg/login/social/azuread_oauth_test.go @@ -530,7 +530,6 @@ func TestSocialAzureAD_UserInfo(t *testing.T) { t.Run(tt.name, func(t *testing.T) { s := &SocialAzureAD{ SocialBase: tt.fields.SocialBase, - allowedGroups: tt.fields.allowedGroups, allowedOrganizations: tt.fields.allowedOrganizations, forceUseGraphAPI: tt.fields.forceUseGraphAPI, cache: cache, @@ -540,6 +539,10 @@ func TestSocialAzureAD_UserInfo(t *testing.T) { s.SocialBase = newSocialBase("azuread", &oauth2.Config{ClientID: "client-id-example"}, &OAuthInfo{}, "", false, *featuremgmt.WithFeatures()) } + if tt.fields.allowedGroups != nil { + s.allowedGroups = tt.fields.allowedGroups + } + if tt.fields.usGovURL { s.SocialBase.Endpoint.AuthURL = usGovAuthURL } else { @@ -710,14 +713,15 @@ func TestSocialAzureAD_SkipOrgRole(t *testing.T) { t.Run(tt.name, func(t *testing.T) { s := &SocialAzureAD{ SocialBase: tt.fields.SocialBase, - allowedGroups: tt.fields.allowedGroups, forceUseGraphAPI: tt.fields.forceUseGraphAPI, skipOrgRoleSync: tt.fields.skipOrgRoleSync, cache: cache, } if tt.fields.SocialBase == nil { - s.SocialBase = newSocialBase("azuread", &oauth2.Config{ClientID: "client-id-example"}, &OAuthInfo{}, "", false, *featuremgmt.WithFeatures()) + s.SocialBase = newSocialBase("azuread", &oauth2.Config{ClientID: "client-id-example"}, &OAuthInfo{ + AllowedGroups: tt.fields.allowedGroups, + }, "", false, *featuremgmt.WithFeatures()) } s.SocialBase.Endpoint.AuthURL = authURL diff --git a/pkg/login/social/gitlab_oauth.go b/pkg/login/social/gitlab_oauth.go index 0bf22db646b..70d2eb23730 100644 --- a/pkg/login/social/gitlab_oauth.go +++ b/pkg/login/social/gitlab_oauth.go @@ -22,7 +22,6 @@ const ( type SocialGitlab struct { *SocialBase - allowedGroups []string apiUrl string skipOrgRoleSync bool } @@ -48,22 +47,6 @@ type userData struct { IsGrafanaAdmin *bool `json:"-"` } -func (s *SocialGitlab) isGroupMember(groups []string) bool { - if len(s.allowedGroups) == 0 { - return true - } - - for _, allowedGroup := range s.allowedGroups { - for _, group := range groups { - if group == allowedGroup { - return true - } - } - } - - return false -} - func (s *SocialGitlab) getGroups(ctx context.Context, client *http.Client) []string { groups := make([]string, 0) nextPage := new(int) diff --git a/pkg/login/social/google_oauth.go b/pkg/login/social/google_oauth.go index 21beaccf449..f4957eb4096 100644 --- a/pkg/login/social/google_oauth.go +++ b/pkg/login/social/google_oauth.go @@ -30,6 +30,7 @@ type googleUserData struct { Email string `json:"email"` Name string `json:"name"` EmailVerified bool `json:"email_verified"` + rawJSON []byte `json:"-"` } func (s *SocialGoogle) UserInfo(ctx context.Context, client *http.Client, token *oauth2.Token) (*BasicUserInfo, error) { @@ -59,6 +60,10 @@ func (s *SocialGoogle) UserInfo(ctx context.Context, client *http.Client, token s.log.Warn("Error retrieving groups", "error", errPage) } + if !s.isGroupMember(groups) { + return nil, errMissingGroupMembership + } + userInfo := &BasicUserInfo{ Id: data.ID, Name: data.Name, @@ -69,6 +74,19 @@ func (s *SocialGoogle) UserInfo(ctx context.Context, client *http.Client, token Groups: groups, } + if !s.skipOrgRoleSync { + role, grafanaAdmin, errRole := s.extractRoleAndAdmin(data.rawJSON, groups) + if errRole != nil { + return nil, errRole + } + + if s.allowAssignGrafanaAdmin { + userInfo.IsGrafanaAdmin = &grafanaAdmin + } + + userInfo.Role = role + } + s.log.Debug("Resolved user info", "data", fmt.Sprintf("%+v", userInfo)) return userInfo, nil @@ -98,6 +116,7 @@ func (s *SocialGoogle) extractFromAPI(ctx context.Context, client *http.Client) Name: data.Name, Email: data.Email, EmailVerified: data.EmailVerified, + rawJSON: response.Body, }, nil } @@ -145,6 +164,8 @@ func (s *SocialGoogle) extractFromToken(ctx context.Context, client *http.Client return nil, fmt.Errorf("Error getting user info: %s", err) } + data.rawJSON = rawJSON + return &data, nil } diff --git a/pkg/login/social/google_oauth_test.go b/pkg/login/social/google_oauth_test.go index 783125d0f45..96aa302d490 100644 --- a/pkg/login/social/google_oauth_test.go +++ b/pkg/login/social/google_oauth_test.go @@ -15,6 +15,7 @@ import ( "golang.org/x/oauth2" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models/roletype" ) func TestSocialGoogle_retrieveGroups(t *testing.T) { @@ -239,8 +240,13 @@ func TestSocialGoogle_UserInfo(t *testing.T) { tokenWithoutID := &oauth2.Token{} type fields struct { - Scopes []string - apiURL string + Scopes []string + apiURL string + allowedGroups []string + roleAttributePath string + roleAttributeStrict bool + allowAssignGrafanaAdmin bool + skipOrgRoleSync bool } type args struct { client *http.Client @@ -257,7 +263,8 @@ func TestSocialGoogle_UserInfo(t *testing.T) { { name: "Success id_token", fields: fields{ - Scopes: []string{}, + Scopes: []string{}, + skipOrgRoleSync: true, }, args: args{ token: tokenWithID, @@ -273,7 +280,8 @@ func TestSocialGoogle_UserInfo(t *testing.T) { { name: "Success id_token - groups requested", fields: fields{ - Scopes: []string{"https://www.googleapis.com/auth/cloud-identity.groups.readonly"}, + Scopes: []string{"https://www.googleapis.com/auth/cloud-identity.groups.readonly"}, + skipOrgRoleSync: true, }, args: args{ token: tokenWithID, @@ -310,7 +318,8 @@ func TestSocialGoogle_UserInfo(t *testing.T) { { name: "Legacy API URL", fields: fields{ - apiURL: legacyAPIURL, + apiURL: legacyAPIURL, + skipOrgRoleSync: true, }, args: args{ token: tokenWithoutID, @@ -340,7 +349,8 @@ func TestSocialGoogle_UserInfo(t *testing.T) { { name: "Legacy API URL - no id provided", fields: fields{ - apiURL: legacyAPIURL, + apiURL: legacyAPIURL, + skipOrgRoleSync: true, }, args: args{ token: tokenWithoutID, @@ -426,7 +436,8 @@ func TestSocialGoogle_UserInfo(t *testing.T) { { name: "Success", fields: fields{ - apiURL: "https://openidconnect.googleapis.com/v1/userinfo", + apiURL: "https://openidconnect.googleapis.com/v1/userinfo", + skipOrgRoleSync: true, }, args: args{ token: tokenWithoutID, @@ -478,6 +489,145 @@ func TestSocialGoogle_UserInfo(t *testing.T) { wantErr: true, wantErrMsg: "email is not verified", }, + { + name: "not in allowed Groups", + fields: fields{ + Scopes: []string{"https://www.googleapis.com/auth/cloud-identity.groups.readonly"}, + allowedGroups: []string{"not-that-one"}, + }, + args: args{ + token: tokenWithID, + client: &http.Client{ + Transport: &roundTripperFunc{ + fn: func(req *http.Request) (*http.Response, error) { + resp := httptest.NewRecorder() + _, _ = resp.WriteString(`{ + "memberships": [ + { + "group": "test-group", + "groupKey": { + "id": "test-group@google.com" + }, + "displayName": "Test Group" + } + ], + "nextPageToken": "" + }`) + return resp.Result(), nil + }, + }, + }, + }, + wantData: &BasicUserInfo{ + Id: "88888888888888", + Login: "test@example.com", + Email: "test@example.com", + Name: "Test User", + Groups: []string{"test-group@google.com"}, + }, + wantErr: true, + wantErrMsg: "user not a member of one of the required groups", + }, + { + name: "Role mapping - strict", + fields: fields{ + Scopes: []string{}, + allowedGroups: []string{}, + roleAttributePath: "this", + roleAttributeStrict: true, + }, + args: args{ + token: tokenWithID, + }, + wantData: &BasicUserInfo{ + Id: "88888888888888", + Login: "test@example.com", + Email: "test@example.com", + Name: "Test User", + Groups: []string{"test-group@google.com"}, + }, + wantErr: true, + wantErrMsg: "idP did not return a role attribute, but role_attribute_strict is set", + }, + { + name: "role mapping from id_token - no allowed assign Grafana Admin", + fields: fields{ + Scopes: []string{}, + allowAssignGrafanaAdmin: false, + roleAttributePath: "email_verified && 'GrafanaAdmin'", + }, + args: args{ + token: tokenWithID, + }, + wantData: &BasicUserInfo{ + Id: "88888888888888", + Login: "test@example.com", + Email: "test@example.com", + Name: "Test User", + Role: roletype.RoleAdmin, + IsGrafanaAdmin: nil, + }, + wantErr: false, + }, + { + name: "role mapping from id_token - allowed assign Grafana Admin", + fields: fields{ + Scopes: []string{}, + allowAssignGrafanaAdmin: true, + roleAttributePath: "email_verified && 'GrafanaAdmin'", + }, + args: args{ + token: tokenWithID, + }, + wantData: &BasicUserInfo{ + Id: "88888888888888", + Login: "test@example.com", + Email: "test@example.com", + Name: "Test User", + Role: roletype.RoleAdmin, + IsGrafanaAdmin: trueBoolPtr(), + }, + wantErr: false, + }, + { + name: "mapping from groups", + fields: fields{ + Scopes: []string{"https://www.googleapis.com/auth/cloud-identity.groups.readonly"}, + roleAttributePath: "contains(groups[*], 'test-group@google.com') && 'Editor'", + }, + args: args{ + token: tokenWithID, + client: &http.Client{ + Transport: &roundTripperFunc{ + fn: func(req *http.Request) (*http.Response, error) { + resp := httptest.NewRecorder() + _, _ = resp.WriteString(`{ + "memberships": [ + { + "group": "test-group", + "groupKey": { + "id": "test-group@google.com" + }, + "displayName": "Test Group" + } + ], + "nextPageToken": "" + }`) + return resp.Result(), nil + }, + }, + }, + }, + wantData: &BasicUserInfo{ + Id: "88888888888888", + Login: "test@example.com", + Email: "test@example.com", + Name: "Test User", + Role: "Editor", + Groups: []string{"test-group@google.com"}, + }, + wantErr: false, + }, } for _, tt := range tests { @@ -485,10 +635,15 @@ func TestSocialGoogle_UserInfo(t *testing.T) { s := &SocialGoogle{ apiUrl: tt.fields.apiURL, SocialBase: &SocialBase{ - Config: &oauth2.Config{Scopes: tt.fields.Scopes}, - log: log.NewNopLogger(), - allowSignup: false, + Config: &oauth2.Config{Scopes: tt.fields.Scopes}, + log: log.NewNopLogger(), + allowSignup: false, + allowedGroups: tt.fields.allowedGroups, + roleAttributePath: tt.fields.roleAttributePath, + roleAttributeStrict: tt.fields.roleAttributeStrict, + allowAssignGrafanaAdmin: tt.fields.allowAssignGrafanaAdmin, }, + skipOrgRoleSync: tt.fields.skipOrgRoleSync, } gotData, err := s.UserInfo(context.Background(), tt.args.client, tt.args.token) diff --git a/pkg/login/social/social.go b/pkg/login/social/social.go index 044e4f5a40a..2d81fd8a97c 100644 --- a/pkg/login/social/social.go +++ b/pkg/login/social/social.go @@ -63,6 +63,7 @@ type OAuthInfo struct { TlsClientKey string `toml:"tls_client_key"` TokenUrl string `toml:"token_url"` AllowedDomains []string `toml:"allowed_domains"` + AllowedGroups []string `toml:"allowed_groups"` Scopes []string `toml:"scopes"` AllowAssignGrafanaAdmin bool `toml:"allow_assign_grafana_admin"` AllowSignup bool `toml:"allow_signup"` @@ -120,6 +121,7 @@ func ProvideService(cfg *setting.Cfg, UseRefreshToken: sec.Key("use_refresh_token").MustBool(false), AllowAssignGrafanaAdmin: sec.Key("allow_assign_grafana_admin").MustBool(false), AutoLogin: sec.Key("auto_login").MustBool(false), + AllowedGroups: util.SplitString(sec.Key("allowed_groups").String()), } // when empty_scopes parameter exists and is true, overwrite scope with empty value @@ -178,7 +180,6 @@ func ProvideService(cfg *setting.Cfg, ss.socialMap["gitlab"] = &SocialGitlab{ SocialBase: newSocialBase(name, &config, info, cfg.AutoAssignOrgRole, cfg.OAuthSkipOrgRoleUpdateSync, *features), apiUrl: info.ApiUrl, - allowedGroups: util.SplitString(sec.Key("allowed_groups").String()), skipOrgRoleSync: cfg.GitLabSkipOrgRoleSync, } } @@ -202,7 +203,6 @@ func ProvideService(cfg *setting.Cfg, SocialBase: newSocialBase(name, &config, info, cfg.AutoAssignOrgRole, cfg.OAuthSkipOrgRoleUpdateSync, *features), cache: cache, allowedOrganizations: util.SplitString(sec.Key("allowed_organizations").String()), - allowedGroups: util.SplitString(sec.Key("allowed_groups").String()), forceUseGraphAPI: sec.Key("force_use_graph_api").MustBool(false), skipOrgRoleSync: cfg.AzureADSkipOrgRoleSync, } @@ -305,6 +305,7 @@ type SocialBase struct { allowSignup bool allowAssignGrafanaAdmin bool allowedDomains []string + allowedGroups []string roleAttributePath string roleAttributeStrict bool @@ -356,9 +357,10 @@ func newSocialBase(name string, allowSignup: info.AllowSignup, allowAssignGrafanaAdmin: info.AllowAssignGrafanaAdmin, allowedDomains: info.AllowedDomains, - autoAssignOrgRole: autoAssignOrgRole, + allowedGroups: info.AllowedGroups, roleAttributePath: info.RoleAttributePath, roleAttributeStrict: info.RoleAttributeStrict, + autoAssignOrgRole: autoAssignOrgRole, skipOrgRoleSync: skipOrgRoleSync, features: features, useRefreshToken: info.UseRefreshToken, @@ -571,6 +573,22 @@ func (ss *SocialService) getUsageStats(ctx context.Context) (map[string]interfac return m, nil } +func (s *SocialBase) isGroupMember(groups []string) bool { + if len(s.allowedGroups) == 0 { + return true + } + + for _, allowedGroup := range s.allowedGroups { + for _, group := range groups { + if group == allowedGroup { + return true + } + } + } + + return false +} + func (s *SocialBase) retrieveRawIDToken(idToken interface{}) ([]byte, error) { tokenString, ok := idToken.(string) if !ok { diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 42260e9eae7..5c86c56fc4d 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -1497,9 +1497,7 @@ func readAuthGithubSettings(cfg *Cfg) { func readAuthGoogleSettings(cfg *Cfg) { sec := cfg.SectionWithEnvOverrides("auth.google") cfg.GoogleAuthEnabled = sec.Key("enabled").MustBool(false) - // FIXME: for now we skip org role sync for google auth - // as we do not sync organization roles from Google - cfg.GoogleSkipOrgRoleSync = true + cfg.GoogleSkipOrgRoleSync = sec.Key("skip_org_role_sync").MustBool(true) } func readAuthGitlabSettings(cfg *Cfg) { From 2a6c16d9699152688029f73ad80b8fe974f55481 Mon Sep 17 00:00:00 2001 From: Andrej Ocenas Date: Tue, 10 Oct 2023 18:13:14 +0200 Subject: [PATCH 021/102] Pyroscope: Change phlare/api module to pyroscope/api module (#76271) --- go.mod | 5 ++-- go.sum | 11 +++++++ .../pyroscopeClient.go | 10 ++++--- .../pyroscopeClient_test.go | 29 ++++++++++++------- 4 files changed, 38 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 6132ce63b1c..451a33dbc83 100644 --- a/go.mod +++ b/go.mod @@ -201,7 +201,7 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common/sigv4 v0.1.0 // indirect github.com/prometheus/exporter-toolkit v0.10.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect + github.com/prometheus/procfs v0.11.0 // indirect github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b // indirect github.com/rs/cors v1.9.0 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect @@ -234,7 +234,7 @@ require ( github.com/armon/go-radix v1.0.0 // @grafana/grafana-app-platform-squad github.com/blugelabs/bluge v0.1.9 // @grafana/backend-platform github.com/blugelabs/bluge_segment_api v0.2.0 // @grafana/backend-platform - github.com/bufbuild/connect-go v1.4.1 // @grafana/observability-traces-and-profiling + github.com/bufbuild/connect-go v1.10.0 // @grafana/observability-traces-and-profiling github.com/dlmiddlecote/sqlstats v1.0.2 // @grafana/backend-platform github.com/drone/drone-cli v1.6.1 // @grafana/grafana-delivery github.com/getkin/kin-openapi v0.120.0 // @grafana/grafana-operator-experience-squad @@ -342,6 +342,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/grafana/pyroscope/api v0.2.0 // indirect github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db // indirect github.com/grafana/sqlds/v2 v2.3.10 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/go.sum b/go.sum index 43db7678d1c..1342b279715 100644 --- a/go.sum +++ b/go.sum @@ -843,6 +843,9 @@ github.com/bsm/ginkgo/v2 v2.5.0 h1:aOAnND1T40wEdAtkGSkvSICWeQ8L3UASX7YVCqQx+eQ= github.com/bsm/gomega v1.20.0 h1:JhAwLmtRzXFTx2AkALSLa8ijZafntmhSoU63Ok18Uq8= github.com/bufbuild/connect-go v1.4.1 h1:6usL3JGjKhxQpvDlizP7u8VfjAr1JkckcAUbrdcbgNY= github.com/bufbuild/connect-go v1.4.1/go.mod h1:9iNvh/NOsfhNBUH5CtvXeVUskQO1xsrEviH7ZArwZ3I= +github.com/bufbuild/connect-go v1.9.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8= +github.com/bufbuild/connect-go v1.10.0 h1:QAJ3G9A1OYQW2Jbk3DeoJbkCxuKArrvZgDt47mjdTbg= +github.com/bufbuild/connect-go v1.10.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/buildkite/yaml v2.1.0+incompatible h1:xirI+ql5GzfikVNDmt+yeiXpf/v1Gt03qXTtT5WXdr8= github.com/buildkite/yaml v2.1.0+incompatible/go.mod h1:UoU8vbcwu1+vjZq01+KrpSeLBgQQIjL/H7Y6KwikUrI= @@ -1298,6 +1301,7 @@ github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:9wScpmSP5A3Bk8V3XHWUcJmYTh+ZnlHVyc+A4oZYS3Y= github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:56xuuqnHyryaerycW3BfssRdxQstACi0Epw/yC5E2xM= github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= @@ -1649,6 +1653,7 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -1724,6 +1729,7 @@ github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10/go.mod h1:79YE0hCXdHa github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1818,6 +1824,8 @@ github.com/grafana/phlare/api v0.1.4-0.20230426005640-f90edba05413 h1:bBzCezZNRy github.com/grafana/phlare/api v0.1.4-0.20230426005640-f90edba05413/go.mod h1:IvwuGG9xa/h96UH/exgvsfy3zE+ZpctkNT9o5aaGdrU= github.com/grafana/prometheus-alertmanager v0.25.1-0.20230918083811-3513be6760b7 h1:7gsywzIb39SYZEp9qOnNaxD4d9OOkAfJGvnRUQUtlTM= github.com/grafana/prometheus-alertmanager v0.25.1-0.20230918083811-3513be6760b7/go.mod h1:0SOsbwZb277OdNdUR+376IxcAYV3Pp+igiAOJdr/98M= +github.com/grafana/pyroscope/api v0.2.0 h1:TzOxL0s6SiaLEy944ZAKgHcx/JDRJXu4O8ObwkqR6p4= +github.com/grafana/pyroscope/api v0.2.0/go.mod h1:nhH+xai9cYFgs6lMy/+L0pKj0d5yCMwji/QAiQFCP+U= github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db h1:7aN5cccjIqCLTzedH7MZzRZt5/lsAHch6Z3L2ZGn5FA= github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= @@ -2596,6 +2604,8 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= +github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/prometheus v0.43.0 h1:18iCSfrbAHbXvYFvR38U1Pt4uZmU9SmDcCpCrBKUiGg= github.com/prometheus/prometheus v0.43.0/go.mod h1:2BA14LgBeqlPuzObSEbh+Y+JwLH2GcqDlJKbF2sA6FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= @@ -2835,6 +2845,7 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/unknwon/bra v0.0.0-20200517080246-1e3013ecaff8 h1:aVGB3YnaS/JNfOW3tiHIlmNmTDg618va+eT0mVomgyI= github.com/unknwon/bra v0.0.0-20200517080246-1e3013ecaff8/go.mod h1:fVle4kNr08ydeohzYafr20oZzbAkhQT39gKK/pFQ5M4= github.com/unknwon/com v1.0.1 h1:3d1LTxD+Lnf3soQiD4Cp/0BRB+Rsa/+RTvz8GMMzIXs= diff --git a/pkg/tsdb/grafana-pyroscope-datasource/pyroscopeClient.go b/pkg/tsdb/grafana-pyroscope-datasource/pyroscopeClient.go index b63879f2c58..3653c455ae2 100644 --- a/pkg/tsdb/grafana-pyroscope-datasource/pyroscopeClient.go +++ b/pkg/tsdb/grafana-pyroscope-datasource/pyroscopeClient.go @@ -6,9 +6,11 @@ import ( "net/http" "strings" + typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" + "github.com/bufbuild/connect-go" - querierv1 "github.com/grafana/phlare/api/gen/proto/go/querier/v1" - "github.com/grafana/phlare/api/gen/proto/go/querier/v1/querierv1connect" + querierv1 "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1" + "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1/querierv1connect" ) type ProfileType struct { @@ -185,7 +187,7 @@ func getUnits(profileTypeID string) string { } func (c *PyroscopeClient) LabelNames(ctx context.Context) ([]string, error) { - resp, err := c.connectClient.LabelNames(ctx, connect.NewRequest(&querierv1.LabelNamesRequest{})) + resp, err := c.connectClient.LabelNames(ctx, connect.NewRequest(&typesv1.LabelNamesRequest{})) if err != nil { return nil, fmt.Errorf("error seding LabelNames request %v", err) } @@ -201,7 +203,7 @@ func (c *PyroscopeClient) LabelNames(ctx context.Context) ([]string, error) { } func (c *PyroscopeClient) LabelValues(ctx context.Context, label string) ([]string, error) { - resp, err := c.connectClient.LabelValues(ctx, connect.NewRequest(&querierv1.LabelValuesRequest{Name: label})) + resp, err := c.connectClient.LabelValues(ctx, connect.NewRequest(&typesv1.LabelValuesRequest{Name: label})) if err != nil { return nil, err } diff --git a/pkg/tsdb/grafana-pyroscope-datasource/pyroscopeClient_test.go b/pkg/tsdb/grafana-pyroscope-datasource/pyroscopeClient_test.go index 4b07ff9cfd2..28c39e2108c 100644 --- a/pkg/tsdb/grafana-pyroscope-datasource/pyroscopeClient_test.go +++ b/pkg/tsdb/grafana-pyroscope-datasource/pyroscopeClient_test.go @@ -5,9 +5,9 @@ import ( "testing" "github.com/bufbuild/connect-go" - googlev1 "github.com/grafana/phlare/api/gen/proto/go/google/v1" - querierv1 "github.com/grafana/phlare/api/gen/proto/go/querier/v1" - typesv1 "github.com/grafana/phlare/api/gen/proto/go/types/v1" + googlev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1" + querierv1 "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1" + typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" "github.com/stretchr/testify/require" ) @@ -68,18 +68,25 @@ type FakePyroscopeConnectClient struct { SendEmptyProfileResponse bool } +func (f *FakePyroscopeConnectClient) LabelValues(ctx context.Context, c *connect.Request[typesv1.LabelValuesRequest]) (*connect.Response[typesv1.LabelValuesResponse], error) { + //TODO implement me + panic("implement me") +} + +func (f *FakePyroscopeConnectClient) LabelNames(ctx context.Context, c *connect.Request[typesv1.LabelNamesRequest]) (*connect.Response[typesv1.LabelNamesResponse], error) { + //TODO implement me + panic("implement me") +} + +func (f *FakePyroscopeConnectClient) Diff(ctx context.Context, c *connect.Request[querierv1.DiffRequest]) (*connect.Response[querierv1.DiffResponse], error) { + //TODO implement me + panic("implement me") +} + func (f *FakePyroscopeConnectClient) ProfileTypes(ctx context.Context, c *connect.Request[querierv1.ProfileTypesRequest]) (*connect.Response[querierv1.ProfileTypesResponse], error) { panic("implement me") } -func (f *FakePyroscopeConnectClient) LabelValues(ctx context.Context, c *connect.Request[querierv1.LabelValuesRequest]) (*connect.Response[querierv1.LabelValuesResponse], error) { - panic("implement me") -} - -func (f *FakePyroscopeConnectClient) LabelNames(context.Context, *connect.Request[querierv1.LabelNamesRequest]) (*connect.Response[querierv1.LabelNamesResponse], error) { - panic("implement me") -} - func (f *FakePyroscopeConnectClient) Series(ctx context.Context, c *connect.Request[querierv1.SeriesRequest]) (*connect.Response[querierv1.SeriesResponse], error) { panic("implement me") } From 56043052714c216a1617cb1a19e0b94714e34bb7 Mon Sep 17 00:00:00 2001 From: Drew Slobodnjak <60050885+drew08t@users.noreply.github.com> Date: Tue, 10 Oct 2023 11:14:56 -0700 Subject: [PATCH 022/102] BarChart: Gradient Mode for Auto Max (#74122) * BarChart: Gradient Mode for Auto Max * add gdev panel * clean --------- Co-authored-by: Leon Sorokin --- .../barchart-thresholds-mappings.json | 93 ++++++++++++++++++- public/app/plugins/panel/barchart/utils.ts | 4 +- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/devenv/dev-dashboards/panel-barchart/barchart-thresholds-mappings.json b/devenv/dev-dashboards/panel-barchart/barchart-thresholds-mappings.json index 4f29f8950e0..6aaa42dddc6 100644 --- a/devenv/dev-dashboards/panel-barchart/barchart-thresholds-mappings.json +++ b/devenv/dev-dashboards/panel-barchart/barchart-thresholds-mappings.json @@ -1121,6 +1121,93 @@ "title": "Hue", "type": "barchart" }, + { + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 100, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 5, + "x": 10, + "y": 18 + }, + "id": 15, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "csvContent": "label,value\nx,3\na,10\nb,20\nc,30", + "datasource": { + "type": "testdata", + "uid": "PD8C576611E62080A" + }, + "refId": "A", + "scenarioId": "csv_content" + } + ], + "title": "Green-Yellow-Red", + "type": "barchart" + }, { "datasource": { "type": "testdata", @@ -1239,7 +1326,7 @@ "type": "barchart" } ], - "schemaVersion": 37, + "schemaVersion": 38, "tags": [ "gdev", "panel-tests", @@ -1257,6 +1344,6 @@ "timezone": "", "title": "BarChart - Thresholds & Mappings", "uid": "2I2uMSB7z", - "version": 32, + "version": 7, "weekStart": "" -} +} \ No newline at end of file diff --git a/public/app/plugins/panel/barchart/utils.ts b/public/app/plugins/panel/barchart/utils.ts index a144caa256a..9783cad0849 100644 --- a/public/app/plugins/panel/barchart/utils.ts +++ b/public/app/plugins/panel/barchart/utils.ts @@ -236,8 +236,8 @@ export const preparePlotConfigBuilder: UPlotConfigPrepFn = ({ thresholds: field.config.thresholds, hardMin: field.config.min, hardMax: field.config.max, - softMin, - softMax, + softMin: customConfig.axisSoftMin, + softMax: customConfig.axisSoftMax, // The following properties are not used in the uPlot config, but are utilized as transport for legend config // PlotLegend currently gets unfiltered DataFrame[], so index must be into that field array, not the prepped frame's which we're iterating here From 3d84956215bd0cd0d6331fed09e91795a2802de3 Mon Sep 17 00:00:00 2001 From: deresolution20 <85902399+deresolution20@users.noreply.github.com> Date: Tue, 10 Oct 2023 13:12:58 -0600 Subject: [PATCH 023/102] Docs: update raw format for data source variable (#75488) * Update index.md Please see this PR for the breaking change to raw. https://github.com/grafana/grafana/pull/69259 * Update docs/sources/dashboards/variables/variable-syntax/index.md Co-authored-by: Isabel <76437239+imatwawana@users.noreply.github.com> * Fixed linting error --------- Co-authored-by: Isabel <76437239+imatwawana@users.noreply.github.com> --- .../sources/dashboards/variables/variable-syntax/index.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/sources/dashboards/variables/variable-syntax/index.md b/docs/sources/dashboards/variables/variable-syntax/index.md index 9ce3feb9c40..2559d87f1c2 100644 --- a/docs/sources/dashboards/variables/variable-syntax/index.md +++ b/docs/sources/dashboards/variables/variable-syntax/index.md @@ -135,12 +135,12 @@ Interpolation result: 'test1.|test2' ### Raw -Turns off data source-specific formatting, such as single quotes in an SQL query. +The raw format for a data source variable returns the UID (unique identifier) of the data source, rather than its name. ```bash -servers = ['test.1', 'test2'] -String to interpolate: '${var_name:raw}' -Interpolation result: 'test.1,test2' +datasourceVariable = 'd7bbe725-9e48-4af8-a0cb-6cb255d873a3' +String to interpolate: '${datasourceVariable:raw}' +Interpolation result: 'd7bbe725-9e48-4af8-a0cb-6cb255d873a3' ``` ### Regex From c26e3d80e35352ab4c7f5a0a4da6c1304ecac421 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 10 Oct 2023 12:46:12 -0700 Subject: [PATCH 024/102] Playlist: Add create+update timestamps to the database (#76295) --- .github/CODEOWNERS | 2 +- pkg/services/playlist/model.go | 13 +- pkg/services/playlist/model_test.go | 64 ------ .../playlist/playlistimpl/playlist.go | 13 +- .../playlist/playlistimpl/sqlx_store.go | 201 ------------------ .../playlist/playlistimpl/sqlx_store_test.go | 16 -- .../playlist/playlistimpl/store_test.go | 39 ++++ .../playlist/playlistimpl/xorm_store.go | 53 ++--- .../sqlstore/migrations/playlist_mig.go | 8 + 9 files changed, 81 insertions(+), 328 deletions(-) delete mode 100644 pkg/services/playlist/model_test.go delete mode 100644 pkg/services/playlist/playlistimpl/sqlx_store.go delete mode 100644 pkg/services/playlist/playlistimpl/sqlx_store_test.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 79de8c6fff9..934e37fdc62 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -114,7 +114,7 @@ /pkg/services/navtree/ @grafana/backend-platform /pkg/services/notifications/ @grafana/backend-platform /pkg/services/org/ @grafana/backend-platform -/pkg/services/playlist/ @grafana/backend-platform +/pkg/services/playlist/ @grafana/grafana-app-platform-squad /pkg/services/plugindashboards/ @grafana/backend-platform /pkg/services/preference/ @grafana/backend-platform /pkg/services/provisioning/ @grafana/backend-platform diff --git a/pkg/services/playlist/model.go b/pkg/services/playlist/model.go index 8c1b564793d..f0a0b4b4ae7 100644 --- a/pkg/services/playlist/model.go +++ b/pkg/services/playlist/model.go @@ -8,9 +8,8 @@ import ( // Typed errors var ( - ErrPlaylistNotFound = errors.New("Playlist not found") - ErrPlaylistFailedGenerateUniqueUid = errors.New("failed to generate unique playlist UID") - ErrCommandValidationFailed = errors.New("command missing required fields") + ErrPlaylistNotFound = errors.New("Playlist not found") + ErrCommandValidationFailed = errors.New("command missing required fields") ) // Playlist model @@ -20,6 +19,12 @@ type Playlist struct { Name string `json:"name" db:"name"` Interval string `json:"interval" db:"interval"` OrgId int64 `json:"-" db:"org_id"` + + // Added for kubernetes migration + synchronization + // Hidden from json because this is used for openapi generation + // Using int64 rather than time.Time to avoid database issues with time support + CreatedAt int64 `json:"-" db:"created_at"` + UpdatedAt int64 `json:"-" db:"updated_at"` } type PlaylistDTO = playlist.Spec @@ -54,6 +59,8 @@ type CreatePlaylistCommand struct { Interval string `json:"interval"` Items []PlaylistItem `json:"items"` OrgId int64 `json:"-"` + // Used to create playlists from kubectl with a known uid/name + UID string `json:"-"` } type DeletePlaylistCommand struct { diff --git a/pkg/services/playlist/model_test.go b/pkg/services/playlist/model_test.go deleted file mode 100644 index f0eb722e10f..00000000000 --- a/pkg/services/playlist/model_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package playlist - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/grafana/grafana/pkg/kinds/playlist" - "github.com/grafana/grafana/pkg/util" - "github.com/stretchr/testify/require" -) - -func TestPlaylistConversion(t *testing.T) { - src := PlaylistDTO{ - Uid: "abc", - Name: "TeamA", - Interval: "10s", - Items: []playlist.Item{ - {Title: util.Pointer("First"), Type: playlist.ItemTypeDashboardByUid, Value: "UID0"}, - {Title: util.Pointer("Second"), Type: playlist.ItemTypeDashboardByTag, Value: "tagA"}, - {Title: util.Pointer("Third"), Type: playlist.ItemTypeDashboardById, Value: "123"}, - }, - } - - dst := PlaylistToResource(src) - - require.Equal(t, "abc", src.Uid) - require.Equal(t, "abc", dst.Metadata.Name) - require.Equal(t, src.Name, dst.Spec.Name) - - out, err := json.MarshalIndent(dst, "", " ") - require.NoError(t, err) - fmt.Printf("%s", string(out)) - require.JSONEq(t, `{ - "apiVersion": "v0-0-alpha", - "kind": "Playlist", - "metadata": { - "name": "abc", - "creationTimestamp": null - }, - "spec": { - "interval": "10s", - "items": [ - { - "title": "First", - "type": "dashboard_by_uid", - "value": "UID0" - }, - { - "title": "Second", - "type": "dashboard_by_tag", - "value": "tagA" - }, - { - "title": "Third", - "type": "dashboard_by_id", - "value": "123" - } - ], - "name": "TeamA", - "uid": "" - } - }`, string(out)) -} diff --git a/pkg/services/playlist/playlistimpl/playlist.go b/pkg/services/playlist/playlistimpl/playlist.go index 054b96a7257..3111d701e58 100644 --- a/pkg/services/playlist/playlistimpl/playlist.go +++ b/pkg/services/playlist/playlistimpl/playlist.go @@ -16,17 +16,8 @@ type Service struct { var _ playlist.Service = &Service{} func ProvideService(db db.DB, toggles featuremgmt.FeatureToggles, objserver entity.EntityStoreServer) playlist.Service { - var sqlstore store - - // 🐢🐢🐢 pick the store - if toggles.IsEnabled(featuremgmt.FlagNewDBLibrary) { - sqlstore = &sqlxStore{ - sess: db.GetSqlxSession(), - } - } else { - sqlstore = &sqlStore{ - db: db, - } + sqlstore := &sqlStore{ + db: db, } return &Service{store: sqlstore} } diff --git a/pkg/services/playlist/playlistimpl/sqlx_store.go b/pkg/services/playlist/playlistimpl/sqlx_store.go deleted file mode 100644 index a6321ae44a0..00000000000 --- a/pkg/services/playlist/playlistimpl/sqlx_store.go +++ /dev/null @@ -1,201 +0,0 @@ -package playlistimpl - -import ( - "context" - "database/sql" - "errors" - - "github.com/grafana/grafana/pkg/services/playlist" - "github.com/grafana/grafana/pkg/services/sqlstore/session" - "github.com/grafana/grafana/pkg/services/star" -) - -type sqlxStore struct { - sess *session.SessionDB -} - -func (s *sqlxStore) Insert(ctx context.Context, cmd *playlist.CreatePlaylistCommand) (*playlist.Playlist, error) { - p := playlist.Playlist{} - var err error - uid, err := newGenerateAndValidateNewPlaylistUid(ctx, s.sess, cmd.OrgId) - if err != nil { - return nil, err - } - - p = playlist.Playlist{ - Name: cmd.Name, - Interval: cmd.Interval, - OrgId: cmd.OrgId, - UID: uid, - } - - err = s.sess.WithTransaction(ctx, func(tx *session.SessionTx) error { - query := `INSERT INTO playlist (name, "interval", org_id, uid) VALUES (?, ?, ?, ?)` - var err error - p.Id, err = tx.ExecWithReturningId(ctx, query, p.Name, p.Interval, p.OrgId, p.UID) - if err != nil { - return err - } - - if len(cmd.Items) > 0 { - playlistItems := make([]playlist.PlaylistItem, 0) - for order, item := range cmd.Items { - playlistItems = append(playlistItems, playlist.PlaylistItem{ - PlaylistId: p.Id, - Type: item.Type, - Value: item.Value, - Order: order + 1, - Title: item.Title, - }) - } - query := `INSERT INTO playlist_item (playlist_id, type, value, title, "order") VALUES (:playlist_id, :type, :value, :title, :order)` - _, err = tx.NamedExec(ctx, query, playlistItems) - if err != nil { - return err - } - } - return nil - }) - - return &p, err -} - -func (s *sqlxStore) Update(ctx context.Context, cmd *playlist.UpdatePlaylistCommand) (*playlist.PlaylistDTO, error) { - dto := playlist.PlaylistDTO{} - - // Get the id of playlist to be updated with orgId and UID - existingPlaylist, err := s.Get(ctx, &playlist.GetPlaylistByUidQuery{UID: cmd.UID, OrgId: cmd.OrgId}) - if err != nil { - return nil, err - } - - // Create object to be update to - p := playlist.Playlist{ - Id: existingPlaylist.Id, - UID: cmd.UID, - OrgId: cmd.OrgId, - Name: cmd.Name, - Interval: cmd.Interval, - } - - err = s.sess.WithTransaction(ctx, func(tx *session.SessionTx) error { - query := `UPDATE playlist SET uid=:uid, org_id=:org_id, name=:name, "interval"=:interval WHERE id=:id` - _, err = tx.NamedExec(ctx, query, p) - if err != nil { - return err - } - - if _, err = tx.Exec(ctx, "DELETE FROM playlist_item WHERE playlist_id = ?", p.Id); err != nil { - return err - } - - playlistItems := make([]playlist.PlaylistItem, 0) - - for index, item := range cmd.Items { - playlistItems = append(playlistItems, playlist.PlaylistItem{ - PlaylistId: p.Id, - Type: item.Type, - Value: item.Value, - Order: index + 1, - Title: item.Title, - }) - } - query = `INSERT INTO playlist_item (playlist_id, type, value, title, "order") VALUES (:playlist_id, :type, :value, :title, :order)` - _, err = tx.NamedExec(ctx, query, playlistItems) - return err - }) - - return &dto, err -} - -func (s *sqlxStore) Get(ctx context.Context, query *playlist.GetPlaylistByUidQuery) (*playlist.Playlist, error) { - if query.UID == "" || query.OrgId == 0 { - return nil, playlist.ErrCommandValidationFailed - } - - p := playlist.Playlist{} - err := s.sess.Get(ctx, &p, "SELECT * FROM playlist WHERE uid=? AND org_id=?", query.UID, query.OrgId) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, playlist.ErrPlaylistNotFound - } - return nil, err - } - return &p, err -} - -func (s *sqlxStore) Delete(ctx context.Context, cmd *playlist.DeletePlaylistCommand) error { - if cmd.UID == "" || cmd.OrgId == 0 { - return playlist.ErrCommandValidationFailed - } - - p := playlist.Playlist{} - if err := s.sess.Get(ctx, &p, "SELECT * FROM playlist WHERE uid=? AND org_id=?", cmd.UID, cmd.OrgId); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil - } - return err - } - - err := s.sess.WithTransaction(ctx, func(tx *session.SessionTx) error { - if _, err := tx.Exec(ctx, "DELETE FROM playlist WHERE uid = ? and org_id = ?", cmd.UID, cmd.OrgId); err != nil { - return err - } - - if _, err := tx.Exec(ctx, "DELETE FROM playlist_item WHERE playlist_id = ?", p.Id); err != nil { - return err - } - return nil - }) - - return err -} - -func (s *sqlxStore) List(ctx context.Context, query *playlist.GetPlaylistsQuery) (playlist.Playlists, error) { - playlists := make(playlist.Playlists, 0) - if query.OrgId == 0 { - return playlists, playlist.ErrCommandValidationFailed - } - - var err error - if query.Name == "" { - err = s.sess.Select( - ctx, &playlists, "SELECT * FROM playlist WHERE org_id = ? LIMIT ?", query.OrgId, query.Limit) - } else { - err = s.sess.Select( - ctx, &playlists, "SELECT * FROM playlist WHERE org_id = ? AND name LIKE ? LIMIT ?", query.OrgId, "%"+query.Name+"%", query.Limit) - } - return playlists, err -} - -func (s *sqlxStore) GetItems(ctx context.Context, query *playlist.GetPlaylistItemsByUidQuery) ([]playlist.PlaylistItem, error) { - var playlistItems = make([]playlist.PlaylistItem, 0) - if query.PlaylistUID == "" || query.OrgId == 0 { - return playlistItems, star.ErrCommandValidationFailed - } - - var p = playlist.Playlist{} - err := s.sess.Get(ctx, &p, "SELECT * FROM playlist WHERE uid=? AND org_id=?", query.PlaylistUID, query.OrgId) - if err != nil { - return playlistItems, err - } - - err = s.sess.Select(ctx, &playlistItems, "SELECT * FROM playlist_item WHERE playlist_id=?", p.Id) - return playlistItems, err -} - -func newGenerateAndValidateNewPlaylistUid(ctx context.Context, sess *session.SessionDB, orgId int64) (string, error) { - for i := 0; i < 3; i++ { - uid := generateNewUid() - p := playlist.Playlist{OrgId: orgId, UID: uid} - err := sess.Get(ctx, &p, "SELECT * FROM playlist WHERE uid=? AND org_id=?", uid, orgId) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return uid, nil - } - return "", err - } - } - - return "", playlist.ErrPlaylistFailedGenerateUniqueUid -} diff --git a/pkg/services/playlist/playlistimpl/sqlx_store_test.go b/pkg/services/playlist/playlistimpl/sqlx_store_test.go deleted file mode 100644 index 5b77bd5b850..00000000000 --- a/pkg/services/playlist/playlistimpl/sqlx_store_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package playlistimpl - -import ( - "testing" - - "github.com/grafana/grafana/pkg/infra/db" -) - -func TestIntegrationSQLxPlaylistDataAccess(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - testIntegrationPlaylistDataAccess(t, func(ss db.DB) store { - return &sqlxStore{sess: ss.GetSqlxSession()} - }) -} diff --git a/pkg/services/playlist/playlistimpl/store_test.go b/pkg/services/playlist/playlistimpl/store_test.go index 1bb2d82d0ba..b2d79b59c83 100644 --- a/pkg/services/playlist/playlistimpl/store_test.go +++ b/pkg/services/playlist/playlistimpl/store_test.go @@ -3,6 +3,7 @@ package playlistimpl import ( "context" "testing" + "time" "github.com/stretchr/testify/require" @@ -15,6 +16,7 @@ type getStore func(db.DB) store func testIntegrationPlaylistDataAccess(t *testing.T, fn getStore) { t.Helper() + start := time.Now().UnixMilli() ss := db.InitTestDB(t) playlistStore := fn(ss) @@ -33,6 +35,8 @@ func testIntegrationPlaylistDataAccess(t *testing.T, fn getStore) { pl, err := playlistStore.Get(context.Background(), get) require.NoError(t, err) require.Equal(t, p.Id, pl.Id) + require.GreaterOrEqual(t, pl.CreatedAt, start) + require.GreaterOrEqual(t, pl.UpdatedAt, start) }) t.Run("Can get playlist items", func(t *testing.T) { @@ -43,6 +47,7 @@ func testIntegrationPlaylistDataAccess(t *testing.T, fn getStore) { }) t.Run("Can update playlist", func(t *testing.T) { + time.Sleep(time.Millisecond * 2) items := []playlist.PlaylistItem{ {Title: "influxdb", Value: "influxdb", Type: "dashboard_by_tag"}, {Title: "Backend response times", Value: "2", Type: "dashboard_by_id"}, @@ -50,6 +55,14 @@ func testIntegrationPlaylistDataAccess(t *testing.T, fn getStore) { query := playlist.UpdatePlaylistCommand{Name: "NYC office ", OrgId: 1, UID: uid, Interval: "10s", Items: items} _, err = playlistStore.Update(context.Background(), &query) require.NoError(t, err) + + // Now check that UpdatedAt has increased + pl, err := playlistStore.Get(context.Background(), &playlist.GetPlaylistByUidQuery{UID: uid, OrgId: 1}) + require.NoError(t, err) + require.Equal(t, p.Id, pl.Id) + require.Equal(t, p.CreatedAt, pl.CreatedAt) + require.Greater(t, pl.UpdatedAt, p.UpdatedAt) + require.Greater(t, pl.UpdatedAt, pl.CreatedAt) }) t.Run("Can remove playlist", func(t *testing.T) { @@ -64,6 +77,32 @@ func testIntegrationPlaylistDataAccess(t *testing.T, fn getStore) { }) }) + t.Run("Can create playlist with known UID", func(t *testing.T) { + items := []playlist.PlaylistItem{ + {Title: "graphite", Value: "graphite", Type: "dashboard_by_tag"}, + {Title: "Backend response times", Value: "3", Type: "dashboard_by_id"}, + } + cmd := playlist.CreatePlaylistCommand{Name: "NYC office", Interval: "10m", OrgId: 1, + Items: items, + UID: "abcd", + } + p, err := playlistStore.Insert(context.Background(), &cmd) + require.NoError(t, err) + require.Equal(t, "abcd", p.UID) + + // Should get an error with an invalid UID + cmd.UID = "invalid uid" + _, err = playlistStore.Insert(context.Background(), &cmd) + require.Error(t, err) + + // cleanup + err = playlistStore.Delete(context.Background(), &playlist.DeletePlaylistCommand{ + OrgId: 1, + UID: "abcd", + }) + require.NoError(t, err) + }) + t.Run("Search playlist", func(t *testing.T) { items := []playlist.PlaylistItem{ {Title: "graphite", Value: "graphite", Type: "dashboard_by_tag"}, diff --git a/pkg/services/playlist/playlistimpl/xorm_store.go b/pkg/services/playlist/playlistimpl/xorm_store.go index 218588dbeba..871252d3a65 100644 --- a/pkg/services/playlist/playlistimpl/xorm_store.go +++ b/pkg/services/playlist/playlistimpl/xorm_store.go @@ -2,6 +2,7 @@ package playlistimpl import ( "context" + "time" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/services/playlist" @@ -13,22 +14,31 @@ type sqlStore struct { db db.DB } +var _ store = &sqlStore{} + func (s *sqlStore) Insert(ctx context.Context, cmd *playlist.CreatePlaylistCommand) (*playlist.Playlist, error) { p := playlist.Playlist{} - err := s.db.WithTransactionalDbSession(ctx, func(sess *db.Session) error { - uid, err := generateAndValidateNewPlaylistUid(sess, cmd.OrgId) + if cmd.UID == "" { + cmd.UID = util.GenerateShortUID() + } else { + err := util.ValidateUID(cmd.UID) if err != nil { - return err + return nil, err } + } + err := s.db.WithTransactionalDbSession(ctx, func(sess *db.Session) error { + ts := time.Now().UnixMilli() p = playlist.Playlist{ - Name: cmd.Name, - Interval: cmd.Interval, - OrgId: cmd.OrgId, - UID: uid, + Name: cmd.Name, + Interval: cmd.Interval, + OrgId: cmd.OrgId, + UID: cmd.UID, + CreatedAt: ts, + UpdatedAt: ts, } - _, err = sess.Insert(&p) + _, err := sess.Insert(&p) if err != nil { return err } @@ -67,6 +77,8 @@ func (s *sqlStore) Update(ctx context.Context, cmd *playlist.UpdatePlaylistComma return err } p.Id = existingPlaylist.Id + p.CreatedAt = existingPlaylist.CreatedAt + p.UpdatedAt = time.Now().UnixMilli() dto = playlist.PlaylistDTO{ Uid: p.UID, @@ -74,7 +86,7 @@ func (s *sqlStore) Update(ctx context.Context, cmd *playlist.UpdatePlaylistComma Interval: p.Interval, } - _, err = sess.Where("id=?", p.Id).Cols("name", "interval").Update(&p) + _, err = sess.Where("id=?", p.Id).Cols("name", "interval", "updated_at").Update(&p) if err != nil { return err } @@ -187,26 +199,3 @@ func (s *sqlStore) GetItems(ctx context.Context, query *playlist.GetPlaylistItem }) return playlistItems, err } - -// generateAndValidateNewPlaylistUid generates a playlistUID and verifies that -// the uid isn't already in use. This is deliberately overly cautious, since users -// can also specify playlist uids during provisioning. -func generateAndValidateNewPlaylistUid(sess *db.Session, orgId int64) (string, error) { - for i := 0; i < 3; i++ { - uid := generateNewUid() - - playlist := playlist.Playlist{OrgId: orgId, UID: uid} - exists, err := sess.Get(&playlist) - if err != nil { - return "", err - } - - if !exists { - return uid, nil - } - } - - return "", playlist.ErrPlaylistFailedGenerateUniqueUid -} - -var generateNewUid func() string = util.GenerateShortUID diff --git a/pkg/services/sqlstore/migrations/playlist_mig.go b/pkg/services/sqlstore/migrations/playlist_mig.go index 9ea365d45c8..443de889215 100644 --- a/pkg/services/sqlstore/migrations/playlist_mig.go +++ b/pkg/services/sqlstore/migrations/playlist_mig.go @@ -33,6 +33,14 @@ func addPlaylistMigrations(mg *Migrator) { {Name: "value", Type: DB_Text, Nullable: false}, {Name: "title", Type: DB_Text, Nullable: false}, })) + + // Add columns used for kubernetes dual write synchronization + mg.AddMigration("Add playlist column created_at", NewAddColumnMigration(playlistV2(), &Column{ + Name: "created_at", Type: DB_BigInt, Nullable: false, Default: "0", + })) + mg.AddMigration("Add playlist column updated_at", NewAddColumnMigration(playlistV2(), &Column{ + Name: "updated_at", Type: DB_BigInt, Nullable: false, Default: "0", + })) } func addPlaylistUIDMigration(mg *Migrator) { From 6184bf21e28e67b25f50b31e19f1e327efa78a92 Mon Sep 17 00:00:00 2001 From: Nathan Marrs Date: Wed, 11 Oct 2023 00:53:56 +0200 Subject: [PATCH 025/102] Canvas: Add standard text config options to button element (#76160) Co-authored-by: Adela Almasan --- .betterer.results | 4 - public/app/features/canvas/element.ts | 3 +- .../app/features/canvas/elements/button.tsx | 74 +++++++++++++++---- .../features/canvas/elements/rectangle.tsx | 29 ++++---- 4 files changed, 79 insertions(+), 31 deletions(-) diff --git a/.betterer.results b/.betterer.results index 115753f74ad..0f4e3f64193 100644 --- a/.betterer.results +++ b/.betterer.results @@ -2907,10 +2907,6 @@ exports[`better eslint`] = { [0, 0, 0, "Styles should be written using objects.", "1"], [0, 0, 0, "Styles should be written using objects.", "2"] ], - "public/app/features/canvas/elements/rectangle.tsx:5381": [ - [0, 0, 0, "Styles should be written using objects.", "0"], - [0, 0, 0, "Styles should be written using objects.", "1"] - ], "public/app/features/canvas/elements/server/server.tsx:5381": [ [0, 0, 0, "Styles should be written using objects.", "0"], [0, 0, 0, "Styles should be written using objects.", "1"], diff --git a/public/app/features/canvas/element.ts b/public/app/features/canvas/element.ts index e156860313a..b7b0bf069dd 100644 --- a/public/app/features/canvas/element.ts +++ b/public/app/features/canvas/element.ts @@ -91,4 +91,5 @@ export interface CanvasElementItem extends RegistryI export const defaultBgColor = '#D9D9D9'; export const defaultTextColor = '#000000'; -export const defaultThemeTextColor = config.theme2.colors.text.primary; +export const defaultLightTextColor = '#F0F4FD'; +export const defaultThemeTextColor = config.theme2.colors.background.primary; diff --git a/public/app/features/canvas/elements/button.tsx b/public/app/features/canvas/elements/button.tsx index 981bc02b36a..e0f3f46f3bf 100644 --- a/public/app/features/canvas/elements/button.tsx +++ b/public/app/features/canvas/elements/button.tsx @@ -3,26 +3,26 @@ import React, { PureComponent } from 'react'; import { GrafanaTheme2 } from '@grafana/data'; import { PluginState } from '@grafana/data/src'; -import { TextDimensionConfig, TextDimensionMode } from '@grafana/schema'; +import { TextDimensionMode } from '@grafana/schema'; import { Button, stylesFactory } from '@grafana/ui'; import { config } from 'app/core/config'; import { DimensionContext } from 'app/features/dimensions/context'; +import { ColorDimensionEditor } from 'app/features/dimensions/editors'; import { TextDimensionEditor } from 'app/features/dimensions/editors/TextDimensionEditor'; import { APIEditor, APIEditorConfig } from 'app/plugins/panel/canvas/editor/element/APIEditor'; import { ButtonStyleConfig, ButtonStyleEditor } from 'app/plugins/panel/canvas/editor/element/ButtonStyleEditor'; import { callApi } from 'app/plugins/panel/canvas/editor/element/utils'; import { HttpRequestMethod } from 'app/plugins/panel/canvas/panelcfg.gen'; -import { CanvasElementItem, CanvasElementProps } from '../element'; +import { CanvasElementItem, CanvasElementProps, defaultLightTextColor } from '../element'; +import { Align, TextConfig, TextData } from '../types'; -interface ButtonData { - text?: string; +interface ButtonData extends Omit { api?: APIEditorConfig; style?: ButtonStyleConfig; } -interface ButtonConfig { - text?: TextDimensionConfig; +interface ButtonConfig extends Omit { api?: APIEditorConfig; style?: ButtonStyleConfig; } @@ -61,9 +61,16 @@ class ButtonDisplay extends PureComponent ({ button: css({ - position: 'absolute', height: '100%', width: '100%', + display: 'grid', + + '> span': { + display: 'inline-grid', + textAlign: data?.align, + fontSize: `${data?.size}px`, + color: data?.color, + }, }), })); @@ -80,8 +87,8 @@ export const buttonItem: CanvasElementItem = { display: ButtonDisplay, defaultSize: { - width: 78, - height: 32, + width: 150, + height: 45, }, getNewOptions: (options) => ({ @@ -91,6 +98,11 @@ export const buttonItem: CanvasElementItem = { mode: TextDimensionMode.Fixed, fixed: 'Button', }, + align: Align.Center, + color: { + fixed: defaultLightTextColor, + }, + size: 14, api: defaultApiConfig, style: defaultStyleConfig, }, @@ -124,10 +136,16 @@ export const buttonItem: CanvasElementItem = { const data: ButtonData = { text: cfg?.text ? ctx.getText(cfg.text).value() : '', + align: cfg.align ?? Align.Center, + size: cfg.size, api: getCfgApi(), style: cfg?.style ?? defaultStyleConfig, }; + if (cfg.color) { + data.color = ctx.getColor(cfg.color).value(); + } + return data; }, @@ -135,6 +153,13 @@ export const buttonItem: CanvasElementItem = { registerOptionsUI: (builder) => { const category = ['Button']; builder + .addCustomEditor({ + category, + id: 'styleSelector', + path: 'config.style', + name: 'Style', + editor: ButtonStyleEditor, + }) .addCustomEditor({ category, id: 'textSelector', @@ -144,10 +169,33 @@ export const buttonItem: CanvasElementItem = { }) .addCustomEditor({ category, - id: 'styleSelector', - path: 'config.style', - name: 'Style', - editor: ButtonStyleEditor, + id: 'config.color', + path: 'config.color', + name: 'Text color', + editor: ColorDimensionEditor, + settings: {}, + defaultValue: {}, + }) + .addRadio({ + category, + path: 'config.align', + name: 'Align text', + settings: { + options: [ + { value: Align.Left, label: 'Left' }, + { value: Align.Center, label: 'Center' }, + { value: Align.Right, label: 'Right' }, + ], + }, + defaultValue: Align.Left, + }) + .addNumberInput({ + category, + path: 'config.size', + name: 'Text size', + settings: { + placeholder: 'Auto', + }, }) .addCustomEditor({ category, diff --git a/public/app/features/canvas/elements/rectangle.tsx b/public/app/features/canvas/elements/rectangle.tsx index af9d427b848..4f0f515dc1a 100644 --- a/public/app/features/canvas/elements/rectangle.tsx +++ b/public/app/features/canvas/elements/rectangle.tsx @@ -15,6 +15,7 @@ class RectangleDisplay extends PureComponent {data?.text} @@ -22,21 +23,23 @@ class RectangleDisplay extends PureComponent ({ - container: css` - position: absolute; - height: 100%; - width: 100%; - display: table; - `, - span: css` - display: table-cell; - vertical-align: ${data?.valign}; - text-align: ${data?.align}; - font-size: ${data?.size}px; - color: ${data?.color}; - `, + container: css({ + position: 'absolute', + height: '100%', + width: '100%', + display: 'table', + }), + span: css({ + display: 'table-cell', + verticalAlign: data?.valign, + textAlign: data?.align, + fontSize: `${data?.size}px`, + color: data?.color, + }), })); + export const rectangleItem: CanvasElementItem = { id: 'rectangle', name: 'Rectangle', From dc8b07d46ffad945715df6035f48ecd6a3f5f723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Farkas?= Date: Wed, 11 Oct 2023 10:38:38 +0200 Subject: [PATCH 026/102] postgres/mysql: improved imports in unit tests (#75980) * postgres/mysql: improved imports in unit tests * simpler mock approach --- .../datasource/mysql/specs/datasource.test.ts | 13 ++++-- .../datasource/postgres/datasource.test.ts | 46 +++++++++++++------ 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/public/app/plugins/datasource/mysql/specs/datasource.test.ts b/public/app/plugins/datasource/mysql/specs/datasource.test.ts index 335823e8594..aaee5eabf0b 100644 --- a/public/app/plugins/datasource/mysql/specs/datasource.test.ts +++ b/public/app/plugins/datasource/mysql/specs/datasource.test.ts @@ -9,20 +9,25 @@ import { FieldType, createDataFrame, } from '@grafana/data'; -import { FetchResponse, setBackendSrv } from '@grafana/runtime'; -import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__ +import { FetchResponse } from '@grafana/runtime'; import { SQLQuery } from 'app/features/plugins/sql/types'; import { makeVariable } from 'app/features/plugins/sql/utils/testHelpers'; import { MySqlDatasource } from '../MySqlDatasource'; import { MySQLOptions } from '../types'; +const fetchMock = jest.fn(); +jest.mock('@grafana/runtime', () => ({ + ...jest.requireActual('@grafana/runtime'), + getBackendSrv: () => ({ + fetch: fetchMock, + }), +})); + describe('MySQLDatasource', () => { const defaultRange = getDefaultTimeRange(); // it does not matter what value this has const setupTestContext = (response: unknown, templateSrv?: unknown) => { jest.clearAllMocks(); - setBackendSrv(backendSrv); - const fetchMock = jest.spyOn(backendSrv, 'fetch'); const instanceSettings = { jsonData: { defaultProject: 'testproject', diff --git a/public/app/plugins/datasource/postgres/datasource.test.ts b/public/app/plugins/datasource/postgres/datasource.test.ts index da814d026a9..f5379945829 100644 --- a/public/app/plugins/datasource/postgres/datasource.test.ts +++ b/public/app/plugins/datasource/postgres/datasource.test.ts @@ -12,28 +12,44 @@ import { LoadingState, createDataFrame, } from '@grafana/data'; -import { FetchResponse } from '@grafana/runtime'; -import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__ +import { + BackendSrv, + DataSourceSrv, + FetchResponse, + getBackendSrv, + setBackendSrv, + getDataSourceSrv, + setDataSourceSrv, +} from '@grafana/runtime'; import { QueryFormat, SQLQuery } from 'app/features/plugins/sql/types'; import { makeVariable } from 'app/features/plugins/sql/utils/testHelpers'; import { PostgresDatasource } from './datasource'; import { PostgresOptions } from './types'; -jest.mock('@grafana/runtime', () => ({ - ...jest.requireActual('@grafana/runtime'), - getBackendSrv: () => backendSrv, -})); +const backendSrv: BackendSrv = { + // this will get mocked below, it only needs to exist + fetch: () => undefined, +} as unknown as BackendSrv; // we cast it so that we do not have to implement all the methods -jest.mock('@grafana/runtime/src/services', () => ({ - ...jest.requireActual('@grafana/runtime/src/services'), - getBackendSrv: () => backendSrv, - getDataSourceSrv: () => { - return { - getInstanceSettings: () => ({ id: 8674 }), - }; - }, -})); +// we type this as `any` to not have to define the whole type +const fakeDataSourceSrv: DataSourceSrv = { + getInstanceSettings: () => ({ id: 8674 }), +} as unknown as DataSourceSrv; + +let origBackendSrv: BackendSrv; +let origDataSourceSrv: DataSourceSrv; +beforeAll(() => { + origBackendSrv = getBackendSrv(); + origDataSourceSrv = getDataSourceSrv(); + setBackendSrv(backendSrv); + setDataSourceSrv(fakeDataSourceSrv); +}); + +afterAll(() => { + setBackendSrv(origBackendSrv); + setDataSourceSrv(origDataSourceSrv); +}); describe('PostgreSQLDatasource', () => { const defaultRange = getDefaultTimeRange(); // it does not matter what value this has From 9101eb219cb3d647055b33243739c59c02c23356 Mon Sep 17 00:00:00 2001 From: Joey <90795735+joey-grafana@users.noreply.github.com> Date: Wed, 11 Oct 2023 09:43:05 +0100 Subject: [PATCH 027/102] Tempo: Improve UX of service graph related warnings (#76134) Improve UX of service graph related warnings --- .../datasource/tempo/ServiceGraphSection.tsx | 53 ++++++++++++------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/public/app/plugins/datasource/tempo/ServiceGraphSection.tsx b/public/app/plugins/datasource/tempo/ServiceGraphSection.tsx index 53bba555f23..fc174bf971d 100644 --- a/public/app/plugins/datasource/tempo/ServiceGraphSection.tsx +++ b/public/app/plugins/datasource/tempo/ServiceGraphSection.tsx @@ -53,15 +53,18 @@ export function ServiceGraphSection({ const ds = dsState.value as PrometheusDatasource; if (!graphDatasourceUid) { - return
    Please set up a service graph datasource in the datasource settings.
    ; + return getWarning( + 'No service graph datasource selected', + 'Please set up a service graph datasource in the datasource settings', + styles + ); } if (graphDatasourceUid && !ds) { - return ( -
    - Service graph datasource is configured but the data source no longer exists. Please configure existing data - source to use the service graph functionality. -
    + return getWarning( + 'No service graph data found', + 'Service graph datasource is configured but the data source no longer exists. Please configure existing data source to use the service graph functionality', + styles ); } @@ -101,24 +104,34 @@ export function ServiceGraphSection({ /> - {hasKeys === false ? ( - - Please ensure that service graph metrics are set up correctly according to the{' '} - - Tempo documentation - - . - - ) : null} + {hasKeys === false + ? getWarning( + 'No service graph data found', + 'Please ensure that service graph metrics are set up correctly', + styles + ) + : null}
    ); } +function getWarning(title: string, description: string, styles: { alert: string; link: string }) { + return ( + + {description} according to the{' '} + + Tempo documentation + + . + + ); +} + function queryToFilter(query: string): AdHocVariableFilter[] { let match; let filters: AdHocVariableFilter[] = []; From d5691e6dd155eacb4462000bcb12607c8f00a12d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Calisto?= Date: Wed, 11 Oct 2023 09:45:24 +0100 Subject: [PATCH 028/102] Live: Allow setting the engine password (#76289) --- conf/defaults.ini | 3 +++ conf/sample.ini | 3 +++ pkg/services/live/live.go | 6 ++++-- pkg/setting/setting.go | 4 +++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index 6712b15c734..d200015e520 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -1525,6 +1525,9 @@ ha_engine = # This option is EXPERIMENTAL. ha_engine_address = "127.0.0.1:6379" +# ha_engine_password allows setting an optional password to authenticate with the engine +ha_engine_password = "" + #################################### Grafana Image Renderer Plugin ########################## [plugin.grafana-image-renderer] # Instruct headless browser instance to use a default timezone when not provided by Grafana, e.g. when rendering panel image of alert. diff --git a/conf/sample.ini b/conf/sample.ini index 6972722789c..126a4aa594f 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -1418,6 +1418,9 @@ # This option is EXPERIMENTAL. ;ha_engine_address = "127.0.0.1:6379" +# ha_engine_password allows setting an optional password to authenticate with the engine +;ha_engine_password = "" + #################################### Grafana Image Renderer Plugin ########################## [plugin.grafana-image-renderer] # Instruct headless browser instance to use a default timezone when not provided by Grafana, e.g. when rendering panel image of alert. diff --git a/pkg/services/live/live.go b/pkg/services/live/live.go index bb6babd1292..eb802c8b236 100644 --- a/pkg/services/live/live.go +++ b/pkg/services/live/live.go @@ -124,8 +124,9 @@ func ProvideService(plugCtxProvider *plugincontext.Provider, cfg *setting.Cfg, r // will be connected over Redis PUB/SUB. Presence will work // globally since kept inside Redis. redisAddress := g.Cfg.LiveHAEngineAddress + redisPassword := g.Cfg.LiveHAEnginePassword redisShardConfigs := []centrifuge.RedisShardConfig{ - {Address: redisAddress}, + {Address: redisAddress, Password: redisPassword}, } var redisShards []*centrifuge.RedisShard for _, redisConf := range redisShardConfigs { @@ -160,7 +161,8 @@ func ProvideService(plugCtxProvider *plugincontext.Provider, cfg *setting.Cfg, r var managedStreamRunner *managedstream.Runner if g.IsHA() { redisClient := redis.NewClient(&redis.Options{ - Addr: g.Cfg.LiveHAEngineAddress, + Addr: g.Cfg.LiveHAEngineAddress, + Password: g.Cfg.LiveHAEnginePassword, }) cmd := redisClient.Ping(context.Background()) if _, err := cmd.Result(); err != nil { diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 5c86c56fc4d..1ea1b0bfe77 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -482,7 +482,8 @@ type Cfg struct { // Zero value means in-memory single node setup. LiveHAEngine string // LiveHAEngineAddress is a connection address for Live HA engine. - LiveHAEngineAddress string + LiveHAEngineAddress string + LiveHAEnginePassword string // LiveAllowedOrigins is a set of origins accepted by Live. If not provided // then Live uses AppURL as the only allowed origin. LiveAllowedOrigins []string @@ -1989,6 +1990,7 @@ func (cfg *Cfg) readLiveSettings(iniFile *ini.File) error { return fmt.Errorf("unsupported live HA engine type: %s", cfg.LiveHAEngine) } cfg.LiveHAEngineAddress = section.Key("ha_engine_address").MustString("127.0.0.1:6379") + cfg.LiveHAEnginePassword = section.Key("ha_engine_password").MustString("") var originPatterns []string allowedOrigins := section.Key("allowed_origins").MustString("") From 1f8b08202eb5bba2171423129fa21e0be23944e8 Mon Sep 17 00:00:00 2001 From: Andres Martinez Gotor Date: Wed, 11 Oct 2023 10:49:30 +0200 Subject: [PATCH 029/102] Chore: Avoid showing plugin version if it's %VERSION% (#75974) --- .../manager/pipeline/bootstrap/steps.go | 17 +++++++++++ .../manager/pipeline/bootstrap/steps_test.go | 29 +++++++++++++++++++ .../pluginsintegration/loader/loader_test.go | 8 ++--- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/pkg/plugins/manager/pipeline/bootstrap/steps.go b/pkg/plugins/manager/pipeline/bootstrap/steps.go index f2d61e58e91..9cfb92ea3cd 100644 --- a/pkg/plugins/manager/pipeline/bootstrap/steps.go +++ b/pkg/plugins/manager/pipeline/bootstrap/steps.go @@ -30,6 +30,7 @@ func DefaultConstructFunc(signatureCalculator plugins.SignatureCalculator, asset func DefaultDecorateFuncs(cfg *config.Cfg) []DecorateFunc { return []DecorateFunc{ AppDefaultNavURLDecorateFunc, + TemplateDecorateFunc, AppChildDecorateFunc(cfg), } } @@ -86,6 +87,22 @@ func AppDefaultNavURLDecorateFunc(_ context.Context, p *plugins.Plugin) (*plugin return p, nil } +// TemplateDecorateFunc is a DecorateFunc that removes the placeholder for the version and last_update fields. +func TemplateDecorateFunc(_ context.Context, p *plugins.Plugin) (*plugins.Plugin, error) { + // %VERSION% and %TODAY% are valid values, according to the plugin schema + // but it's meant to be replaced by the build system with the actual version and date. + // If not, it's the same than not having a version or a date. + if p.Info.Version == "%VERSION%" { + p.Info.Version = "" + } + + if p.Info.Updated == "%TODAY%" { + p.Info.Updated = "" + } + + return p, nil +} + func setDefaultNavURL(p *plugins.Plugin) { // slugify pages for _, include := range p.Includes { diff --git a/pkg/plugins/manager/pipeline/bootstrap/steps_test.go b/pkg/plugins/manager/pipeline/bootstrap/steps_test.go index 670a667a707..795d7f8a5a9 100644 --- a/pkg/plugins/manager/pipeline/bootstrap/steps_test.go +++ b/pkg/plugins/manager/pipeline/bootstrap/steps_test.go @@ -1,6 +1,7 @@ package bootstrap import ( + "context" "testing" "github.com/stretchr/testify/require" @@ -66,6 +67,34 @@ func TestSetDefaultNavURL(t *testing.T) { }) } +func TestTemplateDecorateFunc(t *testing.T) { + t.Run("Removes %VERSION%", func(t *testing.T) { + pluginWithoutVersion := &plugins.Plugin{ + JSONData: plugins.JSONData{ + Info: plugins.Info{ + Version: "%VERSION%", + }, + }, + } + p, err := TemplateDecorateFunc(context.TODO(), pluginWithoutVersion) + require.NoError(t, err) + require.Equal(t, "", p.Info.Version) + }) + + t.Run("Removes %TODAY%", func(t *testing.T) { + pluginWithoutVersion := &plugins.Plugin{ + JSONData: plugins.JSONData{ + Info: plugins.Info{ + Version: "%TODAY%", + }, + }, + } + p, err := TemplateDecorateFunc(context.TODO(), pluginWithoutVersion) + require.NoError(t, err) + require.Equal(t, "", p.Info.Updated) + }) +} + func Test_configureAppChildPlugin(t *testing.T) { t.Run("When setting paths based on core plugin on Windows", func(t *testing.T) { child := &plugins.Plugin{ diff --git a/pkg/services/pluginsintegration/loader/loader_test.go b/pkg/services/pluginsintegration/loader/loader_test.go index d062955e7fa..c4fda2f7b14 100644 --- a/pkg/services/pluginsintegration/loader/loader_test.go +++ b/pkg/services/pluginsintegration/loader/loader_test.go @@ -1335,8 +1335,8 @@ func TestLoader_Load_NestedPlugins(t *testing.T) { }, Screenshots: []plugins.Screenshots{}, Description: "Grafana App Plugin Template", - Version: "%VERSION%", - Updated: "%TODAY%", + Version: "", + Updated: "", }, Dependencies: plugins.Dependencies{ GrafanaVersion: "7.0.0", @@ -1415,8 +1415,8 @@ func TestLoader_Load_NestedPlugins(t *testing.T) { }, Screenshots: []plugins.Screenshots{}, Description: "Grafana Panel Plugin Template", - Version: "%VERSION%", - Updated: "%TODAY%", + Version: "", + Updated: "", }, Dependencies: plugins.Dependencies{ GrafanaDependency: ">=7.0.0", From 608f066ca5d6dc09c021cfb5fb89ecb3af3ec4da Mon Sep 17 00:00:00 2001 From: Oscar Kilhed Date: Wed, 11 Oct 2023 10:50:11 +0200 Subject: [PATCH 030/102] PanelConfig: Add option to calculate min/max per field instead of using the global min/max in the data frame. (#75952) * Add option to calculate min max per field * Fix eslint warnings * Add back hideFromDefaults that went missing * whitespace * Add tests * Refactor range calculation * Rename localMinMax -> fieldMinMax * Remove the lint exceptions Removing these as to not hide these once we get around to fixing the underlying typing issue. * Update docs --- .betterer.results | 16 ++- .../configure-standard-options/index.md | 8 +- .../src/field/fieldOverrides.test.ts | 100 ++++++++++++++++++ .../grafana-data/src/field/fieldOverrides.ts | 38 +++++-- packages/grafana-data/src/types/dataFrame.ts | 3 + .../src/utils/tests/mockStandardProperties.ts | 19 +++- .../core/components/OptionsUI/registry.tsx | 21 +++- .../PanelEditor/getVisualizationOptions.tsx | 14 +-- 8 files changed, 195 insertions(+), 24 deletions(-) diff --git a/.betterer.results b/.betterer.results index 0f4e3f64193..6d5ba9d6a08 100644 --- a/.betterer.results +++ b/.betterer.results @@ -173,7 +173,8 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "3"] ], "packages/grafana-data/src/field/fieldOverrides.ts:5381": [ - [0, 0, 0, "Unexpected any. Specify a different type.", "0"] + [0, 0, 0, "Unexpected any. Specify a different type.", "0"], + [0, 0, 0, "Unexpected any. Specify a different type.", "1"] ], "packages/grafana-data/src/field/overrides/processors.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], @@ -1390,10 +1391,15 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "77"], [0, 0, 0, "Do not use any type assertions.", "78"], [0, 0, 0, "Unexpected any. Specify a different type.", "79"], - [0, 0, 0, "Do not use any type assertions.", "80"], - [0, 0, 0, "Unexpected any. Specify a different type.", "81"], - [0, 0, 0, "Do not use any type assertions.", "82"], - [0, 0, 0, "Unexpected any. Specify a different type.", "83"] + [0, 0, 0, "Unexpected any. Specify a different type.", "80"], + [0, 0, 0, "Do not use any type assertions.", "81"], + [0, 0, 0, "Unexpected any. Specify a different type.", "82"], + [0, 0, 0, "Do not use any type assertions.", "83"], + [0, 0, 0, "Unexpected any. Specify a different type.", "84"], + [0, 0, 0, "Do not use any type assertions.", "85"], + [0, 0, 0, "Unexpected any. Specify a different type.", "86"], + [0, 0, 0, "Do not use any type assertions.", "87"], + [0, 0, 0, "Unexpected any. Specify a different type.", "88"] ], "public/app/core/components/OptionsUI/slider.tsx:5381": [ [0, 0, 0, "Styles should be written using objects.", "0"] diff --git a/docs/sources/panels-visualizations/configure-standard-options/index.md b/docs/sources/panels-visualizations/configure-standard-options/index.md index cfa0c6ca9b7..860f5238f7f 100644 --- a/docs/sources/panels-visualizations/configure-standard-options/index.md +++ b/docs/sources/panels-visualizations/configure-standard-options/index.md @@ -82,11 +82,15 @@ Grafana can sometimes be too aggressive in parsing strings and displaying them a ### Min -Lets you set the minimum value used in percentage threshold calculations. Leave blank for auto calculation based on all series and fields. +Lets you set the minimum value used in percentage threshold calculations. Leave blank to automatically calculate the minimum. ### Max -Lets you set the maximum value used in percentage threshold calculations. Leave blank for auto calculation based on all series and fields. +Lets you set the maximum value used in percentage threshold calculations. Leave blank to automatically calculate the maximum. + +### Field min/max + +By default the calculated min and max will be based on the minimum and maximum, in all series and fields. Turning field min/max on, will calculate the min or max on each field individually, based on the minimum or maximum value of the field. ### Decimals diff --git a/packages/grafana-data/src/field/fieldOverrides.test.ts b/packages/grafana-data/src/field/fieldOverrides.test.ts index fe241167923..8d358a80320 100644 --- a/packages/grafana-data/src/field/fieldOverrides.test.ts +++ b/packages/grafana-data/src/field/fieldOverrides.test.ts @@ -330,6 +330,106 @@ describe('applyFieldOverrides', () => { expect(range.min).toEqual(-20); }); + it('should calculate min/max per field when fieldMinMax is set', () => { + const df = toDataFrame([ + { title: 'AAA', value: 100, value2: 1234 }, + { title: 'BBB', value: -20, value2: null }, + { title: 'CCC', value: 200, value2: 1000 }, + ]); + + const fieldCfgSource: FieldConfigSource = { + defaults: { + fieldMinMax: true, + }, + overrides: [], + }; + const data = applyFieldOverrides({ + data: [df], // the frame + fieldConfig: fieldCfgSource, + replaceVariables: undefined as unknown as InterpolateFunction, + theme: createTheme(), + fieldConfigRegistry: customFieldRegistry, + })[0]; + + const valueColumn1 = data.fields[1]; + const range1 = valueColumn1.state!.range!; + expect(range1.max).toEqual(200); + expect(range1.min).toEqual(-20); + + const valueColumn2 = data.fields[2]; + const range2 = valueColumn2.state!.range!; + expect(range2.max).toEqual(1234); + expect(range2.min).toEqual(1000); + }); + + it('should calculate min/max locally for fields with fieldMinMax and globally for other fields', () => { + const df = toDataFrame({ + fields: [ + { name: 'first', type: FieldType.number, values: [-1, -2] }, + { name: 'second', type: FieldType.number, values: [1, 2] }, + { name: 'third', type: FieldType.number, values: [1000, 2000] }, + ], + }); + + const fieldCfgSource: FieldConfigSource = { + defaults: {}, + overrides: [ + { + matcher: { id: FieldMatcherID.byName, options: 'second' }, + properties: [{ id: 'fieldMinMax', value: true }], + }, + ], + }; + const data = applyFieldOverrides({ + data: [df], // the frame + fieldConfig: fieldCfgSource, + replaceVariables: undefined as unknown as InterpolateFunction, + theme: createTheme(), + fieldConfigRegistry: customFieldRegistry, + })[0]; + + const valueColumn0 = data.fields[0]; + const range0 = valueColumn0.state!.range!; + expect(range0.max).toEqual(2000); + expect(range0.min).toEqual(-2); + + const valueColumn1 = data.fields[1]; + const range1 = valueColumn1.state!.range!; + expect(range1.max).toEqual(2); + expect(range1.min).toEqual(1); + + const valueColumn2 = data.fields[2]; + const range2 = valueColumn2.state!.range!; + expect(range2.max).toEqual(2000); + expect(range2.min).toEqual(-2); + }); + + it('should not calculate min if min is set', () => { + const df = toDataFrame({ + fields: [{ name: 'first', type: FieldType.number, values: [-1, -2] }], + }); + + const fieldCfgSource: FieldConfigSource = { + defaults: { + min: 1, + fieldMinMax: true, + }, + overrides: [], + }; + const data = applyFieldOverrides({ + data: [df], // the frame + fieldConfig: fieldCfgSource, + replaceVariables: undefined as unknown as InterpolateFunction, + theme: createTheme(), + fieldConfigRegistry: customFieldRegistry, + })[0]; + + const valueColumn0 = data.fields[0]; + const range0 = valueColumn0.state!.range!; + expect(range0.max).toEqual(-1); + expect(range0.min).toEqual(1); + }); + it('getLinks should use applied field config', () => { const replaceVariablesCalls: ScopedVars[] = []; diff --git a/packages/grafana-data/src/field/fieldOverrides.ts b/packages/grafana-data/src/field/fieldOverrides.ts index 6fc79b771fd..8316e1a7350 100644 --- a/packages/grafana-data/src/field/fieldOverrides.ts +++ b/packages/grafana-data/src/field/fieldOverrides.ts @@ -40,6 +40,7 @@ import { mapInternalLinkToExplore } from '../utils/dataLinks'; import { FieldConfigOptionsRegistry } from './FieldConfigOptionsRegistry'; import { getDisplayProcessor, getRawDisplayProcessor } from './displayProcessor'; +import { getMinMaxAndDelta } from './scale'; import { standardFieldConfigEditorRegistry } from './standardFieldConfigEditorRegistry'; interface OverrideProps { @@ -166,15 +167,8 @@ export function applyFieldOverrides(options: ApplyFieldOverrideOptions): DataFra } // Set the Min/Max value automatically - let range: NumericRange | undefined = undefined; - if (field.type === FieldType.number) { - if (!globalRange && (!isNumber(config.min) || !isNumber(config.max))) { - globalRange = findNumericFieldMinMax(options.data!); - } - const min = config.min ?? globalRange!.min; - const max = config.max ?? globalRange!.max; - range = { min, max, delta: max! - min! }; - } + const { range, newGlobalRange } = calculateRange(config, field, globalRange, options.data!); + globalRange = newGlobalRange; field.state!.seriesIndex = seriesIndex; field.state!.range = range; @@ -243,6 +237,32 @@ export function applyFieldOverrides(options: ApplyFieldOverrideOptions): DataFra }); } +function calculateRange( + config: FieldConfig, + field: Field, + globalRange: NumericRange | undefined, + data: DataFrame[] +): { range?: { min?: number | null; max?: number | null; delta: number }; newGlobalRange: NumericRange | undefined } { + // Only calculate ranges when the field is a number and one of min/max is set to auto. + if (field.type !== FieldType.number || (isNumber(config.min) && isNumber(config.max))) { + return { newGlobalRange: globalRange }; + } + + // Calculate the min/max from the field. + if (config.fieldMinMax) { + const localRange = getMinMaxAndDelta(field); + const min = config.min ?? localRange.min; + const max = config.max ?? localRange.max; + return { range: { min, max, delta: max! - min! }, newGlobalRange: globalRange }; + } + + // We use the global range if supplied, otherwise we calculate it. + const newGlobalRange = globalRange ?? findNumericFieldMinMax(data); + const min = config.min ?? newGlobalRange!.min; + const max = config.max ?? newGlobalRange!.max; + return { range: { min, max, delta: max! - min! }, newGlobalRange }; +} + // this is a significant optimization for streaming, where we currently re-process all values in the buffer on ech update // via field.display(value). this can potentially be removed once we... // 1. process data packets incrementally and/if cache the results in the streaming datafame (maybe by buffer index) diff --git a/packages/grafana-data/src/types/dataFrame.ts b/packages/grafana-data/src/types/dataFrame.ts index cb971ab9c56..0154d8014c9 100644 --- a/packages/grafana-data/src/types/dataFrame.ts +++ b/packages/grafana-data/src/types/dataFrame.ts @@ -101,6 +101,9 @@ export interface FieldConfig { // Panel Specific Values custom?: TOptions; + + // Calculate min max per field + fieldMinMax?: boolean; } export interface FieldTypeConfig { diff --git a/packages/grafana-data/src/utils/tests/mockStandardProperties.ts b/packages/grafana-data/src/utils/tests/mockStandardProperties.ts index 85ad66e660f..5858cd0a51f 100644 --- a/packages/grafana-data/src/utils/tests/mockStandardProperties.ts +++ b/packages/grafana-data/src/utils/tests/mockStandardProperties.ts @@ -67,6 +67,23 @@ export const mockStandardProperties = () => { shouldApply: () => true, }; + const fieldMinMax = { + id: 'fieldMinMax', + path: 'fieldMinMax', + name: 'localminmax', + description: 'Calculate min/max per field ', + + editor: () => null, + override: () => null, + process: identityOverrideProcessor, + + settings: { + placeholder: 'auto', + }, + + shouldApply: () => true, + }; + const decimals = { id: 'decimals', path: 'decimals', @@ -166,5 +183,5 @@ export const mockStandardProperties = () => { shouldApply: () => true, }; - return [unit, min, max, decimals, title, noValue, thresholds, mappings, links, color]; + return [unit, min, max, fieldMinMax, decimals, title, noValue, thresholds, mappings, links, color]; }; diff --git a/public/app/core/components/OptionsUI/registry.tsx b/public/app/core/components/OptionsUI/registry.tsx index 7f5681609dd..18c1816c8ac 100644 --- a/public/app/core/components/OptionsUI/registry.tsx +++ b/public/app/core/components/OptionsUI/registry.tsx @@ -1,3 +1,4 @@ +import { BooleanFieldSettings } from '@react-awesome-query-builder/ui'; import React from 'react'; import { @@ -27,6 +28,7 @@ import { FieldNamePickerConfigSettings, booleanOverrideProcessor, } from '@grafana/data'; +import { FieldConfig } from '@grafana/schema'; import { RadioButtonGroup, TimeZonePicker, Switch } from '@grafana/ui'; import { FieldNamePicker } from '@grafana/ui/src/components/MatchersUI/FieldNamePicker'; import { ThresholdsValueEditor } from 'app/features/dimensions/editors/ThresholdsEditor/thresholds'; @@ -245,6 +247,23 @@ export const getAllStandardFieldConfigs = () => { category, }; + const fieldMinMax: FieldConfigPropertyItem = { + id: 'fieldMinMax', + path: 'fieldMinMax', + name: 'Field min/max', + description: 'Calculate min max per field', + + editor: standardEditorsRegistry.get('boolean').editor as any, + override: standardEditorsRegistry.get('boolean').editor as any, + process: booleanOverrideProcessor, + + shouldApply: (field) => field.type === FieldType.number, + showIf: (options: FieldConfig) => { + return options.min === undefined || options.max === undefined; + }, + category, + }; + const min: FieldConfigPropertyItem = { id: 'min', path: 'min', @@ -397,5 +416,5 @@ export const getAllStandardFieldConfigs = () => { category, }; - return [unit, min, max, decimals, displayName, color, noValue, links, mappings, thresholds, filterable]; + return [unit, min, max, fieldMinMax, decimals, displayName, color, noValue, links, mappings, thresholds, filterable]; }; diff --git a/public/app/features/dashboard/components/PanelEditor/getVisualizationOptions.tsx b/public/app/features/dashboard/components/PanelEditor/getVisualizationOptions.tsx index 099a5461f6b..5a33695a3e6 100644 --- a/public/app/features/dashboard/components/PanelEditor/getVisualizationOptions.tsx +++ b/public/app/features/dashboard/components/PanelEditor/getVisualizationOptions.tsx @@ -97,12 +97,14 @@ export function getVisualizationOptions(props: OptionPaneRenderProps): OptionsPa * Field options */ for (const fieldOption of plugin.fieldConfigRegistry.list()) { - if ( - fieldOption.isCustom && - fieldOption.showIf && - !fieldOption.showIf(currentFieldConfig.defaults.custom, data?.series) - ) { - continue; + if (fieldOption.isCustom) { + if (fieldOption.showIf && !fieldOption.showIf(currentFieldConfig.defaults.custom, data?.series)) { + continue; + } + } else { + if (fieldOption.showIf && !fieldOption.showIf(currentFieldConfig.defaults, data?.series)) { + continue; + } } if (fieldOption.hideFromDefaults) { From 07b08fc09f1d212cb3712f5081fcfb13b4a3fbf1 Mon Sep 17 00:00:00 2001 From: Joey <90795735+joey-grafana@users.noreply.github.com> Date: Wed, 11 Oct 2023 09:51:48 +0100 Subject: [PATCH 031/102] Tempo: Disable show matches only button when no matches (#76124) * Disable show matches only button when there are no matches * Update tests --- .../TracePageHeader/SearchBar/TracePageSearchBar.tsx | 8 +++----- .../TracePageHeader/SpanFilters/SpanFilters.test.tsx | 12 +++++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/public/app/features/explore/TraceView/components/TracePageHeader/SearchBar/TracePageSearchBar.tsx b/public/app/features/explore/TraceView/components/TracePageHeader/SearchBar/TracePageSearchBar.tsx index ff8b6397e10..86f574891bc 100644 --- a/public/app/features/explore/TraceView/components/TracePageHeader/SearchBar/TracePageSearchBar.tsx +++ b/public/app/features/explore/TraceView/components/TracePageHeader/SearchBar/TracePageSearchBar.tsx @@ -89,12 +89,14 @@ export default memo(function TracePageSearchBar(props: TracePageSearchBarProps) value={showSpanFilterMatchesOnly} onChange={(value) => setShowSpanFilterMatchesOnly(value.currentTarget.checked ?? false)} label="Show matches only switch" + disabled={!spanFilterMatches?.size} /> @@ -139,16 +141,12 @@ export const getStyles = (theme: GrafanaTheme2) => { margin: 0 0 0 25px; vertical-align: middle; align-items: center; - - span { - cursor: pointer; - } `, clearMatchesButton: css` color: ${theme.colors.text.primary}; + &:hover { background: inherit; - color: inherit; } `, nextPrevResult: css` diff --git a/public/app/features/explore/TraceView/components/TracePageHeader/SpanFilters/SpanFilters.test.tsx b/public/app/features/explore/TraceView/components/TracePageHeader/SpanFilters/SpanFilters.test.tsx index 84fd36a0240..05f52f3e6c2 100644 --- a/public/app/features/explore/TraceView/components/TracePageHeader/SpanFilters/SpanFilters.test.tsx +++ b/public/app/features/explore/TraceView/components/TracePageHeader/SpanFilters/SpanFilters.test.tsx @@ -39,11 +39,17 @@ const trace: Trace = { logs: [{ fields: [{ key: 'LogKey1', type: 'string', value: 'LogValue1' }] }], }, ], + processes: { + '1ed38015486087ca': { + serviceName: 'Service0', + tags: [], + }, + }, } as unknown as Trace; describe('SpanFilters', () => { let user: ReturnType; - const SpanFiltersWithProps = ({ showFilters = true }) => { + const SpanFiltersWithProps = ({ showFilters = true, matches }: { showFilters?: boolean; matches?: Set }) => { const [search, setSearch] = useState(defaultFilters); const [showSpanFilterMatchesOnly, setShowSpanFilterMatchesOnly] = useState(false); const props = { @@ -54,7 +60,7 @@ describe('SpanFilters', () => { setShowSpanFilterMatchesOnly, search, setSearch, - spanFilterMatches: undefined, + spanFilterMatches: matches, setFocusedSpanIdForSearch: jest.fn(), datasourceType: 'tempo', }; @@ -197,7 +203,7 @@ describe('SpanFilters', () => { }); it('should allow resetting filters', async () => { - render(); + render(); const clearFiltersButton = screen.getByRole('button', { name: 'Clear filters button' }); expect(clearFiltersButton).toBeInTheDocument(); expect((clearFiltersButton as HTMLButtonElement)['disabled']).toBe(true); From 9d2e8f650b2397a4dce6f7414d3533fabc2ba134 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 10:17:05 +0100 Subject: [PATCH 032/102] Update dependency @rushstack/ts-command-line to v4.16.1 (#76282) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 799e2dcdf90..cc695fb4f66 100644 --- a/package.json +++ b/package.json @@ -418,7 +418,7 @@ "underscore": "1.13.6", "@types/slate": "0.47.11", "@rushstack/rig-package": "0.5.1", - "@rushstack/ts-command-line": "4.15.1", + "@rushstack/ts-command-line": "4.16.1", "ngtemplate-loader/loader-utils": "^2.0.0", "trim": "1.0.1", "semver@~7.0.0": "7.5.4", From a0ab1e4fd7f7892ad09a5962c6822ee0c2120db8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 10:18:54 +0100 Subject: [PATCH 033/102] Update dependency @rollup/plugin-node-resolve to v15.2.3 (#76261) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/grafana-data/package.json | 2 +- packages/grafana-e2e-selectors/package.json | 2 +- packages/grafana-e2e/package.json | 2 +- packages/grafana-flamegraph/package.json | 2 +- packages/grafana-runtime/package.json | 2 +- packages/grafana-schema/package.json | 2 +- packages/grafana-ui/package.json | 2 +- yarn.lock | 24 ++++++++++----------- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index 239b0db9edf..f74e8fa6f1f 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -65,7 +65,7 @@ "@grafana/tsconfig": "^1.2.0-rc1", "@rollup/plugin-commonjs": "25.0.2", "@rollup/plugin-json": "6.0.0", - "@rollup/plugin-node-resolve": "15.1.0", + "@rollup/plugin-node-resolve": "15.2.3", "@testing-library/dom": "9.3.1", "@testing-library/jest-dom": "5.16.5", "@testing-library/react": "14.0.0", diff --git a/packages/grafana-e2e-selectors/package.json b/packages/grafana-e2e-selectors/package.json index 80b1945c29a..32d7a53888f 100644 --- a/packages/grafana-e2e-selectors/package.json +++ b/packages/grafana-e2e-selectors/package.json @@ -40,7 +40,7 @@ }, "devDependencies": { "@rollup/plugin-commonjs": "25.0.2", - "@rollup/plugin-node-resolve": "15.1.0", + "@rollup/plugin-node-resolve": "15.2.3", "@types/node": "18.16.19", "esbuild": "0.18.12", "rimraf": "5.0.1", diff --git a/packages/grafana-e2e/package.json b/packages/grafana-e2e/package.json index b09628caf34..1418371d01c 100644 --- a/packages/grafana-e2e/package.json +++ b/packages/grafana-e2e/package.json @@ -47,7 +47,7 @@ "postpack": "mv package.json.bak package.json" }, "devDependencies": { - "@rollup/plugin-node-resolve": "15.1.0", + "@rollup/plugin-node-resolve": "15.2.3", "@types/chrome-remote-interface": "0.31.10", "@types/lodash": "4.14.195", "@types/node": "18.16.19", diff --git a/packages/grafana-flamegraph/package.json b/packages/grafana-flamegraph/package.json index 4e01d8f2989..a2bbffd9804 100644 --- a/packages/grafana-flamegraph/package.json +++ b/packages/grafana-flamegraph/package.json @@ -60,7 +60,7 @@ "@babel/preset-env": "7.22.9", "@babel/preset-react": "7.22.5", "@grafana/tsconfig": "^1.2.0-rc1", - "@rollup/plugin-node-resolve": "15.1.0", + "@rollup/plugin-node-resolve": "15.2.3", "@testing-library/jest-dom": "^6.1.2", "@testing-library/react": "14.0.0", "@testing-library/user-event": "14.4.3", diff --git a/packages/grafana-runtime/package.json b/packages/grafana-runtime/package.json index 049caa86b13..f4eefc9de58 100644 --- a/packages/grafana-runtime/package.json +++ b/packages/grafana-runtime/package.json @@ -51,7 +51,7 @@ "devDependencies": { "@grafana/tsconfig": "^1.2.0-rc1", "@rollup/plugin-commonjs": "25.0.2", - "@rollup/plugin-node-resolve": "15.1.0", + "@rollup/plugin-node-resolve": "15.2.3", "@testing-library/dom": "9.3.1", "@testing-library/react": "14.0.0", "@testing-library/user-event": "14.4.3", diff --git a/packages/grafana-schema/package.json b/packages/grafana-schema/package.json index 5521c9a87a4..dcb04f54b5c 100644 --- a/packages/grafana-schema/package.json +++ b/packages/grafana-schema/package.json @@ -39,7 +39,7 @@ "@grafana/tsconfig": "^1.2.0-rc1", "@rollup/plugin-commonjs": "25.0.2", "@rollup/plugin-json": "6.0.0", - "@rollup/plugin-node-resolve": "15.1.0", + "@rollup/plugin-node-resolve": "15.2.3", "esbuild": "0.18.12", "glob": "^10.2.7", "rimraf": "5.0.1", diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index 0f74465d555..8268fa39811 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -118,7 +118,7 @@ "@babel/core": "7.23.0", "@grafana/tsconfig": "^1.2.0-rc1", "@mdx-js/react": "1.6.22", - "@rollup/plugin-node-resolve": "15.1.0", + "@rollup/plugin-node-resolve": "15.2.3", "@storybook/addon-a11y": "7.4.5", "@storybook/addon-actions": "7.4.5", "@storybook/addon-docs": "7.4.5", diff --git a/yarn.lock b/yarn.lock index ce7dfd83649..8f332c58ebb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2991,7 +2991,7 @@ __metadata: "@grafana/tsconfig": ^1.2.0-rc1 "@rollup/plugin-commonjs": 25.0.2 "@rollup/plugin-json": 6.0.0 - "@rollup/plugin-node-resolve": 15.1.0 + "@rollup/plugin-node-resolve": 15.2.3 "@testing-library/dom": 9.3.1 "@testing-library/jest-dom": 5.16.5 "@testing-library/react": 14.0.0 @@ -3053,7 +3053,7 @@ __metadata: dependencies: "@grafana/tsconfig": ^1.2.0-rc1 "@rollup/plugin-commonjs": 25.0.2 - "@rollup/plugin-node-resolve": 15.1.0 + "@rollup/plugin-node-resolve": 15.2.3 "@types/node": 18.16.19 esbuild: 0.18.12 rimraf: 5.0.1 @@ -3088,7 +3088,7 @@ __metadata: "@grafana/schema": 10.2.0-pre "@grafana/tsconfig": ^1.2.0-rc1 "@mochajs/json-file-reporter": ^1.2.0 - "@rollup/plugin-node-resolve": 15.1.0 + "@rollup/plugin-node-resolve": 15.2.3 "@types/chrome-remote-interface": 0.31.10 "@types/lodash": 4.14.195 "@types/node": 18.16.19 @@ -3227,7 +3227,7 @@ __metadata: "@grafana/tsconfig": ^1.2.0-rc1 "@grafana/ui": 10.2.0-pre "@leeoniya/ufuzzy": 1.0.8 - "@rollup/plugin-node-resolve": 15.1.0 + "@rollup/plugin-node-resolve": 15.2.3 "@testing-library/jest-dom": ^6.1.2 "@testing-library/react": 14.0.0 "@testing-library/user-event": 14.4.3 @@ -3309,7 +3309,7 @@ __metadata: "@grafana/tsconfig": ^1.2.0-rc1 "@grafana/ui": 10.2.0-pre "@rollup/plugin-commonjs": 25.0.2 - "@rollup/plugin-node-resolve": 15.1.0 + "@rollup/plugin-node-resolve": 15.2.3 "@testing-library/dom": 9.3.1 "@testing-library/react": 14.0.0 "@testing-library/user-event": 14.4.3 @@ -3368,7 +3368,7 @@ __metadata: "@grafana/tsconfig": ^1.2.0-rc1 "@rollup/plugin-commonjs": 25.0.2 "@rollup/plugin-json": 6.0.0 - "@rollup/plugin-node-resolve": 15.1.0 + "@rollup/plugin-node-resolve": 15.2.3 esbuild: 0.18.12 glob: ^10.2.7 rimraf: 5.0.1 @@ -3418,7 +3418,7 @@ __metadata: "@react-aria/overlays": 3.15.0 "@react-aria/utils": 3.18.0 "@react-stately/menu": 3.5.3 - "@rollup/plugin-node-resolve": 15.1.0 + "@rollup/plugin-node-resolve": 15.2.3 "@storybook/addon-a11y": 7.4.5 "@storybook/addon-actions": 7.4.5 "@storybook/addon-docs": 7.4.5 @@ -6893,9 +6893,9 @@ __metadata: languageName: node linkType: hard -"@rollup/plugin-node-resolve@npm:15.1.0": - version: 15.1.0 - resolution: "@rollup/plugin-node-resolve@npm:15.1.0" +"@rollup/plugin-node-resolve@npm:15.2.3": + version: 15.2.3 + resolution: "@rollup/plugin-node-resolve@npm:15.2.3" dependencies: "@rollup/pluginutils": ^5.0.1 "@types/resolve": 1.20.2 @@ -6904,11 +6904,11 @@ __metadata: is-module: ^1.0.0 resolve: ^1.22.1 peerDependencies: - rollup: ^2.78.0||^3.0.0 + rollup: ^2.78.0||^3.0.0||^4.0.0 peerDependenciesMeta: rollup: optional: true - checksum: 83617cdbb90cb780251e8b16dc1671e35bde90b8d4d30e008aefe706b5b643057f6299bdd3226b2a30bf5e4f807a880169de3faa47b9f2ba38d39f294f85f951 + checksum: 730f32c2f8fdddff07cf0fca86a5dac7c475605fb96930197a868c066e62eb6388c557545e4f7d99b7a283411754c9fbf98944ab086b6074e04fc1292e234aa8 languageName: node linkType: hard From 8d9ef716fef86b4bb2a27d61112c4df955e532aa Mon Sep 17 00:00:00 2001 From: Josh Hunt Date: Wed, 11 Oct 2023 09:26:39 +0000 Subject: [PATCH 034/102] I18N: Collect stats on number of untranslated strings in message catalogues (#76272) * I18N: Collect stats on number of untranslated strings in message catalogues * eachLeaf -> eachMessage * use __dirname --- package.json | 1 + scripts/ci-frontend-metrics.sh | 8 +++ scripts/cli/reportI18nStats.mjs | 88 +++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 scripts/cli/reportI18nStats.mjs diff --git a/package.json b/package.json index cc695fb4f66..39cb8f06f81 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "i18n:extract": "yarn run i18next -c public/locales/i18next-parser.config.js 'public/**/*.{tsx,ts}' 'packages/grafana-ui/**/*.{tsx,ts}' && yarn i18n:pseudo", "i18n:compile": "echo 'no i18n compile yet, all good'", "i18n:pseudo": "node ./public/locales/pseudo.js", + "i18n:stats": "node ./scripts/cli/reportI18nStats.mjs", "betterer": "betterer", "betterer:merge": "betterer merge", "betterer:stats": "ts-node --transpile-only --project ./scripts/cli/tsconfig.json ./scripts/cli/reportBettererStats.ts", diff --git a/scripts/ci-frontend-metrics.sh b/scripts/ci-frontend-metrics.sh index 1817697ec16..7fec6a1c3fc 100755 --- a/scripts/ci-frontend-metrics.sh +++ b/scripts/ci-frontend-metrics.sh @@ -46,6 +46,13 @@ do BETTERER_STATS+="\"grafana.ci-code.betterer.${name}\": \"${value}\"," done <<< "$(yarn betterer:stats)" +I18N_STATS="" +while read -r name value +do + I18N_STATS+=$'\n ' + I18N_STATS+="\"grafana.ci-code.i18n.${name}\": \"${value}\"," +done <<< "$(yarn i18n:stats)" + THEME_TOKEN_USAGE="" while read -r name value do @@ -56,6 +63,7 @@ done <<< "$(yarn themes:usage | awk '$4 == "@grafana/theme-token-usage" {print $ echo "Metrics: { $THEME_TOKEN_USAGE $BETTERER_STATS + $I18N_STATS \"grafana.ci-code.strictErrors\": \"${ERROR_COUNT}\", \"grafana.ci-code.accessibilityErrors\": \"${ACCESSIBILITY_ERRORS}\", \"grafana.ci-code.directives\": \"${DIRECTIVES}\", diff --git a/scripts/cli/reportI18nStats.mjs b/scripts/cli/reportI18nStats.mjs new file mode 100644 index 00000000000..b34b5074ea6 --- /dev/null +++ b/scripts/cli/reportI18nStats.mjs @@ -0,0 +1,88 @@ +/// @ts-check + +import { readdir, stat, readFile } from 'fs/promises'; +import path, { dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const LOCALES_DIR = path.resolve(dirname(fileURLToPath(import.meta.url)), '..', '..', 'public', 'locales'); + +const locales = await readdir(LOCALES_DIR); + +/** + * @type {Array<{ language: string, untranslatedCount: number, translatedCount: number }>} + */ +const stats = []; + +for (const fileName of locales) { + const filePath = path.join(LOCALES_DIR, fileName, 'grafana.json'); + if (!(await exists(filePath))) { + continue; + } + + const messages = await readFile(filePath); + const parsedMessages = JSON.parse(messages.toString()); + + let translatedCount = 0; + let untranslatedCount = 0; + + eachMessage(parsedMessages, (value) => { + if (value === '') { + untranslatedCount += 1; + } else { + translatedCount += 1; + } + }); + + stats.push({ + language: fileName, + translatedCount, + untranslatedCount, + }); +} + +for (const stat of stats) { + logStat(`untranslated.${stat.language}`, stat.untranslatedCount); + logStat(`translated.${stat.language}`, stat.translatedCount); +} + +/** + * @param {string} filePath + */ +async function exists(filePath) { + try { + await stat(filePath); + return true; + } catch (err) { + if (err.code === 'ENOENT' || err.code === 'ENOTDIR') { + return false; + } + throw err; + } +} + +/** + * @param {unknown} value + * @param {(v: string) => void} callback + */ +function eachMessage(value, callback) { + if (typeof value === 'object') { + for (const key in value) { + const element = value[key]; + eachMessage(element, callback); + } + } else if (typeof value === 'string') { + callback(value); + } else { + throw new Error(`Unknown value ${value} in eachMessage`); + } +} + +/** + * @param {string} name + * @param {string | number} value + */ +function logStat(name, value) { + // Note that this output format must match the parsing in ci-frontend-metrics.sh + // which expects the two values to be separated by a space + console.log(`${name} ${value}`); +} From 9035a498ca6d6aeaf25568d23fe8adf0294a88cb Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Wed, 11 Oct 2023 11:44:36 +0200 Subject: [PATCH 035/102] Chore: Export closestIdx function from @grafana/data (#76176) * export calculateFieldDisplayName function * export closestIdx function * Use getFieldDisplayName --- packages/grafana-data/src/dataframe/index.ts | 2 +- public/app/features/live/data/amendTimeSeries.ts | 2 +- .../app/plugins/datasource/prometheus/result_transformer.ts | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/grafana-data/src/dataframe/index.ts b/packages/grafana-data/src/dataframe/index.ts index 07217a3ce12..b1362fc3e38 100644 --- a/packages/grafana-data/src/dataframe/index.ts +++ b/packages/grafana-data/src/dataframe/index.ts @@ -8,4 +8,4 @@ export * from './ArrayDataFrame'; export * from './DataFrameJSON'; export * from './frameComparisons'; export { anySeriesWithTimeField, isTimeSeriesFrame, isTimeSeriesFrames } from './utils'; -export { StreamingDataFrame, StreamingFrameAction, type StreamingFrameOptions } from './StreamingDataFrame'; +export { StreamingDataFrame, StreamingFrameAction, type StreamingFrameOptions, closestIdx } from './StreamingDataFrame'; diff --git a/public/app/features/live/data/amendTimeSeries.ts b/public/app/features/live/data/amendTimeSeries.ts index cafc0714ec5..ead006ef89b 100644 --- a/public/app/features/live/data/amendTimeSeries.ts +++ b/public/app/features/live/data/amendTimeSeries.ts @@ -1,4 +1,4 @@ -import { closestIdx } from '@grafana/data/src/dataframe/StreamingDataFrame'; +import { closestIdx } from '@grafana/data'; export type Table = [times: number[], ...values: any[][]]; diff --git a/public/app/plugins/datasource/prometheus/result_transformer.ts b/public/app/plugins/datasource/prometheus/result_transformer.ts index 94ce874ba23..59308d9208c 100644 --- a/public/app/plugins/datasource/prometheus/result_transformer.ts +++ b/public/app/plugins/datasource/prometheus/result_transformer.ts @@ -14,13 +14,13 @@ import { FieldType, formatLabels, getDisplayProcessor, + getFieldDisplayName, Labels, + renderLegendFormat, ScopedVars, TIME_SERIES_TIME_FIELD_NAME, TIME_SERIES_VALUE_FIELD_NAME, - renderLegendFormat, } from '@grafana/data'; -import { calculateFieldDisplayName } from '@grafana/data/src/field/fieldState'; import { config, FetchResponse, getDataSourceSrv, getTemplateSrv } from '@grafana/runtime'; import { @@ -79,7 +79,7 @@ export function transformV2( f.fields.forEach((field) => { if (field.labels?.__name__ && field.labels?.__name__ === field.name) { const fieldCopy = { ...field, name: TIME_SERIES_VALUE_FIELD_NAME }; - field.config.displayNameFromDS = calculateFieldDisplayName(fieldCopy, f, response.data); + field.config.displayNameFromDS = getFieldDisplayName(fieldCopy, f, response.data); } }); } From 58ba11ecbd60863e2bfc6e32f0399ed4feb3927a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 11:18:30 +0100 Subject: [PATCH 036/102] Update dependency @testing-library/user-event to v14.5.1 (#76329) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 2 +- packages/grafana-data/package.json | 2 +- packages/grafana-flamegraph/package.json | 2 +- packages/grafana-runtime/package.json | 2 +- packages/grafana-ui/package.json | 2 +- yarn.lock | 18 +++++++++--------- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 39cb8f06f81..d4ffd95e9f3 100644 --- a/package.json +++ b/package.json @@ -93,7 +93,7 @@ "@testing-library/dom": "9.3.1", "@testing-library/jest-dom": "5.16.5", "@testing-library/react": "14.0.0", - "@testing-library/user-event": "14.4.3", + "@testing-library/user-event": "14.5.1", "@types/angular": "1.8.5", "@types/angular-route": "1.7.3", "@types/chance": "^1.1.3", diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index f74e8fa6f1f..5aac56b3acc 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -69,7 +69,7 @@ "@testing-library/dom": "9.3.1", "@testing-library/jest-dom": "5.16.5", "@testing-library/react": "14.0.0", - "@testing-library/user-event": "14.4.3", + "@testing-library/user-event": "14.5.1", "@types/dompurify": "^2", "@types/history": "4.7.11", "@types/jest": "29.5.4", diff --git a/packages/grafana-flamegraph/package.json b/packages/grafana-flamegraph/package.json index a2bbffd9804..74da889295b 100644 --- a/packages/grafana-flamegraph/package.json +++ b/packages/grafana-flamegraph/package.json @@ -63,7 +63,7 @@ "@rollup/plugin-node-resolve": "15.2.3", "@testing-library/jest-dom": "^6.1.2", "@testing-library/react": "14.0.0", - "@testing-library/user-event": "14.4.3", + "@testing-library/user-event": "14.5.1", "@types/d3": "^7", "@types/jest": "^29.5.4", "@types/lodash": "4.14.195", diff --git a/packages/grafana-runtime/package.json b/packages/grafana-runtime/package.json index f4eefc9de58..e1735a53209 100644 --- a/packages/grafana-runtime/package.json +++ b/packages/grafana-runtime/package.json @@ -54,7 +54,7 @@ "@rollup/plugin-node-resolve": "15.2.3", "@testing-library/dom": "9.3.1", "@testing-library/react": "14.0.0", - "@testing-library/user-event": "14.4.3", + "@testing-library/user-event": "14.5.1", "@types/angular": "1.8.5", "@types/history": "4.7.11", "@types/jest": "29.5.4", diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index 8268fa39811..4c768d327a2 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -137,7 +137,7 @@ "@testing-library/dom": "9.3.1", "@testing-library/jest-dom": "5.16.5", "@testing-library/react": "14.0.0", - "@testing-library/user-event": "14.4.3", + "@testing-library/user-event": "14.5.1", "@types/common-tags": "^1.8.0", "@types/d3": "7.4.0", "@types/hoist-non-react-statics": "3.3.1", diff --git a/yarn.lock b/yarn.lock index 8f332c58ebb..ffcd198a749 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2995,7 +2995,7 @@ __metadata: "@testing-library/dom": 9.3.1 "@testing-library/jest-dom": 5.16.5 "@testing-library/react": 14.0.0 - "@testing-library/user-event": 14.4.3 + "@testing-library/user-event": 14.5.1 "@types/d3-interpolate": ^3.0.0 "@types/dompurify": ^2 "@types/history": 4.7.11 @@ -3230,7 +3230,7 @@ __metadata: "@rollup/plugin-node-resolve": 15.2.3 "@testing-library/jest-dom": ^6.1.2 "@testing-library/react": 14.0.0 - "@testing-library/user-event": 14.4.3 + "@testing-library/user-event": 14.5.1 "@types/d3": ^7 "@types/jest": ^29.5.4 "@types/lodash": 4.14.195 @@ -3312,7 +3312,7 @@ __metadata: "@rollup/plugin-node-resolve": 15.2.3 "@testing-library/dom": 9.3.1 "@testing-library/react": 14.0.0 - "@testing-library/user-event": 14.4.3 + "@testing-library/user-event": 14.5.1 "@types/angular": 1.8.5 "@types/history": 4.7.11 "@types/jest": 29.5.4 @@ -3437,7 +3437,7 @@ __metadata: "@testing-library/dom": 9.3.1 "@testing-library/jest-dom": 5.16.5 "@testing-library/react": 14.0.0 - "@testing-library/user-event": 14.4.3 + "@testing-library/user-event": 14.5.1 "@types/common-tags": ^1.8.0 "@types/d3": 7.4.0 "@types/hoist-non-react-statics": 3.3.1 @@ -8484,12 +8484,12 @@ __metadata: languageName: node linkType: hard -"@testing-library/user-event@npm:14.4.3": - version: 14.4.3 - resolution: "@testing-library/user-event@npm:14.4.3" +"@testing-library/user-event@npm:14.5.1": + version: 14.5.1 + resolution: "@testing-library/user-event@npm:14.5.1" peerDependencies: "@testing-library/dom": ">=7.21.4" - checksum: 852c48ea6db1c9471b18276617c84fec4320771e466cd58339a732ca3fd73ad35e5a43ae14f51af51a8d0a150dcf60fcaab049ef367871207bea8f92c4b8195e + checksum: 3e6bc9fd53dfe2f3648190193ed2fd4bca2a1bfb47f68810df3b33f05412526e5fd5c4ef9dc5375635e0f4cdf1859916867b597eed22bda1321e04242ea6c519 languageName: node linkType: hard @@ -17688,7 +17688,7 @@ __metadata: "@testing-library/jest-dom": 5.16.5 "@testing-library/react": 14.0.0 "@testing-library/react-hooks": ^8.0.1 - "@testing-library/user-event": 14.4.3 + "@testing-library/user-event": 14.5.1 "@types/angular": 1.8.5 "@types/angular-route": 1.7.3 "@types/chance": ^1.1.3 From abd2c9c287630e8aebdec94714af3ce97f1fab11 Mon Sep 17 00:00:00 2001 From: Ashley Harrison Date: Wed, 11 Oct 2023 11:33:15 +0100 Subject: [PATCH 037/102] Navigation: Hide docked menu and dock button on mobile (#76334) * hide docked menu and dock button on mobile * add logic to handle hamburger menu button on mobile --- .../core/components/AppChrome/AppChrome.tsx | 27 +++++++++++++++++-- .../AppChrome/DockedMegaMenu/MegaMenu.tsx | 5 ++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/public/app/core/components/AppChrome/AppChrome.tsx b/public/app/core/components/AppChrome/AppChrome.tsx index ae02e558f86..4dcc1c114db 100644 --- a/public/app/core/components/AppChrome/AppChrome.tsx +++ b/public/app/core/components/AppChrome/AppChrome.tsx @@ -3,7 +3,7 @@ import classNames from 'classnames'; import React, { PropsWithChildren } from 'react'; import { GrafanaTheme2, PageLayoutType } from '@grafana/data'; -import { useStyles2, LinkButton } from '@grafana/ui'; +import { useStyles2, LinkButton, useTheme2 } from '@grafana/ui'; import config from 'app/core/config'; import { useGrafana } from 'app/core/context/GrafanaContext'; import { CommandPalette } from 'app/features/commandPalette/CommandPalette'; @@ -23,6 +23,7 @@ export function AppChrome({ children }: Props) { const { chrome } = useGrafana(); const state = chrome.useState(); const searchBarHidden = state.searchBarHidden || state.kioskMode === KioskMode.TV; + const theme = useTheme2(); const styles = useStyles2(getStyles); const contentClass = cx({ @@ -31,6 +32,23 @@ export function AppChrome({ children }: Props) { [styles.contentChromeless]: state.chromeless, }); + const handleMegaMenu = () => { + switch (state.megaMenu) { + case 'closed': + chrome.setMegaMenu('open'); + break; + case 'open': + chrome.setMegaMenu('closed'); + break; + case 'docked': + // on desktop, clicking the button when the menu is docked should close the menu + // on mobile, the docked menu is hidden, so clicking the button should open the menu + const isDesktop = window.innerWidth > theme.breakpoints.values.md; + isDesktop ? chrome.setMegaMenu('closed') : chrome.setMegaMenu('open'); + break; + } + }; + // Chromeless routes are without topNav, mega menu, search & command palette // We check chromeless twice here instead of having a separate path so {children} // doesn't get re-mounted when chromeless goes from true to false. @@ -54,7 +72,7 @@ export function AppChrome({ children }: Props) { pageNav={state.pageNav} actions={state.actions} onToggleSearchBar={chrome.onToggleSearchBar} - onToggleMegaMenu={() => chrome.setMegaMenu(state.megaMenu === 'closed' ? 'open' : 'closed')} + onToggleMegaMenu={handleMegaMenu} onToggleKioskMode={chrome.onToggleKioskMode} />
    @@ -110,7 +128,12 @@ const getStyles = (theme: GrafanaTheme2) => { background: theme.colors.background.primary, borderRight: `1px solid ${theme.colors.border.weak}`, borderTop: `1px solid ${theme.colors.border.weak}`, + display: 'none', zIndex: theme.zIndex.navbarFixed, + + [theme.breakpoints.up('md')]: { + display: 'block', + }, }), topNav: css({ display: 'flex', diff --git a/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.tsx b/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.tsx index d4459cb11ad..0c7b5ce2f07 100644 --- a/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.tsx +++ b/public/app/core/components/AppChrome/DockedMegaMenu/MegaMenu.tsx @@ -116,6 +116,11 @@ const getStyles = (theme: GrafanaTheme2) => ({ }, }), dockMenuButton: css({ + display: 'none', marginRight: theme.spacing(2), + + [theme.breakpoints.up('md')]: { + display: 'inline-flex', + }, }), }); From 50504ba00803a291ea24e8a4a48b14494b97c744 Mon Sep 17 00:00:00 2001 From: Giuseppe Guerra Date: Wed, 11 Oct 2023 12:42:32 +0200 Subject: [PATCH 038/102] Plugins: Chore: Renamed instrumentation middleware to metrics middleware (#76186) * Plugins: Chore: Renamed instrumentation middleware to metrics middleware * Removed repeated logger attributes in middleware and contextual logger * renamed loggerParams to logParams * PR review suggestion * Add contextual logger middleware * Removed unused params from logRequest * Removed unwanted changes * Safer FromContext method * Removed traceID from logParams --- pkg/plugins/log/fake.go | 6 ++ pkg/plugins/log/ifaces.go | 35 ++++++---- pkg/plugins/log/logger.go | 12 ++++ .../contextual_logger_middleware.go | 69 +++++++++++++++++++ .../clientmiddleware/logger_middleware.go | 33 +++------ ...on_middleware.go => metrics_middleware.go} | 53 +++++--------- ...are_test.go => metrics_middleware_test.go} | 2 +- .../pluginsintegration/pluginsintegration.go | 3 +- 8 files changed, 136 insertions(+), 77 deletions(-) create mode 100644 pkg/services/pluginsintegration/clientmiddleware/contextual_logger_middleware.go rename pkg/services/pluginsintegration/clientmiddleware/{instrumentation_middleware.go => metrics_middleware.go} (67%) rename pkg/services/pluginsintegration/clientmiddleware/{instrumentation_middleware_test.go => metrics_middleware_test.go} (98%) diff --git a/pkg/plugins/log/fake.go b/pkg/plugins/log/fake.go index 41af06d21e8..2a4f0163ac2 100644 --- a/pkg/plugins/log/fake.go +++ b/pkg/plugins/log/fake.go @@ -1,5 +1,7 @@ package log +import "context" + var _ Logger = (*TestLogger)(nil) type TestLogger struct { @@ -41,6 +43,10 @@ func (f *TestLogger) Error(msg string, ctx ...any) { f.ErrorLogs.Ctx = ctx } +func (f *TestLogger) FromContext(_ context.Context) Logger { + return NewTestLogger() +} + type Logs struct { Calls int Message string diff --git a/pkg/plugins/log/ifaces.go b/pkg/plugins/log/ifaces.go index 2d3033194d3..14e603b97e4 100644 --- a/pkg/plugins/log/ifaces.go +++ b/pkg/plugins/log/ifaces.go @@ -1,35 +1,40 @@ package log +import "context" + // Logger is the default logger type Logger interface { // New returns a new contextual Logger that has this logger's context plus the given context. - New(ctx ...interface{}) Logger + New(ctx ...any) Logger // Debug logs a message with debug level and key/value pairs, if any. - Debug(msg string, ctx ...interface{}) + Debug(msg string, ctx ...any) // Info logs a message with info level and key/value pairs, if any. - Info(msg string, ctx ...interface{}) + Info(msg string, ctx ...any) // Warn logs a message with warning level and key/value pairs, if any. - Warn(msg string, ctx ...interface{}) + Warn(msg string, ctx ...any) // Error logs a message with error level and key/value pairs, if any. - Error(msg string, ctx ...interface{}) + Error(msg string, ctx ...any) + + // FromContext returns a new contextual Logger that has this logger's context plus the given context. + FromContext(ctx context.Context) Logger } // PrettyLogger is used primarily to facilitate logging/user feedback for both // the grafana-cli and the grafana backend when managing plugin installs type PrettyLogger interface { - Successf(format string, args ...interface{}) - Failuref(format string, args ...interface{}) + Successf(format string, args ...any) + Failuref(format string, args ...any) - Info(args ...interface{}) - Infof(format string, args ...interface{}) - Debug(args ...interface{}) - Debugf(format string, args ...interface{}) - Warn(args ...interface{}) - Warnf(format string, args ...interface{}) - Error(args ...interface{}) - Errorf(format string, args ...interface{}) + Info(args ...any) + Infof(format string, args ...any) + Debug(args ...any) + Debugf(format string, args ...any) + Warn(args ...any) + Warnf(format string, args ...any) + Error(args ...any) + Errorf(format string, args ...any) } diff --git a/pkg/plugins/log/logger.go b/pkg/plugins/log/logger.go index d0fbd855303..58ac225cf64 100644 --- a/pkg/plugins/log/logger.go +++ b/pkg/plugins/log/logger.go @@ -1,6 +1,8 @@ package log import ( + "context" + "github.com/grafana/grafana/pkg/infra/log" ) @@ -42,3 +44,13 @@ func (d *grafanaInfraLogWrapper) Warn(msg string, ctx ...any) { func (d *grafanaInfraLogWrapper) Error(msg string, ctx ...any) { d.l.Error(msg, ctx...) } + +func (d *grafanaInfraLogWrapper) FromContext(ctx context.Context) Logger { + concreteInfraLogger, ok := d.l.FromContext(ctx).(*log.ConcreteLogger) + if !ok { + return d.New() + } + return &grafanaInfraLogWrapper{ + l: concreteInfraLogger, + } +} diff --git a/pkg/services/pluginsintegration/clientmiddleware/contextual_logger_middleware.go b/pkg/services/pluginsintegration/clientmiddleware/contextual_logger_middleware.go new file mode 100644 index 00000000000..cf97590294b --- /dev/null +++ b/pkg/services/pluginsintegration/clientmiddleware/contextual_logger_middleware.go @@ -0,0 +1,69 @@ +package clientmiddleware + +import ( + "context" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/plugins" +) + +// NewContextualLoggerMiddleware creates a new plugins.ClientMiddleware that adds +// a contextual logger to the request context. +func NewContextualLoggerMiddleware() plugins.ClientMiddleware { + return plugins.ClientMiddlewareFunc(func(next plugins.Client) plugins.Client { + return &ContextualLoggerMiddleware{ + next: next, + } + }) +} + +type ContextualLoggerMiddleware struct { + next plugins.Client +} + +// instrumentContext adds a contextual logger with plugin and request details to the given context. +func instrumentContext(ctx context.Context, endpoint string, pCtx backend.PluginContext) context.Context { + p := []any{"endpoint", endpoint, "pluginId", pCtx.PluginID} + if pCtx.DataSourceInstanceSettings != nil { + p = append(p, "dsName", pCtx.DataSourceInstanceSettings.Name) + p = append(p, "dsUID", pCtx.DataSourceInstanceSettings.UID) + } + if pCtx.User != nil { + p = append(p, "uname", pCtx.User.Login) + } + return log.WithContextualAttributes(ctx, p) +} + +func (m *ContextualLoggerMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { + ctx = instrumentContext(ctx, endpointQueryData, req.PluginContext) + return m.next.QueryData(ctx, req) +} + +func (m *ContextualLoggerMiddleware) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { + ctx = instrumentContext(ctx, endpointCallResource, req.PluginContext) + return m.next.CallResource(ctx, req, sender) +} + +func (m *ContextualLoggerMiddleware) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { + ctx = instrumentContext(ctx, endpointCheckHealth, req.PluginContext) + return m.next.CheckHealth(ctx, req) +} + +func (m *ContextualLoggerMiddleware) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) { + ctx = instrumentContext(ctx, endpointCollectMetrics, req.PluginContext) + return m.next.CollectMetrics(ctx, req) +} + +func (m *ContextualLoggerMiddleware) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) { + return m.next.SubscribeStream(ctx, req) +} + +func (m *ContextualLoggerMiddleware) PublishStream(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) { + return m.next.PublishStream(ctx, req) +} + +func (m *ContextualLoggerMiddleware) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error { + return m.next.RunStream(ctx, req, sender) +} diff --git a/pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go b/pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go index 716df10bb8e..8230997e236 100644 --- a/pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go +++ b/pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go @@ -6,8 +6,8 @@ import ( "time" "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/plugins" plog "github.com/grafana/grafana/pkg/plugins/log" "github.com/grafana/grafana/pkg/setting" @@ -33,10 +33,11 @@ type LoggerMiddleware struct { logger plog.Logger } -func (m *LoggerMiddleware) logRequest(ctx context.Context, pluginCtx backend.PluginContext, endpoint string, fn func(ctx context.Context) error) error { +func (m *LoggerMiddleware) logRequest(ctx context.Context, fn func(ctx context.Context) error) error { status := statusOK start := time.Now() timeBeforePluginRequest := log.TimeSinceStart(ctx, start) + err := fn(ctx) if err != nil { status = statusError @@ -48,31 +49,13 @@ func (m *LoggerMiddleware) logRequest(ctx context.Context, pluginCtx backend.Plu logParams := []any{ "status", status, "duration", time.Since(start), - "pluginId", pluginCtx.PluginID, - "endpoint", endpoint, "eventName", "grafana-data-egress", "time_before_plugin_request", timeBeforePluginRequest, } - - if pluginCtx.User != nil { - logParams = append(logParams, "uname", pluginCtx.User.Login) - } - - traceID := tracing.TraceIDFromContext(ctx, false) - if traceID != "" { - logParams = append(logParams, "traceID", traceID) - } - - if pluginCtx.DataSourceInstanceSettings != nil { - logParams = append(logParams, "dsName", pluginCtx.DataSourceInstanceSettings.Name) - logParams = append(logParams, "dsUID", pluginCtx.DataSourceInstanceSettings.UID) - } - if status == statusError { logParams = append(logParams, "error", err) } - - m.logger.Info("Plugin Request Completed", logParams...) + m.logger.FromContext(ctx).Info("Plugin Request Completed", logParams...) return err } @@ -82,7 +65,7 @@ func (m *LoggerMiddleware) QueryData(ctx context.Context, req *backend.QueryData } var resp *backend.QueryDataResponse - err := m.logRequest(ctx, req.PluginContext, endpointQueryData, func(ctx context.Context) (innerErr error) { + err := m.logRequest(ctx, func(ctx context.Context) (innerErr error) { resp, innerErr = m.next.QueryData(ctx, req) return innerErr }) @@ -95,7 +78,7 @@ func (m *LoggerMiddleware) CallResource(ctx context.Context, req *backend.CallRe return m.next.CallResource(ctx, req, sender) } - err := m.logRequest(ctx, req.PluginContext, endpointCallResource, func(ctx context.Context) (innerErr error) { + err := m.logRequest(ctx, func(ctx context.Context) (innerErr error) { innerErr = m.next.CallResource(ctx, req, sender) return innerErr }) @@ -109,7 +92,7 @@ func (m *LoggerMiddleware) CheckHealth(ctx context.Context, req *backend.CheckHe } var resp *backend.CheckHealthResult - err := m.logRequest(ctx, req.PluginContext, endpointCheckHealth, func(ctx context.Context) (innerErr error) { + err := m.logRequest(ctx, func(ctx context.Context) (innerErr error) { resp, innerErr = m.next.CheckHealth(ctx, req) return innerErr }) @@ -123,7 +106,7 @@ func (m *LoggerMiddleware) CollectMetrics(ctx context.Context, req *backend.Coll } var resp *backend.CollectMetricsResult - err := m.logRequest(ctx, req.PluginContext, endpointCollectMetrics, func(ctx context.Context) (innerErr error) { + err := m.logRequest(ctx, func(ctx context.Context) (innerErr error) { resp, innerErr = m.next.CollectMetrics(ctx, req) return innerErr }) diff --git a/pkg/services/pluginsintegration/clientmiddleware/instrumentation_middleware.go b/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware.go similarity index 67% rename from pkg/services/pluginsintegration/clientmiddleware/instrumentation_middleware.go rename to pkg/services/pluginsintegration/clientmiddleware/metrics_middleware.go index ddbaaeaa3a1..fa765b80053 100644 --- a/pkg/services/pluginsintegration/clientmiddleware/instrumentation_middleware.go +++ b/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware.go @@ -8,13 +8,12 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/prometheus/client_golang/prometheus" - "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins/manager/registry" ) -// pluginMetrics contains the prometheus metrics used by the InstrumentationMiddleware. +// pluginMetrics contains the prometheus metrics used by the MetricsMiddleware. type pluginMetrics struct { pluginRequestCounter *prometheus.CounterVec pluginRequestDuration *prometheus.HistogramVec @@ -22,17 +21,15 @@ type pluginMetrics struct { pluginRequestDurationSeconds *prometheus.HistogramVec } -// InstrumentationMiddleware is a middleware that instruments plugin requests. +// MetricsMiddleware is a middleware that instruments plugin requests. // It tracks requests count, duration and size as prometheus metrics. -// It also enriches the [context.Context] with a contextual logger containing plugin and request details. -// For those reasons, this middleware should live at the top of the middleware stack. -type InstrumentationMiddleware struct { +type MetricsMiddleware struct { pluginMetrics pluginRegistry registry.Service next plugins.Client } -func newInstrumentationMiddleware(promRegisterer prometheus.Registerer, pluginRegistry registry.Service) *InstrumentationMiddleware { +func newMetricsMiddleware(promRegisterer prometheus.Registerer, pluginRegistry registry.Service) *MetricsMiddleware { pluginRequestCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "grafana", Name: "plugin_request_total", @@ -64,7 +61,7 @@ func newInstrumentationMiddleware(promRegisterer prometheus.Registerer, pluginRe pluginRequestSize, pluginRequestDurationSeconds, ) - return &InstrumentationMiddleware{ + return &MetricsMiddleware{ pluginMetrics: pluginMetrics{ pluginRequestCounter: pluginRequestCounter, pluginRequestDuration: pluginRequestDuration, @@ -75,9 +72,9 @@ func newInstrumentationMiddleware(promRegisterer prometheus.Registerer, pluginRe } } -// NewInstrumentationMiddleware returns a new InstrumentationMiddleware. -func NewInstrumentationMiddleware(promRegisterer prometheus.Registerer, pluginRegistry registry.Service) plugins.ClientMiddleware { - imw := newInstrumentationMiddleware(promRegisterer, pluginRegistry) +// NewMetricsMiddleware returns a new MetricsMiddleware. +func NewMetricsMiddleware(promRegisterer prometheus.Registerer, pluginRegistry registry.Service) plugins.ClientMiddleware { + imw := newMetricsMiddleware(promRegisterer, pluginRegistry) return plugins.ClientMiddlewareFunc(func(next plugins.Client) plugins.Client { imw.next = next return imw @@ -85,7 +82,7 @@ func NewInstrumentationMiddleware(promRegisterer prometheus.Registerer, pluginRe } // pluginTarget returns the value for the "target" Prometheus label for the given plugin ID. -func (m *InstrumentationMiddleware) pluginTarget(ctx context.Context, pluginID string) (string, error) { +func (m *MetricsMiddleware) pluginTarget(ctx context.Context, pluginID string) (string, error) { p, exists := m.pluginRegistry.Plugin(ctx, pluginID) if !exists { return "", plugins.ErrPluginNotRegistered @@ -93,21 +90,8 @@ func (m *InstrumentationMiddleware) pluginTarget(ctx context.Context, pluginID s return string(p.Target()), nil } -// instrumentContext adds a contextual logger with plugin and request details to the given context. -func instrumentContext(ctx context.Context, endpoint string, pCtx backend.PluginContext) context.Context { - p := []any{"endpoint", endpoint, "pluginId", pCtx.PluginID} - if pCtx.DataSourceInstanceSettings != nil { - p = append(p, "dsName", pCtx.DataSourceInstanceSettings.Name) - p = append(p, "dsUID", pCtx.DataSourceInstanceSettings.UID) - } - if pCtx.User != nil { - p = append(p, "uname", pCtx.User.Login) - } - return log.WithContextualAttributes(ctx, p) -} - // instrumentPluginRequestSize tracks the size of the given request in the m.pluginRequestSize metric. -func (m *InstrumentationMiddleware) instrumentPluginRequestSize(ctx context.Context, pluginCtx backend.PluginContext, endpoint string, requestSize float64) error { +func (m *MetricsMiddleware) instrumentPluginRequestSize(ctx context.Context, pluginCtx backend.PluginContext, endpoint string, requestSize float64) error { target, err := m.pluginTarget(ctx, pluginCtx.PluginID) if err != nil { return err @@ -117,7 +101,7 @@ func (m *InstrumentationMiddleware) instrumentPluginRequestSize(ctx context.Cont } // instrumentPluginRequest increments the m.pluginRequestCounter metric and tracks the duration of the given request. -func (m *InstrumentationMiddleware) instrumentPluginRequest(ctx context.Context, pluginCtx backend.PluginContext, endpoint string, fn func(context.Context) error) error { +func (m *MetricsMiddleware) instrumentPluginRequest(ctx context.Context, pluginCtx backend.PluginContext, endpoint string, fn func(context.Context) error) error { target, err := m.pluginTarget(ctx, pluginCtx.PluginID) if err != nil { return err @@ -126,7 +110,6 @@ func (m *InstrumentationMiddleware) instrumentPluginRequest(ctx context.Context, status := statusOK start := time.Now() - ctx = instrumentContext(ctx, endpoint, pluginCtx) err = fn(ctx) if err != nil { status = statusError @@ -158,7 +141,7 @@ func (m *InstrumentationMiddleware) instrumentPluginRequest(ctx context.Context, return err } -func (m *InstrumentationMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { +func (m *MetricsMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { var requestSize float64 for _, v := range req.Queries { requestSize += float64(len(v.JSON)) @@ -174,7 +157,7 @@ func (m *InstrumentationMiddleware) QueryData(ctx context.Context, req *backend. return resp, err } -func (m *InstrumentationMiddleware) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { +func (m *MetricsMiddleware) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { if err := m.instrumentPluginRequestSize(ctx, req.PluginContext, endpointCallResource, float64(len(req.Body))); err != nil { return err } @@ -183,7 +166,7 @@ func (m *InstrumentationMiddleware) CallResource(ctx context.Context, req *backe }) } -func (m *InstrumentationMiddleware) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { +func (m *MetricsMiddleware) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { var result *backend.CheckHealthResult err := m.instrumentPluginRequest(ctx, req.PluginContext, endpointCheckHealth, func(ctx context.Context) (innerErr error) { result, innerErr = m.next.CheckHealth(ctx, req) @@ -192,7 +175,7 @@ func (m *InstrumentationMiddleware) CheckHealth(ctx context.Context, req *backen return result, err } -func (m *InstrumentationMiddleware) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) { +func (m *MetricsMiddleware) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) { var result *backend.CollectMetricsResult err := m.instrumentPluginRequest(ctx, req.PluginContext, endpointCollectMetrics, func(ctx context.Context) (innerErr error) { result, innerErr = m.next.CollectMetrics(ctx, req) @@ -201,14 +184,14 @@ func (m *InstrumentationMiddleware) CollectMetrics(ctx context.Context, req *bac return result, err } -func (m *InstrumentationMiddleware) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) { +func (m *MetricsMiddleware) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) { return m.next.SubscribeStream(ctx, req) } -func (m *InstrumentationMiddleware) PublishStream(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) { +func (m *MetricsMiddleware) PublishStream(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) { return m.next.PublishStream(ctx, req) } -func (m *InstrumentationMiddleware) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error { +func (m *MetricsMiddleware) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error { return m.next.RunStream(ctx, req, sender) } diff --git a/pkg/services/pluginsintegration/clientmiddleware/instrumentation_middleware_test.go b/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go similarity index 98% rename from pkg/services/pluginsintegration/clientmiddleware/instrumentation_middleware_test.go rename to pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go index 38dfea37df2..f9321fb22e3 100644 --- a/pkg/services/pluginsintegration/clientmiddleware/instrumentation_middleware_test.go +++ b/pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go @@ -75,7 +75,7 @@ func TestInstrumentationMiddleware(t *testing.T) { JSONData: plugins.JSONData{ID: pluginID, Backend: true}, })) - mw := newInstrumentationMiddleware(promRegistry, pluginsRegistry) + mw := newMetricsMiddleware(promRegistry, pluginsRegistry) cdt := clienttest.NewClientDecoratorTest(t, clienttest.WithMiddlewares( plugins.ClientMiddlewareFunc(func(next plugins.Client) plugins.Client { mw.next = next diff --git a/pkg/services/pluginsintegration/pluginsintegration.go b/pkg/services/pluginsintegration/pluginsintegration.go index cc09121be9f..7114142b66b 100644 --- a/pkg/services/pluginsintegration/pluginsintegration.go +++ b/pkg/services/pluginsintegration/pluginsintegration.go @@ -156,7 +156,8 @@ func CreateMiddlewares(cfg *setting.Cfg, oAuthTokenService oauthtoken.OAuthToken skipCookiesNames := []string{cfg.LoginCookieName} middlewares := []plugins.ClientMiddleware{ clientmiddleware.NewTracingMiddleware(tracer), - clientmiddleware.NewInstrumentationMiddleware(promRegisterer, registry), + clientmiddleware.NewMetricsMiddleware(promRegisterer, registry), + clientmiddleware.NewContextualLoggerMiddleware(), clientmiddleware.NewLoggerMiddleware(cfg, log.New("plugin.instrumentation")), clientmiddleware.NewTracingHeaderMiddleware(), clientmiddleware.NewClearAuthHeadersMiddleware(), From 5f6fea33f293ac57625305afa74027654716f452 Mon Sep 17 00:00:00 2001 From: Joao Silva <100691367+JoaoSilvaGrafana@users.noreply.github.com> Date: Wed, 11 Oct 2023 11:43:25 +0100 Subject: [PATCH 039/102] Chore: Improve typings in OptionsUI and Explore (#76245) --- .betterer.results | 37 +++++-------------- .../core/components/OptionsUI/registry.tsx | 10 ++--- .../explore/ExploreQueryInspector.tsx | 3 +- .../features/explore/TraceView/TraceView.tsx | 8 ++-- 4 files changed, 21 insertions(+), 37 deletions(-) diff --git a/.betterer.results b/.betterer.results index 6d5ba9d6a08..bb112843ee0 100644 --- a/.betterer.results +++ b/.betterer.results @@ -1331,11 +1331,11 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "17"], [0, 0, 0, "Do not use any type assertions.", "18"], [0, 0, 0, "Unexpected any. Specify a different type.", "19"], - [0, 0, 0, "Do not use any type assertions.", "20"], - [0, 0, 0, "Unexpected any. Specify a different type.", "21"], - [0, 0, 0, "Do not use any type assertions.", "22"], - [0, 0, 0, "Unexpected any. Specify a different type.", "23"], - [0, 0, 0, "Do not use any type assertions.", "24"], + [0, 0, 0, "Unexpected any. Specify a different type.", "20"], + [0, 0, 0, "Do not use any type assertions.", "21"], + [0, 0, 0, "Unexpected any. Specify a different type.", "22"], + [0, 0, 0, "Do not use any type assertions.", "23"], + [0, 0, 0, "Unexpected any. Specify a different type.", "24"], [0, 0, 0, "Unexpected any. Specify a different type.", "25"], [0, 0, 0, "Do not use any type assertions.", "26"], [0, 0, 0, "Unexpected any. Specify a different type.", "27"], @@ -1386,20 +1386,10 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "72"], [0, 0, 0, "Do not use any type assertions.", "73"], [0, 0, 0, "Unexpected any. Specify a different type.", "74"], - [0, 0, 0, "Unexpected any. Specify a different type.", "75"], - [0, 0, 0, "Do not use any type assertions.", "76"], - [0, 0, 0, "Unexpected any. Specify a different type.", "77"], - [0, 0, 0, "Do not use any type assertions.", "78"], - [0, 0, 0, "Unexpected any. Specify a different type.", "79"], - [0, 0, 0, "Unexpected any. Specify a different type.", "80"], - [0, 0, 0, "Do not use any type assertions.", "81"], - [0, 0, 0, "Unexpected any. Specify a different type.", "82"], - [0, 0, 0, "Do not use any type assertions.", "83"], - [0, 0, 0, "Unexpected any. Specify a different type.", "84"], - [0, 0, 0, "Do not use any type assertions.", "85"], - [0, 0, 0, "Unexpected any. Specify a different type.", "86"], - [0, 0, 0, "Do not use any type assertions.", "87"], - [0, 0, 0, "Unexpected any. Specify a different type.", "88"] + [0, 0, 0, "Do not use any type assertions.", "75"], + [0, 0, 0, "Unexpected any. Specify a different type.", "76"], + [0, 0, 0, "Do not use any type assertions.", "77"], + [0, 0, 0, "Unexpected any. Specify a different type.", "78"] ], "public/app/core/components/OptionsUI/slider.tsx:5381": [ [0, 0, 0, "Styles should be written using objects.", "0"] @@ -3811,9 +3801,6 @@ exports[`better eslint`] = { "public/app/features/explore/ExplorePaneContainer.tsx:5381": [ [0, 0, 0, "Styles should be written using objects.", "0"] ], - "public/app/features/explore/ExploreQueryInspector.tsx:5381": [ - [0, 0, 0, "Do not use any type assertions.", "0"] - ], "public/app/features/explore/FeatureTogglePage.tsx:5381": [ [0, 0, 0, "Styles should be written using objects.", "0"] ], @@ -3994,11 +3981,7 @@ exports[`better eslint`] = { "public/app/features/explore/TraceView/TraceView.tsx:5381": [ [0, 0, 0, "Styles should be written using objects.", "0"], [0, 0, 0, "Do not use any type assertions.", "1"], - [0, 0, 0, "Do not use any type assertions.", "2"], - [0, 0, 0, "Do not use any type assertions.", "3"], - [0, 0, 0, "Do not use any type assertions.", "4"], - [0, 0, 0, "Unexpected any. Specify a different type.", "5"], - [0, 0, 0, "Do not use any type assertions.", "6"] + [0, 0, 0, "Do not use any type assertions.", "2"] ], "public/app/features/explore/TraceView/components/TracePageHeader/Actions/ActionButton.tsx:5381": [ [0, 0, 0, "Styles should be written using objects.", "0"] diff --git a/public/app/core/components/OptionsUI/registry.tsx b/public/app/core/components/OptionsUI/registry.tsx index 18c1816c8ac..2a3f6c65f4d 100644 --- a/public/app/core/components/OptionsUI/registry.tsx +++ b/public/app/core/components/OptionsUI/registry.tsx @@ -69,7 +69,7 @@ export const getAllOptionEditors = () => { id: 'text', name: 'Text', description: 'Allows string values input', - editor: StringValueEditor as any, + editor: StringValueEditor, }; const strings: StandardEditorsRegistryItem = { @@ -155,28 +155,28 @@ export const getAllOptionEditors = () => { id: 'timezone', name: 'Time zone', description: 'Time zone selection', - editor: TimeZonePicker as any, + editor: TimeZonePicker, }; const fieldName: StandardEditorsRegistryItem = { id: 'field-name', name: 'Field name', description: 'Allows selecting a field name from a data frame', - editor: FieldNamePicker as any, + editor: FieldNamePicker, }; const dashboardPicker: StandardEditorsRegistryItem = { id: 'dashboard-uid', name: 'Dashboard', description: 'Select dashboard', - editor: DashboardPicker as any, + editor: DashboardPicker, }; const mappings: StandardEditorsRegistryItem = { id: 'mappings', name: 'Mappings', description: 'Allows defining value mappings', - editor: ValueMappingsEditor as any, + editor: ValueMappingsEditor, }; const thresholds: StandardEditorsRegistryItem = { diff --git a/public/app/features/explore/ExploreQueryInspector.tsx b/public/app/features/explore/ExploreQueryInspector.tsx index 83e0a8f257f..1af76715975 100644 --- a/public/app/features/explore/ExploreQueryInspector.tsx +++ b/public/app/features/explore/ExploreQueryInspector.tsx @@ -3,6 +3,7 @@ import { connect, ConnectedProps } from 'react-redux'; import { CoreApp, LoadingState, TimeZone } from '@grafana/data'; import { reportInteraction } from '@grafana/runtime/src'; +import { defaultTimeZone } from '@grafana/schema'; import { TabbedContainer, TabConfig } from '@grafana/ui'; import { ExploreDrawer } from 'app/features/explore/ExploreDrawer'; import { InspectDataTab } from 'app/features/inspector/InspectDataTab'; @@ -39,7 +40,7 @@ export function ExploreQueryInspector(props: Props) { label: 'Stats', value: 'stats', icon: 'chart-line', - content: , + content: , }; const jsonTab: TabConfig = { diff --git a/public/app/features/explore/TraceView/TraceView.tsx b/public/app/features/explore/TraceView/TraceView.tsx index 80166258735..8f973234ef1 100644 --- a/public/app/features/explore/TraceView/TraceView.tsx +++ b/public/app/features/explore/TraceView/TraceView.tsx @@ -18,7 +18,7 @@ import { import { getTemplateSrv } from '@grafana/runtime'; import { DataQuery } from '@grafana/schema'; import { useStyles2 } from '@grafana/ui'; -import { getTraceToLogsOptions, TraceToLogsData } from 'app/core/components/TraceToLogs/TraceToLogsSettings'; +import { getTraceToLogsOptions } from 'app/core/components/TraceToLogs/TraceToLogsSettings'; import { TraceToMetricsData } from 'app/core/components/TraceToMetrics/TraceToMetricsSettings'; import { getDatasourceSrv } from 'app/features/plugins/datasource_srv'; import { getTimeZone } from 'app/features/profile/state/selectors'; @@ -112,8 +112,9 @@ export function TraceView(props: Props) { ); const instanceSettings = getDatasourceSrv().getInstanceSettings(datasource?.name); - const traceToLogsOptions = getTraceToLogsOptions(instanceSettings?.jsonData as TraceToLogsData); - const traceToMetricsOptions = (instanceSettings?.jsonData as TraceToMetricsData)?.tracesToMetrics; + const traceToLogsOptions = getTraceToLogsOptions(instanceSettings?.jsonData); + const traceToMetrics: TraceToMetricsData | undefined = instanceSettings?.jsonData; + const traceToMetricsOptions = traceToMetrics?.tracesToMetrics; const spanBarOptions: SpanBarOptionsData | undefined = instanceSettings?.jsonData; const createSpanLink = useMemo( @@ -261,7 +262,6 @@ function useFocusSpanLink(options: { link, internalLink: link.internal!, scopedVars: {}, - range: {} as any, field: {} as Field, onClickFn: sameTrace ? () => setFocusedSpanId(focusedSpanId === spanId ? undefined : spanId) From ae84ec38d5fa9d50443f34ac7c67c6de40aba7d8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 14:03:14 +0300 Subject: [PATCH 040/102] Update dependency chrome-remote-interface to v0.33.0 (#76337) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 2 +- packages/grafana-e2e/package.json | 2 +- yarn.lock | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index d4ffd95e9f3..3a518ac2984 100644 --- a/package.json +++ b/package.json @@ -158,7 +158,7 @@ "blob-polyfill": "7.0.20220408", "browserslist": "^4.21.4", "chance": "^1.0.10", - "chrome-remote-interface": "0.32.2", + "chrome-remote-interface": "0.33.0", "codeowners": "^5.1.1", "copy-webpack-plugin": "11.0.0", "css-loader": "6.8.1", diff --git a/packages/grafana-e2e/package.json b/packages/grafana-e2e/package.json index 1418371d01c..c4dc1a38049 100644 --- a/packages/grafana-e2e/package.json +++ b/packages/grafana-e2e/package.json @@ -69,7 +69,7 @@ "@mochajs/json-file-reporter": "^1.2.0", "babel-loader": "9.1.3", "blink-diff": "1.0.13", - "chrome-remote-interface": "0.32.2", + "chrome-remote-interface": "0.33.0", "commander": "8.3.0", "cypress": "9.5.1", "cypress-file-upload": "5.0.8", diff --git a/yarn.lock b/yarn.lock index ffcd198a749..cd01323f18e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3095,7 +3095,7 @@ __metadata: "@types/uuid": 9.0.2 babel-loader: 9.1.3 blink-diff: 1.0.13 - chrome-remote-interface: 0.32.2 + chrome-remote-interface: 0.33.0 commander: 8.3.0 cypress: 9.5.1 cypress-file-upload: 5.0.8 @@ -12642,15 +12642,15 @@ __metadata: languageName: node linkType: hard -"chrome-remote-interface@npm:0.32.2": - version: 0.32.2 - resolution: "chrome-remote-interface@npm:0.32.2" +"chrome-remote-interface@npm:0.33.0": + version: 0.33.0 + resolution: "chrome-remote-interface@npm:0.33.0" dependencies: commander: 2.11.x ws: ^7.2.0 bin: chrome-remote-interface: bin/client.js - checksum: 8a9fab249675eb1301b50ed99258aba8a844c07b03b5b4481d395ed72086aab80a492da81108e191a95da82a5930464c9d3ea359f331de56cd75f8e303f3a120 + checksum: 8e874759fe0c2343b9363f3f79aa2f9073668ea7a0c964178887055e3bf9742251060c7d92c07ffa4a5371a0498ec9c120d4899f256cd74bebd67fc1c3383c3f languageName: node linkType: hard @@ -17773,7 +17773,7 @@ __metadata: calculate-size: 1.1.1 centrifuge: 4.0.1 chance: ^1.0.10 - chrome-remote-interface: 0.32.2 + chrome-remote-interface: 0.33.0 classnames: 2.3.2 codeowners: ^5.1.1 combokeys: ^3.0.0 From 0ec15de905bf433e4818a2e9181fa5df17bd360a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 11:08:41 +0000 Subject: [PATCH 041/102] Update dependency @types/node to v18.18.4 (#76336) * Update dependency @types/node to v18.18.4 * fix types --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Ashley Harrison --- package.json | 2 +- packages/grafana-data/package.json | 2 +- packages/grafana-e2e-selectors/package.json | 2 +- packages/grafana-e2e/package.json | 2 +- packages/grafana-ui/package.json | 2 +- .../timeseries/plugins/ExemplarMarker.tsx | 2 +- .../plugins/annotations/AnnotationMarker.tsx | 2 +- yarn.lock | 25 +++++++------------ 8 files changed, 16 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index 3a518ac2984..35c6ca3b028 100644 --- a/package.json +++ b/package.json @@ -118,7 +118,7 @@ "@types/lucene": "^2", "@types/marked": "5.0.1", "@types/mousetrap": "1.6.11", - "@types/node": "18.16.16", + "@types/node": "18.18.4", "@types/node-forge": "^1", "@types/ol-ext": "npm:@siedlerchr/types-ol-ext@3.2.0", "@types/papaparse": "5.3.7", diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index 5aac56b3acc..0ef93b17550 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -76,7 +76,7 @@ "@types/jquery": "3.5.16", "@types/lodash": "4.14.195", "@types/marked": "5.0.1", - "@types/node": "18.16.19", + "@types/node": "18.18.4", "@types/papaparse": "5.3.7", "@types/react": "18.2.15", "@types/react-dom": "18.2.7", diff --git a/packages/grafana-e2e-selectors/package.json b/packages/grafana-e2e-selectors/package.json index 32d7a53888f..8f5c597e1d5 100644 --- a/packages/grafana-e2e-selectors/package.json +++ b/packages/grafana-e2e-selectors/package.json @@ -41,7 +41,7 @@ "devDependencies": { "@rollup/plugin-commonjs": "25.0.2", "@rollup/plugin-node-resolve": "15.2.3", - "@types/node": "18.16.19", + "@types/node": "18.18.4", "esbuild": "0.18.12", "rimraf": "5.0.1", "rollup": "2.79.1", diff --git a/packages/grafana-e2e/package.json b/packages/grafana-e2e/package.json index c4dc1a38049..9948eefc6b0 100644 --- a/packages/grafana-e2e/package.json +++ b/packages/grafana-e2e/package.json @@ -50,7 +50,7 @@ "@rollup/plugin-node-resolve": "15.2.3", "@types/chrome-remote-interface": "0.31.10", "@types/lodash": "4.14.195", - "@types/node": "18.16.19", + "@types/node": "18.18.4", "@types/uuid": "9.0.2", "esbuild": "0.18.12", "rollup": "2.79.1", diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index 4c768d327a2..a5419a25e03 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -146,7 +146,7 @@ "@types/jquery": "3.5.16", "@types/lodash": "4.14.195", "@types/mock-raf": "1.0.3", - "@types/node": "18.16.19", + "@types/node": "18.18.4", "@types/prismjs": "1.26.0", "@types/react": "18.2.15", "@types/react-beautiful-dnd": "13.1.4", diff --git a/public/app/plugins/panel/timeseries/plugins/ExemplarMarker.tsx b/public/app/plugins/panel/timeseries/plugins/ExemplarMarker.tsx index 9646e08a527..0a60cb13cd7 100644 --- a/public/app/plugins/panel/timeseries/plugins/ExemplarMarker.tsx +++ b/public/app/plugins/panel/timeseries/plugins/ExemplarMarker.tsx @@ -57,7 +57,7 @@ export const ExemplarMarker = ({ }, ], }); - const popoverRenderTimeout = useRef(); + const popoverRenderTimeout = useRef(); useEffect(() => { if ( diff --git a/public/app/plugins/panel/timeseries/plugins/annotations/AnnotationMarker.tsx b/public/app/plugins/panel/timeseries/plugins/annotations/AnnotationMarker.tsx index 0b111de0cef..4c5f312f171 100644 --- a/public/app/plugins/panel/timeseries/plugins/annotations/AnnotationMarker.tsx +++ b/public/app/plugins/panel/timeseries/plugins/annotations/AnnotationMarker.tsx @@ -45,7 +45,7 @@ export function AnnotationMarker({ annotation, timeZone, width }: Props) { const [tooltipRef, setTooltipRef] = useState(null); const [editorRef, setEditorRef] = useState(null); - const popoverRenderTimeout = useRef(); + const popoverRenderTimeout = useRef(); const popper = usePopper(markerRef, tooltipRef, POPPER_CONFIG); const editorPopper = usePopper(markerRef, editorRef, POPPER_CONFIG); diff --git a/yarn.lock b/yarn.lock index cd01323f18e..5d512a1a0e9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3003,7 +3003,7 @@ __metadata: "@types/jquery": 3.5.16 "@types/lodash": 4.14.195 "@types/marked": 5.0.1 - "@types/node": 18.16.19 + "@types/node": 18.18.4 "@types/papaparse": 5.3.7 "@types/react": 18.2.15 "@types/react-dom": 18.2.7 @@ -3054,7 +3054,7 @@ __metadata: "@grafana/tsconfig": ^1.2.0-rc1 "@rollup/plugin-commonjs": 25.0.2 "@rollup/plugin-node-resolve": 15.2.3 - "@types/node": 18.16.19 + "@types/node": 18.18.4 esbuild: 0.18.12 rimraf: 5.0.1 rollup: 2.79.1 @@ -3091,7 +3091,7 @@ __metadata: "@rollup/plugin-node-resolve": 15.2.3 "@types/chrome-remote-interface": 0.31.10 "@types/lodash": 4.14.195 - "@types/node": 18.16.19 + "@types/node": 18.18.4 "@types/uuid": 9.0.2 babel-loader: 9.1.3 blink-diff: 1.0.13 @@ -3446,7 +3446,7 @@ __metadata: "@types/jquery": 3.5.16 "@types/lodash": 4.14.195 "@types/mock-raf": 1.0.3 - "@types/node": 18.16.19 + "@types/node": 18.18.4 "@types/prismjs": 1.26.0 "@types/react": 18.2.15 "@types/react-beautiful-dnd": 13.1.4 @@ -9433,17 +9433,10 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:*, @types/node@npm:18.16.19, @types/node@npm:>=13.7.0": - version: 18.16.19 - resolution: "@types/node@npm:18.16.19" - checksum: 63c31f09616508aa7135380a4c79470a897b75f9ff3a70eb069e534dfabdec3f32fb0f9df5939127f1086614d980ddea0fa5e8cc29a49103c4f74cd687618aaf - languageName: node - linkType: hard - -"@types/node@npm:18.16.16": - version: 18.16.16 - resolution: "@types/node@npm:18.16.16" - checksum: 0efad726dd1e0bef71c392c708fc5d78c5b39c46b0ac5186fee74de4ccb1b2e847b3fa468da67d62812f56569da721b15bf31bdc795e6c69b56c73a45079ed2d +"@types/node@npm:*, @types/node@npm:18.18.4, @types/node@npm:>=13.7.0": + version: 18.18.4 + resolution: "@types/node@npm:18.18.4" + checksum: 4901e91c4cc479bb58acbcd79236a97a0ad6db4a53cb1f4ba4cf32af15324c61b16faa6e31c1b09bf538a20feb5f5274239157ce5237f5741db0b9ab71e69c52 languageName: node linkType: hard @@ -17713,7 +17706,7 @@ __metadata: "@types/lucene": ^2 "@types/marked": 5.0.1 "@types/mousetrap": 1.6.11 - "@types/node": 18.16.16 + "@types/node": 18.18.4 "@types/node-forge": ^1 "@types/ol-ext": "npm:@siedlerchr/types-ol-ext@3.2.0" "@types/papaparse": 5.3.7 From 555acdf180f2251705cb7f60300b79e3bfa870c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 11 Oct 2023 13:18:24 +0200 Subject: [PATCH 042/102] Drawer: Add 1 grid unit padding between title and tabs (#76249) --- packages/grafana-ui/src/components/Drawer/Drawer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grafana-ui/src/components/Drawer/Drawer.tsx b/packages/grafana-ui/src/components/Drawer/Drawer.tsx index c167cdfcf65..6359ea814ad 100644 --- a/packages/grafana-ui/src/components/Drawer/Drawer.tsx +++ b/packages/grafana-ui/src/components/Drawer/Drawer.tsx @@ -302,7 +302,7 @@ const getStyles = (theme: GrafanaTheme2) => { tabsWrapper: css({ label: 'drawer-tabs', paddingLeft: theme.spacing(2), - margin: theme.spacing(0, -1, -3, -3), + margin: theme.spacing(1, -1, -3, -3), }), }; }; From 6968f4d6ff8e281dbe7c5d9bf3d435023997540b Mon Sep 17 00:00:00 2001 From: Dominik Prokop Date: Wed, 11 Oct 2023 13:24:18 +0200 Subject: [PATCH 043/102] DashboardScene: Share snapshot (#76132) * Dashboard snapshot creation logic and tests * Add variable type to serialized json, add refresh prop to datasource var * WIP Snapshots tab UI * Use Grafana snapshot query for a generated snapshot * Make anno snapshots backwards compatible * Share snapshot tab UI * Single panel snapshot * Remove unused param * Snap update * Ts fix * One more snap fix * Basic rows support * Add tests for basic rows * Bettererupdate --- .betterer.results | 5 +- .../test/__mocks__/pluginMocks.ts | 2 +- .../src/veneer/dashboard.types.ts | 3 + .../transformSceneToSaveModel.test.ts.snap | 2 + .../dashboard-scene/serialization/const.ts | 4 + .../serialization/dataLayersToAnnotations.ts | 20 + .../sceneVariablesSetToVariables.test.ts | 5 + .../sceneVariablesSetToVariables.ts | 4 +- .../testfiles/snapshotable_dashboard.json | 360 +++++++++++++++ .../testfiles/snapshotable_with_rows.json | 425 ++++++++++++++++++ .../transformSceneToSaveModel.test.ts | 320 ++++++++++++- .../transformSceneToSaveModel.ts | 120 ++++- .../dashboard-scene/sharing/ShareModal.tsx | 2 +- .../sharing/ShareSnapshotTab.tsx | 237 +++++++++- .../components/ShareModal/ShareSnapshot.tsx | 2 + 15 files changed, 1479 insertions(+), 32 deletions(-) create mode 100644 public/app/features/dashboard-scene/serialization/const.ts create mode 100644 public/app/features/dashboard-scene/serialization/dataLayersToAnnotations.ts create mode 100644 public/app/features/dashboard-scene/serialization/testfiles/snapshotable_dashboard.json create mode 100644 public/app/features/dashboard-scene/serialization/testfiles/snapshotable_with_rows.json diff --git a/.betterer.results b/.betterer.results index bb112843ee0..abfc6463bfe 100644 --- a/.betterer.results +++ b/.betterer.results @@ -3009,7 +3009,10 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "1"], [0, 0, 0, "Unexpected any. Specify a different type.", "2"], [0, 0, 0, "Unexpected any. Specify a different type.", "3"], - [0, 0, 0, "Unexpected any. Specify a different type.", "4"] + [0, 0, 0, "Unexpected any. Specify a different type.", "4"], + [0, 0, 0, "Unexpected any. Specify a different type.", "5"], + [0, 0, 0, "Unexpected any. Specify a different type.", "6"], + [0, 0, 0, "Unexpected any. Specify a different type.", "7"] ], "public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.ts:5381": [ [0, 0, 0, "Do not use any type assertions.", "0"], diff --git a/packages/grafana-data/test/__mocks__/pluginMocks.ts b/packages/grafana-data/test/__mocks__/pluginMocks.ts index 71268a244c9..226d6eb87cb 100644 --- a/packages/grafana-data/test/__mocks__/pluginMocks.ts +++ b/packages/grafana-data/test/__mocks__/pluginMocks.ts @@ -61,7 +61,7 @@ export function getPanelPlugin( }, screenshots: [], updated: '', - version: '', + version: '1', }, hideFromList: options.hideFromList === true, module: options.module ?? '', diff --git a/packages/grafana-schema/src/veneer/dashboard.types.ts b/packages/grafana-schema/src/veneer/dashboard.types.ts index 1d3522622ea..c444b0e7840 100644 --- a/packages/grafana-schema/src/veneer/dashboard.types.ts +++ b/packages/grafana-schema/src/veneer/dashboard.types.ts @@ -36,6 +36,9 @@ export interface AnnotationQuery extends Omit { datasource?: DataSourceRef | null; target?: TQuery; + // TODO: When migrating to snapshot queries, remove this property. + // With snapshot queries annotations become a part of the panel snapshot data. + snapshotData?: unknown; } export interface AnnotationContainer extends Omit { diff --git a/public/app/features/dashboard-scene/serialization/__snapshots__/transformSceneToSaveModel.test.ts.snap b/public/app/features/dashboard-scene/serialization/__snapshots__/transformSceneToSaveModel.test.ts.snap index f71c10f518e..3e9a5012a51 100644 --- a/public/app/features/dashboard-scene/serialization/__snapshots__/transformSceneToSaveModel.test.ts.snap +++ b/public/app/features/dashboard-scene/serialization/__snapshots__/transformSceneToSaveModel.test.ts.snap @@ -209,6 +209,7 @@ exports[`transformSceneToSaveModel Given a scene with rows Should transform back "name": "server", "options": [], "query": "A,B,C,D,E,F,E,G,H,I,J,K,L", + "type": "custom", }, { "current": { @@ -226,6 +227,7 @@ exports[`transformSceneToSaveModel Given a scene with rows Should transform back "name": "pod", "options": [], "query": "Bob : 1, Rob : 2,Sod : 3, Hod : 4, Cod : 5", + "type": "custom", }, ], }, diff --git a/public/app/features/dashboard-scene/serialization/const.ts b/public/app/features/dashboard-scene/serialization/const.ts new file mode 100644 index 00000000000..f7e03e4328b --- /dev/null +++ b/public/app/features/dashboard-scene/serialization/const.ts @@ -0,0 +1,4 @@ +export const GRAFANA_DATASOURCE_REF = { + name: 'grafana', + uid: 'grafana', +}; diff --git a/public/app/features/dashboard-scene/serialization/dataLayersToAnnotations.ts b/public/app/features/dashboard-scene/serialization/dataLayersToAnnotations.ts new file mode 100644 index 00000000000..1261ec4157a --- /dev/null +++ b/public/app/features/dashboard-scene/serialization/dataLayersToAnnotations.ts @@ -0,0 +1,20 @@ +import { SceneDataLayerProvider, dataLayers } from '@grafana/scenes'; +import { AnnotationQuery } from '@grafana/schema'; + +export function dataLayersToAnnotations(layers: SceneDataLayerProvider[]) { + const annotations: AnnotationQuery[] = []; + for (const layer of layers) { + if (!(layer instanceof dataLayers.AnnotationsDataLayer)) { + continue; + } + const result = { + ...layer.state.query, + enable: Boolean(layer.state.isEnabled), + hide: Boolean(layer.state.isHidden), + }; + + annotations.push(result); + } + + return annotations; +} diff --git a/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.test.ts b/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.test.ts index fb528d04bca..33df731b3f0 100644 --- a/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.test.ts +++ b/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.test.ts @@ -122,6 +122,7 @@ describe('sceneVariablesSetToVariables', () => { "query": "query", "refresh": 1, "regex": "", + "type": "query", } `); }); @@ -165,7 +166,9 @@ describe('sceneVariablesSetToVariables', () => { "name": "test", "options": [], "query": "fake-std", + "refresh": 1, "regex": "", + "type": "datasource", } `); }); @@ -214,6 +217,7 @@ describe('sceneVariablesSetToVariables', () => { "name": "test", "options": [], "query": "test,test1,test2", + "type": "custom", } `); }); @@ -245,6 +249,7 @@ describe('sceneVariablesSetToVariables', () => { "name": "test", "query": "constant value", "skipUrlSync": true, + "type": "constant", } `); }); diff --git a/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.ts b/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.ts index e11ad014301..17e69a1ac4b 100644 --- a/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.ts +++ b/public/app/features/dashboard-scene/serialization/sceneVariablesSetToVariables.ts @@ -1,5 +1,5 @@ import { SceneVariableSet, QueryVariable, CustomVariable, DataSourceVariable, ConstantVariable } from '@grafana/scenes'; -import { VariableModel, VariableHide, VariableSort } from '@grafana/schema'; +import { VariableModel, VariableHide, VariableRefresh, VariableSort } from '@grafana/schema'; export function sceneVariablesSetToVariables(set: SceneVariableSet) { const variables: VariableModel[] = []; @@ -10,6 +10,7 @@ export function sceneVariablesSetToVariables(set: SceneVariableSet) { description: variable.state.description, skipUrlSync: Boolean(variable.state.skipUrlSync), hide: variable.state.hide || VariableHide.dontHide, + type: variable.state.type, }; if (variable instanceof QueryVariable) { variables.push({ @@ -58,6 +59,7 @@ export function sceneVariablesSetToVariables(set: SceneVariableSet) { }, options: [], regex: variable.state.regex, + refresh: VariableRefresh.onDashboardLoad, query: variable.state.pluginId, multi: variable.state.isMulti, allValue: variable.state.allValue, diff --git a/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_dashboard.json b/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_dashboard.json new file mode 100644 index 00000000000..3d4528fc997 --- /dev/null +++ b/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_dashboard.json @@ -0,0 +1,360 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "enable": true, + "iconColor": "red", + "name": "New annotation", + "target": { + "lines": 10, + "refId": "Anno1111", + "scenarioId": "annotations" + } + } + ] + }, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "New link", + "tooltip": "", + "type": "link", + "url": "https://test.link.co" + } + ], + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2306, + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "refId": "A", + "scenarioId": "random_walk", + "seriesCount": 1 + } + ], + "title": "Panel 1", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.2.0-pre", + "targets": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "refId": "B", + "scenarioId": "random_walk", + "seriesCount": 1 + } + ], + "title": "Panel 2", + "transformations": [ + { + "id": "reduce", + "options": {} + } + ], + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + }, + "panelId": 1, + "refId": "A" + } + ], + "title": "Panel 3", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "annotations", + "value": "annotations" + }, + "datasource": { + "type": "prometheus", + "uid": "gdev-prometheus" + }, + "definition": "label_values(backend)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "query0", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(backend)", + "refId": "VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Snapshots repro", + "uid": "d6657f14-9c6a-4eaa-9593-54c9c46b6394", + "version": 4, + "weekStart": "" +} diff --git a/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_with_rows.json b/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_with_rows.json new file mode 100644 index 00000000000..408c5fc18cd --- /dev/null +++ b/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_with_rows.json @@ -0,0 +1,425 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2312, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "refId": "A", + "scenarioId": "random_walk", + "seriesCount": 1 + } + ], + "title": "Panel 1", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "panels": [], + "title": "Expanded row", + "type": "row" + }, + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "refId": "B", + "scenarioId": "random_walk", + "seriesCount": 1 + } + ], + "title": "Panel 2", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "refId": "C", + "scenarioId": "random_walk", + "seriesCount": 1 + } + ], + "title": "Panel 3", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 4, + "panels": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "title": "Panel inside colapsed row", + "type": "timeseries" + } + ], + "title": "Collapsed row", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Snapshots - basic rows", + "uid": "a3b23921-287d-4f90-9a14-c04228057dfa", + "version": 4, + "weekStart": "" + } diff --git a/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts b/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts index af8e41af6e0..fd3e42565ae 100644 --- a/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts +++ b/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts @@ -1,3 +1,20 @@ +import { advanceTo } from 'jest-date-mock'; +import { map, of } from 'rxjs'; + +import { + DataFrame, + DataQueryRequest, + DataSourceApi, + dateTime, + FieldType, + PanelData, + standardTransformersRegistry, + StandardVariableQuery, + toDataFrame, + VariableSupportType, +} from '@grafana/data'; +import { getPanelPlugin } from '@grafana/data/test/__mocks__/pluginMocks'; +import { setPluginImportUtils } from '@grafana/runtime'; import { MultiValueVariable, SceneDataLayers, @@ -5,22 +22,121 @@ import { SceneGridLayout, SceneGridRow, SceneVariable, + VizPanel, } from '@grafana/scenes'; -import { Panel, RowPanel } from '@grafana/schema'; +import { Dashboard, LoadingState, Panel, RowPanel, VariableRefresh } from '@grafana/schema'; import { PanelModel } from 'app/features/dashboard/state'; +import { getTimeRange } from 'app/features/dashboard/utils/timeRange'; +import { reduceTransformRegistryItem } from 'app/features/transformers/editors/ReduceTransformerEditor'; import { SHARED_DASHBOARD_QUERY } from 'app/plugins/datasource/dashboard'; import { RowRepeaterBehavior } from '../scene/RowRepeaterBehavior'; +import { activateFullSceneTree } from '../utils/test-utils'; +import { getVizPanelKeyForPanelId } from '../utils/utils'; +import { GRAFANA_DATASOURCE_REF } from './const'; import dashboard_to_load1 from './testfiles/dashboard_to_load1.json'; import repeatingRowsAndPanelsDashboardJson from './testfiles/repeating_rows_and_panels.json'; +import snapshotableDashboardJson from './testfiles/snapshotable_dashboard.json'; +import snapshotableWithRowsDashboardJson from './testfiles/snapshotable_with_rows.json'; import { buildGridItemForLibPanel, buildGridItemForPanel, transformSaveModelToScene, } from './transformSaveModelToScene'; -import { gridItemToPanel, transformSceneToSaveModel } from './transformSceneToSaveModel'; +import { gridItemToPanel, transformSceneToSaveModel, trimDashboardForSnapshot } from './transformSceneToSaveModel'; +standardTransformersRegistry.setInit(() => [reduceTransformRegistryItem]); +setPluginImportUtils({ + importPanelPlugin: (id: string) => Promise.resolve(getPanelPlugin({})), + getPanelPluginFromCache: (id: string) => undefined, +}); + +const AFrame = toDataFrame({ + name: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [100, 200, 300] }, + { name: 'values', type: FieldType.number, values: [1, 2, 3] }, + ], +}); + +const BFrame = toDataFrame({ + name: 'B', + fields: [ + { name: 'time', type: FieldType.time, values: [100, 200, 300] }, + { name: 'values', type: FieldType.number, values: [10, 20, 30] }, + ], +}); + +const CFrame = toDataFrame({ + name: 'C', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000] }, + { name: 'values', type: FieldType.number, values: [100, 200, 300] }, + ], +}); + +const AnnoFrame = toDataFrame({ + fields: [ + { name: 'time', values: [1, 2, 2, 5, 5] }, + { name: 'id', values: ['1', '2', '2', '5', '5'] }, + { name: 'text', values: ['t1', 't2', 't3', 't4', 't5'] }, + ], +}); + +const VariableQueryFrame = toDataFrame({ + fields: [{ name: 'text', type: FieldType.string, values: ['val1', 'val2', 'val11'] }], +}); + +const testSeries: Record = { + A: AFrame, + B: BFrame, + C: CFrame, + Anno: AnnoFrame, + VariableQuery: VariableQueryFrame, +}; + +const runRequestMock = jest.fn().mockImplementation((ds: DataSourceApi, request: DataQueryRequest) => { + const result: PanelData = { + state: LoadingState.Loading, + series: [], + timeRange: request.range, + }; + + return of([]).pipe( + map(() => { + result.state = LoadingState.Done; + + const refId = request.targets[0].refId; + result.series = [testSeries[refId]]; + + return result; + }) + ); +}); +jest.mock('@grafana/runtime', () => ({ + ...jest.requireActual('@grafana/runtime'), + getDataSourceSrv: () => ({ + get: () => ({ + getRef: () => ({ type: 'mock-ds', uid: 'mock-uid' }), + variables: { + getType: () => VariableSupportType.Standard, + toDataQuery: (q: StandardVariableQuery) => q, + }, + }), + }), + getRunRequest: () => (ds: DataSourceApi, request: DataQueryRequest) => { + return runRequestMock(ds, request); + }, + config: { + panels: [], + theme2: { + visualization: { + getColorByName: jest.fn().mockReturnValue('red'), + }, + }, + }, +})); describe('transformSceneToSaveModel', () => { describe('Given a simple scene', () => { it('Should transform back to peristed model', () => { @@ -360,6 +476,206 @@ describe('transformSceneToSaveModel', () => { }); }); }); + + describe('Snapshots', () => { + const fakeCurrentDate = dateTime('2023-01-01T20:00:00.000Z').toDate(); + + beforeEach(() => { + advanceTo(fakeCurrentDate); + }); + + it('attaches snapshot data to panels using Grafana snapshot query', async () => { + const scene = transformSaveModelToScene({ dashboard: snapshotableDashboardJson as any, meta: {} }); + + activateFullSceneTree(scene); + + await new Promise((r) => setTimeout(r, 1)); + + const snapshot = transformSceneToSaveModel(scene, true); + + expect(snapshot.panels?.length).toBe(3); + + // Regular panel with SceneQueryRunner + // @ts-expect-error + expect(snapshot.panels?.[0].datasource).toEqual(GRAFANA_DATASOURCE_REF); + // @ts-expect-error + expect(snapshot.panels?.[0].targets?.[0].datasource).toEqual(GRAFANA_DATASOURCE_REF); + // @ts-expect-error + expect(snapshot.panels?.[0].targets?.[0].snapshot[0].data).toEqual({ + values: [ + [100, 200, 300], + [1, 2, 3], + ], + }); + + // Panel with transformations + // @ts-expect-error + expect(snapshot.panels?.[1].datasource).toEqual(GRAFANA_DATASOURCE_REF); + // @ts-expect-error + expect(snapshot.panels?.[1].targets?.[0].datasource).toEqual(GRAFANA_DATASOURCE_REF); + // @ts-expect-error + expect(snapshot.panels?.[1].targets?.[0].snapshot[0].data).toEqual({ + values: [ + [100, 200, 300], + [10, 20, 30], + ], + }); + // @ts-expect-error + expect(snapshot.panels?.[1].transformations).toEqual([ + { + id: 'reduce', + options: {}, + }, + ]); + + // Panel with a shared query (dahsboard query) + // @ts-expect-error + expect(snapshot.panels?.[2].datasource).toEqual(GRAFANA_DATASOURCE_REF); + // @ts-expect-error + expect(snapshot.panels?.[2].targets?.[0].datasource).toEqual(GRAFANA_DATASOURCE_REF); + // @ts-expect-error + expect(snapshot.panels?.[2].targets?.[0].snapshot[0].data).toEqual({ + values: [ + [100, 200, 300], + [1, 2, 3], + ], + }); + }); + + it('handles basic rows', async () => { + const scene = transformSaveModelToScene({ dashboard: snapshotableWithRowsDashboardJson as any, meta: {} }); + + activateFullSceneTree(scene); + + await new Promise((r) => setTimeout(r, 1)); + + const snapshot = transformSceneToSaveModel(scene, true); + + expect(snapshot.panels?.length).toBe(5); + + // @ts-expect-error + expect(snapshot.panels?.[0].targets?.[0].datasource).toEqual(GRAFANA_DATASOURCE_REF); + // @ts-expect-error + expect(snapshot.panels?.[0].targets?.[0].snapshot[0].data).toEqual({ + values: [ + [100, 200, 300], + [1, 2, 3], + ], + }); + + // @ts-expect-error + expect(snapshot.panels?.[1].targets).toBeUndefined(); + // @ts-expect-error + expect(snapshot.panels?.[1].panels).toEqual([]); + // @ts-expect-error + expect(snapshot.panels?.[1].collapsed).toEqual(false); + + // @ts-expect-error + expect(snapshot.panels?.[2].targets?.[0].datasource).toEqual(GRAFANA_DATASOURCE_REF); + // @ts-expect-error + expect(snapshot.panels?.[2].targets?.[0].snapshot[0].data).toEqual({ + values: [ + [100, 200, 300], + [10, 20, 30], + ], + }); + // @ts-expect-error + expect(snapshot.panels?.[3].targets?.[0].datasource).toEqual(GRAFANA_DATASOURCE_REF); + // @ts-expect-error + expect(snapshot.panels?.[3].targets?.[0].snapshot[0].data).toEqual({ + values: [ + [1000, 2000, 3000], + [100, 200, 300], + ], + }); + + // @ts-expect-error + expect(snapshot.panels?.[4].targets).toBeUndefined(); + // @ts-expect-error + expect(snapshot.panels?.[4].panels).toHaveLength(1); + // @ts-expect-error + expect(snapshot.panels?.[4].collapsed).toEqual(true); + }); + + describe('trimDashboardForSnapshot', () => { + let snapshot: Dashboard = {} as Dashboard; + + beforeEach(() => { + const scene = transformSaveModelToScene({ dashboard: snapshotableDashboardJson as any, meta: {} }); + activateFullSceneTree(scene); + snapshot = transformSceneToSaveModel(scene, true); + }); + + it('should not mutate provided dashboard', () => { + const result = trimDashboardForSnapshot('Snap title', getTimeRange({ from: 'now-6h', to: 'now' }), snapshot); + expect(result).not.toBe(snapshot); + }); + + it('should apply provided title and absolute time range', async () => { + const result = trimDashboardForSnapshot('Snap title', getTimeRange({ from: 'now-6h', to: 'now' }), snapshot); + + expect(result.title).toBe('Snap title'); + expect(result.time).toBeDefined(); + expect(result.time!.from).toEqual('2023-01-01T14:00:00.000Z'); + expect(result.time!.to).toEqual('2023-01-01T20:00:00.000Z'); + }); + + it('should remove queries from annotations and attach empty snapshotData', () => { + expect(snapshot.annotations?.list?.[0].target).toBeDefined(); + expect(snapshot.annotations?.list?.[1].target).toBeDefined(); + + const result = trimDashboardForSnapshot('Snap title', getTimeRange({ from: 'now-6h', to: 'now' }), snapshot); + + expect(result.annotations?.list?.length).toBe(2); + expect(result.annotations?.list?.[0].target).toBeUndefined(); + expect(result.annotations?.list?.[0].snapshotData).toEqual([]); + expect(result.annotations?.list?.[1].target).toBeUndefined(); + expect(result.annotations?.list?.[1].snapshotData).toEqual([]); + }); + it('should remove queries from variables', () => { + expect(snapshot.templating?.list?.length).toBe(1); + + const result = trimDashboardForSnapshot('Snap title', getTimeRange({ from: 'now-6h', to: 'now' }), snapshot); + + expect(result.templating?.list?.length).toBe(1); + expect(result.templating?.list?.[0].query).toBe(''); + expect(result.templating?.list?.[0].refresh).toBe(VariableRefresh.never); + expect(result.templating?.list?.[0].options).toHaveLength(1); + expect(result.templating?.list?.[0].options?.[0]).toEqual({ + text: 'annotations', + value: 'annotations', + }); + }); + + it('should snapshot a single panel when provided', () => { + const vizPanel = new VizPanel({ + key: getVizPanelKeyForPanelId(2), + }); + + const result = trimDashboardForSnapshot( + 'Snap title', + getTimeRange({ from: 'now-6h', to: 'now' }), + snapshot, + vizPanel + ); + + expect(snapshot.panels?.length).toBe(3); + expect(result.panels?.length).toBe(1); + // @ts-expect-error + expect(result.panels?.[0].gridPos).toEqual({ w: 24, x: 0, y: 0, h: 20 }); + }); + + // TODO: Uncomment when we support links + // it('should remove links', async () => { + // const scene = transformSaveModelToScene({ dashboard: snapshotableDashboardJson as any, meta: {} }); + // activateFullSceneTree(scene); + // const snapshot = transformSceneToSaveModel(scene, true); + // expect(snapshot.links?.length).toBe(1); + // const result = trimDashboardForSnapshot('Snap title', getTimeRange({ from: 'now-6h', to: 'now' }), snapshot); + // expect(result.links?.length).toBe(0); + // }); + }); + }); }); export function buildGridItemFromPanelSchema(panel: Partial): SceneGridItemLike { diff --git a/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.ts b/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.ts index bac01eaaaac..795a7a50b67 100644 --- a/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.ts +++ b/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.ts @@ -1,3 +1,4 @@ +import { isEmptyObject, TimeRange } from '@grafana/data'; import { SceneDataLayers, SceneGridItem, @@ -5,8 +6,6 @@ import { SceneGridLayout, SceneGridRow, VizPanel, - dataLayers, - SceneDataLayerProvider, SceneQueryRunner, SceneDataTransformer, SceneVariableSet, @@ -21,9 +20,12 @@ import { Panel, RowPanel, VariableModel, + VariableRefresh, } from '@grafana/schema'; import { sortedDeepCloneWithoutNulls } from 'app/core/utils/object'; +import { getPanelDataFrames } from 'app/features/dashboard/components/HelpWizard/utils'; import { SHARED_DASHBOARD_QUERY } from 'app/plugins/datasource/dashboard'; +import { GrafanaQueryType } from 'app/plugins/datasource/grafana/types'; import { DashboardScene } from '../scene/DashboardScene'; import { LibraryVizPanel } from '../scene/LibraryVizPanel'; @@ -33,9 +35,11 @@ import { RowRepeaterBehavior } from '../scene/RowRepeaterBehavior'; import { ShareQueryDataProvider } from '../scene/ShareQueryDataProvider'; import { getPanelIdForVizPanel } from '../utils/utils'; +import { GRAFANA_DATASOURCE_REF } from './const'; +import { dataLayersToAnnotations } from './dataLayersToAnnotations'; import { sceneVariablesSetToVariables } from './sceneVariablesSetToVariables'; -export function transformSceneToSaveModel(scene: DashboardScene): Dashboard { +export function transformSceneToSaveModel(scene: DashboardScene, isSnapshot = false): Dashboard { const state = scene.state; const timeRange = state.$timeRange!.state; const data = state.$data; @@ -48,7 +52,7 @@ export function transformSceneToSaveModel(scene: DashboardScene): Dashboard { if (body instanceof SceneGridLayout) { for (const child of body.state.children) { if (child instanceof SceneGridItem) { - panels.push(gridItemToPanel(child)); + panels.push(gridItemToPanel(child, isSnapshot)); } if (child instanceof SceneGridRow) { @@ -56,7 +60,7 @@ export function transformSceneToSaveModel(scene: DashboardScene): Dashboard { if (child.state.key!.indexOf('-clone-') > 0) { continue; } - gridRowToSaveModel(child, panels); + gridRowToSaveModel(child, panels, isSnapshot); } } } @@ -64,6 +68,7 @@ export function transformSceneToSaveModel(scene: DashboardScene): Dashboard { let annotations: AnnotationQuery[] = []; if (data instanceof SceneDataLayers) { const layers = data.state.layers; + annotations = dataLayersToAnnotations(layers); } @@ -106,7 +111,7 @@ export function transformSceneToSaveModel(scene: DashboardScene): Dashboard { return sortedDeepCloneWithoutNulls(dashboard); } -export function gridItemToPanel(gridItem: SceneGridItemLike): Panel { +export function gridItemToPanel(gridItem: SceneGridItemLike, isSnapshot = false): Panel { let vizPanel: VizPanel | undefined; let x = 0, y = 0, @@ -164,7 +169,7 @@ export function gridItemToPanel(gridItem: SceneGridItemLike): Panel { options: vizPanel.state.options, fieldConfig: (vizPanel.state.fieldConfig as FieldConfigSource) ?? { defaults: {}, overrides: [] }, transformations: [], - transparent: false, + transparent: vizPanel.state.displayMode === 'transparent', }; const panelTime = vizPanel.state.$timeRange; @@ -227,8 +232,23 @@ export function gridItemToPanel(gridItem: SceneGridItemLike): Panel { panel.transformations = dataProvider.state.transformations as DataTransformerConfig[]; } - if (vizPanel.state.displayMode === 'transparent') { - panel.transparent = true; + if (dataProvider && isSnapshot) { + panel.datasource = GRAFANA_DATASOURCE_REF; + + let data = getPanelDataFrames(dataProvider.state.data); + if (dataProvider instanceof SceneDataTransformer) { + // For transformations the non-transformed data is snapshoted + data = getPanelDataFrames(dataProvider.state.$data!.state.data); + } + + panel.targets = [ + { + refId: 'A', + datasource: panel.datasource, + queryType: GrafanaQueryType.Snapshot, + snapshot: data, + }, + ]; } if (gridItem instanceof PanelRepeaterGridItem) { @@ -240,7 +260,7 @@ export function gridItemToPanel(gridItem: SceneGridItemLike): Panel { return panel; } -export function gridRowToSaveModel(gridRow: SceneGridRow, panelsArray: Array) { +export function gridRowToSaveModel(gridRow: SceneGridRow, panelsArray: Array, isSnapshot = false) { const rowPanel: RowPanel = { type: 'row', id: getPanelIdForVizPanel(gridRow), @@ -265,7 +285,7 @@ export function gridRowToSaveModel(gridRow: SceneGridRow, panelsArray: Array gridItemToPanel(c, isSnapshot)); if (gridRow.state.isCollapsed) { rowPanel.panels = panelsInsideRow; @@ -274,19 +294,75 @@ export function gridRowToSaveModel(gridRow: SceneGridRow, panelsArray: Array p.id === getPanelIdForVizPanel(panel)); + if (singlePanel) { + // @ts-expect-error Due to legacy panels types. Id is present on such panels too. + singlePanel.gridPos = { w: 24, x: 0, y: 0, h: 20 }; + result = { + ...result, + panels: [singlePanel], + }; + } + } + + // Remove links from all panels + result.panels?.forEach((panel) => { + if ('links' in panel) { + panel.links = []; + } + }); + + // Remove annotation queries, attach snapshotData: [] for backwards compatibility + if (result.annotations) { + const annotations = result.annotations.list?.filter((annotation) => annotation.enable) || []; + const trimedAnnotations = annotations.map((annotation) => { + return { + name: annotation.name, + enable: annotation.enable, + iconColor: annotation.iconColor, + type: annotation.type, + // @ts-expect-error + builtIn: annotation.builtIn, + hide: annotation.hide, + // TODO: Remove when we migrate snapshots to snapshot queries. + // For now leaving this in here to avoid annotation queries in snapshots. + // Annotations per panel are part of the snapshot query, so we don't need to store them here. + snapshotData: [], + }; + }); + + result.annotations.list = trimedAnnotations; + } + + if (result.templating) { + result.templating.list?.forEach((variable) => { + if ('query' in variable) { + variable.query = ''; + } + if ('options' in variable) { + variable.options = variable.current && !isEmptyObject(variable.current) ? [variable.current] : []; + } + + if ('refresh' in variable) { + variable.refresh = VariableRefresh.never; + } }); } - return annotations; + return result; } diff --git a/public/app/features/dashboard-scene/sharing/ShareModal.tsx b/public/app/features/dashboard-scene/sharing/ShareModal.tsx index 4bc3b19e84b..e6ac04955a5 100644 --- a/public/app/features/dashboard-scene/sharing/ShareModal.tsx +++ b/public/app/features/dashboard-scene/sharing/ShareModal.tsx @@ -46,7 +46,7 @@ export class ShareModal extends SceneObjectBase implements Moda } if (contextSrv.isSignedIn && config.snapshotEnabled) { - tabs.push(new ShareSnapshotTab({ panelRef, modalRef: this.getRef() })); + tabs.push(new ShareSnapshotTab({ panelRef, dashboardRef, modalRef: this.getRef() })); } this.setState({ tabs }); diff --git a/public/app/features/dashboard-scene/sharing/ShareSnapshotTab.tsx b/public/app/features/dashboard-scene/sharing/ShareSnapshotTab.tsx index fa3562c8cc8..591966f7fb4 100644 --- a/public/app/features/dashboard-scene/sharing/ShareSnapshotTab.tsx +++ b/public/app/features/dashboard-scene/sharing/ShareSnapshotTab.tsx @@ -1,20 +1,249 @@ import React from 'react'; +import useAsyncFn from 'react-use/lib/useAsyncFn'; -import { SceneComponentProps, SceneObjectBase, SceneObjectRef, VizPanel } from '@grafana/scenes'; -import { t } from 'app/core/internationalization'; +import { SelectableValue } from '@grafana/data'; +import { getBackendSrv } from '@grafana/runtime'; +import { SceneComponentProps, sceneGraph, SceneObjectBase, SceneObjectRef, VizPanel } from '@grafana/scenes'; +import { Button, ClipboardButton, Field, Input, Modal, RadioButtonGroup } from '@grafana/ui'; +import { t, Trans } from 'app/core/internationalization'; +import { trackDashboardSharingActionPerType } from 'app/features/dashboard/components/ShareModal/analytics'; +import { shareDashboardType } from 'app/features/dashboard/components/ShareModal/utils'; + +import { DashboardScene } from '../scene/DashboardScene'; +import { transformSceneToSaveModel, trimDashboardForSnapshot } from '../serialization/transformSceneToSaveModel'; import { SceneShareTabState } from './types'; +const SNAPSHOTS_API_ENDPOINT = '/api/snapshots'; +const DEFAULT_EXPIRE_OPTION: SelectableValue = { + label: t('share-modal.snapshot.expire-never', `Never`), + value: 0, +}; + +const EXPIRE_OPTIONS = [ + DEFAULT_EXPIRE_OPTION, + { + label: t('share-modal.snapshot.expire-hour', `1 Hour`), + value: 60 * 60, + }, + { + label: t('share-modal.snapshot.expire-day', `1 Day`), + value: 60 * 60 * 24, + }, + { + label: t('share-modal.snapshot.expire-week', `7 Days`), + value: 60 * 60 * 24 * 7, + }, +]; + +type SnapshotSharingOptions = { + externalEnabled: boolean; + externalSnapshotName: string; + externalSnapshotURL: string; + snapshotEnabled: boolean; +}; export interface ShareSnapshotTabState extends SceneShareTabState { panelRef?: SceneObjectRef; + dashboardRef: SceneObjectRef; + snapshotName?: string; + selectedExpireOption?: SelectableValue; + + snapshotSharingOptions?: SnapshotSharingOptions; } export class ShareSnapshotTab extends SceneObjectBase { + static Component = ShareSnapshoTabRenderer; + + public constructor(state: ShareSnapshotTabState) { + super({ + ...state, + snapshotName: state.dashboardRef.resolve().state.title, + selectedExpireOption: DEFAULT_EXPIRE_OPTION, + }); + + this.addActivationHandler(() => { + this._onActivate(); + }); + } + + private _onActivate() { + getBackendSrv() + .get('/api/snapshot/shared-options') + .then((shareOptions: SnapshotSharingOptions) => { + if (this.isActive) { + this.setState({ + snapshotSharingOptions: shareOptions, + }); + } + }); + } + public getTabLabel() { return t('share-modal.tab-title.snapshot', 'Snapshot'); } - static Component = ({ model }: SceneComponentProps) => { - return
    Snapshot
    ; + public onSnasphotNameChange = (snapshotName: string) => { + this.setState({ snapshotName: snapshotName.trim() }); + }; + + public onExpireChange = (option: number) => { + this.setState({ + selectedExpireOption: EXPIRE_OPTIONS.find((o) => o.value === option), + }); + }; + + private prepareSnapshot() { + const timeRange = sceneGraph.getTimeRange(this); + const { dashboardRef, panelRef } = this.state; + const saveModel = transformSceneToSaveModel(dashboardRef.resolve(), true); + + return trimDashboardForSnapshot( + this.state.snapshotName || '', + timeRange.state.value, + saveModel, + panelRef?.resolve() + ); + } + + public onSnapshotCreate = async (external = false) => { + const { selectedExpireOption } = this.state; + const snapshot = this.prepareSnapshot(); + + // TODO + // snapshot.snapshot = { + // originalUrl: window.location.href, + // }; + + const cmdData = { + dashboard: snapshot, + name: snapshot.title, + expires: selectedExpireOption?.value, + external, + }; + + try { + const results: { deleteUrl: string; url: string } = await getBackendSrv().post(SNAPSHOTS_API_ENDPOINT, cmdData); + return results; + } finally { + trackDashboardSharingActionPerType(external ? 'publish_snapshot' : 'local_snapshot', shareDashboardType.snapshot); + } }; } + +function ShareSnapshoTabRenderer({ model }: SceneComponentProps) { + const { snapshotName, selectedExpireOption, modalRef, snapshotSharingOptions } = model.useState(); + + const [snapshotResult, createSnapshot] = useAsyncFn(async (external = false) => { + return model.onSnapshotCreate(external); + }); + + const [deleteSnapshotResult, deleteSnapshot] = useAsyncFn(async (url: string) => { + return await getBackendSrv().get(url); + }); + + // If snapshot has been deleted - show message and allow to close modal + if (deleteSnapshotResult.value) { + return ( + + The snapshot has been deleted. If you have already accessed it once, then it might take up to an hour before + before it is removed from browser caches or CDN caches. + + ); + } + + return ( + <> + {/* Before snapshot has been created show configuration */} + {!Boolean(snapshotResult.value) && ( + <> +
    +

    + + A snapshot is an instant way to share an interactive dashboard publicly. When created, we strip + sensitive data like queries (metric, template, and annotation) and panel links, leaving only the visible + metric data and series names embedded in your dashboard. + +

    +

    + + Keep in mind, your snapshot can be viewed by anyone that has the link and can access the URL. + Share wisely. + +

    +
    + + + model.onSnasphotNameChange(e.target.value)} + /> + + + + + id="expire-select-input" + options={EXPIRE_OPTIONS} + value={selectedExpireOption?.value} + onChange={model.onExpireChange} + /> + + + + + + {snapshotSharingOptions?.externalEnabled && ( + + )} + + + + )} + + {/* When snapshot has been created - show link and allow copy/deletion */} + {snapshotResult.value && ( + <> + + snapshotResult.value!.url}> + Copy + + } + /> + + +
    + Did you make a mistake?   + +
    + + )} + + ); +} diff --git a/public/app/features/dashboard/components/ShareModal/ShareSnapshot.tsx b/public/app/features/dashboard/components/ShareModal/ShareSnapshot.tsx index 10bbf50b0e4..1923a0fb30f 100644 --- a/public/app/features/dashboard/components/ShareModal/ShareSnapshot.tsx +++ b/public/app/features/dashboard/components/ShareModal/ShareSnapshot.tsx @@ -98,6 +98,7 @@ export class ShareSnapshot extends PureComponent { saveSnapshot = async (dashboard: DashboardModel, external?: boolean) => { const { snapshotExpires } = this.state; const dash = this.dashboard.getSaveModelClone(); + this.scrubDashboard(dash); const cmdData = { @@ -141,6 +142,7 @@ export class ShareSnapshot extends PureComponent { // remove annotation queries const annotations = dash.annotations.list.filter((annotation) => annotation.enable); + dash.annotations.list = annotations.map((annotation) => { return { name: annotation.name, From 3fc925364fef8e0b0e50de30dcda2c5b3c0c0bfd Mon Sep 17 00:00:00 2001 From: Karl Persson Date: Wed, 11 Oct 2023 13:50:38 +0200 Subject: [PATCH 044/102] Siningkeys: Fix test setup (#76333) * Don't use integration tests for service and init store only once for integration tests * Set one key as expired in test --- .../signingkeysimpl/service_test.go | 17 +++------ .../signingkeys/signingkeystore/store_test.go | 36 ++++++------------- 2 files changed, 15 insertions(+), 38 deletions(-) diff --git a/pkg/services/signingkeys/signingkeysimpl/service_test.go b/pkg/services/signingkeys/signingkeysimpl/service_test.go index 239c255f622..8bca186d0b9 100644 --- a/pkg/services/signingkeys/signingkeysimpl/service_test.go +++ b/pkg/services/signingkeys/signingkeysimpl/service_test.go @@ -18,7 +18,6 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/api/routing" - "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/localcache" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/remotecache" @@ -45,14 +44,10 @@ func getPrivateKey(t *testing.T, svc *Service) []byte { return bytes } -func TestIntegrationEmbeddedKeyService_GetJWKS_OnlyPublicKeyShared(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - +func TestEmbeddedKeyService_GetJWKS_OnlyPublicKeyShared(t *testing.T) { svc := &Service{ log: log.NewNopLogger(), - store: signingkeystore.NewSigningKeyStore(db.InitTestDB(t)), + store: signingkeystore.NewFakeStore(), secretsService: secretstest.NewFakeSecretsService(), remoteCache: remotecache.NewFakeCacheStorage(), localCache: localcache.New(privateKeyTTL, 10*time.Hour), @@ -87,15 +82,11 @@ func TestIntegrationEmbeddedKeyService_GetJWKS_OnlyPublicKeyShared(t *testing.T) } } -func TestIntegrationEmbeddedKeyService_GetOrCreatePrivateKey(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - +func TestEmbeddedKeyService_GetOrCreatePrivateKey(t *testing.T) { cacheStorage := remotecache.NewFakeCacheStorage() svc := &Service{ log: log.NewNopLogger(), - store: signingkeystore.NewSigningKeyStore(db.InitTestDB(t)), + store: signingkeystore.NewFakeStore(), secretsService: secretstest.NewFakeSecretsService(), remoteCache: cacheStorage, localCache: localcache.New(privateKeyTTL, 10*time.Hour), diff --git a/pkg/services/signingkeys/signingkeystore/store_test.go b/pkg/services/signingkeys/signingkeystore/store_test.go index 54e587dcd40..2ddebe006cb 100644 --- a/pkg/services/signingkeys/signingkeystore/store_test.go +++ b/pkg/services/signingkeys/signingkeystore/store_test.go @@ -17,23 +17,15 @@ func TestIntegrationSigningKeyStore(t *testing.T) { t.Skip("skipping integration test") } - setup := func() (context.Context, *Store) { - return context.Background(), NewSigningKeyStore(db.InitTestDB(t)) - } + ctx, store := context.Background(), NewSigningKeyStore(db.InitTestDB(t)) t.Run("Should successfully add new singing key", func(_ *testing.T) { - ctx, store := setup() key, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", AddedAt: time.Now().UTC(), PrivateKey: []byte{}}, false) require.NoError(t, err) assert.Equal(t, "1", key.KeyID) }) t.Run("Should return old key if already exists", func(_ *testing.T) { - ctx, store := setup() - key, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().UTC()}, false) - require.NoError(t, err) - assert.Equal(t, "1", key.KeyID) - // try to add the same key again with a different AddedAt key2, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().Add(10 * time.Minute).UTC()}, false) require.ErrorIs(t, err, signingkeys.ErrSigningKeyAlreadyExists) @@ -41,38 +33,32 @@ func TestIntegrationSigningKeyStore(t *testing.T) { }) t.Run("Should update old key when force is true", func(t *testing.T) { - ctx, store := setup() - key, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().UTC()}, false) + key, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "2", PrivateKey: []byte{}, AddedAt: time.Now().UTC()}, false) require.NoError(t, err) - assert.Equal(t, "1", key.KeyID) + assert.Equal(t, "2", key.KeyID) // try to add the same key again with a different AddedAt and force is true - key2, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().Add(10 * time.Minute).UTC()}, true) + key2, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "2", PrivateKey: []byte{}, AddedAt: time.Now().Add(10 * time.Minute).UTC()}, true) require.NoError(t, err) - assert.Equal(t, "1", key2.KeyID) + assert.Equal(t, "2", key2.KeyID) assert.NotEqual(t, key.AddedAt, key2.AddedAt) }) t.Run("Should update old key when expired", func(t *testing.T) { - ctx, store := setup() - key, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().UTC(), ExpiresAt: &time.Time{}}, false) + key, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "3", PrivateKey: []byte{}, AddedAt: time.Now().UTC(), ExpiresAt: &time.Time{}}, false) require.NoError(t, err) - assert.Equal(t, "1", key.KeyID) + assert.Equal(t, "3", key.KeyID) // try to add the same key again with a different AddedAt and force is false - key2, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().Add(10 * time.Minute).UTC()}, false) + key2, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "3", PrivateKey: []byte{}, AddedAt: time.Now().Add(10 * time.Minute).UTC()}, false) require.NoError(t, err) - assert.Equal(t, "1", key2.KeyID) + assert.Equal(t, "3", key2.KeyID) assert.NotEqual(t, key.AddedAt, key2.AddedAt) }) t.Run("List should return all keys that are not expired", func(t *testing.T) { - ctx, store := setup() - _, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "1", PrivateKey: []byte{}, AddedAt: time.Now().UTC()}, false) - require.NoError(t, err) - _, err = store.Add(ctx, &signingkeys.SigningKey{KeyID: "2", PrivateKey: []byte{}, AddedAt: time.Now().UTC(), ExpiresAt: &time.Time{}}, false) - require.NoError(t, err) - _, err = store.Add(ctx, &signingkeys.SigningKey{KeyID: "3", PrivateKey: []byte{}, AddedAt: time.Now().UTC()}, false) + // expire key 3 + _, err := store.Add(ctx, &signingkeys.SigningKey{KeyID: "3", PrivateKey: []byte{}, AddedAt: time.Now().UTC(), ExpiresAt: &time.Time{}}, true) require.NoError(t, err) keys, err := store.List(ctx) From 42fb42a90d54f39209f11a7e764ce31b9d405511 Mon Sep 17 00:00:00 2001 From: Khushi Jain <57278642+khushijain21@users.noreply.github.com> Date: Wed, 11 Oct 2023 17:29:13 +0530 Subject: [PATCH 045/102] PublicDashboards: Add validation deletion (#75336) --- pkg/services/publicdashboards/api/api.go | 9 ++- .../public_dashboard_service_mock.go | 10 +-- .../publicdashboards/publicdashboard.go | 2 +- .../publicdashboards/service/service.go | 15 ++++- .../publicdashboards/service/service_test.go | 63 +++++++++++++------ 5 files changed, 72 insertions(+), 27 deletions(-) diff --git a/pkg/services/publicdashboards/api/api.go b/pkg/services/publicdashboards/api/api.go index 708918f14f2..8ab2b0172e7 100644 --- a/pkg/services/publicdashboards/api/api.go +++ b/pkg/services/publicdashboards/api/api.go @@ -220,10 +220,15 @@ func (api *Api) UpdatePublicDashboard(c *contextmodel.ReqContext) response.Respo func (api *Api) DeletePublicDashboard(c *contextmodel.ReqContext) response.Response { uid := web.Params(c.Req)[":uid"] if !validation.IsValidShortUID(uid) { - return response.Err(ErrInvalidUid.Errorf("UpdatePublicDashboard: invalid Uid %s", uid)) + return response.Err(ErrInvalidUid.Errorf("DeletePublicDashboard: invalid Uid %s", uid)) } - err := api.PublicDashboardService.Delete(c.Req.Context(), uid) + dashboardUid := web.Params(c.Req)[":dashboardUid"] + if !validation.IsValidShortUID(dashboardUid) { + return response.Err(ErrInvalidUid.Errorf("DeletePublicDashboard: invalid dashboard Uid %s", dashboardUid)) + } + + err := api.PublicDashboardService.Delete(c.Req.Context(), uid, dashboardUid) if err != nil { return response.Err(err) } diff --git a/pkg/services/publicdashboards/public_dashboard_service_mock.go b/pkg/services/publicdashboards/public_dashboard_service_mock.go index 4e8e03362dd..949e0d9b3d3 100644 --- a/pkg/services/publicdashboards/public_dashboard_service_mock.go +++ b/pkg/services/publicdashboards/public_dashboard_service_mock.go @@ -49,13 +49,13 @@ func (_m *FakePublicDashboardService) Create(ctx context.Context, u *user.Signed return r0, r1 } -// Delete provides a mock function with given fields: ctx, uid -func (_m *FakePublicDashboardService) Delete(ctx context.Context, uid string) error { - ret := _m.Called(ctx, uid) +// Delete provides a mock function with given fields: ctx, uid, dashboardUid +func (_m *FakePublicDashboardService) Delete(ctx context.Context, uid string, dashboardUid string) error { + ret := _m.Called(ctx, uid, dashboardUid) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { - r0 = rf(ctx, uid) + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, uid, dashboardUid) } else { r0 = ret.Error(0) } diff --git a/pkg/services/publicdashboards/publicdashboard.go b/pkg/services/publicdashboards/publicdashboard.go index df070738b07..d6fe478c87b 100644 --- a/pkg/services/publicdashboards/publicdashboard.go +++ b/pkg/services/publicdashboards/publicdashboard.go @@ -26,7 +26,7 @@ type Service interface { Find(ctx context.Context, uid string) (*PublicDashboard, error) Create(ctx context.Context, u *user.SignedInUser, dto *SavePublicDashboardDTO) (*PublicDashboard, error) Update(ctx context.Context, u *user.SignedInUser, dto *SavePublicDashboardDTO) (*PublicDashboard, error) - Delete(ctx context.Context, uid string) error + Delete(ctx context.Context, uid string, dashboardUid string) error DeleteByDashboard(ctx context.Context, dashboard *dashboards.Dashboard) error GetMetricRequest(ctx context.Context, dashboard *dashboards.Dashboard, publicDashboard *PublicDashboard, panelId int64, reqDTO PublicDashboardQueryDTO) (dtos.MetricRequest, error) diff --git a/pkg/services/publicdashboards/service/service.go b/pkg/services/publicdashboards/service/service.go index a091e33931b..b67c9b2a2cc 100644 --- a/pkg/services/publicdashboards/service/service.go +++ b/pkg/services/publicdashboards/service/service.go @@ -332,7 +332,20 @@ func (pd *PublicDashboardServiceImpl) GetOrgIdByAccessToken(ctx context.Context, return pd.store.GetOrgIdByAccessToken(ctx, accessToken) } -func (pd *PublicDashboardServiceImpl) Delete(ctx context.Context, uid string) error { +func (pd *PublicDashboardServiceImpl) Delete(ctx context.Context, uid string, dashboardUid string) error { + // get existing public dashboard if exists + existingPubdash, err := pd.store.Find(ctx, uid) + if err != nil { + return ErrInternalServerError.Errorf("Delete: failed to find public dashboard by uid: %s: %w", uid, err) + } + if existingPubdash == nil { + return ErrPublicDashboardNotFound.Errorf("Delete: public dashboard not found by uid: %s", uid) + } + + // validate the public dashboard belongs to the dashboard + if existingPubdash.DashboardUid != dashboardUid { + return ErrInvalidUid.Errorf("Delete: the public dashboard does not belong to the dashboard") + } return pd.serviceWrapper.Delete(ctx, uid) } diff --git a/pkg/services/publicdashboards/service/service_test.go b/pkg/services/publicdashboards/service/service_test.go index dd2fa4fe6f4..4a85e60c020 100644 --- a/pkg/services/publicdashboards/service/service_test.go +++ b/pkg/services/publicdashboards/service/service_test.go @@ -1295,36 +1295,64 @@ func assertOldValueIfNull(t *testing.T, expectedValue bool, oldValue bool, nulla } func TestDeletePublicDashboard(t *testing.T) { - testCases := []struct { - Name string + pubdash := &PublicDashboard{Uid: "2", OrgId: 1, DashboardUid: "uid"} + + type mockFindResponse struct { + PublicDashboard *PublicDashboard + Err error + } + + type mockDeleteResponse struct { AffectedRowsResp int64 - ExpectedErrResp error StoreRespErr error + } + + testCases := []struct { + Name string + ExpectedErrResp error + mockFindStore *mockFindResponse + mockDeleteStore *mockDeleteResponse }{ { - Name: "Successfully deletes a public dashboards", - AffectedRowsResp: 1, - ExpectedErrResp: nil, - StoreRespErr: nil, + Name: "Successfully deletes a public dashboard", + ExpectedErrResp: nil, + mockFindStore: &mockFindResponse{pubdash, nil}, + mockDeleteStore: &mockDeleteResponse{1, nil}, }, { - Name: "Public dashboard not found", - AffectedRowsResp: 0, - ExpectedErrResp: nil, - StoreRespErr: nil, + Name: "Public dashboard not found", + ExpectedErrResp: ErrInternalServerError.Errorf("Delete: failed to find public dashboard by uid: pubdashUID: error"), + mockFindStore: &mockFindResponse{pubdash, errors.New("error")}, + mockDeleteStore: &mockDeleteResponse{0, nil}, }, { - Name: "Database error", - AffectedRowsResp: 0, - ExpectedErrResp: ErrInternalServerError.Errorf("Delete: failed to delete a public dashboard by Uid: uid db error!"), - StoreRespErr: errors.New("db error!"), + Name: "Public dashboard not found by UID", + ExpectedErrResp: ErrPublicDashboardNotFound.Errorf("Delete: public dashboard not found by uid: pubdashUID"), + mockFindStore: &mockFindResponse{nil, nil}, + mockDeleteStore: &mockDeleteResponse{0, nil}, + }, + { + Name: "Public dashboard UID does not belong to the dashboard", + ExpectedErrResp: ErrInvalidUid.Errorf("Delete: the public dashboard does not belong to the dashboard"), + mockFindStore: &mockFindResponse{&PublicDashboard{Uid: "2", OrgId: 1, DashboardUid: "wrong"}, nil}, + mockDeleteStore: &mockDeleteResponse{0, nil}, + }, + + { + Name: "Failed to delete - Database error", + ExpectedErrResp: ErrInternalServerError.Errorf("Delete: failed to delete a public dashboard by Uid: pubdashUID db error!"), + mockFindStore: &mockFindResponse{pubdash, nil}, + mockDeleteStore: &mockDeleteResponse{1, errors.New("db error!")}, }, } for _, tt := range testCases { t.Run(tt.Name, func(t *testing.T) { store := NewFakePublicDashboardStore(t) - store.On("Delete", mock.Anything, mock.Anything).Return(tt.AffectedRowsResp, tt.StoreRespErr) + store.On("Find", mock.Anything, mock.Anything).Return(tt.mockFindStore.PublicDashboard, tt.mockFindStore.Err) + if tt.ExpectedErrResp == nil || tt.mockDeleteStore.StoreRespErr != nil { + store.On("Delete", mock.Anything, mock.Anything).Return(tt.mockDeleteStore.AffectedRowsResp, tt.mockDeleteStore.StoreRespErr) + } serviceWrapper := &PublicDashboardServiceWrapperImpl{ log: log.New("test.logger"), store: store, @@ -1335,10 +1363,9 @@ func TestDeletePublicDashboard(t *testing.T) { serviceWrapper: serviceWrapper, } - err := service.Delete(context.Background(), "uid") + err := service.Delete(context.Background(), "pubdashUID", "uid") if tt.ExpectedErrResp != nil { assert.Equal(t, tt.ExpectedErrResp.Error(), err.Error()) - assert.Equal(t, tt.ExpectedErrResp.Error(), err.Error()) } else { assert.NoError(t, err) } From d17f25d8c4588150cd499d057d6342f5d689e5a6 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 11 Oct 2023 14:09:10 +0200 Subject: [PATCH 046/102] Chore: Remove reference to storybook 6 (#75815) remove reference to storybook 6 --- packages/grafana-ui/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/grafana-ui/README.md b/packages/grafana-ui/README.md index 00b712ebfdc..c3291db3415 100644 --- a/packages/grafana-ui/README.md +++ b/packages/grafana-ui/README.md @@ -19,7 +19,3 @@ See [package source](https://github.com/grafana/grafana/tree/main/packages/grafa ## Development For development purposes we suggest using `yarn link` that will create symlink to @grafana/ui lib. To do so navigate to `packages/grafana-ui` and run `yarn link`. Then, navigate to your project and run `yarn link @grafana/ui` to use the linked version of the lib. To unlink follow the same procedure, but use `yarn unlink` instead. - -### Storybook 6.x migration - -We've upgraded Storybook to version 6 and with that we will convert to using [controls](https://storybook.js.org/docs/react/essentials/controls) instead of knobs for manipulating components. Controls will not require as much coding as knobs do. Please refer to the [storybook style-guide](https://github.com/grafana/grafana/blob/main/contribute/style-guides/storybook.md#contrls) for further information. From e46e66313df07264eed6e83aef50f9d6ceca64c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 11 Oct 2023 14:27:20 +0200 Subject: [PATCH 047/102] DashboardScene: Support for Angular panels (#76072) * Could work * it's working * remove comment * Update comment * Make options work * Progress * Update panel class name * Update public/app/angular/panel/AngularPanelReactWrapper.tsx Co-authored-by: Dominik Prokop * Fixes --------- Co-authored-by: Dominik Prokop --- public/app/angular/AngularApp.ts | 4 + .../panel/AngularPanelReactWrapper.tsx | 126 ++++++++++++++++++ .../transformSaveModelToScene.ts | 2 +- .../DashboardModelCompatibilityWrapper.ts | 58 ++++++++ .../app/features/plugins/importPanelPlugin.ts | 17 ++- 5 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 public/app/angular/panel/AngularPanelReactWrapper.tsx create mode 100644 public/app/features/dashboard-scene/utils/DashboardModelCompatibilityWrapper.ts diff --git a/public/app/angular/AngularApp.ts b/public/app/angular/AngularApp.ts index 37df54085e2..045f1d27985 100644 --- a/public/app/angular/AngularApp.ts +++ b/public/app/angular/AngularApp.ts @@ -14,12 +14,14 @@ import { config } from 'app/core/config'; import { contextSrv } from 'app/core/services/context_srv'; import { DashboardLoaderSrv } from 'app/features/dashboard/services/DashboardLoaderSrv'; import { getTimeSrv } from 'app/features/dashboard/services/TimeSrv'; +import { setAngularPanelReactWrapper } from 'app/features/plugins/importPanelPlugin'; import { buildImportMap } from 'app/features/plugins/loader/utils'; import * as sdk from 'app/plugins/sdk'; import { registerAngularDirectives } from './angular_wrappers'; import { initAngularRoutingBridge } from './bridgeReactAngularRouting'; import { monkeyPatchInjectorWithPreAssignedBindings } from './injectorMonkeyPatch'; +import { getAngularPanelReactWrapper } from './panel/AngularPanelReactWrapper'; import { promiseToDigest } from './promiseToDigest'; import { registerComponents } from './registerComponents'; @@ -56,6 +58,8 @@ export class AngularApp { init() { const app = angular.module('grafana', []); + setAngularPanelReactWrapper(getAngularPanelReactWrapper); + app.config([ '$controllerProvider', '$compileProvider', diff --git a/public/app/angular/panel/AngularPanelReactWrapper.tsx b/public/app/angular/panel/AngularPanelReactWrapper.tsx new file mode 100644 index 00000000000..d676a431ef8 --- /dev/null +++ b/public/app/angular/panel/AngularPanelReactWrapper.tsx @@ -0,0 +1,126 @@ +import React, { ComponentType, useEffect, useRef } from 'react'; +import { Observable, ReplaySubject } from 'rxjs'; + +import { EventBusSrv, PanelData, PanelPlugin, PanelProps, FieldConfigSource } from '@grafana/data'; +import { AngularComponent, getAngularLoader, RefreshEvent } from '@grafana/runtime'; +import { DashboardModelCompatibilityWrapper } from 'app/features/dashboard-scene/utils/DashboardModelCompatibilityWrapper'; +import { GetDataOptions } from 'app/features/query/state/PanelQueryRunner'; +import { RenderEvent } from 'app/types/events'; + +interface AngularScopeProps { + panel: PanelModelCompatibilityWrapper; + dashboard: DashboardModelCompatibilityWrapper; + queryRunner: FakeQueryRunner; + size: { + height: number; + width: number; + }; +} + +export function getAngularPanelReactWrapper(plugin: PanelPlugin): ComponentType { + return function AngularWrapper(props: PanelProps) { + const divRef = useRef(null); + const angularState = useRef(); + const angularComponent = useRef(); + + useEffect(() => { + if (!divRef.current) { + return; + } + + const loader = getAngularLoader(); + const template = ''; + const queryRunner = new FakeQueryRunner(); + const fakePanel = new PanelModelCompatibilityWrapper(plugin, props, queryRunner); + + angularState.current = { + // @ts-ignore + panel: fakePanel, + // @ts-ignore + dashboard: new DashboardModelCompatibilityWrapper(), + size: { width: props.width, height: props.height }, + queryRunner: queryRunner, + }; + + angularComponent.current = loader.load(divRef.current, angularState.current, template); + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // Re-render angular panel when dimensions change + useEffect(() => { + if (!angularComponent.current) { + return; + } + + angularState.current!.size.height = props.height; + angularState.current!.size.width = props.width; + angularState.current!.panel.events.publish(new RenderEvent()); + }, [props.width, props.height]); + + // Pass new data to angular panel + useEffect(() => { + if (!angularState.current?.panel) { + return; + } + + angularState.current.queryRunner.forwardNewData(props.data); + }, [props.data]); + + return
    ; + }; +} + +class PanelModelCompatibilityWrapper { + id: number; + type: string; + title: string; + plugin: PanelPlugin; + events: EventBusSrv; + queryRunner: FakeQueryRunner; + fieldConfig: FieldConfigSource; + options: Record; + + constructor(plugin: PanelPlugin, props: PanelProps, queryRunner: FakeQueryRunner) { + // Assign legacy "root" level options + if (props.options.angularOptions) { + Object.assign(this, props.options.angularOptions); + } + + this.id = props.id; + this.type = plugin.meta.id; + this.title = props.title; + this.fieldConfig = props.fieldConfig; + this.options = props.options; + + this.plugin = plugin; + this.events = new EventBusSrv(); + this.queryRunner = queryRunner; + } + + refresh() { + this.events.publish(new RefreshEvent()); + } + + render() { + this.events.publish(new RenderEvent()); + } + + getQueryRunner() { + return this.queryRunner; + } +} + +class FakeQueryRunner { + private subject = new ReplaySubject(1); + + getData(options: GetDataOptions): Observable { + return this.subject; + } + + forwardNewData(data: PanelData) { + this.subject.next(data); + } + + run() {} +} diff --git a/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts b/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts index 690a7bc735d..feb796f7b89 100644 --- a/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts +++ b/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts @@ -57,7 +57,7 @@ export interface DashboardLoaderState { export function transformSaveModelToScene(rsp: DashboardDTO): DashboardScene { // Just to have migrations run const oldModel = new DashboardModel(rsp.dashboard, rsp.meta, { - autoMigrateOldPanels: true, + autoMigrateOldPanels: false, }); // Setting for built-in annotations query to run diff --git a/public/app/features/dashboard-scene/utils/DashboardModelCompatibilityWrapper.ts b/public/app/features/dashboard-scene/utils/DashboardModelCompatibilityWrapper.ts new file mode 100644 index 00000000000..33adcc149e2 --- /dev/null +++ b/public/app/features/dashboard-scene/utils/DashboardModelCompatibilityWrapper.ts @@ -0,0 +1,58 @@ +import { DashboardCursorSync, dateTimeFormat, DateTimeInput, EventBusSrv } from '@grafana/data'; +import { behaviors, sceneGraph } from '@grafana/scenes'; + +import { DashboardScene } from '../scene/DashboardScene'; + +/** + * Will move this to make it the main way we remain somewhat compatible with getDashboardSrv().getCurrent + */ +export class DashboardModelCompatibilityWrapper { + events = new EventBusSrv(); + panelInitialized() {} + + public getTimezone() { + const time = sceneGraph.getTimeRange(window.__grafanaSceneContext); + return time.getTimeZone(); + } + + public sharedTooltipModeEnabled() { + return this._getSyncMode() > 0; + } + + public sharedCrosshairModeOnly() { + return this._getSyncMode() > 1; + } + + private _getSyncMode() { + const dashboard = this.getDashboardScene(); + + if (dashboard.state.$behaviors) { + for (const behavior of dashboard.state.$behaviors) { + if (behavior instanceof behaviors.CursorSync) { + return behavior.state.sync > 0; + } + } + } + + return DashboardCursorSync.Off; + } + + private getDashboardScene(): DashboardScene { + if (window.__grafanaSceneContext instanceof DashboardScene) { + return window.__grafanaSceneContext; + } + + throw new Error('Dashboard scene not found'); + } + + public otherPanelInFullscreen(panel: unknown) { + return false; + } + + public formatDate(date: DateTimeInput, format?: string) { + return dateTimeFormat(date, { + format, + timeZone: this.getTimezone(), + }); + } +} diff --git a/public/app/features/plugins/importPanelPlugin.ts b/public/app/features/plugins/importPanelPlugin.ts index 87e215e09f9..053090a1fc2 100644 --- a/public/app/features/plugins/importPanelPlugin.ts +++ b/public/app/features/plugins/importPanelPlugin.ts @@ -1,4 +1,6 @@ -import { PanelPlugin, PanelPluginMeta } from '@grafana/data'; +import { ComponentType } from 'react'; + +import { PanelPlugin, PanelPluginMeta, PanelProps } from '@grafana/data'; import config from 'app/core/config'; import { getPanelPluginLoadError } from '../panel/components/PanelPluginError'; @@ -70,9 +72,14 @@ function getPanelPlugin(meta: PanelPluginMeta): Promise { } throw new Error('missing export: plugin or PanelCtrl'); }) - .then((plugin) => { + .then((plugin: PanelPlugin) => { plugin.meta = meta; panelPluginCache[meta.id] = plugin; + + if (!plugin.panel && plugin.angularPanelCtrl) { + plugin.panel = getAngularPanelReactWrapper(plugin); + } + return plugin; }) .catch((err) => { @@ -81,3 +88,9 @@ function getPanelPlugin(meta: PanelPluginMeta): Promise { return getPanelPluginLoadError(meta, err); }); } + +let getAngularPanelReactWrapper = (plugin: PanelPlugin): ComponentType | null => null; + +export function setAngularPanelReactWrapper(wrapper: typeof getAngularPanelReactWrapper) { + getAngularPanelReactWrapper = wrapper; +} From 1528d6f5c404c24e552913b903e32f3b337e11eb Mon Sep 17 00:00:00 2001 From: Karl Persson Date: Wed, 11 Oct 2023 14:27:43 +0200 Subject: [PATCH 048/102] Authn: Prevent empty username and email during sync (#76330) * Move errors to error file * Move check for both empty username and email to user service * Move check for empty email and username to user service Update * Wrap inner error * Set username in test --- pkg/api/admin_users.go | 9 +------ pkg/api/user.go | 9 +------ .../authn/authnimpl/sync/user_sync.go | 2 +- pkg/services/quota/quotaimpl/quota_test.go | 1 + pkg/services/user/error.go | 24 +++++++++++++++++++ pkg/services/user/model.go | 13 ---------- pkg/services/user/userimpl/user.go | 19 +++++++++++++++ pkg/services/user/userimpl/user_test.go | 20 ++++++++++++++++ 8 files changed, 67 insertions(+), 30 deletions(-) create mode 100644 pkg/services/user/error.go diff --git a/pkg/api/admin_users.go b/pkg/api/admin_users.go index 9030ba71876..6539fb3de0b 100644 --- a/pkg/api/admin_users.go +++ b/pkg/api/admin_users.go @@ -57,13 +57,6 @@ func (hs *HTTPServer) AdminCreateUser(c *contextmodel.ReqContext) response.Respo OrgID: form.OrgId, } - if len(cmd.Login) == 0 { - cmd.Login = cmd.Email - if len(cmd.Login) == 0 { - return response.Error(400, "Validation error, need specify either username or email", nil) - } - } - if len(cmd.Password) < 4 { return response.Error(400, "Password is missing or too short", nil) } @@ -78,7 +71,7 @@ func (hs *HTTPServer) AdminCreateUser(c *contextmodel.ReqContext) response.Respo return response.Error(http.StatusPreconditionFailed, fmt.Sprintf("User with email '%s' or username '%s' already exists", form.Email, form.Login), err) } - return response.Error(http.StatusInternalServerError, "failed to create user", err) + return response.ErrOrFallback(http.StatusInternalServerError, "failed to create user", err) } metrics.MApiAdminUserCreate.Inc() diff --git a/pkg/api/user.go b/pkg/api/user.go index 0437baa5fbf..32b7db52597 100644 --- a/pkg/api/user.go +++ b/pkg/api/user.go @@ -221,18 +221,11 @@ func (hs *HTTPServer) handleUpdateUser(ctx context.Context, cmd user.UpdateUserC return response.Error(http.StatusForbidden, "User info cannot be updated for external Users", nil) } - if len(cmd.Login) == 0 { - cmd.Login = cmd.Email - if len(cmd.Login) == 0 { - return response.Error(http.StatusBadRequest, "Validation error, need to specify either username or email", nil) - } - } - if err := hs.userService.Update(ctx, &cmd); err != nil { if errors.Is(err, user.ErrCaseInsensitive) { return response.Error(http.StatusConflict, "Update would result in user login conflict", err) } - return response.Error(http.StatusInternalServerError, "Failed to update user", err) + return response.ErrOrFallback(http.StatusInternalServerError, "Failed to update user", err) } return response.Success("User updated") diff --git a/pkg/services/authn/authnimpl/sync/user_sync.go b/pkg/services/authn/authnimpl/sync/user_sync.go index 97bc58482a8..0b0a5b09b91 100644 --- a/pkg/services/authn/authnimpl/sync/user_sync.go +++ b/pkg/services/authn/authnimpl/sync/user_sync.go @@ -91,7 +91,7 @@ func (s *UserSync) SyncUserHook(ctx context.Context, id *authn.Identity, _ *auth usr, errCreate = s.createUser(ctx, id) if errCreate != nil { s.log.FromContext(ctx).Error("Failed to create user", "error", errCreate, "auth_module", id.AuthenticatedBy, "auth_id", id.AuthID) - return errSyncUserInternal.Errorf("unable to create user") + return errSyncUserInternal.Errorf("unable to create user: %w", errCreate) } } else { // update user diff --git a/pkg/services/quota/quotaimpl/quota_test.go b/pkg/services/quota/quotaimpl/quota_test.go index 9a3a1cf7d03..42960015213 100644 --- a/pkg/services/quota/quotaimpl/quota_test.go +++ b/pkg/services/quota/quotaimpl/quota_test.go @@ -98,6 +98,7 @@ func TestIntegrationQuotaCommandsAndQueries(t *testing.T) { u, err := userService.Create(context.Background(), &user.CreateUserCommand{ Name: "TestUser", + Login: "TestUser", SkipOrgSetup: true, }) require.NoError(t, err) diff --git a/pkg/services/user/error.go b/pkg/services/user/error.go new file mode 100644 index 00000000000..ba70fd24baf --- /dev/null +++ b/pkg/services/user/error.go @@ -0,0 +1,24 @@ +package user + +import ( + "errors" + + "github.com/grafana/grafana/pkg/util/errutil" +) + +var ( + ErrCaseInsensitive = errors.New("case insensitive conflict") + ErrUserNotFound = errors.New("user not found") + ErrUserAlreadyExists = errors.New("user already exists") + ErrLastGrafanaAdmin = errors.New("cannot remove last grafana admin") + ErrProtectedUser = errors.New("cannot adopt protected user") + ErrNoUniqueID = errors.New("identifying id not found") + ErrLastSeenUpToDate = errors.New("last seen is already up to date") + ErrUpdateInvalidID = errors.New("unable to update invalid id") +) + +var ( + ErrEmptyUsernameAndEmail = errutil.BadRequest( + "user.empty-username-and-email", errutil.WithPublicMessage("Need to specify either username or email"), + ) +) diff --git a/pkg/services/user/model.go b/pkg/services/user/model.go index 9f8f816772a..81b184286ab 100644 --- a/pkg/services/user/model.go +++ b/pkg/services/user/model.go @@ -1,7 +1,6 @@ package user import ( - "errors" "fmt" "strings" "time" @@ -20,18 +19,6 @@ const ( HelpFlagDashboardHelp1 ) -// Typed errors -var ( - ErrCaseInsensitive = errors.New("case insensitive conflict") - ErrUserNotFound = errors.New("user not found") - ErrUserAlreadyExists = errors.New("user already exists") - ErrLastGrafanaAdmin = errors.New("cannot remove last grafana admin") - ErrProtectedUser = errors.New("cannot adopt protected user") - ErrNoUniqueID = errors.New("identifying id not found") - ErrLastSeenUpToDate = errors.New("last seen is already up to date") - ErrUpdateInvalidID = errors.New("unable to update invalid id") -) - type User struct { ID int64 `xorm:"pk autoincr 'id'"` Version int diff --git a/pkg/services/user/userimpl/user.go b/pkg/services/user/userimpl/user.go index 621546db957..fced32139d9 100644 --- a/pkg/services/user/userimpl/user.go +++ b/pkg/services/user/userimpl/user.go @@ -98,6 +98,15 @@ func (s *Service) Usage(ctx context.Context, _ *quota.ScopeParameters) (*quota.M } func (s *Service) Create(ctx context.Context, cmd *user.CreateUserCommand) (*user.User, error) { + if len(cmd.Login) == 0 { + cmd.Login = cmd.Email + } + + // if login is still empty both email and login field is missing + if len(cmd.Login) == 0 { + return nil, user.ErrEmptyUsernameAndEmail.Errorf("user cannot be created with empty username and email") + } + cmdOrg := org.GetOrgIDForNewUserCommand{ Email: cmd.Email, Login: cmd.Login, @@ -215,10 +224,20 @@ func (s *Service) GetByEmail(ctx context.Context, query *user.GetUserByEmailQuer } func (s *Service) Update(ctx context.Context, cmd *user.UpdateUserCommand) error { + if len(cmd.Login) == 0 { + cmd.Login = cmd.Email + } + + // if login is still empty both email and login field is missing + if len(cmd.Login) == 0 { + return user.ErrEmptyUsernameAndEmail.Errorf("user cannot be created with empty username and email") + } + if s.cfg.CaseInsensitiveLogin { cmd.Login = strings.ToLower(cmd.Login) cmd.Email = strings.ToLower(cmd.Email) } + return s.store.Update(ctx, cmd) } diff --git a/pkg/services/user/userimpl/user_test.go b/pkg/services/user/userimpl/user_test.go index 43c9c5d3b5d..dc6bcbbc6e1 100644 --- a/pkg/services/user/userimpl/user_test.go +++ b/pkg/services/user/userimpl/user_test.go @@ -38,6 +38,16 @@ func TestUserService(t *testing.T) { require.NoError(t, err) }) + t.Run("create user should fail when username and email are empty", func(t *testing.T) { + _, err := userService.Create(context.Background(), &user.CreateUserCommand{ + Email: "", + Login: "", + Name: "name", + }) + + require.ErrorIs(t, err, user.ErrEmptyUsernameAndEmail) + }) + t.Run("get user by ID", func(t *testing.T) { userService.cfg = setting.NewCfg() userService.cfg.CaseInsensitiveLogin = false @@ -88,6 +98,16 @@ func TestUserService(t *testing.T) { require.NoError(t, err) }) + t.Run("update user should fail with empty username and password", func(t *testing.T) { + err := userService.Update(context.Background(), &user.UpdateUserCommand{ + Email: "", + Login: "", + Name: "name", + }) + + require.ErrorIs(t, err, user.ErrEmptyUsernameAndEmail) + }) + t.Run("GetByID - email conflict", func(t *testing.T) { userService.cfg.CaseInsensitiveLogin = true userStore.ExpectedError = errors.New("email conflict") From 123f34c73d2b7a6a4cf7da128e02e624d8068bb6 Mon Sep 17 00:00:00 2001 From: Giuseppe Guerra Date: Wed, 11 Oct 2023 14:43:31 +0200 Subject: [PATCH 049/102] Chore: Bump grafana-plugin-sdk-go to v0.180.0 (#76349) Bump grafana-plugin-sdk-go to v0.180.0 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 451a33dbc83..7f055116c51 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( github.com/grafana/cuetsy v0.1.10 // @grafana/grafana-as-code github.com/grafana/grafana-aws-sdk v0.19.1 // @grafana/aws-datasources github.com/grafana/grafana-azure-sdk-go v1.9.0 // @grafana/backend-platform - github.com/grafana/grafana-plugin-sdk-go v0.179.0 // @grafana/plugins-platform-backend + github.com/grafana/grafana-plugin-sdk-go v0.180.0 // @grafana/plugins-platform-backend github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // @grafana/backend-platform github.com/hashicorp/go-hclog v1.5.0 // @grafana/plugins-platform-backend github.com/hashicorp/go-plugin v1.4.9 // @grafana/plugins-platform-backend diff --git a/go.sum b/go.sum index 1342b279715..5daad8e9c43 100644 --- a/go.sum +++ b/go.sum @@ -1818,6 +1818,8 @@ github.com/grafana/grafana-plugin-sdk-go v0.94.0/go.mod h1:3VXz4nCv6wH5SfgB3mlW3 github.com/grafana/grafana-plugin-sdk-go v0.114.0/go.mod h1:D7x3ah+1d4phNXpbnOaxa/osSaZlwh9/ZUnGGzegRbk= github.com/grafana/grafana-plugin-sdk-go v0.179.0 h1:gaow50UDWLhvVlrPye4rESajpRItEecKi2+i+TE1r7Q= github.com/grafana/grafana-plugin-sdk-go v0.179.0/go.mod h1:fPX9spPWEzyUg0BLTQbdXCCq7PVSJZGNVKfNpiTQUts= +github.com/grafana/grafana-plugin-sdk-go v0.180.0 h1:+5aFolBj2H7ze7oejIUjrkmZ9lSydnsbo3e4Kn1CN1Y= +github.com/grafana/grafana-plugin-sdk-go v0.180.0/go.mod h1:fPX9spPWEzyUg0BLTQbdXCCq7PVSJZGNVKfNpiTQUts= github.com/grafana/kindsys v0.0.0-20230508162304-452481b63482 h1:1YNoeIhii4UIIQpCPU+EXidnqf449d0C3ZntAEt4KSo= github.com/grafana/kindsys v0.0.0-20230508162304-452481b63482/go.mod h1:GNcfpy5+SY6RVbNGQW264gC0r336Dm+0zgQ5vt6+M8Y= github.com/grafana/phlare/api v0.1.4-0.20230426005640-f90edba05413 h1:bBzCezZNRyYlJpXTkyZdY4fpPxHZUdyeyRWzhtw/P6I= From bd23757a4224413f87d37e73cba69dc9325703e4 Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Wed, 11 Oct 2023 15:16:21 +0200 Subject: [PATCH 050/102] Chore: Reorder InfluxDB languages in combobox (#76352) Reorder the languages in combobox --- .../influxdb/components/editor/config/ConfigEditor.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/influxdb/components/editor/config/ConfigEditor.tsx b/public/app/plugins/datasource/influxdb/components/editor/config/ConfigEditor.tsx index f5e91daa356..87970d4db16 100644 --- a/public/app/plugins/datasource/influxdb/components/editor/config/ConfigEditor.tsx +++ b/public/app/plugins/datasource/influxdb/components/editor/config/ConfigEditor.tsx @@ -29,17 +29,18 @@ const versions: Array> = [ { label: 'Flux', value: InfluxVersion.Flux, - description: 'Advanced data scripting and query language. Supported in InfluxDB 2.x and 1.8+', + description: 'Supported in InfluxDB 2.x and 1.8+', }, ]; const versionsWithSQL: Array> = [ - ...versions, + { ...versions[0] }, { label: 'SQL', value: InfluxVersion.SQL, description: 'Native SQL language. Supported in InfluxDB 3.0', }, + { ...versions[1] }, ]; export type Props = DataSourcePluginOptionsEditorProps; From ddb9b64128025d5eedf029dbfb2c546329b3a0bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Jamr=C3=B3z?= Date: Wed, 11 Oct 2023 15:18:43 +0200 Subject: [PATCH 051/102] Explore: Remove redundant TODO (#76262) Remove redundant todo --- public/app/features/explore/utils/links.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/public/app/features/explore/utils/links.ts b/public/app/features/explore/utils/links.ts index cf3ed748e95..fe686153aeb 100644 --- a/public/app/features/explore/utils/links.ts +++ b/public/app/features/explore/utils/links.ts @@ -213,7 +213,6 @@ export const getFieldLinksForExplore = (options: { range, field, // Don't track internal links without split view as they are used only in Dashboards - // TODO: It should be revisited in #66570 onClickFn: options.splitOpenFn ? (options) => splitFnWithTracking(options) : undefined, replaceVariables: getTemplateSrv().replace.bind(getTemplateSrv()), }); From 14d01e2b6e939da335a62f926014fefd48fb0536 Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Wed, 11 Oct 2023 15:18:56 +0200 Subject: [PATCH 052/102] Prometheus: Remove prometheusResourceBrowserCache feature toggle (#76172) * Remove prometheusResourceBrowserCache feature toggle * Remove prometheusResourceBrowserCache feature toggle from prometheus datasource * Update tests --- .../feature-toggles/index.md | 1 - .../src/types/featureToggles.gen.ts | 1 - pkg/services/featuremgmt/registry.go | 8 - pkg/services/featuremgmt/toggles_gen.csv | 1 - pkg/services/featuremgmt/toggles_gen.go | 4 - .../prometheus/configuration/PromSettings.tsx | 69 +++++---- .../datasource/prometheus/datasource.test.ts | 47 +----- .../datasource/prometheus/datasource.tsx | 5 - .../prometheus/language_provider.test.ts | 47 ------ .../prometheus/language_provider.ts | 145 +----------------- .../prometheus/metric_find_query.test.ts | 4 +- 11 files changed, 49 insertions(+), 283 deletions(-) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 8412aa950d1..1e1d429e4bf 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -35,7 +35,6 @@ Some features are enabled by default. You can disable these feature by setting t | `logsContextDatasourceUi` | Allow datasource to provide custom UI for context view | Yes | | `gcomOnlyExternalOrgRoleSync` | Prohibits a user from changing organization roles synced with Grafana Cloud auth provider | | | `prometheusMetricEncyclopedia` | Adds the metrics explorer component to the Prometheus query builder as an option in metric select | Yes | -| `prometheusResourceBrowserCache` | Displays browser caching options in Prometheus data source configuration | Yes | | `prometheusDataplane` | Changes responses to from Prometheus to be compliant with the dataplane specification. In particular it sets the numeric Field.Name from 'Value' to the value of the `__name__` label when present. | Yes | | `lokiMetricDataplane` | Changes metric responses from Loki to be compliant with the dataplane specification. | Yes | | `dataplaneFrontendFallback` | Support dataplane contract field name change for transformations and field name matchers where the name is different | Yes | diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 4acfe8c7aed..35a753964bc 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -66,7 +66,6 @@ export interface FeatureToggles { gcomOnlyExternalOrgRoleSync?: boolean; prometheusMetricEncyclopedia?: boolean; timeSeriesTable?: boolean; - prometheusResourceBrowserCache?: boolean; influxdbBackendMigration?: boolean; clientTokenRotation?: boolean; prometheusDataplane?: boolean; diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index c42da0ee7bc..28354e32caf 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -331,14 +331,6 @@ var ( FrontendOnly: true, Owner: appO11ySquad, }, - { - Name: "prometheusResourceBrowserCache", - Description: "Displays browser caching options in Prometheus data source configuration", - Stage: FeatureStageGeneralAvailability, - FrontendOnly: true, - Expression: "true", // turned on by default - Owner: grafanaObservabilityMetricsSquad, - }, { Name: "influxdbBackendMigration", Description: "Query InfluxDB InfluxQL without the proxy", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 0409ef6b370..651b7a48fa9 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -47,7 +47,6 @@ individualCookiePreferences,experimental,@grafana/backend-platform,false,false,f gcomOnlyExternalOrgRoleSync,GA,@grafana/grafana-authnz-team,false,false,false,false prometheusMetricEncyclopedia,GA,@grafana/observability-metrics,false,false,false,true timeSeriesTable,experimental,@grafana/app-o11y,false,false,false,true -prometheusResourceBrowserCache,GA,@grafana/observability-metrics,false,false,false,true influxdbBackendMigration,preview,@grafana/observability-metrics,false,false,false,true clientTokenRotation,experimental,@grafana/grafana-authnz-team,false,false,false,false prometheusDataplane,GA,@grafana/observability-metrics,false,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 95da20c5195..820b0db4bdf 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -199,10 +199,6 @@ const ( // Enable time series table transformer & sparkline cell type FlagTimeSeriesTable = "timeSeriesTable" - // FlagPrometheusResourceBrowserCache - // Displays browser caching options in Prometheus data source configuration - FlagPrometheusResourceBrowserCache = "prometheusResourceBrowserCache" - // FlagInfluxdbBackendMigration // Query InfluxDB InfluxQL without the proxy FlagInfluxdbBackendMigration = "influxdbBackendMigration" diff --git a/public/app/plugins/datasource/prometheus/configuration/PromSettings.tsx b/public/app/plugins/datasource/prometheus/configuration/PromSettings.tsx index 7a5eae28637..4d0ce1931c8 100644 --- a/public/app/plugins/datasource/prometheus/configuration/PromSettings.tsx +++ b/public/app/plugins/datasource/prometheus/configuration/PromSettings.tsx @@ -12,7 +12,6 @@ import { ConfigSubSection } from '@grafana/experimental'; import { getBackendSrv } from '@grafana/runtime/src'; import { InlineField, Input, Select, Switch, useTheme2 } from '@grafana/ui'; -import config from '../../../../core/config'; import { useUpdateDatasource } from '../../../../features/datasources/state'; import { PromApplication, PromBuildInfoResponse } from '../../../../types/unified-alerting-dto'; import { QueryEditorMode } from '../querybuilder/shared/types'; @@ -198,7 +197,12 @@ export const PromSettings = (props: Props) => { spellCheck={false} placeholder="15s" onChange={onChangeHandler('timeInterval', options, onOptionsChange)} - onBlur={(e) => updateValidDuration({ ...validDuration, timeInterval: e.currentTarget.value })} + onBlur={(e) => + updateValidDuration({ + ...validDuration, + timeInterval: e.currentTarget.value, + }) + } /> {validateInput(validDuration.timeInterval, DURATION_REGEX, durationError)} @@ -222,7 +226,12 @@ export const PromSettings = (props: Props) => { onChange={onChangeHandler('queryTimeout', options, onOptionsChange)} spellCheck={false} placeholder="60s" - onBlur={(e) => updateValidDuration({ ...validDuration, queryTimeout: e.currentTarget.value })} + onBlur={(e) => + updateValidDuration({ + ...validDuration, + queryTimeout: e.currentTarget.value, + }) + } /> {validateInput(validDuration.queryTimeout, DURATION_REGEX, durationError)} @@ -362,33 +371,32 @@ export const PromSettings = (props: Props) => {
    )}
  • - {config.featureToggles.prometheusResourceBrowserCache && ( -
    -
    - - Sets the browser caching level for editor queries. Higher cache settings are recommended for high - cardinality data sources. - + +
    +
    + + Sets the browser caching level for editor queries. Higher cache settings are recommended for high + cardinality data sources. + + } + interactive={true} + disabled={options.readOnly} + > + o.value === options.jsonData.cacheLevel) ?? PrometheusCacheLevel.Low - } - /> - -
    + /> +
    - )} +
    @@ -431,7 +439,10 @@ export const PromSettings = (props: Props) => { <> - updateValidDuration({ ...validDuration, incrementalQueryOverlapWindow: e.currentTarget.value }) + updateValidDuration({ + ...validDuration, + incrementalQueryOverlapWindow: e.currentTarget.value, + }) } className="width-20" value={options.jsonData.incrementalQueryOverlapWindow ?? defaultPrometheusQueryOverlapWindow} diff --git a/public/app/plugins/datasource/prometheus/datasource.test.ts b/public/app/plugins/datasource/prometheus/datasource.test.ts index e717b5089be..afdaf6a19ff 100644 --- a/public/app/plugins/datasource/prometheus/datasource.test.ts +++ b/public/app/plugins/datasource/prometheus/datasource.test.ts @@ -14,14 +14,12 @@ import { getFieldDisplayName, LoadingState, toDataFrame, + VariableHide, } from '@grafana/data'; -import { config } from '@grafana/runtime'; import { TimeSrv } from 'app/features/dashboard/services/TimeSrv'; import { TemplateSrv } from 'app/features/templating/template_srv'; import { QueryOptions } from 'app/types'; -import { VariableHide } from '../../../features/variables/types'; - import { alignRange, extractRuleMappingFromGroups, @@ -440,50 +438,7 @@ describe('PrometheusDatasource', () => { }); }); - // Remove when prometheusResourceBrowserCache is removed - describe('When prometheusResourceBrowserCache feature flag is off, there should be no change to the query intervals ', () => { - beforeEach(() => { - config.featureToggles.prometheusResourceBrowserCache = false; - }); - - it('test default 1 minute quantization', () => { - const dataSource = new PrometheusDatasource( - { - ...instanceSettings, - jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.Low }, - }, - templateSrvStub as unknown as TemplateSrv, - timeSrvStub as unknown as TimeSrv - ); - const quantizedRange = dataSource.getAdjustedInterval(); - const oldRange = dataSource.getTimeRangeParams(); - // For "1 minute" the window is unchanged - expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(60); - expect(parseInt(oldRange.end, 10) - parseInt(oldRange.start, 10)).toBe(60); - }); - - it('test 10 minute quantization', () => { - const dataSource = new PrometheusDatasource( - { - ...instanceSettings, - jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.Medium }, - }, - templateSrvStub as unknown as TemplateSrv, - timeSrvStub as unknown as TimeSrv - ); - const quantizedRange = dataSource.getAdjustedInterval(); - const oldRange = dataSource.getTimeRangeParams(); - - expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(60); - expect(parseInt(oldRange.end, 10) - parseInt(oldRange.start, 10)).toBe(60); - }); - }); - describe('Test query range snapping', () => { - beforeEach(() => { - config.featureToggles.prometheusResourceBrowserCache = true; - }); - it('test default 1 minute quantization', () => { const dataSource = new PrometheusDatasource( { diff --git a/public/app/plugins/datasource/prometheus/datasource.tsx b/public/app/plugins/datasource/prometheus/datasource.tsx index bb0e2c01894..e5ea02a88e9 100644 --- a/public/app/plugins/datasource/prometheus/datasource.tsx +++ b/public/app/plugins/datasource/prometheus/datasource.tsx @@ -46,8 +46,6 @@ import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv'; import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv'; import { PromApiFeatures, PromApplication } from 'app/types/unified-alerting-dto'; -import config from '../../../core/config'; - import { addLabelToQuery } from './add_label_to_query'; import { AnnotationQueryEditor } from './components/AnnotationQueryEditor'; import PrometheusLanguageProvider from './language_provider'; @@ -1227,9 +1225,6 @@ export class PrometheusDatasource * Returns the adjusted "snapped" interval parameters */ getAdjustedInterval(): { start: string; end: string } { - if (!config.featureToggles.prometheusResourceBrowserCache) { - return this.getTimeRangeParams(); - } const range = this.timeSrv.timeRange(); return getRangeSnapInterval(this.cacheLevel, range); } diff --git a/public/app/plugins/datasource/prometheus/language_provider.test.ts b/public/app/plugins/datasource/prometheus/language_provider.test.ts index 406d2db26ac..51d821bc2fb 100644 --- a/public/app/plugins/datasource/prometheus/language_provider.test.ts +++ b/public/app/plugins/datasource/prometheus/language_provider.test.ts @@ -2,7 +2,6 @@ import { Editor as SlateEditor } from 'slate'; import Plain from 'slate-plain-serializer'; import { AbstractLabelOperator, dateTime, HistoryItem, TimeRange } from '@grafana/data'; -import { config } from '@grafana/runtime'; import { SearchFunctionType } from '@grafana/ui'; import { Label } from './components/monaco-query-field/monaco-completion-provider/situation'; @@ -98,53 +97,7 @@ describe('Language completion provider', () => { }); }); - // @todo clean up prometheusResourceBrowserCache feature flag - describe('getSeriesLabelsDeprecatedLRU', () => { - beforeEach(() => { - config.featureToggles.prometheusResourceBrowserCache = false; - }); - it('should call labels endpoint', () => { - const languageProvider = new LanguageProvider({ - ...defaultDatasource, - hasLabelsMatchAPISupport: () => true, - } as PrometheusDatasource); - const getSeriesLabels = languageProvider.getSeriesLabels; - const requestSpy = jest.spyOn(languageProvider, 'request'); - - const labelName = 'job'; - const labelValue = 'grafana'; - getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]); - expect(requestSpy).toHaveBeenCalled(); - expect(requestSpy).toHaveBeenCalledWith(`/api/v1/labels`, [], { - end: toPrometheusTimeString, - 'match[]': '{job="grafana"}', - start: fromPrometheusTimeString, - }); - }); - - it('should call series endpoint', () => { - const languageProvider = new LanguageProvider({ - ...defaultDatasource, - getAdjustedInterval: () => getRangeSnapInterval(PrometheusCacheLevel.None, getMockQuantizedTimeRangeParams()), - } as PrometheusDatasource); - const getSeriesLabels = languageProvider.getSeriesLabels; - const requestSpy = jest.spyOn(languageProvider, 'request'); - - const labelName = 'job'; - const labelValue = 'grafana'; - getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]); - expect(requestSpy).toHaveBeenCalled(); - expect(requestSpy).toHaveBeenCalledWith('/api/v1/series', [], { - end: toPrometheusTimeString, - 'match[]': '{job="grafana"}', - start: fromPrometheusTimeString, - }); - }); - }); describe('getSeriesLabels', () => { - beforeEach(() => { - config.featureToggles.prometheusResourceBrowserCache = true; - }); it('should call labels endpoint', () => { const languageProvider = new LanguageProvider({ ...defaultDatasource, diff --git a/public/app/plugins/datasource/prometheus/language_provider.ts b/public/app/plugins/datasource/prometheus/language_provider.ts index a1397517365..265b6332e02 100644 --- a/public/app/plugins/datasource/prometheus/language_provider.ts +++ b/public/app/plugins/datasource/prometheus/language_provider.ts @@ -1,5 +1,4 @@ import { chain, difference, once } from 'lodash'; -import { LRUCache } from 'lru-cache'; import Prism from 'prismjs'; import { Value } from 'slate'; @@ -11,7 +10,7 @@ import { HistoryItem, LanguageProvider, } from '@grafana/data'; -import { BackendSrvRequest, config } from '@grafana/runtime'; +import { BackendSrvRequest } from '@grafana/runtime'; import { CompletionItem, CompletionItemGroup, SearchFunctionType, TypeaheadInput, TypeaheadOutput } from '@grafana/ui'; import { Label } from './components/monaco-query-field/monaco-completion-provider/situation'; @@ -23,7 +22,6 @@ import { parseSelector, processHistogramMetrics, processLabels, - roundSecToMin, toPromLikeQuery, } from './language_utils'; import PromqlSyntax, { FUNCTIONS, RATE_RANGES } from './promql'; @@ -116,13 +114,7 @@ export default class PromQlLanguageProvider extends LanguageProvider { datasource: PrometheusDatasource; labelKeys: string[] = []; declare labelFetchTs: number; - /** - * Cache for labels of series. This is bit simplistic in the sense that it just counts responses each as a 1 and does - * not account for different size of a response. If that is needed a `length` function can be added in the options. - * 10 as a max size is totally arbitrary right now. - */ - private labelsCache = new LRUCache>({ max: 10 }); - private labelValuesCache = new LRUCache({ max: 10 }); + constructor(datasource: PrometheusDatasource, initialValues?: Partial) { super(); @@ -135,11 +127,8 @@ export default class PromQlLanguageProvider extends LanguageProvider { } getDefaultCacheHeaders() { - // @todo clean up prometheusResourceBrowserCache feature flag - if (config.featureToggles.prometheusResourceBrowserCache) { - if (this.datasource.cacheLevel !== PrometheusCacheLevel.None) { - return buildCacheHeaders(this.datasource.getCacheDurationInMinutes() * 60); - } + if (this.datasource.cacheLevel !== PrometheusCacheLevel.None) { + return buildCacheHeaders(this.datasource.getCacheDurationInMinutes() * 60); } return; } @@ -177,10 +166,7 @@ export default class PromQlLanguageProvider extends LanguageProvider { }; async loadMetricsMetadata() { - // @todo clean up prometheusResourceBrowserCache feature flag - const headers = config.featureToggles.prometheusResourceBrowserCache - ? buildCacheHeaders(this.datasource.getDaysToCacheMetadata() * secondsInDay) - : {}; + const headers = buildCacheHeaders(this.datasource.getDaysToCacheMetadata() * secondsInDay); this.metricsMetadata = fixSummariesMetadata( await this.request( '/api/v1/metadata', @@ -574,11 +560,6 @@ export default class PromQlLanguageProvider extends LanguageProvider { ...(interpolatedMatch && { 'match[]': interpolatedMatch }), }; - // @todo clean up prometheusResourceBrowserCache feature flag - if (!config.featureToggles.prometheusResourceBrowserCache) { - return await this.fetchSeriesValuesLRUCache(interpolatedName, range, name, urlParams); - } - const value = await this.request( `/api/v1/label/${interpolatedName}/values`, [], @@ -588,39 +569,6 @@ export default class PromQlLanguageProvider extends LanguageProvider { return value ?? []; }; - /** - * @deprecated - * @todo clean up prometheusResourceBrowserCache feature flag - * @param interpolatedName - * @param range - * @param name - * @param urlParams - * @private - */ - private async fetchSeriesValuesLRUCache( - interpolatedName: string | null, - range: { start: string; end: string }, - name: string, - urlParams: { start: string; end: string } - ) { - const cacheParams = new URLSearchParams({ - 'match[]': interpolatedName ?? '', - start: roundSecToMin(parseInt(range.start, 10)).toString(), - end: roundSecToMin(parseInt(range.end, 10)).toString(), - name: name, - }); - - const cacheKey = `/api/v1/label/?${cacheParams.toString()}/values`; - let value: string[] | undefined = this.labelValuesCache.get(cacheKey); - if (!value) { - value = await this.request(`/api/v1/label/${interpolatedName}/values`, [], urlParams); - if (value) { - this.labelValuesCache.set(cacheKey, value); - } - } - return value ?? []; - } - /** * Gets series labels * Function to replace old getSeries calls in a way that will provide faster endpoints for new prometheus instances, @@ -662,53 +610,11 @@ export default class PromQlLanguageProvider extends LanguageProvider { }; const url = `/api/v1/series`; - if (!config.featureToggles.prometheusResourceBrowserCache) { - return await this.fetchSeriesLabelsLRUCache(interpolatedName, range, withName, url, urlParams); - } - const data = await this.request(url, [], urlParams, this.getDefaultCacheHeaders()); const { values } = processLabels(data, withName); return values; }; - /** - * @deprecated - * @param interpolatedName - * @param range - * @param withName - * @param url - * @param urlParams - * @private - */ - private async fetchSeriesLabelsLRUCache( - interpolatedName: string, - range: { start: string; end: string }, - withName: boolean | undefined, - url: string, - urlParams: { start: string; 'match[]': string; end: string } - ) { - // Cache key is a bit different here. We add the `withName` param and also round up to a minute the intervals. - // The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every - // millisecond while still actually getting all the keys for the correct interval. This still can create problems - // when user does not the newest values for a minute if already cached. - const cacheParams = new URLSearchParams({ - 'match[]': interpolatedName, - start: roundSecToMin(parseInt(range.start, 10)).toString(), - end: roundSecToMin(parseInt(range.end, 10)).toString(), - withName: withName ? 'true' : 'false', - }); - - const cacheKey = `/api/v1/series?${cacheParams.toString()}`; - let value = this.labelsCache.get(cacheKey); - if (!value) { - const data = await this.request(url, [], urlParams); - const { values } = processLabels(data, withName); - value = values; - this.labelsCache.set(cacheKey, value); - } - return value; - } - /** * Fetch labels for a series using /labels endpoint. This is cached by its args but also by the global timeRange currently selected as * they can change over requested time. @@ -723,53 +629,12 @@ export default class PromQlLanguageProvider extends LanguageProvider { 'match[]': interpolatedName, }; const url = `/api/v1/labels`; - if (!config.featureToggles.prometheusResourceBrowserCache) { - return await this.fetchSeriesLabelMatchLRUCache(interpolatedName, range, withName, url, urlParams); - } const data: string[] = await this.request(url, [], urlParams, this.getDefaultCacheHeaders()); // Convert string array to Record return data.reduce((ac, a) => ({ ...ac, [a]: '' }), {}); }; - /** - * @deprecated - * @param interpolatedName - * @param range - * @param withName - * @param url - * @param urlParams - * @private - */ - private async fetchSeriesLabelMatchLRUCache( - interpolatedName: string, - range: { start: string; end: string }, - withName: boolean | undefined, - url: string, - urlParams: { start: string; 'match[]': string; end: string } - ) { - // Cache key is a bit different here. We add the `withName` param and also round up to a minute the intervals. - // The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every - // millisecond while still actually getting all the keys for the correct interval. This still can create problems - // when user does not the newest values for a minute if already cached. - const cacheParams = new URLSearchParams({ - 'match[]': interpolatedName, - start: roundSecToMin(parseInt(range.start, 10)).toString(), - end: roundSecToMin(parseInt(range.end, 10)).toString(), - withName: withName ? 'true' : 'false', - }); - - const cacheKey = `${url}?${cacheParams.toString()}`; - let value = this.labelsCache.get(cacheKey); - if (!value) { - const data: string[] = await this.request(url, [], urlParams); - // Convert string array to Record - value = data.reduce((ac, a) => ({ ...ac, [a]: '' }), {}); - this.labelsCache.set(cacheKey, value); - } - return value; - } - /** * Fetch series for a selector. Use this for raw results. Use fetchSeriesLabels() to get labels. * @param match diff --git a/public/app/plugins/datasource/prometheus/metric_find_query.test.ts b/public/app/plugins/datasource/prometheus/metric_find_query.test.ts index 8089abe9149..2acf59a6228 100644 --- a/public/app/plugins/datasource/prometheus/metric_find_query.test.ts +++ b/public/app/plugins/datasource/prometheus/metric_find_query.test.ts @@ -91,7 +91,9 @@ describe('PrometheusMetricFindQuery', () => { url: `/api/datasources/uid/ABCDEF/resources/api/v1/labels?start=${raw.from.unix()}&end=${raw.to.unix()}`, hideFromInspector: true, showErrorAlert: false, - headers: {}, + headers: { + 'X-Grafana-Cache': 'private, max-age=60', + }, }); }); From effd2a118903d38e00e2ac6b32c5cad106ff8a46 Mon Sep 17 00:00:00 2001 From: Ashley Harrison Date: Wed, 11 Oct 2023 14:37:36 +0100 Subject: [PATCH 053/102] Navigation: Split admin into subsections behind `navAdminSubsections` feature toggle (#76280) * split admin into subsections behind feature toggle * make authentication a section header * add translations for subsections * rename translation keys * add subtitles --- pkg/api/index.go | 2 +- pkg/services/navtree/models.go | 113 ++++++++++++++---- .../app/core/utils/navBarItem-translations.ts | 12 ++ public/app/routes/routes.tsx | 14 +-- public/locales/de-DE/grafana.json | 12 ++ public/locales/en-US/grafana.json | 12 ++ public/locales/es-ES/grafana.json | 12 ++ public/locales/fr-FR/grafana.json | 12 ++ public/locales/pseudo-LOCALE/grafana.json | 12 ++ public/locales/zh-Hans/grafana.json | 12 ++ 10 files changed, 181 insertions(+), 32 deletions(-) diff --git a/pkg/api/index.go b/pkg/api/index.go index 966f6af846d..db692c272f6 100644 --- a/pkg/api/index.go +++ b/pkg/api/index.go @@ -166,7 +166,7 @@ func (hs *HTTPServer) setIndexViewData(c *contextmodel.ReqContext) (*dtos.IndexV hs.HooksService.RunIndexDataHooks(&data, c) - data.NavTree.ApplyAdminIA() + data.NavTree.ApplyAdminIA(hs.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagNavAdminSubsections)) data.NavTree.Sort() return &data, nil diff --git a/pkg/services/navtree/models.go b/pkg/services/navtree/models.go index 8b2e7e9c746..1054d87802e 100644 --- a/pkg/services/navtree/models.go +++ b/pkg/services/navtree/models.go @@ -38,6 +38,9 @@ const ( NavIDMonitoring = "monitoring" NavIDReporting = "reports" NavIDApps = "apps" + NavIDCfgGeneral = "cfg/general" + NavIDCfgPlugins = "cfg/plugins" + NavIDCfgAccess = "cfg/access" ) type NavLink struct { @@ -115,33 +118,103 @@ func Sort(nodes []*NavLink) { } } -func (root *NavTreeRoot) ApplyAdminIA() { +func (root *NavTreeRoot) ApplyAdminIA(navAdminSubsectionsEnabled bool) { orgAdminNode := root.FindById(NavIDCfg) if orgAdminNode != nil { adminNodeLinks := []*NavLink{} - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("datasources")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("plugins")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("global-users")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("teams")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("serviceaccounts")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("apikeys")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("org-settings")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("authentication")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("server-settings")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("global-orgs")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("feature-toggles")) + if navAdminSubsectionsEnabled { + generalNodeLinks := []*NavLink{} + generalNodeLinks = AppendIfNotNil(generalNodeLinks, root.FindById("upgrading")) // TODO does this even exist + generalNodeLinks = AppendIfNotNil(generalNodeLinks, root.FindById("licensing")) + generalNodeLinks = AppendIfNotNil(generalNodeLinks, root.FindById("org-settings")) + generalNodeLinks = AppendIfNotNil(generalNodeLinks, root.FindById("server-settings")) + generalNodeLinks = AppendIfNotNil(generalNodeLinks, root.FindById("global-orgs")) + generalNodeLinks = AppendIfNotNil(generalNodeLinks, root.FindById("feature-toggles")) + generalNodeLinks = AppendIfNotNil(generalNodeLinks, root.FindById("storage")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("upgrading")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("licensing")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("recordedQueries")) // enterprise only - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("correlations")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("plugin-page-grafana-cloud-link-app")) + generalNode := &NavLink{ + Text: "General", + SubTitle: "Manage default preferences and settings across Grafana", + Id: NavIDCfgGeneral, + Url: "/admin/general", + Icon: "shield", + Children: generalNodeLinks, + } - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("ldap")) - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("standalone-plugin-page-/a/grafana-auth-app")) // Cloud Access Policies - adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("storage")) + pluginsNodeLinks := []*NavLink{} + pluginsNodeLinks = AppendIfNotNil(pluginsNodeLinks, root.FindById("plugins")) + pluginsNodeLinks = AppendIfNotNil(pluginsNodeLinks, root.FindById("datasources")) + pluginsNodeLinks = AppendIfNotNil(pluginsNodeLinks, root.FindById("recordedQueries")) + pluginsNodeLinks = AppendIfNotNil(pluginsNodeLinks, root.FindById("correlations")) + pluginsNodeLinks = AppendIfNotNil(pluginsNodeLinks, root.FindById("plugin-page-grafana-cloud-link-app")) + + pluginsNode := &NavLink{ + Text: "Plugins and data", + SubTitle: "Install plugins and define the relationships between data", + Id: NavIDCfgPlugins, + Url: "/admin/plugins", + Icon: "shield", + Children: pluginsNodeLinks, + } + + accessNodeLinks := []*NavLink{} + accessNodeLinks = AppendIfNotNil(accessNodeLinks, root.FindById("global-users")) + accessNodeLinks = AppendIfNotNil(accessNodeLinks, root.FindById("teams")) + accessNodeLinks = AppendIfNotNil(accessNodeLinks, root.FindById("standalone-plugin-page-/a/grafana-auth-app")) + accessNodeLinks = AppendIfNotNil(accessNodeLinks, root.FindById("serviceaccounts")) + accessNodeLinks = AppendIfNotNil(accessNodeLinks, root.FindById("apikeys")) + + usersNode := &NavLink{ + Text: "Users and access", + SubTitle: "Configure access for individual users, teams, and service accounts", + Id: NavIDCfgAccess, + Url: "/admin/access", + Icon: "shield", + Children: accessNodeLinks, + } + + if len(generalNode.Children) > 0 { + adminNodeLinks = append(adminNodeLinks, generalNode) + } + + if len(pluginsNode.Children) > 0 { + adminNodeLinks = append(adminNodeLinks, pluginsNode) + } + + if len(usersNode.Children) > 0 { + adminNodeLinks = append(adminNodeLinks, usersNode) + } + + authenticationNode := root.FindById("authentication") + if authenticationNode != nil { + authenticationNode.IsSection = true + adminNodeLinks = append(adminNodeLinks, authenticationNode) + } + } else { + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("datasources")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("plugins")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("global-users")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("teams")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("serviceaccounts")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("apikeys")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("org-settings")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("authentication")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("server-settings")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("global-orgs")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("feature-toggles")) + + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("upgrading")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("licensing")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("recordedQueries")) // enterprise only + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("correlations")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("plugin-page-grafana-cloud-link-app")) + + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("ldap")) + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("standalone-plugin-page-/a/grafana-auth-app")) // Cloud Access Policies + adminNodeLinks = AppendIfNotNil(adminNodeLinks, root.FindById("storage")) + } if len(adminNodeLinks) > 0 { orgAdminNode.Children = adminNodeLinks diff --git a/public/app/core/utils/navBarItem-translations.ts b/public/app/core/utils/navBarItem-translations.ts index 48f5e3d493f..2c08370ad4f 100644 --- a/public/app/core/utils/navBarItem-translations.ts +++ b/public/app/core/utils/navBarItem-translations.ts @@ -74,6 +74,12 @@ export function getNavTitle(navId: string | undefined) { return t('nav.alerting-admin.title', 'Admin'); case 'cfg': return t('nav.config.title', 'Administration'); + case 'cfg/general': + return t('nav.config-general.title', 'General'); + case 'cfg/plugins': + return t('nav.config-plugins.title', 'Plugins and data'); + case 'cfg/access': + return t('nav.config-access.title', 'Users and access'); case 'datasources': return t('nav.datasources.title', 'Data sources'); case 'authentication': @@ -230,6 +236,12 @@ export function getNavSubTitle(navId: string | undefined) { 'nav.admin.subtitle', 'Manage server-wide settings and access to resources such as organizations, users, and licenses' ); + case 'cfg/general': + return t('nav.config-general.subtitle', 'Manage default preferences and settings across Grafana'); + case 'cfg/plugins': + return t('nav.config-plugins.subtitle', 'Install plugins and define the relationships between data'); + case 'cfg/access': + return t('nav.config-access.subtitle', 'Configure access for individual users, teams, and service accounts'); case 'apps': return t('nav.apps.subtitle', 'App plugins that extend the Grafana experience'); case 'monitoring': diff --git a/public/app/routes/routes.tsx b/public/app/routes/routes.tsx index dce15c3584e..b46e9e8f86d 100644 --- a/public/app/routes/routes.tsx +++ b/public/app/routes/routes.tsx @@ -197,15 +197,15 @@ export function getAppRoutes(): RouteDescriptor[] { }, { path: '/admin/general', - component: () => , + component: () => , }, { path: '/admin/plugins', - component: () => , + component: () => , }, { path: '/admin/access', - component: () => , + component: () => , }, { path: '/org', @@ -295,14 +295,6 @@ export function getAppRoutes(): RouteDescriptor[] { ) : () => , }, - { - path: '/admin/access', - component: () => , - }, - { - path: '/admin/config', - component: () => , - }, { path: '/admin/settings', component: SafeDynamicImport( diff --git a/public/locales/de-DE/grafana.json b/public/locales/de-DE/grafana.json index 61e16dbea00..9f6aa35d392 100644 --- a/public/locales/de-DE/grafana.json +++ b/public/locales/de-DE/grafana.json @@ -517,6 +517,18 @@ "config": { "title": "Verwaltung" }, + "config-access": { + "subtitle": "", + "title": "" + }, + "config-general": { + "subtitle": "", + "title": "" + }, + "config-plugins": { + "subtitle": "", + "title": "" + }, "connect-data": { "title": "" }, diff --git a/public/locales/en-US/grafana.json b/public/locales/en-US/grafana.json index 601a0ea8b87..40f90682684 100644 --- a/public/locales/en-US/grafana.json +++ b/public/locales/en-US/grafana.json @@ -517,6 +517,18 @@ "config": { "title": "Administration" }, + "config-access": { + "subtitle": "Configure access for individual users, teams, and service accounts", + "title": "Users and access" + }, + "config-general": { + "subtitle": "Manage default preferences and settings across Grafana", + "title": "General" + }, + "config-plugins": { + "subtitle": "Install plugins and define the relationships between data", + "title": "Plugins and data" + }, "connect-data": { "title": "Connect data" }, diff --git a/public/locales/es-ES/grafana.json b/public/locales/es-ES/grafana.json index 671d2059313..0fa848259d1 100644 --- a/public/locales/es-ES/grafana.json +++ b/public/locales/es-ES/grafana.json @@ -523,6 +523,18 @@ "config": { "title": "Administración" }, + "config-access": { + "subtitle": "", + "title": "" + }, + "config-general": { + "subtitle": "", + "title": "" + }, + "config-plugins": { + "subtitle": "", + "title": "" + }, "connect-data": { "title": "" }, diff --git a/public/locales/fr-FR/grafana.json b/public/locales/fr-FR/grafana.json index 48997a11644..13a678be1b2 100644 --- a/public/locales/fr-FR/grafana.json +++ b/public/locales/fr-FR/grafana.json @@ -523,6 +523,18 @@ "config": { "title": "Administration" }, + "config-access": { + "subtitle": "", + "title": "" + }, + "config-general": { + "subtitle": "", + "title": "" + }, + "config-plugins": { + "subtitle": "", + "title": "" + }, "connect-data": { "title": "" }, diff --git a/public/locales/pseudo-LOCALE/grafana.json b/public/locales/pseudo-LOCALE/grafana.json index 5af63ced102..12d900228c1 100644 --- a/public/locales/pseudo-LOCALE/grafana.json +++ b/public/locales/pseudo-LOCALE/grafana.json @@ -517,6 +517,18 @@ "config": { "title": "Åđmįʼnįşŧřäŧįőʼn" }, + "config-access": { + "subtitle": "Cőʼnƒįģūřę äččęşş ƒőř įʼnđįvįđūäľ ūşęřş, ŧęämş, äʼnđ şęřvįčę äččőūʼnŧş", + "title": "Ůşęřş äʼnđ äččęşş" + }, + "config-general": { + "subtitle": "Mäʼnäģę đęƒäūľŧ přęƒęřęʼnčęş äʼnđ şęŧŧįʼnģş äčřőşş Ğřäƒäʼnä", + "title": "Ğęʼnęřäľ" + }, + "config-plugins": { + "subtitle": "Ĩʼnşŧäľľ pľūģįʼnş äʼnđ đęƒįʼnę ŧĥę řęľäŧįőʼnşĥįpş þęŧŵęęʼn đäŧä", + "title": "Pľūģįʼnş äʼnđ đäŧä" + }, "connect-data": { "title": "Cőʼnʼnęčŧ đäŧä" }, diff --git a/public/locales/zh-Hans/grafana.json b/public/locales/zh-Hans/grafana.json index 23bfbaee5cb..27adff77cae 100644 --- a/public/locales/zh-Hans/grafana.json +++ b/public/locales/zh-Hans/grafana.json @@ -511,6 +511,18 @@ "config": { "title": "管理" }, + "config-access": { + "subtitle": "", + "title": "" + }, + "config-general": { + "subtitle": "", + "title": "" + }, + "config-plugins": { + "subtitle": "", + "title": "" + }, "connect-data": { "title": "" }, From 755805597bc1032508a3d8b26fe31de2dcf622b9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 14:40:04 +0100 Subject: [PATCH 054/102] Update dependency core-js to v3.33.0 (#76341) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 2 +- packages/grafana-ui/package.json | 2 +- yarn.lock | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 35c6ca3b028..fdbf162f159 100644 --- a/package.json +++ b/package.json @@ -306,7 +306,7 @@ "combokeys": "^3.0.0", "comlink": "4.4.1", "common-tags": "1.8.2", - "core-js": "3.31.1", + "core-js": "3.33.0", "d3": "7.8.5", "d3-force": "3.0.0", "d3-scale-chromatic": "3.0.0", diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index a5419a25e03..0c2c120a453 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -66,7 +66,7 @@ "ansicolor": "1.1.100", "calculate-size": "1.1.1", "classnames": "2.3.2", - "core-js": "3.31.1", + "core-js": "3.33.0", "d3": "7.8.5", "date-fns": "2.30.0", "hoist-non-react-statics": "3.3.2", diff --git a/yarn.lock b/yarn.lock index 5d512a1a0e9..0d9cdf2b00a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3469,7 +3469,7 @@ __metadata: calculate-size: 1.1.1 classnames: 2.3.2 common-tags: 1.8.2 - core-js: 3.31.1 + core-js: 3.33.0 css-loader: 6.8.1 csstype: 3.1.2 d3: 7.8.5 @@ -13389,10 +13389,10 @@ __metadata: languageName: node linkType: hard -"core-js@npm:3.31.1, core-js@npm:^3.6.0, core-js@npm:^3.8.3": - version: 3.31.1 - resolution: "core-js@npm:3.31.1" - checksum: 14519213a63c55cf188bdd2f4dece54583feaf6b90e75d6c65e07f509cd487055bf64898aeda7c97c36029ac1ea2f2ed8e4b02281553f6a257e7143a32a14015 +"core-js@npm:3.33.0, core-js@npm:^3.6.0, core-js@npm:^3.8.3": + version: 3.33.0 + resolution: "core-js@npm:3.33.0" + checksum: dd62217935ac281faf6f833bb306fb891162919fcf9c1f0c975b1b91e82ac09a940f5deb5950bbb582739ceef716e8bd7e4f9eab8328932fb029d3bc2ecb2881 languageName: node linkType: hard @@ -17773,7 +17773,7 @@ __metadata: comlink: 4.4.1 common-tags: 1.8.2 copy-webpack-plugin: 11.0.0 - core-js: 3.31.1 + core-js: 3.33.0 css-loader: 6.8.1 css-minimizer-webpack-plugin: 5.0.1 cypress: 13.1.0 From 6b52bb9c27c0345c74e16b4799df72f017dc7884 Mon Sep 17 00:00:00 2001 From: Horst Gutmann Date: Wed, 11 Oct 2023 15:49:24 +0200 Subject: [PATCH 055/102] Chore: Upgrade Go to 1.20.10 (#76359) --- .drone.yml | 202 ++++++++++---------- .github/workflows/alerting-swagger-gen.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/pr-codeql-analysis-go.yml | 2 +- .github/workflows/publish-kinds-next.yml | 2 +- .github/workflows/publish-kinds-release.yml | 2 +- .github/workflows/verify-kinds.yml | 2 +- Dockerfile | 2 +- Makefile | 2 +- scripts/build/ci-build/Dockerfile | 2 +- scripts/drone/variables.star | 2 +- 11 files changed, 111 insertions(+), 111 deletions(-) diff --git a/.drone.yml b/.drone.yml index b7921ee08f8..9af5f9fb069 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,7 +24,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build verify-drone @@ -74,14 +74,14 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - go install github.com/bazelbuild/buildtools/buildifier@latest - buildifier --lint=warn -mode=check -r . depends_on: - compile-build-cmd - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: lint-starlark trigger: event: @@ -316,7 +316,7 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' @@ -325,21 +325,21 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: test-backend - commands: - apk add --update build-base @@ -348,7 +348,7 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: test-backend-integration trigger: event: @@ -397,7 +397,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - apk add --update curl jq bash @@ -424,7 +424,7 @@ steps: - apk add --update make - make gen-go depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - apk add --update make build-base @@ -433,16 +433,16 @@ steps: - wire-install environment: CGO_ENABLED: "1" - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: lint-backend - commands: - go run scripts/modowners/modowners.go check go.mod - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: validate-modfile - commands: - apk add --update make - make swagger-validate - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: validate-openapi-spec trigger: event: @@ -498,7 +498,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -508,7 +508,7 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' @@ -517,14 +517,14 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - yarn install --immutable @@ -587,7 +587,7 @@ steps: token: from_secret: drone_token - commands: - - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.8 + - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10 --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD > packages.txt depends_on: @@ -871,7 +871,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME @@ -885,7 +885,7 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' @@ -894,14 +894,14 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - dockerize -wait tcp://postgres:5432 -timeout 120s @@ -922,7 +922,7 @@ steps: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - dockerize -wait tcp://mysql57:3306 -timeout 120s @@ -943,7 +943,7 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: mysql-5.7-integration-tests - commands: - dockerize -wait tcp://mysql80:3306 -timeout 120s @@ -964,7 +964,7 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: mysql-8.0-integration-tests - commands: - dockerize -wait tcp://redis:6379 -timeout 120s @@ -979,7 +979,7 @@ steps: - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s @@ -994,7 +994,7 @@ steps: - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: memcached-integration-tests - commands: - dockerize -wait tcp://mimir:8080 -timeout 120s @@ -1007,7 +1007,7 @@ steps: depends_on: - wire-install - wait-for-remote-alertmanager - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: remote-alertmanager-integration-tests trigger: event: @@ -1093,7 +1093,7 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue trigger: event: @@ -1133,7 +1133,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - apt-get update -yq && apt-get install shellcheck @@ -1240,7 +1240,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -1251,7 +1251,7 @@ steps: - CODEGEN_VERIFY=1 make gen-cue depends_on: - clone-enterprise - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' @@ -1261,14 +1261,14 @@ steps: - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: - clone-enterprise - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - apk add --update build-base @@ -1276,7 +1276,7 @@ steps: - go test -v -run=^$ -benchmem -timeout=1h -count=8 -bench=. ${GO_PACKAGES} depends_on: - wire-install - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: sqlite-benchmark-integration-tests - commands: - apk add --update build-base @@ -1288,7 +1288,7 @@ steps: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: postgres-benchmark-integration-tests - commands: - apk add --update build-base @@ -1299,7 +1299,7 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: mysql-5.7-benchmark-integration-tests - commands: - apk add --update build-base @@ -1310,7 +1310,7 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: mysql-8.0-benchmark-integration-tests trigger: event: @@ -1386,7 +1386,7 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue trigger: branch: main @@ -1558,7 +1558,7 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' @@ -1567,21 +1567,21 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: test-backend - commands: - apk add --update build-base @@ -1590,7 +1590,7 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: test-backend-integration trigger: branch: main @@ -1634,13 +1634,13 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - apk add --update make - make gen-go depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - apk add --update make build-base @@ -1649,16 +1649,16 @@ steps: - wire-install environment: CGO_ENABLED: "1" - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: lint-backend - commands: - go run scripts/modowners/modowners.go check go.mod - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: validate-modfile - commands: - apk add --update make - make swagger-validate - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: validate-openapi-spec - commands: - ./bin/build verify-drone @@ -1714,7 +1714,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - '# It is required that code generated from Thema/CUE be committed and in sync @@ -1724,7 +1724,7 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' @@ -1733,14 +1733,14 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - yarn install --immutable @@ -1771,7 +1771,7 @@ steps: image: node:18.12.0-alpine name: build-frontend-packages - commands: - - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.8 + - /src/grafana-build package --distro=linux/amd64,linux/arm64 --go-version=1.20.10 --yarn-cache=$$YARN_CACHE_FOLDER --build-id=$$DRONE_BUILD_NUMBER --grafana-dir=$$PWD > packages.txt depends_on: @@ -2153,7 +2153,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - echo $DRONE_RUNNER_NAME @@ -2167,7 +2167,7 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' @@ -2176,14 +2176,14 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - dockerize -wait tcp://postgres:5432 -timeout 120s @@ -2204,7 +2204,7 @@ steps: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - dockerize -wait tcp://mysql57:3306 -timeout 120s @@ -2225,7 +2225,7 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: mysql-5.7-integration-tests - commands: - dockerize -wait tcp://mysql80:3306 -timeout 120s @@ -2246,7 +2246,7 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: mysql-8.0-integration-tests - commands: - dockerize -wait tcp://redis:6379 -timeout 120s @@ -2261,7 +2261,7 @@ steps: - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s @@ -2276,7 +2276,7 @@ steps: - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: memcached-integration-tests - commands: - dockerize -wait tcp://mimir:8080 -timeout 120s @@ -2289,7 +2289,7 @@ steps: depends_on: - wire-install - wait-for-remote-alertmanager - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: remote-alertmanager-integration-tests trigger: branch: main @@ -2511,7 +2511,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build artifacts docker fetch --edition oss @@ -2607,7 +2607,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build artifacts packages --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} @@ -2676,7 +2676,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - yarn install --immutable @@ -2705,7 +2705,7 @@ steps: NPM_TOKEN: from_secret: npm_token failure: ignore - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: release-npm-packages trigger: event: @@ -2741,7 +2741,7 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - depends_on: - compile-build-cmd @@ -2846,7 +2846,7 @@ steps: from_secret: gcp_key_base64 GITHUB_TOKEN: from_secret: github_token - GO_VERSION: 1.20.8 + GO_VERSION: 1.20.10 GPG_PASSPHRASE: from_secret: packages_gpg_passphrase GPG_PRIVATE_KEY: @@ -2902,13 +2902,13 @@ steps: depends_on: [] environment: CGO_ENABLED: 0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: compile-build-cmd - commands: - ./bin/build whatsnew-checker depends_on: - compile-build-cmd - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: whats-new-checker trigger: event: @@ -3008,7 +3008,7 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' @@ -3017,21 +3017,21 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: test-backend - commands: - apk add --update build-base @@ -3040,7 +3040,7 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: test-backend-integration trigger: event: @@ -3095,7 +3095,7 @@ steps: from_secret: gcp_key_base64 GITHUB_TOKEN: from_secret: github_token - GO_VERSION: 1.20.8 + GO_VERSION: 1.20.10 GPG_PASSPHRASE: from_secret: packages_gpg_passphrase GPG_PRIVATE_KEY: @@ -3276,7 +3276,7 @@ steps: from_secret: gcp_key_base64 GITHUB_TOKEN: from_secret: github_token - GO_VERSION: 1.20.8 + GO_VERSION: 1.20.10 GPG_PASSPHRASE: from_secret: packages_gpg_passphrase GPG_PRIVATE_KEY: @@ -3421,7 +3421,7 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' @@ -3430,21 +3430,21 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - apk add --update build-base shared-mime-info shared-mime-info-lang - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: test-backend - commands: - apk add --update build-base @@ -3453,7 +3453,7 @@ steps: | grep -o '\(.*\)/' | sort -u) depends_on: - wire-install - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: test-backend-integration trigger: cron: @@ -3506,7 +3506,7 @@ steps: from_secret: gcp_key_base64 GITHUB_TOKEN: from_secret: github_token - GO_VERSION: 1.20.8 + GO_VERSION: 1.20.10 GPG_PASSPHRASE: from_secret: packages_gpg_passphrase GPG_PRIVATE_KEY: @@ -3650,7 +3650,7 @@ steps: from_secret: gcp_key_base64 GITHUB_TOKEN: from_secret: github_token - GO_VERSION: 1.20.8 + GO_VERSION: 1.20.10 GPG_PASSPHRASE: from_secret: packages_gpg_passphrase GPG_PRIVATE_KEY: @@ -3746,20 +3746,20 @@ steps: - commands: [] depends_on: - clone - image: golang:1.20.8-windowsservercore-1809 + image: golang:1.20.10-windowsservercore-1809 name: windows-init - commands: - go install github.com/google/wire/cmd/wire@v0.5.0 - wire gen -tags oss ./pkg/server depends_on: - windows-init - image: golang:1.20.8-windowsservercore-1809 + image: golang:1.20.10-windowsservercore-1809 name: wire-install - commands: - go test -tags requires_buildifer -short -covermode=atomic -timeout=5m ./pkg/... depends_on: - wire-install - image: golang:1.20.8-windowsservercore-1809 + image: golang:1.20.10-windowsservercore-1809 name: test-backend trigger: event: @@ -3851,7 +3851,7 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-cue depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-cue - commands: - '# It is required that generated jsonnet is committed and in sync with its inputs.' @@ -3860,14 +3860,14 @@ steps: - apk add --update make - CODEGEN_VERIFY=1 make gen-jsonnet depends_on: [] - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: verify-gen-jsonnet - commands: - apk add --update make - make gen-go depends_on: - verify-gen-cue - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: wire-install - commands: - dockerize -wait tcp://postgres:5432 -timeout 120s @@ -3888,7 +3888,7 @@ steps: GRAFANA_TEST_DB: postgres PGPASSWORD: grafanatest POSTGRES_HOST: postgres - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: postgres-integration-tests - commands: - dockerize -wait tcp://mysql57:3306 -timeout 120s @@ -3909,7 +3909,7 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql57 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: mysql-5.7-integration-tests - commands: - dockerize -wait tcp://mysql80:3306 -timeout 120s @@ -3930,7 +3930,7 @@ steps: environment: GRAFANA_TEST_DB: mysql MYSQL_HOST: mysql80 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: mysql-8.0-integration-tests - commands: - dockerize -wait tcp://redis:6379 -timeout 120s @@ -3945,7 +3945,7 @@ steps: - wait-for-redis environment: REDIS_URL: redis://redis:6379/0 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: redis-integration-tests - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s @@ -3960,7 +3960,7 @@ steps: - wait-for-memcached environment: MEMCACHED_HOSTS: memcached:11211 - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: memcached-integration-tests - commands: - dockerize -wait tcp://mimir:8080 -timeout 120s @@ -3973,7 +3973,7 @@ steps: depends_on: - wire-install - wait-for-remote-alertmanager - image: golang:1.20.8-alpine + image: golang:1.20.10-alpine name: remote-alertmanager-integration-tests trigger: event: @@ -4379,7 +4379,7 @@ steps: path: /root/.docker/ - commands: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine/git:2.40.1 - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.8-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.10-alpine - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM node:18.12.0-alpine - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM google/cloud-sdk:431.0.0 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-deploy:1.3.3 @@ -4413,7 +4413,7 @@ steps: path: /root/.docker/ - commands: - trivy --exit-code 1 --severity HIGH,CRITICAL alpine/git:2.40.1 - - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.8-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.10-alpine - trivy --exit-code 1 --severity HIGH,CRITICAL node:18.12.0-alpine - trivy --exit-code 1 --severity HIGH,CRITICAL google/cloud-sdk:431.0.0 - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-deploy:1.3.3 @@ -4661,6 +4661,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 68c3d5cfb08c1e5384f39ea57badfd6dbb46943a87d54571665ec29a1257b707 +hmac: efc8706531a40b72753d483d3d8a7ba4941b5f54e0b21a68bcf47556a97a9ede ... diff --git a/.github/workflows/alerting-swagger-gen.yml b/.github/workflows/alerting-swagger-gen.yml index a90f9b47bf0..f42389ceeed 100644 --- a/.github/workflows/alerting-swagger-gen.yml +++ b/.github/workflows/alerting-swagger-gen.yml @@ -16,7 +16,7 @@ jobs: - name: Set go version uses: actions/setup-go@v4 with: - go-version: '1.20.8' + go-version: '1.20.10' - name: Build swagger run: | make -C pkg/services/ngalert/api/tooling post.json api.json diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6033215ebc8..60fe93679cf 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -47,7 +47,7 @@ jobs: name: Set go version uses: actions/setup-go@v4 with: - go-version: '1.20.8' + go-version: '1.20.10' # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/pr-codeql-analysis-go.yml b/.github/workflows/pr-codeql-analysis-go.yml index aa0ff0e03d2..26b45ad5814 100644 --- a/.github/workflows/pr-codeql-analysis-go.yml +++ b/.github/workflows/pr-codeql-analysis-go.yml @@ -26,7 +26,7 @@ jobs: - name: Set go version uses: actions/setup-go@v4 with: - go-version: '1.20.8' + go-version: '1.20.10' # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/publish-kinds-next.yml b/.github/workflows/publish-kinds-next.yml index 0a636c62676..97c70377ac5 100644 --- a/.github/workflows/publish-kinds-next.yml +++ b/.github/workflows/publish-kinds-next.yml @@ -36,7 +36,7 @@ jobs: - name: "Setup Go" uses: "actions/setup-go@v4" with: - go-version: '1.20.8' + go-version: '1.20.10' - name: "Verify kinds" run: go run .github/workflows/scripts/kinds/verify-kinds.go diff --git a/.github/workflows/publish-kinds-release.yml b/.github/workflows/publish-kinds-release.yml index 9f734f40d7c..30516f062d8 100644 --- a/.github/workflows/publish-kinds-release.yml +++ b/.github/workflows/publish-kinds-release.yml @@ -39,7 +39,7 @@ jobs: - name: "Setup Go" uses: "actions/setup-go@v4" with: - go-version: '1.20.8' + go-version: '1.20.10' - name: "Verify kinds" run: go run .github/workflows/scripts/kinds/verify-kinds.go diff --git a/.github/workflows/verify-kinds.yml b/.github/workflows/verify-kinds.yml index e9cc5681b2e..7ea1bb46a1c 100644 --- a/.github/workflows/verify-kinds.yml +++ b/.github/workflows/verify-kinds.yml @@ -18,7 +18,7 @@ jobs: - name: "Setup Go" uses: "actions/setup-go@v4" with: - go-version: '1.20.8' + go-version: '1.20.10' - name: "Verify kinds" run: go run .github/workflows/scripts/kinds/verify-kinds.go diff --git a/Dockerfile b/Dockerfile index 982fa08efab..22e1f102583 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG BASE_IMAGE=alpine:3.18.3 ARG JS_IMAGE=node:18-alpine3.18 ARG JS_PLATFORM=linux/amd64 -ARG GO_IMAGE=golang:1.20.8-alpine3.18 +ARG GO_IMAGE=golang:1.20.10-alpine3.18 ARG GO_SRC=go-builder ARG JS_SRC=js-builder diff --git a/Makefile b/Makefile index 00fc748fd91..54287b0ff09 100644 --- a/Makefile +++ b/Makefile @@ -238,7 +238,7 @@ build-docker-full-ubuntu: ## Build Docker image based on Ubuntu for development. --build-arg COMMIT_SHA=$$(git rev-parse HEAD) \ --build-arg BUILD_BRANCH=$$(git rev-parse --abbrev-ref HEAD) \ --build-arg BASE_IMAGE=ubuntu:22.04 \ - --build-arg GO_IMAGE=golang:1.20.8 \ + --build-arg GO_IMAGE=golang:1.20.10 \ --tag grafana/grafana$(TAG_SUFFIX):dev-ubuntu \ $(DOCKER_BUILD_ARGS) diff --git a/scripts/build/ci-build/Dockerfile b/scripts/build/ci-build/Dockerfile index f1c544676fe..a504b12f1c2 100644 --- a/scripts/build/ci-build/Dockerfile +++ b/scripts/build/ci-build/Dockerfile @@ -108,7 +108,7 @@ RUN rm dockerize-linux-amd64-v${DOCKERIZE_VERSION}.tar.gz # Use old Debian (LTS into 2024) in order to ensure binary compatibility with older glibc's. FROM debian:buster-20220822 -ENV GOVERSION=1.20.8 \ +ENV GOVERSION=1.20.10 \ PATH=/usr/local/go/bin:$PATH \ GOPATH=/go \ NODEVERSION=18.12.0-1nodesource1 \ diff --git a/scripts/drone/variables.star b/scripts/drone/variables.star index a4f4d1eaecd..f4d1b97b322 100644 --- a/scripts/drone/variables.star +++ b/scripts/drone/variables.star @@ -3,7 +3,7 @@ global variables """ grabpl_version = "v3.0.42" -golang_version = "1.20.8" +golang_version = "1.20.10" # nodejs_version should match what's in ".nvmrc", but without the v prefix. nodejs_version = "18.12.0" From 05e12e787bc002eccb30a3085aaa397f4fc11e6f Mon Sep 17 00:00:00 2001 From: George Robinson Date: Wed, 11 Oct 2023 14:51:20 +0100 Subject: [PATCH 056/102] Alerting: Add provenance field to /api/v1/provisioning/alert-rules (#76252) This commit adds the missing Provenance field to responses for /api/v1/provisioning/alert-rules. --- pkg/services/ngalert/api/api_provisioning.go | 6 ++-- pkg/services/ngalert/api/compat.go | 4 +-- .../ngalert/provisioning/alert_rules.go | 21 +++++++---- .../api/alerting/api_provisioning_test.go | 36 +++++++++++++++++++ 4 files changed, 55 insertions(+), 12 deletions(-) diff --git a/pkg/services/ngalert/api/api_provisioning.go b/pkg/services/ngalert/api/api_provisioning.go index 14733cbefd5..bd825349c28 100644 --- a/pkg/services/ngalert/api/api_provisioning.go +++ b/pkg/services/ngalert/api/api_provisioning.go @@ -57,7 +57,7 @@ type MuteTimingService interface { } type AlertRuleService interface { - GetAlertRules(ctx context.Context, orgID int64) ([]*alerting_models.AlertRule, error) + GetAlertRules(ctx context.Context, orgID int64) ([]*alerting_models.AlertRule, map[string]alerting_models.Provenance, error) GetAlertRule(ctx context.Context, orgID int64, ruleUID string) (alerting_models.AlertRule, alerting_models.Provenance, error) CreateAlertRule(ctx context.Context, rule alerting_models.AlertRule, provenance alerting_models.Provenance, userID int64) (alerting_models.AlertRule, error) UpdateAlertRule(ctx context.Context, rule alerting_models.AlertRule, provenance alerting_models.Provenance) (alerting_models.AlertRule, error) @@ -300,11 +300,11 @@ func (srv *ProvisioningSrv) RouteDeleteMuteTiming(c *contextmodel.ReqContext, na } func (srv *ProvisioningSrv) RouteGetAlertRules(c *contextmodel.ReqContext) response.Response { - rules, err := srv.alertRules.GetAlertRules(c.Req.Context(), c.SignedInUser.GetOrgID()) + rules, provenances, err := srv.alertRules.GetAlertRules(c.Req.Context(), c.SignedInUser.GetOrgID()) if err != nil { return ErrResp(http.StatusInternalServerError, err, "") } - return response.JSON(http.StatusOK, ProvisionedAlertRuleFromAlertRules(rules)) + return response.JSON(http.StatusOK, ProvisionedAlertRuleFromAlertRules(rules, provenances)) } func (srv *ProvisioningSrv) RouteRouteGetAlertRule(c *contextmodel.ReqContext, UID string) response.Response { diff --git a/pkg/services/ngalert/api/compat.go b/pkg/services/ngalert/api/compat.go index fc5a01ba083..899609a7dc3 100644 --- a/pkg/services/ngalert/api/compat.go +++ b/pkg/services/ngalert/api/compat.go @@ -55,10 +55,10 @@ func ProvisionedAlertRuleFromAlertRule(rule models.AlertRule, provenance models. } // ProvisionedAlertRuleFromAlertRules converts a collection of models.AlertRule to definitions.ProvisionedAlertRules with provenance status models.ProvenanceNone -func ProvisionedAlertRuleFromAlertRules(rules []*models.AlertRule) definitions.ProvisionedAlertRules { +func ProvisionedAlertRuleFromAlertRules(rules []*models.AlertRule, provenances map[string]models.Provenance) definitions.ProvisionedAlertRules { result := make([]definitions.ProvisionedAlertRule, 0, len(rules)) for _, r := range rules { - result = append(result, ProvisionedAlertRuleFromAlertRule(*r, models.ProvenanceNone)) + result = append(result, ProvisionedAlertRuleFromAlertRule(*r, provenances[r.UID])) } return result } diff --git a/pkg/services/ngalert/provisioning/alert_rules.go b/pkg/services/ngalert/provisioning/alert_rules.go index 0c354d0d08c..043f7d8d989 100644 --- a/pkg/services/ngalert/provisioning/alert_rules.go +++ b/pkg/services/ngalert/provisioning/alert_rules.go @@ -45,16 +45,23 @@ func NewAlertRuleService(ruleStore RuleStore, } } -func (service *AlertRuleService) GetAlertRules(ctx context.Context, orgID int64) ([]*models.AlertRule, error) { +func (service *AlertRuleService) GetAlertRules(ctx context.Context, orgID int64) ([]*models.AlertRule, map[string]models.Provenance, error) { q := models.ListAlertRulesQuery{ OrgID: orgID, } rules, err := service.ruleStore.ListAlertRules(ctx, &q) if err != nil { - return nil, err + return nil, nil, err } - // TODO: GET provenance - return rules, nil + provenances := make(map[string]models.Provenance) + if len(rules) > 0 { + resourceType := rules[0].ResourceType() + provenances, err = service.provenanceStore.GetProvenances(ctx, orgID, resourceType) + if err != nil { + return nil, nil, err + } + } + return rules, provenances, nil } func (service *AlertRuleService) GetAlertRule(ctx context.Context, orgID int64, ruleUID string) (models.AlertRule, models.Provenance, error) { @@ -62,15 +69,15 @@ func (service *AlertRuleService) GetAlertRule(ctx context.Context, orgID int64, OrgID: orgID, UID: ruleUID, } - rules, err := service.ruleStore.GetAlertRuleByUID(ctx, query) + rule, err := service.ruleStore.GetAlertRuleByUID(ctx, query) if err != nil { return models.AlertRule{}, models.ProvenanceNone, err } - provenance, err := service.provenanceStore.GetProvenance(ctx, rules, orgID) + provenance, err := service.provenanceStore.GetProvenance(ctx, rule, orgID) if err != nil { return models.AlertRule{}, models.ProvenanceNone, err } - return *rules, provenance, nil + return *rule, provenance, nil } type AlertRuleWithFolderTitle struct { diff --git a/pkg/tests/api/alerting/api_provisioning_test.go b/pkg/tests/api/alerting/api_provisioning_test.go index 4bef914bb2a..9a8c424abe5 100644 --- a/pkg/tests/api/alerting/api_provisioning_test.go +++ b/pkg/tests/api/alerting/api_provisioning_test.go @@ -2,13 +2,16 @@ package alerting import ( "bytes" + "encoding/json" "fmt" "io" "net/http" + "sort" "testing" "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/user" "github.com/grafana/grafana/pkg/tests/testinfra" @@ -43,6 +46,11 @@ func TestIntegrationProvisioning(t *testing.T) { Login: "admin", }) + apiClient := newAlertingApiClient(grafanaListedAddr, "editor", "editor") + // Create the namespace we'll save our alerts to. + namespaceUID := "default" + apiClient.CreateFolder(t, namespaceUID, namespaceUID) + t.Run("when provisioning notification policies", func(t *testing.T) { url := fmt.Sprintf("http://%s/api/v1/provisioning/policies", grafanaListedAddr) body := ` @@ -333,6 +341,34 @@ func TestIntegrationProvisioning(t *testing.T) { require.Equal(t, 200, resp.StatusCode) }) }) + + t.Run("when provisioning alert rules", func(t *testing.T) { + url := fmt.Sprintf("http://%s/api/v1/provisioning/alert-rules", grafanaListedAddr) + body := `{"orgID":1,"folderUID":"default","ruleGroup":"Test Group","title":"Provisioned","condition":"A","data":[{"refId":"A","queryType":"","relativeTimeRange":{"from":600,"to":0},"datasourceUid":"f558c85f-66ad-4fd1-b31d-7979e6c93db4","model":{"editorMode":"code","exemplar":false,"expr":"sum(rate(low_card[5m])) \u003e 0","format":"time_series","instant":true,"intervalMs":1000,"legendFormat":"__auto","maxDataPoints":43200,"range":false,"refId":"A"}}],"noDataState":"NoData","execErrState":"Error","for":"0s"}` + req := createTestRequest("POST", url, "admin", body) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, 201, resp.StatusCode) + + // We want to check the provenances of both provisioned and non-provisioned rules + createRule(t, apiClient, namespaceUID) + + req = createTestRequest("GET", url, "admin", "") + resp, err = http.DefaultClient.Do(req) + require.NoError(t, err) + + var rules definitions.ProvisionedAlertRules + require.NoError(t, json.NewDecoder(resp.Body).Decode(&rules)) + require.NoError(t, resp.Body.Close()) + + require.Len(t, rules, 2) + sort.Slice(rules, func(i, j int) bool { + return rules[i].ID < rules[j].ID + }) + require.Equal(t, definitions.Provenance("api"), rules[0].Provenance) + require.Equal(t, definitions.Provenance(""), rules[1].Provenance) + }) } func createTestRequest(method string, url string, user string, body string) *http.Request { From bcf850f8bf0b8a6b517e5ad25d7f630b8a4aca52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 11 Oct 2023 15:56:10 +0200 Subject: [PATCH 057/102] DashboardScene: Set maxDataPointsFromWidth to true (#76319) * DashboardScene: Set maxDataPointsFromWidth to true * fix --- .../transformSaveModelToScene.test.ts | 14 +++++++------- .../utils/createPanelDataProvider.ts | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.test.ts b/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.test.ts index d2f3d6f1992..3e275ffafb8 100644 --- a/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.test.ts +++ b/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.test.ts @@ -13,7 +13,6 @@ import { SceneGridItem, SceneGridLayout, SceneGridRow, - SceneQueryRunner, VizPanel, } from '@grafana/scenes'; import { DashboardCursorSync, defaultDashboard, Panel, RowPanel, VariableType } from '@grafana/schema'; @@ -26,6 +25,7 @@ import { PanelRepeaterGridItem } from '../scene/PanelRepeaterGridItem'; import { PanelTimeRange } from '../scene/PanelTimeRange'; import { RowRepeaterBehavior } from '../scene/RowRepeaterBehavior'; import { ShareQueryDataProvider } from '../scene/ShareQueryDataProvider'; +import { getQueryRunnerFor } from '../utils/utils'; import dashboard_to_load1 from './testfiles/dashboard_to_load1.json'; import repeatingRowsAndPanelsDashboardJson from './testfiles/repeating_rows_and_panels.json'; @@ -280,12 +280,12 @@ describe('transformSaveModelToScene', () => { expect(vizPanel.state.options).toEqual(panel.options); expect(vizPanel.state.fieldConfig).toEqual(panel.fieldConfig); expect(vizPanel.state.pluginVersion).toBe('1.0.0'); - expect(((vizPanel.state.$data as SceneDataTransformer)?.state.$data as SceneQueryRunner).state.queries).toEqual( - panel.targets - ); - expect( - ((vizPanel.state.$data as SceneDataTransformer)?.state.$data as SceneQueryRunner).state.maxDataPoints - ).toEqual(100); + + const queryRunner = getQueryRunnerFor(vizPanel)!; + expect(queryRunner.state.queries).toEqual(panel.targets); + expect(queryRunner.state.maxDataPoints).toEqual(100); + expect(queryRunner.state.maxDataPointsFromWidth).toEqual(true); + expect((vizPanel.state.$data as SceneDataTransformer)?.state.transformations).toEqual(panel.transformations); }); diff --git a/public/app/features/dashboard-scene/utils/createPanelDataProvider.ts b/public/app/features/dashboard-scene/utils/createPanelDataProvider.ts index 1ee8fc2c44e..12a6d721db2 100644 --- a/public/app/features/dashboard-scene/utils/createPanelDataProvider.ts +++ b/public/app/features/dashboard-scene/utils/createPanelDataProvider.ts @@ -25,6 +25,7 @@ export function createPanelDataProvider(panel: PanelModel): SceneDataProvider | datasource: panel.datasource ?? undefined, queries: panel.targets, maxDataPoints: panel.maxDataPoints ?? undefined, + maxDataPointsFromWidth: true, dataLayerFilter: { panelId: panel.id, }, From 72a1de855cfa4c7a7347df6c437d65ac4dd0c318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 11 Oct 2023 15:56:42 +0200 Subject: [PATCH 058/102] Schema: Clean up / correct panel schema (#76346) * Schema: Clean up / correct panel schema * fixes --- .../kinds/core/dashboard/schema-reference.md | 68 +++++++++---------- kinds/dashboard/dashboard_kind.cue | 12 ++-- .../raw/dashboard/x/dashboard_types.gen.ts | 12 ++-- .../src/veneer/dashboard.types.ts | 2 +- pkg/kinds/dashboard/dashboard_spec_gen.go | 12 ++-- .../features/alerting/TestRuleResult.test.tsx | 4 +- .../transformSaveModelToScene.test.ts | 14 ++-- .../transformSceneToSaveModel.test.ts | 4 +- .../DashboardPrompt/DashboardPrompt.test.tsx | 4 +- .../dashboard/components/GenAI/utils.test.ts | 4 +- .../dashboard/state/DashboardMigrator.test.ts | 17 +---- .../dashboard/state/DashboardModel.test.ts | 24 +++---- .../state/__fixtures__/dashboardFixtures.ts | 10 +-- .../features/dashboard/state/reducers.test.ts | 4 +- .../dashboard/utils/panelMerge.test.ts | 8 +-- .../dashboard/DashboardQueryEditor.test.tsx | 8 +-- 16 files changed, 93 insertions(+), 114 deletions(-) diff --git a/docs/sources/developers/kinds/core/dashboard/schema-reference.md b/docs/sources/developers/kinds/core/dashboard/schema-reference.md index 9b6a5348238..37206a725c2 100644 --- a/docs/sources/developers/kinds/core/dashboard/schema-reference.md +++ b/docs/sources/developers/kinds/core/dashboard/schema-reference.md @@ -71,13 +71,13 @@ extraFields is reserved for any fields that are pulled from the API server metad | Property | Type | Required | Default | Description | |------------------------|---------------------------------------------|----------|-----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `editable` | boolean | **Yes** | `true` | Whether a dashboard is editable or not. | -| `graphTooltip` | integer | **Yes** | `0` | 0 for no shared crosshair or tooltip (default).
    1 for shared crosshair.
    2 for shared crosshair AND shared tooltip.
    Possible values are: `0`, `1`, `2`. | | `schemaVersion` | uint16 | **Yes** | `36` | Version of the JSON schema, incremented each time a Grafana update brings
    changes to said schema. | | `annotations` | [AnnotationContainer](#annotationcontainer) | No | | Contains the list of annotations that are associated with the dashboard.
    Annotations are used to overlay event markers and overlay event tags on graphs.
    Grafana comes with a native annotation store and the ability to add annotation events directly from the graph panel or via the HTTP API.
    See https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/annotate-visualizations/ | | `description` | string | No | | Description of dashboard. | +| `editable` | boolean | No | `true` | Whether a dashboard is editable or not. | | `fiscalYearStartMonth` | integer | No | `0` | The month that the fiscal year starts on. 0 = January, 11 = December
    Constraint: `>=0 & <12`. | | `gnetId` | string | No | | ID of a dashboard imported from the https://grafana.com/grafana/dashboards/ portal | +| `graphTooltip` | integer | No | `0` | 0 for no shared crosshair or tooltip (default).
    1 for shared crosshair.
    2 for shared crosshair AND shared tooltip.
    Possible values are: `0`, `1`, `2`. | | `id` | integer or null | No | | Unique numeric identifier for the dashboard.
    `id` is internal to a specific Grafana instance. `uid` should be used to identify a dashboard across Grafana instances. | | `links` | [DashboardLink](#dashboardlink)[] | No | | Links with references to other dashboards or external websites. | | `liveNow` | boolean | No | | When set to true, the dashboard will redraw panels at an interval matching the pixel width.
    This will keep data "moving left" regardless of the query refresh rate. This setting helps
    avoid dashboards presenting stale live data | @@ -191,9 +191,9 @@ Sensitive information stripped: queries (metric, template,annotation) and panel ### Panels -| Property | Type | Required | Default | Description | -|----------|-----------------------------------------------------------------------------------------------------------------------|----------|---------|-------------| -| `object` | Possible types are: [Panel](#panel), [RowPanel](#rowpanel), [GraphPanel](#graphpanel), [HeatmapPanel](#heatmappanel). | | | +| Property | Type | Required | Default | Description | +|----------|-------------------------------------------------------------------------------------------------------------|----------|---------|-------------| +| `object` | Possible types are: [](#), [RowPanel](#rowpanel), [GraphPanel](#graphpanel), [HeatmapPanel](#heatmappanel). | | | ### DataTransformerConfig @@ -443,19 +443,37 @@ Library panels streamline reuse of panels across multiple dashboards. | `name` | string | **Yes** | | Library panel name | | `uid` | string | **Yes** | | Library panel uid | +### RowPanel + +Row panel + +| Property | Type | Required | Default | Description | +|--------------|---------------------------------|----------|---------|-----------------------------------------------------------------------------------------------------------------------------------------| +| `collapsed` | boolean | **Yes** | `false` | Whether this row should be collapsed or not. | +| `id` | uint32 | **Yes** | | Unique identifier of the panel. Generated by Grafana when creating a new panel. It must be unique within a dashboard, but not globally. | +| `panels` | [panels](#panels)[] | **Yes** | | List of panels in the row | +| `type` | string | **Yes** | | The panel type
    Possible values are: `row`. | +| `datasource` | [DataSourceRef](#datasourceref) | No | | Ref to a DataSource instance | +| `gridPos` | [GridPos](#gridpos) | No | | Position and dimensions of a panel in the grid | +| `repeat` | string | No | | Name of template variable to repeat for. | +| `title` | string | No | | Row title | + +### Panels + +| Property | Type | Required | Default | Description | +|----------|------------------------------------------------------------------------------------------------|----------|---------|-------------| +| `object` | Possible types are: [Panel](#panel), [GraphPanel](#graphpanel), [HeatmapPanel](#heatmappanel). | | | + ### Panel Dashboard panels are the basic visualization building blocks. | Property | Type | Required | Default | Description | |--------------------|---------------------------------------------------|----------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `fieldConfig` | [FieldConfigSource](#fieldconfigsource) | **Yes** | | The data model used in Grafana, namely the data frame, is a columnar-oriented table structure that unifies both time series and table query results.
    Each column within this structure is called a field. A field can represent a single time series or table column.
    Field options allow you to change how the data is displayed in your visualizations. | -| `options` | [object](#options) | **Yes** | | It depends on the panel plugin. They are specified by the Options field in panel plugin schemas. | -| `transformations` | [DataTransformerConfig](#datatransformerconfig)[] | **Yes** | | List of transformations that are applied to the panel data before rendering.
    When there are multiple transformations, Grafana applies them in the order they are listed.
    Each transformation creates a result set that then passes on to the next transformation in the processing pipeline. | -| `transparent` | boolean | **Yes** | `false` | Whether to display the panel without a background. | | `type` | string | **Yes** | | The panel plugin type id. This is used to find the plugin to display the panel.
    Constraint: `length >=1`. | | `datasource` | [DataSourceRef](#datasourceref) | No | | Ref to a DataSource instance | | `description` | string | No | | Panel description. | +| `fieldConfig` | [FieldConfigSource](#fieldconfigsource) | No | | The data model used in Grafana, namely the data frame, is a columnar-oriented table structure that unifies both time series and table query results.
    Each column within this structure is called a field. A field can represent a single time series or table column.
    Field options allow you to change how the data is displayed in your visualizations. | | `gridPos` | [GridPos](#gridpos) | No | | Position and dimensions of a panel in the grid | | `hideTimeOverride` | boolean | No | | Controls if the timeFrom or timeShift overrides are shown in the panel header | | `id` | uint32 | No | | Unique identifier of the panel. Generated by Grafana when creating a new panel. It must be unique within a dashboard, but not globally. | @@ -464,6 +482,7 @@ Dashboard panels are the basic visualization building blocks. | `links` | [DashboardLink](#dashboardlink)[] | No | | Panel links. | | `maxDataPoints` | number | No | | The maximum number of data points that the panel queries are retrieving. | | `maxPerRow` | number | No | | Option for repeated panels that controls max items per row
    Only relevant for horizontally repeated panels | +| `options` | [object](#options) | No | | It depends on the panel plugin. They are specified by the Options field in panel plugin schemas. | | `pluginVersion` | string | No | | The version of the plugin that is used for this panel. This is used to find the plugin to display the panel and to migrate old panel configs. | | `repeatDirection` | string | No | `h` | Direction to repeat in if 'repeat' is set.
    `h` for horizontal, `v` for vertical.
    Possible values are: `h`, `v`. | | `repeat` | string | No | | Name of template variable to repeat for. | @@ -472,6 +491,8 @@ Dashboard panels are the basic visualization building blocks. | `timeFrom` | string | No | | Overrides the relative time range for individual panels,
    which causes them to be different than what is selected in
    the dashboard time picker in the top-right corner of the dashboard. You can use this to show metrics from different
    time periods or days on the same dashboard.
    The value is formatted as time operation like: `now-5m` (Last 5 minutes), `now/d` (the day so far),
    `now-5d/d`(Last 5 days), `now/w` (This week so far), `now-2y/y` (Last 2 years).
    Note: Panel time overrides have no effect when the dashboard’s time range is absolute.
    See: https://grafana.com/docs/grafana/latest/panels-visualizations/query-transform-data/#query-options | | `timeShift` | string | No | | Overrides the time range for individual panels by shifting its start and end relative to the time picker.
    For example, you can shift the time range for the panel to be two hours earlier than the dashboard time picker setting `2h`.
    Note: Panel time overrides have no effect when the dashboard’s time range is absolute.
    See: https://grafana.com/docs/grafana/latest/panels-visualizations/query-transform-data/#query-options | | `title` | string | No | | Panel title. | +| `transformations` | [DataTransformerConfig](#datatransformerconfig)[] | No | | List of transformations that are applied to the panel data before rendering.
    When there are multiple transformations, Grafana applies them in the order they are listed.
    Each transformation creates a result set that then passes on to the next transformation in the processing pipeline. | +| `transparent` | boolean | No | `false` | Whether to display the panel without a background. | ### FieldConfigSource @@ -560,27 +581,6 @@ It depends on the panel plugin. They are specified by the Options field in panel | Property | Type | Required | Default | Description | |----------|------|----------|---------|-------------| -### RowPanel - -Row panel - -| Property | Type | Required | Default | Description | -|--------------|---------------------------------|----------|---------|-----------------------------------------------------------------------------------------------------------------------------------------| -| `collapsed` | boolean | **Yes** | `false` | Whether this row should be collapsed or not. | -| `id` | uint32 | **Yes** | | Unique identifier of the panel. Generated by Grafana when creating a new panel. It must be unique within a dashboard, but not globally. | -| `panels` | [panels](#panels)[] | **Yes** | | List of panels in the row | -| `type` | string | **Yes** | | The panel type
    Possible values are: `row`. | -| `datasource` | [DataSourceRef](#datasourceref) | No | | Ref to a DataSource instance | -| `gridPos` | [GridPos](#gridpos) | No | | Position and dimensions of a panel in the grid | -| `repeat` | string | No | | Name of template variable to repeat for. | -| `title` | string | No | | Row title | - -### Panels - -| Property | Type | Required | Default | Description | -|----------|------------------------------------------------------------------------------------------------|----------|---------|-------------| -| `object` | Possible types are: [Panel](#panel), [GraphPanel](#graphpanel), [HeatmapPanel](#heatmappanel). | | | - ### GraphPanel Support for legacy graph panel. @@ -597,13 +597,10 @@ Dashboard panels are the basic visualization building blocks. | Property | Type | Required | Default | Description | |--------------------|---------------------------------------------------|----------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `fieldConfig` | [FieldConfigSource](#fieldconfigsource) | **Yes** | | The data model used in Grafana, namely the data frame, is a columnar-oriented table structure that unifies both time series and table query results.
    Each column within this structure is called a field. A field can represent a single time series or table column.
    Field options allow you to change how the data is displayed in your visualizations. | -| `options` | [options](#options) | **Yes** | | It depends on the panel plugin. They are specified by the Options field in panel plugin schemas. | -| `transformations` | [DataTransformerConfig](#datatransformerconfig)[] | **Yes** | | List of transformations that are applied to the panel data before rendering.
    When there are multiple transformations, Grafana applies them in the order they are listed.
    Each transformation creates a result set that then passes on to the next transformation in the processing pipeline. | -| `transparent` | boolean | **Yes** | `false` | Whether to display the panel without a background. | | `type` | string | **Yes** | | The panel plugin type id. This is used to find the plugin to display the panel.
    Constraint: `length >=1`. | | `datasource` | [DataSourceRef](#datasourceref) | No | | Ref to a DataSource instance | | `description` | string | No | | Panel description. | +| `fieldConfig` | [FieldConfigSource](#fieldconfigsource) | No | | The data model used in Grafana, namely the data frame, is a columnar-oriented table structure that unifies both time series and table query results.
    Each column within this structure is called a field. A field can represent a single time series or table column.
    Field options allow you to change how the data is displayed in your visualizations. | | `gridPos` | [GridPos](#gridpos) | No | | Position and dimensions of a panel in the grid | | `hideTimeOverride` | boolean | No | | Controls if the timeFrom or timeShift overrides are shown in the panel header | | `id` | uint32 | No | | Unique identifier of the panel. Generated by Grafana when creating a new panel. It must be unique within a dashboard, but not globally. | @@ -612,6 +609,7 @@ Dashboard panels are the basic visualization building blocks. | `links` | [DashboardLink](#dashboardlink)[] | No | | Panel links. | | `maxDataPoints` | number | No | | The maximum number of data points that the panel queries are retrieving. | | `maxPerRow` | number | No | | Option for repeated panels that controls max items per row
    Only relevant for horizontally repeated panels | +| `options` | [options](#options) | No | | It depends on the panel plugin. They are specified by the Options field in panel plugin schemas. | | `pluginVersion` | string | No | | The version of the plugin that is used for this panel. This is used to find the plugin to display the panel and to migrate old panel configs. | | `repeatDirection` | string | No | `h` | Direction to repeat in if 'repeat' is set.
    `h` for horizontal, `v` for vertical.
    Possible values are: `h`, `v`. | | `repeat` | string | No | | Name of template variable to repeat for. | @@ -620,6 +618,8 @@ Dashboard panels are the basic visualization building blocks. | `timeFrom` | string | No | | Overrides the relative time range for individual panels,
    which causes them to be different than what is selected in
    the dashboard time picker in the top-right corner of the dashboard. You can use this to show metrics from different
    time periods or days on the same dashboard.
    The value is formatted as time operation like: `now-5m` (Last 5 minutes), `now/d` (the day so far),
    `now-5d/d`(Last 5 days), `now/w` (This week so far), `now-2y/y` (Last 2 years).
    Note: Panel time overrides have no effect when the dashboard’s time range is absolute.
    See: https://grafana.com/docs/grafana/latest/panels-visualizations/query-transform-data/#query-options | | `timeShift` | string | No | | Overrides the time range for individual panels by shifting its start and end relative to the time picker.
    For example, you can shift the time range for the panel to be two hours earlier than the dashboard time picker setting `2h`.
    Note: Panel time overrides have no effect when the dashboard’s time range is absolute.
    See: https://grafana.com/docs/grafana/latest/panels-visualizations/query-transform-data/#query-options | | `title` | string | No | | Panel title. | +| `transformations` | [DataTransformerConfig](#datatransformerconfig)[] | No | | List of transformations that are applied to the panel data before rendering.
    When there are multiple transformations, Grafana applies them in the order they are listed.
    Each transformation creates a result set that then passes on to the next transformation in the processing pipeline. | +| `transparent` | boolean | No | `false` | Whether to display the panel without a background. | ### Templating diff --git a/kinds/dashboard/dashboard_kind.cue b/kinds/dashboard/dashboard_kind.cue index 20bb612354a..0b32ebff3da 100644 --- a/kinds/dashboard/dashboard_kind.cue +++ b/kinds/dashboard/dashboard_kind.cue @@ -42,11 +42,11 @@ lineage: schemas: [{ timezone?: string | *"browser" // Whether a dashboard is editable or not. - editable: bool | *true + editable?: bool | *true // Configuration of dashboard cursor sync behavior. // Accepted values are 0 (sync turned off), 1 (shared crosshair), 2 (shared crosshair and tooltip). - graphTooltip: #DashboardCursorSync + graphTooltip?: #DashboardCursorSync // Time range for dashboard. // Accepted values are relative time strings like {from: 'now-6h', to: 'now'} or absolute time strings like {from: '2020-07-10T08:00:00.000Z', to: '2020-07-10T14:00:00.000Z'}. @@ -515,7 +515,7 @@ lineage: schemas: [{ description?: string // Whether to display the panel without a background. - transparent: bool | *false + transparent?: bool | *false // The datasource used in all targets. datasource?: #DataSourceRef @@ -543,7 +543,7 @@ lineage: schemas: [{ // List of transformations that are applied to the panel data before rendering. // When there are multiple transformations, Grafana applies them in the order they are listed. // Each transformation creates a result set that then passes on to the next transformation in the processing pipeline. - transformations: [...#DataTransformerConfig] + transformations?: [...#DataTransformerConfig] // The min time interval setting defines a lower limit for the $__interval and $__interval_ms variables. // This value must be formatted as a number followed by a valid time @@ -574,10 +574,10 @@ lineage: schemas: [{ libraryPanel?: #LibraryPanelRef // It depends on the panel plugin. They are specified by the Options field in panel plugin schemas. - options: {...} @grafanamaturity(NeedsExpertReview) + options?: {...} @grafanamaturity(NeedsExpertReview) // Field options allow you to change how the data is displayed in your visualizations. - fieldConfig: #FieldConfigSource + fieldConfig?: #FieldConfigSource } @cuetsy(kind="interface") @grafana(TSVeneer="type") @grafanamaturity(NeedsExpertReview) // The data model used in Grafana, namely the data frame, is a columnar-oriented table structure that unifies both time series and table query results. diff --git a/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts b/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts index e86774a7f7b..99c9b16090d 100644 --- a/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts +++ b/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts @@ -644,7 +644,7 @@ export interface Panel { /** * Field options allow you to change how the data is displayed in your visualizations. */ - fieldConfig: FieldConfigSource; + fieldConfig?: FieldConfigSource; /** * Grid position. */ @@ -684,7 +684,7 @@ export interface Panel { /** * It depends on the panel plugin. They are specified by the Options field in panel plugin schemas. */ - options: Record; + options?: Record; /** * The version of the plugin that is used for this panel. This is used to find the plugin to display the panel and to migrate old panel configs. */ @@ -733,11 +733,11 @@ export interface Panel { * When there are multiple transformations, Grafana applies them in the order they are listed. * Each transformation creates a result set that then passes on to the next transformation in the processing pipeline. */ - transformations: Array; + transformations?: Array; /** * Whether to display the panel without a background. */ - transparent: boolean; + transparent?: boolean; /** * The panel plugin type id. This is used to find the plugin to display the panel. */ @@ -991,7 +991,7 @@ export interface Dashboard { /** * Whether a dashboard is editable or not. */ - editable: boolean; + editable?: boolean; /** * The month that the fiscal year starts on. 0 = January, 11 = December */ @@ -1004,7 +1004,7 @@ export interface Dashboard { * Configuration of dashboard cursor sync behavior. * Accepted values are 0 (sync turned off), 1 (shared crosshair), 2 (shared crosshair and tooltip). */ - graphTooltip: DashboardCursorSync; + graphTooltip?: DashboardCursorSync; /** * Unique numeric identifier for the dashboard. * `id` is internal to a specific Grafana instance. `uid` should be used to identify a dashboard across Grafana instances. diff --git a/packages/grafana-schema/src/veneer/dashboard.types.ts b/packages/grafana-schema/src/veneer/dashboard.types.ts index c444b0e7840..3f85370614e 100644 --- a/packages/grafana-schema/src/veneer/dashboard.types.ts +++ b/packages/grafana-schema/src/veneer/dashboard.types.ts @@ -7,7 +7,7 @@ export type { CommonDataSourceRef as DataSourceRef }; export interface Panel, TCustomFieldConfig = Record> extends Omit { - fieldConfig: FieldConfigSource; + fieldConfig?: FieldConfigSource; } export interface RowPanel extends Omit { diff --git a/pkg/kinds/dashboard/dashboard_spec_gen.go b/pkg/kinds/dashboard/dashboard_spec_gen.go index 145ae289ca3..ca5b81891a3 100644 --- a/pkg/kinds/dashboard/dashboard_spec_gen.go +++ b/pkg/kinds/dashboard/dashboard_spec_gen.go @@ -529,7 +529,7 @@ type Panel struct { // The data model used in Grafana, namely the data frame, is a columnar-oriented table structure that unifies both time series and table query results. // Each column within this structure is called a field. A field can represent a single time series or table column. // Field options allow you to change how the data is displayed in your visualizations. - FieldConfig FieldConfigSource `json:"fieldConfig"` + FieldConfig *FieldConfigSource `json:"fieldConfig,omitempty"` // Position and dimensions of a panel in the grid GridPos *GridPos `json:"gridPos,omitempty"` @@ -562,7 +562,7 @@ type Panel struct { MaxPerRow *float32 `json:"maxPerRow,omitempty"` // It depends on the panel plugin. They are specified by the Options field in panel plugin schemas. - Options map[string]any `json:"options"` + Options map[string]any `json:"options,omitempty"` // The version of the plugin that is used for this panel. This is used to find the plugin to display the panel and to migrate old panel configs. PluginVersion *string `json:"pluginVersion,omitempty"` @@ -602,10 +602,10 @@ type Panel struct { // List of transformations that are applied to the panel data before rendering. // When there are multiple transformations, Grafana applies them in the order they are listed. // Each transformation creates a result set that then passes on to the next transformation in the processing pipeline. - Transformations []DataTransformerConfig `json:"transformations"` + Transformations []DataTransformerConfig `json:"transformations,omitempty"` // Whether to display the panel without a background. - Transparent bool `json:"transparent"` + Transparent *bool `json:"transparent,omitempty"` // The panel plugin type id. This is used to find the plugin to display the panel. Type string `json:"type"` @@ -734,7 +734,7 @@ type Spec struct { Description *string `json:"description,omitempty"` // Whether a dashboard is editable or not. - Editable bool `json:"editable"` + Editable *bool `json:"editable,omitempty"` // The month that the fiscal year starts on. 0 = January, 11 = December FiscalYearStartMonth *int `json:"fiscalYearStartMonth,omitempty"` @@ -745,7 +745,7 @@ type Spec struct { // 0 for no shared crosshair or tooltip (default). // 1 for shared crosshair. // 2 for shared crosshair AND shared tooltip. - GraphTooltip CursorSync `json:"graphTooltip"` + GraphTooltip *CursorSync `json:"graphTooltip,omitempty"` // Unique numeric identifier for the dashboard. // `id` is internal to a specific Grafana instance. `uid` should be used to identify a dashboard across Grafana instances. diff --git a/public/app/features/alerting/TestRuleResult.test.tsx b/public/app/features/alerting/TestRuleResult.test.tsx index 2fbbe94c5b0..9b44afa9f0f 100644 --- a/public/app/features/alerting/TestRuleResult.test.tsx +++ b/public/app/features/alerting/TestRuleResult.test.tsx @@ -2,7 +2,7 @@ import { render, screen } from '@testing-library/react'; import React from 'react'; import { PanelModel } from '../dashboard/state'; -import { createDashboardModelFixture, createPanelJSONFixture } from '../dashboard/state/__fixtures__/dashboardFixtures'; +import { createDashboardModelFixture, createPanelSaveModel } from '../dashboard/state/__fixtures__/dashboardFixtures'; import { TestRuleResult } from './TestRuleResult'; @@ -22,7 +22,7 @@ jest.mock('@grafana/runtime', () => { const props: React.ComponentProps = { panel: new PanelModel({ id: 1 }), dashboard: createDashboardModelFixture({ - panels: [createPanelJSONFixture({ id: 1 })], + panels: [createPanelSaveModel({ id: 1 })], }), }; diff --git a/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.test.ts b/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.test.ts index 3e275ffafb8..21470a4c2f7 100644 --- a/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.test.ts +++ b/public/app/features/dashboard-scene/serialization/transformSaveModelToScene.test.ts @@ -17,7 +17,7 @@ import { } from '@grafana/scenes'; import { DashboardCursorSync, defaultDashboard, Panel, RowPanel, VariableType } from '@grafana/schema'; import { DashboardModel, PanelModel } from 'app/features/dashboard/state'; -import { createPanelJSONFixture } from 'app/features/dashboard/state/__fixtures__/dashboardFixtures'; +import { createPanelSaveModel } from 'app/features/dashboard/state/__fixtures__/dashboardFixtures'; import { SHARED_DASHBOARD_QUERY } from 'app/plugins/datasource/dashboard'; import { DASHBOARD_DATASOURCE_PLUGIN_ID } from 'app/plugins/datasource/dashboard/types'; @@ -112,12 +112,12 @@ describe('transformSaveModelToScene', () => { describe('when organizing panels as scene children', () => { it('should create panels within collapsed rows', () => { - const panel = createPanelJSONFixture({ + const panel = createPanelSaveModel({ title: 'test', gridPos: { x: 1, y: 0, w: 12, h: 8 }, }) as Panel; - const row = createPanelJSONFixture({ + const row = createPanelSaveModel({ title: 'test', type: 'row', gridPos: { x: 0, y: 0, w: 12, h: 1 }, @@ -146,7 +146,7 @@ describe('transformSaveModelToScene', () => { }); it('should create panels within expanded row', () => { - const panelOutOfRow = createPanelJSONFixture({ + const panelOutOfRow = createPanelSaveModel({ title: 'Out of a row', gridPos: { h: 8, @@ -155,7 +155,7 @@ describe('transformSaveModelToScene', () => { y: 0, }, }); - const rowWithPanel = createPanelJSONFixture({ + const rowWithPanel = createPanelSaveModel({ title: 'Row with panel', type: 'row', id: 10, @@ -169,7 +169,7 @@ describe('transformSaveModelToScene', () => { // This panels array is not used if the row is not collapsed panels: [], }); - const panelInRow = createPanelJSONFixture({ + const panelInRow = createPanelSaveModel({ gridPos: { h: 8, w: 12, @@ -178,7 +178,7 @@ describe('transformSaveModelToScene', () => { }, title: 'In row 1', }); - const emptyRow = createPanelJSONFixture({ + const emptyRow = createPanelSaveModel({ collapsed: false, gridPos: { h: 1, diff --git a/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts b/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts index fd3e42565ae..04163e6fbcb 100644 --- a/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts +++ b/public/app/features/dashboard-scene/serialization/transformSceneToSaveModel.test.ts @@ -374,7 +374,7 @@ describe('transformSceneToSaveModel', () => { const result = gridItemToPanel(panel); - expect(result.transformations.length).toBe(1); + expect(result.transformations?.length).toBe(1); expect(result.maxDataPoints).toBe(100); expect(result.targets?.length).toBe(2); @@ -458,7 +458,7 @@ describe('transformSceneToSaveModel', () => { const result = gridItemToPanel(panel); - expect(result.transformations.length).toBe(1); + expect(result.transformations?.length).toBe(1); expect(result.targets?.length).toBe(1); expect(result.targets?.[0]).toEqual({ diff --git a/public/app/features/dashboard/components/DashboardPrompt/DashboardPrompt.test.tsx b/public/app/features/dashboard/components/DashboardPrompt/DashboardPrompt.test.tsx index cc768c3013d..94a437bf93d 100644 --- a/public/app/features/dashboard/components/DashboardPrompt/DashboardPrompt.test.tsx +++ b/public/app/features/dashboard/components/DashboardPrompt/DashboardPrompt.test.tsx @@ -2,14 +2,14 @@ import { getPanelPlugin } from '@grafana/data/test/__mocks__/pluginMocks'; import { ContextSrv, setContextSrv } from '../../../../core/services/context_srv'; import { PanelModel } from '../../state/PanelModel'; -import { createDashboardModelFixture, createPanelJSONFixture } from '../../state/__fixtures__/dashboardFixtures'; +import { createDashboardModelFixture, createPanelSaveModel } from '../../state/__fixtures__/dashboardFixtures'; import { hasChanges, ignoreChanges } from './DashboardPrompt'; function getDefaultDashboardModel() { return createDashboardModelFixture({ panels: [ - createPanelJSONFixture({ + createPanelSaveModel({ id: 1, type: 'graph', gridPos: { x: 0, y: 0, w: 24, h: 6 }, diff --git a/public/app/features/dashboard/components/GenAI/utils.test.ts b/public/app/features/dashboard/components/GenAI/utils.test.ts index 51ccb924bd1..c108aaf9ae4 100644 --- a/public/app/features/dashboard/components/GenAI/utils.test.ts +++ b/public/app/features/dashboard/components/GenAI/utils.test.ts @@ -1,4 +1,4 @@ -import { createDashboardModelFixture, createPanelJSONFixture } from '../../state/__fixtures__/dashboardFixtures'; +import { createDashboardModelFixture, createPanelSaveModel } from '../../state/__fixtures__/dashboardFixtures'; import { openai } from './llms'; import { getDashboardChanges, isLLMPluginEnabled, sanitizeReply } from './utils'; @@ -21,7 +21,7 @@ describe('getDashboardChanges', () => { const deprecatedVersion = 37; const dashboard = createDashboardModelFixture({ schemaVersion: deprecatedVersion, - panels: [createPanelJSONFixture({ title: 'Panel 1', options: deprecatedOptions })], + panels: [createPanelSaveModel({ title: 'Panel 1', options: deprecatedOptions })], }); // Update title for the first panel diff --git a/public/app/features/dashboard/state/DashboardMigrator.test.ts b/public/app/features/dashboard/state/DashboardMigrator.test.ts index c502d8ff682..cea0683ac3b 100644 --- a/public/app/features/dashboard/state/DashboardMigrator.test.ts +++ b/public/app/features/dashboard/state/DashboardMigrator.test.ts @@ -128,7 +128,6 @@ describe('DashboardModel', () => { { type: 'row', panels: [ - // @ts-expect-error { id: 501, type: 'table', @@ -845,7 +844,6 @@ describe('DashboardModel', () => { '# Angular Text Panel\n# $constant\n\nFor markdown syntax help: [commonmark.org/help](https://commonmark.org/help/)\n\n## $text\n\n', mode: 'markdown', }, - // @ts-expect-error { id: 3, type: 'text2', @@ -856,7 +854,6 @@ describe('DashboardModel', () => { '# React Text Panel from scratch\n# $constant\n\nFor markdown syntax help: [commonmark.org/help](https://commonmark.org/help/)\n\n## $text', }, }, - // @ts-expect-error { id: 4, type: 'text2', @@ -1594,7 +1591,6 @@ describe('DashboardModel', () => { ], }, panels: [ - // @ts-expect-error { gridPos: { h: 8, @@ -1717,7 +1713,6 @@ describe('DashboardModel', () => { title: 'DynamoDB', type: 'row', panels: [ - // @ts-expect-error { gridPos: { h: 8, @@ -1772,7 +1767,6 @@ describe('DashboardModel', () => { title: 'Panel Title', type: 'timeseries', }, - // @ts-expect-error { gridPos: { h: 8, @@ -1982,8 +1976,8 @@ describe('DashboardModel', () => { describe('when migrating time series axis visibility', () => { test('preserves x axis visibility', () => { const model = new DashboardModel({ + schemaVersion: 25, panels: [ - // @ts-expect-error { type: 'timeseries', fieldConfig: { @@ -2204,7 +2198,6 @@ describe('when migrating table cell display mode to cell options', () => { beforeEach(() => { model = new DashboardModel({ panels: [ - // @ts-expect-error { id: 1, type: 'table', @@ -2216,9 +2209,8 @@ describe('when migrating table cell display mode to cell options', () => { inspect: false, }, }, - } as unknown as FieldConfigSource, // missing overrides + } as unknown as FieldConfigSource, // missing overrides on purpose }, - // @ts-expect-error { id: 2, type: 'table', @@ -2233,7 +2225,6 @@ describe('when migrating table cell display mode to cell options', () => { overrides: [], }, }, - // @ts-expect-error { id: 3, type: 'table', @@ -2248,7 +2239,6 @@ describe('when migrating table cell display mode to cell options', () => { overrides: [], }, }, - // @ts-expect-error { id: 4, type: 'table', @@ -2263,7 +2253,6 @@ describe('when migrating table cell display mode to cell options', () => { overrides: [], }, }, - // @ts-expect-error { id: 5, type: 'table', @@ -2278,7 +2267,6 @@ describe('when migrating table cell display mode to cell options', () => { overrides: [], }, }, - // @ts-expect-error { id: 6, type: 'table', @@ -2346,7 +2334,6 @@ describe('when migrating table cell display mode to cell options', () => { ], }, }, - // @ts-expect-error { id: 7, type: 'table', diff --git a/public/app/features/dashboard/state/DashboardModel.test.ts b/public/app/features/dashboard/state/DashboardModel.test.ts index 58e14ca32bd..7db392e38d7 100644 --- a/public/app/features/dashboard/state/DashboardModel.test.ts +++ b/public/app/features/dashboard/state/DashboardModel.test.ts @@ -15,7 +15,7 @@ import { PanelModel } from '../state/PanelModel'; import { createAnnotationJSONFixture, createDashboardModelFixture, - createPanelJSONFixture, + createPanelSaveModel, createVariableJSONFixture, } from './__fixtures__/dashboardFixtures'; @@ -77,7 +77,7 @@ describe('DashboardModel', () => { beforeEach(() => { model = createDashboardModelFixture({ - panels: [createPanelJSONFixture({ id: 5 })], + panels: [createPanelSaveModel({ id: 5 })], }); }); @@ -92,10 +92,10 @@ describe('DashboardModel', () => { beforeEach(() => { model = createDashboardModelFixture({ panels: [ - createPanelJSONFixture({ id: 6 }), - createPanelJSONFixture({ id: 2 }), - createPanelJSONFixture({}), // undefined - createPanelJSONFixture({ id: 2 }), + createPanelSaveModel({ id: 6 }), + createPanelSaveModel({ id: 2 }), + createPanelSaveModel({}), // undefined + createPanelSaveModel({ id: 2 }), ], }); }); @@ -217,7 +217,7 @@ describe('DashboardModel', () => { model = createDashboardModelFixture({ schemaVersion: 1, panels: [ - createPanelJSONFixture({ + createPanelSaveModel({ type: 'graph', targets: [ { @@ -418,11 +418,11 @@ describe('DashboardModel', () => { beforeEach(() => { dashboard = createDashboardModelFixture({ panels: [ - createPanelJSONFixture({ id: 1, type: 'graph', gridPos: { x: 0, y: 0, w: 24, h: 2 } }), - createPanelJSONFixture({ id: 2, type: 'row', gridPos: { x: 0, y: 2, w: 24, h: 2 } }), - createPanelJSONFixture({ id: 3, type: 'graph', gridPos: { x: 0, y: 4, w: 12, h: 2 } }), - createPanelJSONFixture({ id: 4, type: 'graph', gridPos: { x: 12, y: 4, w: 12, h: 2 } }), - createPanelJSONFixture({ id: 5, type: 'row', gridPos: { x: 0, y: 6, w: 24, h: 2 } }), + createPanelSaveModel({ id: 1, type: 'graph', gridPos: { x: 0, y: 0, w: 24, h: 2 } }), + createPanelSaveModel({ id: 2, type: 'row', gridPos: { x: 0, y: 2, w: 24, h: 2 } }), + createPanelSaveModel({ id: 3, type: 'graph', gridPos: { x: 0, y: 4, w: 12, h: 2 } }), + createPanelSaveModel({ id: 4, type: 'graph', gridPos: { x: 12, y: 4, w: 12, h: 2 } }), + createPanelSaveModel({ id: 5, type: 'row', gridPos: { x: 0, y: 6, w: 24, h: 2 } }), ], }); dashboard.toggleRow(dashboard.panels[1]); diff --git a/public/app/features/dashboard/state/__fixtures__/dashboardFixtures.ts b/public/app/features/dashboard/state/__fixtures__/dashboardFixtures.ts index f7d56806dfe..d1c6f323c5a 100644 --- a/public/app/features/dashboard/state/__fixtures__/dashboardFixtures.ts +++ b/public/app/features/dashboard/state/__fixtures__/dashboardFixtures.ts @@ -29,16 +29,8 @@ export function createDashboardModelFixture( return new DashboardModel(dashboardJson, meta, { getVariablesFromState }); } -export function createPanelJSONFixture(panelInput: Partial = {}): Panel { +export function createPanelSaveModel(panelInput: Partial = {}): Panel { return { - fieldConfig: { - defaults: {}, - overrides: [], - }, - options: {}, - repeatDirection: 'h', - transformations: [], - transparent: false, type: 'timeseries', ...panelInput, }; diff --git a/public/app/features/dashboard/state/reducers.test.ts b/public/app/features/dashboard/state/reducers.test.ts index 7cdd4300dff..f2bd07126c5 100644 --- a/public/app/features/dashboard/state/reducers.test.ts +++ b/public/app/features/dashboard/state/reducers.test.ts @@ -1,6 +1,6 @@ import { DashboardInitPhase, DashboardState, OrgRole, PermissionLevel } from 'app/types'; -import { createDashboardModelFixture, createPanelJSONFixture } from './__fixtures__/dashboardFixtures'; +import { createDashboardModelFixture, createPanelSaveModel } from './__fixtures__/dashboardFixtures'; import { dashboardInitCompleted, dashboardInitFailed, @@ -37,7 +37,7 @@ describe('dashboard reducer', () => { dashboardInitCompleted( createDashboardModelFixture({ title: 'My dashboard', - panels: [createPanelJSONFixture({ id: 1 }), createPanelJSONFixture({ id: 2 })], + panels: [createPanelSaveModel({ id: 1 }), createPanelSaveModel({ id: 2 })], }) ) ); diff --git a/public/app/features/dashboard/utils/panelMerge.test.ts b/public/app/features/dashboard/utils/panelMerge.test.ts index d20092310dc..4aa420ee76d 100644 --- a/public/app/features/dashboard/utils/panelMerge.test.ts +++ b/public/app/features/dashboard/utils/panelMerge.test.ts @@ -2,7 +2,7 @@ import { PanelModel } from '@grafana/data'; import { FieldColorModeId, ThresholdsMode } from '@grafana/schema/src'; import { DashboardModel } from '../state/DashboardModel'; -import { createDashboardModelFixture, createPanelJSONFixture } from '../state/__fixtures__/dashboardFixtures'; +import { createDashboardModelFixture, createPanelSaveModel } from '../state/__fixtures__/dashboardFixtures'; describe('Merge dashboard panels', () => { describe('simple changes', () => { @@ -13,15 +13,15 @@ describe('Merge dashboard panels', () => { dashboard = createDashboardModelFixture({ title: 'simple title', panels: [ - createPanelJSONFixture({ + createPanelSaveModel({ id: 1, type: 'timeseries', }), - createPanelJSONFixture({ + createPanelSaveModel({ id: 2, type: 'timeseries', }), - createPanelJSONFixture({ + createPanelSaveModel({ id: 3, type: 'table', fieldConfig: { diff --git a/public/app/plugins/datasource/dashboard/DashboardQueryEditor.test.tsx b/public/app/plugins/datasource/dashboard/DashboardQueryEditor.test.tsx index 8693b19e70d..c165132ac63 100644 --- a/public/app/plugins/datasource/dashboard/DashboardQueryEditor.test.tsx +++ b/public/app/plugins/datasource/dashboard/DashboardQueryEditor.test.tsx @@ -10,7 +10,7 @@ import { DashboardModel } from 'app/features/dashboard/state'; import { createDashboardModelFixture, - createPanelJSONFixture, + createPanelSaveModel, } from '../../../features/dashboard/state/__fixtures__/dashboardFixtures'; import { DashboardQueryEditor } from './DashboardQueryEditor'; @@ -49,19 +49,19 @@ describe('DashboardQueryEditor', () => { beforeEach(() => { mockDashboard = createDashboardModelFixture({ panels: [ - createPanelJSONFixture({ + createPanelSaveModel({ targets: [], type: 'timeseries', id: 1, title: 'My first panel', }), - createPanelJSONFixture({ + createPanelSaveModel({ targets: [], id: 2, type: 'timeseries', title: 'Another panel', }), - createPanelJSONFixture({ + createPanelSaveModel({ datasource: { uid: SHARED_DASHBOARD_QUERY, }, From 79b2974215cf6e61d82ec0ba6c8e02de78976c6c Mon Sep 17 00:00:00 2001 From: Jo Date: Wed, 11 Oct 2023 15:57:03 +0200 Subject: [PATCH 059/102] Teams: Allow Services to register deletes (#76325) * add team RegisterDelete * fix-docs --- docs/sources/developers/http_api/team_sync.md | 4 ++-- pkg/services/team/team.go | 1 + pkg/services/team/teamimpl/store.go | 13 +++++++++++-- pkg/services/team/teamimpl/team.go | 6 +++++- pkg/services/team/teamtest/team.go | 3 +++ 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/sources/developers/http_api/team_sync.md b/docs/sources/developers/http_api/team_sync.md index 30a787294dc..eb8622d0054 100644 --- a/docs/sources/developers/http_api/team_sync.md +++ b/docs/sources/developers/http_api/team_sync.md @@ -113,7 +113,7 @@ Status Codes: ## Remove External Group -`DELETE /api/teams/:teamId/groups/:groupId` +`DELETE /api/teams/:teamId/groups?groupId=external-group-id` **Required permissions** @@ -126,7 +126,7 @@ See note in the [introduction]({{< ref "#external-group-synchronization-api" >}} **Example Request**: ```http -DELETE /api/teams/1/groups/cn=editors,ou=groups,dc=grafana,dc=org HTTP/1.1 +DELETE /api/teams/1/groups?groupId=cn%3Deditors%2Cou%3Dgroups%2Cdc%3Dgrafana%2Cdc%3Dorg HTTP/1.1 Accept: application/json Content-Type: application/json Authorization: Basic YWRtaW46YWRtaW4= diff --git a/pkg/services/team/team.go b/pkg/services/team/team.go index 594b224f60b..22e4c77db59 100644 --- a/pkg/services/team/team.go +++ b/pkg/services/team/team.go @@ -20,4 +20,5 @@ type Service interface { RemoveUsersMemberships(tx context.Context, userID int64) error GetUserTeamMemberships(ctx context.Context, orgID, userID int64, external bool) ([]*TeamMemberDTO, error) GetTeamMembers(ctx context.Context, query *GetTeamMembersQuery) ([]*TeamMemberDTO, error) + RegisterDelete(query string) } diff --git a/pkg/services/team/teamimpl/store.go b/pkg/services/team/teamimpl/store.go index f2f830a5565..c600ecfb443 100644 --- a/pkg/services/team/teamimpl/store.go +++ b/pkg/services/team/teamimpl/store.go @@ -30,11 +30,13 @@ type store interface { RemoveMember(ctx context.Context, cmd *team.RemoveTeamMemberCommand) error GetMemberships(ctx context.Context, orgID, userID int64, external bool) ([]*team.TeamMemberDTO, error) GetMembers(ctx context.Context, query *team.GetTeamMembersQuery) ([]*team.TeamMemberDTO, error) + RegisterDelete(query string) } type xormStore struct { - db db.DB - cfg *setting.Cfg + db db.DB + cfg *setting.Cfg + deletes []string } func getFilteredUsers(signedInUser identity.Requester, hiddenUsers map[string]struct{}) []string { @@ -142,6 +144,8 @@ func (ss *xormStore) Delete(ctx context.Context, cmd *team.DeleteTeamCommand) er "DELETE FROM team_role WHERE org_id=? and team_id = ?", } + deletes = append(deletes, ss.deletes...) + for _, sql := range deletes { _, err := sess.Exec(sql, cmd.OrgID, cmd.ID) if err != nil { @@ -567,3 +571,8 @@ func (ss *xormStore) getTeamMembers(ctx context.Context, query *team.GetTeamMemb } return queryResult, nil } + +// RegisterDelete registers a delete query to be executed when the transaction is committed +func (ss *xormStore) RegisterDelete(query string) { + ss.deletes = append(ss.deletes, query) +} diff --git a/pkg/services/team/teamimpl/team.go b/pkg/services/team/teamimpl/team.go index 5bbef67adb2..5895447bc4b 100644 --- a/pkg/services/team/teamimpl/team.go +++ b/pkg/services/team/teamimpl/team.go @@ -14,7 +14,7 @@ type Service struct { } func ProvideService(db db.DB, cfg *setting.Cfg) team.Service { - return &Service{store: &xormStore{db: db, cfg: cfg}} + return &Service{store: &xormStore{db: db, cfg: cfg, deletes: []string{}}} } func (s *Service) CreateTeam(name, email string, orgID int64) (team.Team, error) { @@ -68,3 +68,7 @@ func (s *Service) GetUserTeamMemberships(ctx context.Context, orgID, userID int6 func (s *Service) GetTeamMembers(ctx context.Context, query *team.GetTeamMembersQuery) ([]*team.TeamMemberDTO, error) { return s.store.GetMembers(ctx, query) } + +func (s *Service) RegisterDelete(query string) { + s.store.RegisterDelete(query) +} diff --git a/pkg/services/team/teamtest/team.go b/pkg/services/team/teamtest/team.go index e0a295d859c..71ae3073036 100644 --- a/pkg/services/team/teamtest/team.go +++ b/pkg/services/team/teamtest/team.go @@ -72,3 +72,6 @@ func (s *FakeService) GetUserTeamMemberships(ctx context.Context, orgID, userID func (s *FakeService) GetTeamMembers(ctx context.Context, query *team.GetTeamMembersQuery) ([]*team.TeamMemberDTO, error) { return s.ExpectedMembers, s.ExpectedError } + +func (s *FakeService) RegisterDelete(query string) { +} From 2049ffc71856b1f0b835f022e78a5a948a9c0ae3 Mon Sep 17 00:00:00 2001 From: Kyle Cunningham Date: Wed, 11 Oct 2023 09:34:19 -0500 Subject: [PATCH 060/102] Chore: Add grafana-data transformations to BI squad in CODEOWNERS (#76314) Add transformations directory to bi-squad --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 934e37fdc62..5aa99db1f04 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -323,6 +323,7 @@ /packages/grafana-ui/src/components/VizRepeater/ @grafana/dataviz-squad /packages/grafana-ui/src/components/VizTooltip/ @grafana/dataviz-squad /packages/grafana-ui/src/utils/storybook/ @grafana/plugins-platform-frontend +/packages/grafana-data/src/transformations/ @grafana/grafana-bi-squad /packages/grafana-data/src/**/*logs* @grafana/observability-logs /packages/grafana-schema/src/**/*tempo* @grafana/observability-traces-and-profiling /packages/grafana-flamegraph/ @grafana/observability-traces-and-profiling From c2cbb0a924db1763b48ce98b4ab6b52db21bc73b Mon Sep 17 00:00:00 2001 From: Aaron Godin Date: Wed, 11 Oct 2023 09:42:09 -0500 Subject: [PATCH 061/102] noBasicRole feature: Change RolePickerInput to not show the "None" role (#76300) Change RolePickerInput to only show the basic role when it's not the None role (no basic role feature) --- .../core/components/RolePicker/RolePickerInput.tsx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/public/app/core/components/RolePicker/RolePickerInput.tsx b/public/app/core/components/RolePicker/RolePickerInput.tsx index df3ddfadedb..355daa65b46 100644 --- a/public/app/core/components/RolePicker/RolePickerInput.tsx +++ b/public/app/core/components/RolePicker/RolePickerInput.tsx @@ -49,18 +49,22 @@ export const RolePickerInput = ({ onQueryChange(query); }; - const numberOfRoles = appliedRoles.length; + const showBasicRoleOnLabel = showBasicRole && basicRole !== 'None'; return !isFocused ? ( // TODO: fix keyboard a11y // eslint-disable-next-line jsx-a11y/no-static-element-interactions
    - {showBasicRole && {basicRole}} - + {showBasicRoleOnLabel && {basicRole}} +
    ) : (
    - {showBasicRole && {basicRole}} + {showBasicRoleOnLabel && {basicRole}} {appliedRoles.map((role) => ( {role.displayName || role.name} ))} From 46dc898a877e3f56adb3931780a58cc63037f586 Mon Sep 17 00:00:00 2001 From: Dominik Prokop Date: Wed, 11 Oct 2023 16:56:50 +0200 Subject: [PATCH 062/102] Fix container width not being set after latest scenes update (#76372) * Fix container width after latest scenes update * typecheck --- public/app/features/dashboard-scene/utils/test-utils.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/app/features/dashboard-scene/utils/test-utils.ts b/public/app/features/dashboard-scene/utils/test-utils.ts index aa56667363c..269e2423082 100644 --- a/public/app/features/dashboard-scene/utils/test-utils.ts +++ b/public/app/features/dashboard-scene/utils/test-utils.ts @@ -52,6 +52,12 @@ export function activateFullSceneTree(scene: SceneObject): SceneDeactivationHand const deactivationHandlers: SceneDeactivationHandler[] = []; scene.forEachChild((child) => { + // For query runners which by default use the container width for maxDataPoints calculation we are setting a width. + // In real life this is done by the React component when VizPanel is rendered. + if ('setContainerWidth' in child) { + // @ts-expect-error + child.setContainerWidth(500); + } deactivationHandlers.push(activateFullSceneTree(child)); }); From 7329e2dc62255e5e7a368d6900dbfce81c49d182 Mon Sep 17 00:00:00 2001 From: Andrej Ocenas Date: Wed, 11 Oct 2023 17:57:01 +0200 Subject: [PATCH 063/102] Flamegraph: Fix inefficient regex generating error on some function names (#76377) --- packages/grafana-flamegraph/src/FlameGraph/colors.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/grafana-flamegraph/src/FlameGraph/colors.ts b/packages/grafana-flamegraph/src/FlameGraph/colors.ts index b8b8a472ff8..5e7a5c868e1 100644 --- a/packages/grafana-flamegraph/src/FlameGraph/colors.ts +++ b/packages/grafana-flamegraph/src/FlameGraph/colors.ts @@ -112,9 +112,9 @@ export function getBarColorByDiff( // the language from the backend and use the right regex but right now we just try all of them from most to least // specific. const matchers = [ - ['phpspy', /^(?(.*\/)*)(?.*\.php+)(?.*)$/], - ['pyspy', /^(?(.*\/)*)(?.*\.py+)(?.*)$/], - ['rbspy', /^(?(.*\/)*)(?.*\.rb+)(?.*)$/], + ['phpspy', /^(?([^\/]*\/)*)(?.*\.php+)(?.*)$/], + ['pyspy', /^(?([^\/]*\/)*)(?.*\.py+)(?.*)$/], + ['rbspy', /^(?([^\/]*\/)*)(?.*\.rb+)(?.*)$/], [ 'nodespy', /^(\.\/node_modules\/)?(?[^/]*)(?.*\.?(jsx?|tsx?)?):(?.*):(?.*)$/, From bf7fae4bd322c977c13a6751aeab21bf5e5e800b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 11 Oct 2023 18:04:54 +0200 Subject: [PATCH 064/102] Tempo: Fix type errors that appeared when removing the "any" type from DataQueryResponseData (#75600) * Tempo: Improving typing of data types * last fix * Fix --- .../testfiles/snapshotable_dashboard.json | 4 +- .../testfiles/snapshotable_with_rows.json | 836 +++++++++--------- .../datasource/tempo/datasource.test.ts | 74 +- .../plugins/datasource/tempo/datasource.ts | 56 +- .../datasource/tempo/graphTransform.ts | 44 +- .../datasource/tempo/metricsSummary.ts | 1 + .../datasource/tempo/resultTransformer.ts | 69 +- .../app/plugins/datasource/tempo/variables.ts | 4 +- .../datasource/zipkin/utils/transforms.ts | 20 +- 9 files changed, 580 insertions(+), 528 deletions(-) diff --git a/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_dashboard.json b/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_dashboard.json index 3d4528fc997..b3fee37e368 100644 --- a/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_dashboard.json +++ b/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_dashboard.json @@ -193,9 +193,7 @@ "footer": { "countRows": false, "fields": "", - "reducer": [ - "sum" - ], + "reducer": ["sum"], "show": false }, "showHeader": true diff --git a/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_with_rows.json b/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_with_rows.json index 408c5fc18cd..a2fbce92e55 100644 --- a/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_with_rows.json +++ b/public/app/features/dashboard-scene/serialization/testfiles/snapshotable_with_rows.json @@ -1,425 +1,425 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 2312, - "links": [], - "liveNow": false, - "panels": [ + "annotations": { + "list": [ { + "builtIn": 1, "datasource": { - "type": "grafana-testdata-datasource", - "uid": "gdev-testdata" + "type": "grafana", + "uid": "-- Grafana --" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "grafana-testdata-datasource", - "uid": "gdev-testdata" - }, - "refId": "A", - "scenarioId": "random_walk", - "seriesCount": 1 - } - ], - "title": "Panel 1", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 2, - "panels": [], - "title": "Expanded row", - "type": "row" - }, - { - "datasource": { - "type": "grafana-testdata-datasource", - "uid": "gdev-testdata" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "grafana-testdata-datasource", - "uid": "gdev-testdata" - }, - "refId": "B", - "scenarioId": "random_walk", - "seriesCount": 1 - } - ], - "title": "Panel 2", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-testdata-datasource", - "uid": "gdev-testdata" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "grafana-testdata-datasource", - "uid": "gdev-testdata" - }, - "refId": "C", - "scenarioId": "random_walk", - "seriesCount": 1 - } - ], - "title": "Panel 3", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 4, - "panels": [ - { - "datasource": { - "type": "grafana-testdata-datasource", - "uid": "gdev-testdata" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "title": "Panel inside colapsed row", - "type": "timeseries" - } - ], - "title": "Collapsed row", - "type": "row" + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" } - ], - "refresh": "", - "schemaVersion": 38, - "tags": [], - "templating": { - "list": [] + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2312, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "refId": "A", + "scenarioId": "random_walk", + "seriesCount": 1 + } + ], + "title": "Panel 1", + "type": "timeseries" }, - "time": { - "from": "now-6h", - "to": "now" + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "panels": [], + "title": "Expanded row", + "type": "row" }, - "timepicker": {}, - "timezone": "", - "title": "Snapshots - basic rows", - "uid": "a3b23921-287d-4f90-9a14-c04228057dfa", - "version": 4, - "weekStart": "" - } + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "refId": "B", + "scenarioId": "random_walk", + "seriesCount": 1 + } + ], + "title": "Panel 2", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "refId": "C", + "scenarioId": "random_walk", + "seriesCount": 1 + } + ], + "title": "Panel 3", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 4, + "panels": [ + { + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "gdev-testdata" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "title": "Panel inside colapsed row", + "type": "timeseries" + } + ], + "title": "Collapsed row", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Snapshots - basic rows", + "uid": "a3b23921-287d-4f90-9a14-c04228057dfa", + "version": 4, + "weekStart": "" +} diff --git a/public/app/plugins/datasource/tempo/datasource.test.ts b/public/app/plugins/datasource/tempo/datasource.test.ts index 9091e9f61d3..13051aa8534 100644 --- a/public/app/plugins/datasource/tempo/datasource.test.ts +++ b/public/app/plugins/datasource/tempo/datasource.test.ts @@ -232,11 +232,11 @@ describe('Tempo data source', () => { expect(response.data).toHaveLength(2); const nodesFrame = response.data[0]; expect(nodesFrame.name).toBe('Nodes'); - expect(nodesFrame.meta.preferredVisualisationType).toBe('nodeGraph'); + expect(nodesFrame.meta?.preferredVisualisationType).toBe('nodeGraph'); const edgesFrame = response.data[1]; expect(edgesFrame.name).toBe('Edges'); - expect(edgesFrame.meta.preferredVisualisationType).toBe('nodeGraph'); + expect(edgesFrame.meta?.preferredVisualisationType).toBe('nodeGraph'); }); it('should build search query correctly', () => { @@ -486,67 +486,67 @@ describe('Tempo service graph view', () => { expect(response.data[0].fields[1].values.length).toBe(2); expect(response.data[0].fields[1].values[0]).toBe(12.75164671814457); expect(response.data[0].fields[1].values[1]).toBe(12.121331111401608); - expect(response.data[0].fields[1].config.decimals).toBe(2); - expect(response.data[0].fields[1].config.links[0].title).toBe('Rate'); - expect(response.data[0].fields[1].config.links[0].internal.query.expr).toBe( + expect(response.data[0].fields[1]?.config?.decimals).toBe(2); + expect(response.data[0].fields[1]?.config?.links?.[0]?.title).toBe('Rate'); + expect(response.data[0].fields[1]?.config?.links?.[0]?.internal?.query.expr).toBe( 'sum(rate(traces_spanmetrics_calls_total{span_name="${__data.fields[0]}"}[$__rate_interval]))' ); - expect(response.data[0].fields[1].config.links[0].internal.query.range).toBe(true); - expect(response.data[0].fields[1].config.links[0].internal.query.exemplar).toBe(true); - expect(response.data[0].fields[1].config.links[0].internal.query.instant).toBe(false); + expect(response.data[0].fields[1]?.config?.links?.[0]?.internal?.query.range).toBe(true); + expect(response.data[0].fields[1]?.config?.links?.[0]?.internal?.query.exemplar).toBe(true); + expect(response.data[0].fields[1]?.config?.links?.[0]?.internal?.query.instant).toBe(false); expect(response.data[0].fields[2].values.length).toBe(2); expect(response.data[0].fields[2].values[0]).toBe(12.75164671814457); expect(response.data[0].fields[2].values[1]).toBe(12.121331111401608); - expect(response.data[0].fields[2].config.color.mode).toBe('continuous-BlPu'); - expect(response.data[0].fields[2].config.custom.cellOptions.mode).toBe(BarGaugeDisplayMode.Lcd); - expect(response.data[0].fields[2].config.custom.cellOptions.type).toBe(TableCellDisplayMode.Gauge); - expect(response.data[0].fields[2].config.decimals).toBe(3); + expect(response.data[0].fields[2]?.config?.color?.mode).toBe('continuous-BlPu'); + expect(response.data[0].fields[2]?.config?.custom.cellOptions.mode).toBe(BarGaugeDisplayMode.Lcd); + expect(response.data[0].fields[2]?.config?.custom.cellOptions.type).toBe(TableCellDisplayMode.Gauge); + expect(response.data[0].fields[2]?.config?.decimals).toBe(3); expect(response.data[0].fields[3].name).toBe('Error Rate'); expect(response.data[0].fields[3].values.length).toBe(2); expect(response.data[0].fields[3].values[0]).toBe(3.75164671814457); expect(response.data[0].fields[3].values[1]).toBe(3.121331111401608); - expect(response.data[0].fields[3].config.decimals).toBe(2); - expect(response.data[0].fields[3].config.links[0].title).toBe('Error Rate'); - expect(response.data[0].fields[3].config.links[0].internal.query.expr).toBe( + expect(response.data[0].fields[3]?.config?.decimals).toBe(2); + expect(response.data[0].fields[3]?.config?.links?.[0]?.title).toBe('Error Rate'); + expect(response.data[0].fields[3]?.config?.links?.[0]?.internal?.query.expr).toBe( 'sum(rate(traces_spanmetrics_calls_total{status_code="STATUS_CODE_ERROR",span_name="${__data.fields[0]}"}[$__rate_interval]))' ); - expect(response.data[0].fields[3].config.links[0].internal.query.range).toBe(true); - expect(response.data[0].fields[3].config.links[0].internal.query.exemplar).toBe(true); - expect(response.data[0].fields[3].config.links[0].internal.query.instant).toBe(false); + expect(response.data[0].fields[3]?.config?.links?.[0]?.internal?.query.range).toBe(true); + expect(response.data[0].fields[3]?.config?.links?.[0]?.internal?.query.exemplar).toBe(true); + expect(response.data[0].fields[3]?.config?.links?.[0]?.internal?.query.instant).toBe(false); expect(response.data[0].fields[4].values.length).toBe(2); expect(response.data[0].fields[4].values[0]).toBe(3.75164671814457); expect(response.data[0].fields[4].values[1]).toBe(3.121331111401608); - expect(response.data[0].fields[4].config.color.mode).toBe('continuous-RdYlGr'); - expect(response.data[0].fields[4].config.custom.cellOptions.mode).toBe(BarGaugeDisplayMode.Lcd); - expect(response.data[0].fields[4].config.custom.cellOptions.type).toBe(TableCellDisplayMode.Gauge); - expect(response.data[0].fields[4].config.decimals).toBe(3); + expect(response.data[0].fields[4]?.config?.color?.mode).toBe('continuous-RdYlGr'); + expect(response.data[0].fields[4]?.config?.custom.cellOptions.mode).toBe(BarGaugeDisplayMode.Lcd); + expect(response.data[0].fields[4]?.config?.custom.cellOptions.type).toBe(TableCellDisplayMode.Gauge); + expect(response.data[0].fields[4]?.config?.decimals).toBe(3); expect(response.data[0].fields[5].name).toBe('Duration (p90)'); expect(response.data[0].fields[5].values.length).toBe(2); expect(response.data[0].fields[5].values[0]).toBe('0'); expect(response.data[0].fields[5].values[1]).toBe(0.12003505696757232); - expect(response.data[0].fields[5].config.unit).toBe('s'); - expect(response.data[0].fields[5].config.links[0].title).toBe('Duration'); - expect(response.data[0].fields[5].config.links[0].internal.query.expr).toBe( + expect(response.data[0].fields[5]?.config?.unit).toBe('s'); + expect(response.data[0].fields[5]?.config?.links?.[0]?.title).toBe('Duration'); + expect(response.data[0].fields[5]?.config?.links?.[0]?.internal?.query.expr).toBe( 'histogram_quantile(.9, sum(rate(traces_spanmetrics_latency_bucket{span_name="${__data.fields[0]}"}[$__rate_interval])) by (le))' ); - expect(response.data[0].fields[5].config.links[0].internal.query.range).toBe(true); - expect(response.data[0].fields[5].config.links[0].internal.query.exemplar).toBe(true); - expect(response.data[0].fields[5].config.links[0].internal.query.instant).toBe(false); + expect(response.data[0].fields[5]?.config?.links?.[0]?.internal?.query.range).toBe(true); + expect(response.data[0].fields[5]?.config?.links?.[0]?.internal?.query.exemplar).toBe(true); + expect(response.data[0].fields[5]?.config?.links?.[0]?.internal?.query.instant).toBe(false); - expect(response.data[0].fields[6].config.links[0].url).toBe(''); - expect(response.data[0].fields[6].config.links[0].title).toBe('Tempo'); - expect(response.data[0].fields[6].config.links[0].internal.query.queryType).toBe('traceqlSearch'); - expect(response.data[0].fields[6].config.links[0].internal.query.filters[0].value).toBe('${__data.fields[0]}'); + expect(response.data[0].fields[6]?.config?.links?.[0].url).toBe(''); + expect(response.data[0].fields[6]?.config?.links?.[0].title).toBe('Tempo'); + expect(response.data[0].fields[6]?.config?.links?.[0].internal.query.queryType).toBe('traceqlSearch'); + expect(response.data[0].fields[6]?.config?.links?.[0].internal.query.filters[0].value).toBe('${__data.fields[0]}'); // Service graph expect(response.data[1].name).toBe('Nodes'); expect(response.data[1].fields[0].values.length).toBe(3); - expect(response.data[1].fields[0].config.links.length).toBeGreaterThan(0); - expect(response.data[1].fields[0].config.links).toEqual(serviceGraphLinks); + expect(response.data[1].fields[0]?.config?.links?.length).toBeGreaterThan(0); + expect(response.data[1].fields[0]?.config?.links).toEqual(serviceGraphLinks); expect(response.data[2].name).toBe('Edges'); expect(response.data[2].fields[0].values.length).toBe(2); @@ -804,7 +804,7 @@ describe('Tempo service graph view', () => { fields: [ { name: 'Time', - type: 'time', + type: FieldType.time, config: {}, values: [1653828275000, 1653828275000, 1653828275000, 1653828275000, 1653828275000], }, @@ -813,12 +813,14 @@ describe('Tempo service graph view', () => { config: { filterable: true, }, - type: 'string', + type: FieldType.string, values: ['HTTP Client', 'HTTP GET', 'HTTP GET - root', 'HTTP POST', 'HTTP POST - post'], }, ], + values: [], }, ]; + const objToAlign = { 'HTTP GET - root': { value: '0.1234', diff --git a/public/app/plugins/datasource/tempo/datasource.ts b/public/app/plugins/datasource/tempo/datasource.ts index b9d40dbb218..f539a54c3dd 100644 --- a/public/app/plugins/datasource/tempo/datasource.ts +++ b/public/app/plugins/datasource/tempo/datasource.ts @@ -6,13 +6,13 @@ import semver from 'semver'; import { CoreApp, DataFrame, + DataFrameDTO, DataQueryRequest, DataQueryResponse, DataQueryResponseData, DataSourceApi, DataSourceInstanceSettings, dateTime, - Field, FieldType, isValidGoDuration, LoadingState, @@ -86,6 +86,17 @@ const featuresToTempoVersion = { // This is the last minor version of Tempo that does not expose the endpoint for build information. const defaultTempoVersion = '2.1.0'; +interface ServiceMapQueryResponse { + nodes: DataFrame; + edges: DataFrame; +} + +interface ServiceMapQueryResponseWithRates { + rates: Array; + nodes: DataFrame; + edges: DataFrame; +} + export class TempoDatasource extends DataSourceWithBackend { tracesToLogs?: TraceToLogsOptions; serviceMap?: { @@ -783,7 +794,11 @@ function queryPrometheus(request: DataQueryRequest, datasourceUid: st ); } -function serviceMapQuery(request: DataQueryRequest, datasourceUid: string, tempoDatasourceUid: string) { +function serviceMapQuery( + request: DataQueryRequest, + datasourceUid: string, + tempoDatasourceUid: string +): Observable { const serviceMapRequest = makePromServiceMapRequest(request); return queryPrometheus(serviceMapRequest, datasourceUid).pipe( @@ -849,7 +864,8 @@ function serviceMapQuery(request: DataQueryRequest, datasourceUid: s } return { - data: [nodes, edges], + nodes, + edges, state: LoadingState.Done, }; }) @@ -858,9 +874,9 @@ function serviceMapQuery(request: DataQueryRequest, datasourceUid: s function rateQuery( request: DataQueryRequest, - serviceMapResponse: DataQueryResponse, + serviceMapResponse: ServiceMapQueryResponse, datasourceUid: string -) { +): Observable { const serviceMapRequest = makePromServiceMapRequest(request); serviceMapRequest.targets = makeServiceGraphViewRequest([buildExpr(rateMetric, defaultTableFilter, request)]); @@ -872,8 +888,9 @@ function rateQuery( throw new Error(getErrorMessage(errorRes.error?.message)); } return { - data: [responses[0]?.data ?? [], serviceMapResponse.data[0], serviceMapResponse.data[1]], - state: LoadingState.Done, + rates: responses[0]?.data ?? [], + nodes: serviceMapResponse.nodes, + edges: serviceMapResponse.edges, }; }) ); @@ -883,7 +900,7 @@ function rateQuery( // -> which determine the errorRate/duration span_name(s) we need to query function errorAndDurationQuery( request: DataQueryRequest, - rateResponse: DataQueryResponse, + rateResponse: ServiceMapQueryResponseWithRates, datasourceUid: string, tempoDatasourceUid: string ) { @@ -892,14 +909,14 @@ function errorAndDurationQuery( let durationsBySpanName: string[] = []; let labels = []; - if (rateResponse.data[0][0] && request.app === CoreApp.Explore) { - const spanNameField = rateResponse.data[0][0].fields.find((field: Field) => field.name === 'span_name'); + if (rateResponse.rates[0] && request.app === CoreApp.Explore) { + const spanNameField = rateResponse.rates[0].fields.find((field) => field.name === 'span_name'); if (spanNameField && spanNameField.values) { labels = spanNameField.values; } - } else if (rateResponse.data[0]) { - rateResponse.data[0].map((df: DataFrame) => { - const spanNameLabels = df.fields.find((field: Field) => field.labels?.['span_name']); + } else if (rateResponse.rates) { + rateResponse.rates.map((df: DataFrame | DataFrameDTO) => { + const spanNameLabels = df.fields.find((field) => field.labels?.['span_name']); if (spanNameLabels) { labels.push(spanNameLabels.labels?.['span_name']); } @@ -941,13 +958,13 @@ function errorAndDurationQuery( if (serviceGraphView.fields.length === 0) { return { - data: [rateResponse.data[1], rateResponse.data[2]], + data: [rateResponse.nodes, rateResponse.edges], state: LoadingState.Done, }; } return { - data: [serviceGraphView, rateResponse.data[1], rateResponse.data[2]], + data: [serviceGraphView, rateResponse.nodes, rateResponse.edges], state: LoadingState.Done, }; }) @@ -1078,7 +1095,7 @@ function makePromServiceMapRequest(options: DataQueryRequest): DataQ function getServiceGraphView( request: DataQueryRequest, - rateResponse: DataQueryResponse, + rateResponse: ServiceMapQueryResponseWithRates, secondResponse: DataQueryResponse, errorRateBySpanName: string, durationsBySpanName: string[], @@ -1086,14 +1103,15 @@ function getServiceGraphView( tempoDatasourceUid: string ) { let df: any = { fields: [] }; - const rate = rateResponse.data[0]?.filter((x: { refId: string }) => { + + const rate = rateResponse.rates.filter((x) => { return x.refId === buildExpr(rateMetric, defaultTableFilter, request); }); const errorRate = secondResponse.data.filter((x) => { return x.refId === errorRateBySpanName; }); const duration = secondResponse.data.filter((x) => { - return durationsBySpanName.includes(x.refId); + return durationsBySpanName.includes(x.refId ?? ''); }); if (rate.length > 0 && rate[0].fields?.length > 2) { @@ -1193,7 +1211,7 @@ function getServiceGraphView( if (d.fields.length > 1) { const delimiter = d.refId?.includes('span_name=~"') ? 'span_name=~"' : 'span_name="'; const name = d.refId?.split(delimiter)[1].split('"}')[0]; - durationObj[name] = { value: d.fields[1].values[0] }; + durationObj[name!] = { value: d.fields[1].values[0] }; } }); if (Object.keys(durationObj).length > 0) { diff --git a/public/app/plugins/datasource/tempo/graphTransform.ts b/public/app/plugins/datasource/tempo/graphTransform.ts index 181225e4351..4b7b4b56fed 100644 --- a/public/app/plugins/datasource/tempo/graphTransform.ts +++ b/public/app/plugins/datasource/tempo/graphTransform.ts @@ -8,6 +8,7 @@ import { NodeGraphDataFrameFieldNames as Fields, TimeRange, FieldType, + toDataFrame, } from '@grafana/data'; import { getNonOverlappingDuration, getStats, makeFrames, makeSpanMap } from '../../../core/utils/tracing'; @@ -189,39 +190,53 @@ function createServiceMapDataFrames() { } const nodes = createDF('Nodes', [ - { name: Fields.id, type: FieldType.string }, - { name: Fields.title, type: FieldType.string, config: { displayName: 'Service name' } }, - { name: Fields.subTitle, type: FieldType.string, config: { displayName: 'Service namespace' } }, - { name: Fields.mainStat, type: FieldType.number, config: { unit: 'ms/r', displayName: 'Average response time' } }, + { name: Fields.id, type: FieldType.string, values: [] }, + { name: Fields.title, type: FieldType.string, config: { displayName: 'Service name' }, values: [] }, + { name: Fields.subTitle, type: FieldType.string, config: { displayName: 'Service namespace' }, values: [] }, + { + name: Fields.mainStat, + type: FieldType.number, + config: { unit: 'ms/r', displayName: 'Average response time' }, + values: [], + }, { name: Fields.secondaryStat, type: FieldType.number, config: { unit: 'r/sec', displayName: 'Requests per second' }, + values: [], }, { name: Fields.arc + 'success', type: FieldType.number, config: { displayName: 'Success', color: { fixedColor: 'green', mode: FieldColorModeId.Fixed } }, + values: [], }, { name: Fields.arc + 'failed', type: FieldType.number, config: { displayName: 'Failed', color: { fixedColor: 'red', mode: FieldColorModeId.Fixed } }, + values: [], }, ]); const edges = createDF('Edges', [ - { name: Fields.id, type: FieldType.string }, - { name: Fields.source, type: FieldType.string }, - { name: AdditionalEdgeFields.sourceName, type: FieldType.string }, - { name: AdditionalEdgeFields.sourceNamespace, type: FieldType.string }, - { name: Fields.target, type: FieldType.string }, - { name: AdditionalEdgeFields.targetName, type: FieldType.string }, - { name: AdditionalEdgeFields.targetNamespace, type: FieldType.string }, - { name: Fields.mainStat, type: FieldType.number, config: { unit: 'ms/r', displayName: 'Average response time' } }, + { name: Fields.id, type: FieldType.string, values: [] }, + { name: Fields.source, type: FieldType.string, values: [] }, + { name: AdditionalEdgeFields.sourceName, type: FieldType.string, values: [] }, + { name: AdditionalEdgeFields.sourceNamespace, type: FieldType.string, values: [] }, + { name: Fields.target, type: FieldType.string, values: [] }, + { name: AdditionalEdgeFields.targetName, type: FieldType.string, values: [] }, + { name: AdditionalEdgeFields.targetNamespace, type: FieldType.string, values: [] }, + { + name: Fields.mainStat, + type: FieldType.number, + config: { unit: 'ms/r', displayName: 'Average response time' }, + values: [], + }, { name: Fields.secondaryStat, type: FieldType.number, config: { unit: 'r/sec', displayName: 'Requests per second' }, + values: [], }, ]); @@ -234,8 +249,9 @@ function createServiceMapDataFrames() { * @param responses */ function getMetricFrames(responses: DataQueryResponse[]): Record { - return responses[0].data.reduce>((acc, frame) => { - acc[frame.refId] = new DataFrameView(frame); + return responses[0].data.reduce>((acc, frameDTO) => { + const frame = toDataFrame(frameDTO); + acc[frame.refId ?? 'A'] = new DataFrameView(frame); return acc; }, {}); } diff --git a/public/app/plugins/datasource/tempo/metricsSummary.ts b/public/app/plugins/datasource/tempo/metricsSummary.ts index 68898a8d840..e0b9c4ea23b 100644 --- a/public/app/plugins/datasource/tempo/metricsSummary.ts +++ b/public/app/plugins/datasource/tempo/metricsSummary.ts @@ -60,6 +60,7 @@ export function createTableFrameFromMetricsSummaryQuery( name: `${series.key}`, type: FieldType.string, config: getConfig(series, configQuery, instanceSettings), + values: [], }; }); }); diff --git a/public/app/plugins/datasource/tempo/resultTransformer.ts b/public/app/plugins/datasource/tempo/resultTransformer.ts index 08c86e59cdc..e63cc21bc78 100644 --- a/public/app/plugins/datasource/tempo/resultTransformer.ts +++ b/public/app/plugins/datasource/tempo/resultTransformer.ts @@ -16,6 +16,8 @@ import { createDataFrame, getDisplayProcessor, createTheme, + DataFrameDTO, + toDataFrame, } from '@grafana/data'; import { SearchTableType } from './dataquery.gen'; @@ -23,7 +25,7 @@ import { createGraphFrames } from './graphTransform'; import { Span, SpanAttributes, Spanset, TraceSearchMetadata } from './types'; export function createTableFrame( - logsFrame: DataFrame, + logsFrame: DataFrame | DataFrameDTO, datasourceUid: string, datasourceName: string, traceRegexs: string[] @@ -38,6 +40,7 @@ export function createTableFrame( width: 200, }, }, + values: [], }, { name: 'traceID', @@ -59,10 +62,12 @@ export function createTableFrame( }, ], }, + values: [], }, { name: 'Message', type: FieldType.string, + values: [], }, ], meta: { @@ -80,7 +85,7 @@ export function createTableFrame( for (let field of logsFrame.fields) { let hasMatch = false; if (field.type === FieldType.string) { - const values = field.values; + const values = field.values!; for (let i = 0; i < values.length; i++) { const line = values[i]; if (line) { @@ -88,7 +93,7 @@ export function createTableFrame( const match = line.match(traceRegex); if (match) { const traceId = match[1]; - const time = timeField ? timeField.values[i] : null; + const time = timeField ? timeField.values![i] : null; tableFrame.fields[0].values.push(time); tableFrame.fields[1].values.push(traceId); tableFrame.fields[2].values.push(line); @@ -226,23 +231,23 @@ export function transformFromOTLP( ): DataQueryResponse { const frame = new MutableDataFrame({ fields: [ - { name: 'traceID', type: FieldType.string }, - { name: 'spanID', type: FieldType.string }, - { name: 'parentSpanID', type: FieldType.string }, - { name: 'operationName', type: FieldType.string }, - { name: 'serviceName', type: FieldType.string }, - { name: 'kind', type: FieldType.string }, - { name: 'statusCode', type: FieldType.number }, - { name: 'statusMessage', type: FieldType.string }, - { name: 'instrumentationLibraryName', type: FieldType.string }, - { name: 'instrumentationLibraryVersion', type: FieldType.string }, - { name: 'traceState', type: FieldType.string }, - { name: 'serviceTags', type: FieldType.other }, - { name: 'startTime', type: FieldType.number }, - { name: 'duration', type: FieldType.number }, - { name: 'logs', type: FieldType.other }, - { name: 'references', type: FieldType.other }, - { name: 'tags', type: FieldType.other }, + { name: 'traceID', type: FieldType.string, values: [] }, + { name: 'spanID', type: FieldType.string, values: [] }, + { name: 'parentSpanID', type: FieldType.string, values: [] }, + { name: 'operationName', type: FieldType.string, values: [] }, + { name: 'serviceName', type: FieldType.string, values: [] }, + { name: 'kind', type: FieldType.string, values: [] }, + { name: 'statusCode', type: FieldType.number, values: [] }, + { name: 'statusMessage', type: FieldType.string, values: [] }, + { name: 'instrumentationLibraryName', type: FieldType.string, values: [] }, + { name: 'instrumentationLibraryVersion', type: FieldType.string, values: [] }, + { name: 'traceState', type: FieldType.string, values: [] }, + { name: 'serviceTags', type: FieldType.other, values: [] }, + { name: 'startTime', type: FieldType.number, values: [] }, + { name: 'duration', type: FieldType.number, values: [] }, + { name: 'logs', type: FieldType.other, values: [] }, + { name: 'references', type: FieldType.other, values: [] }, + { name: 'tags', type: FieldType.other, values: [] }, ], meta: { preferredVisualisationType: 'trace', @@ -487,7 +492,7 @@ function getOTLPReferences( } export function transformTrace(response: DataQueryResponse, nodeGraph = false): DataQueryResponse { - const frame: DataFrame = response.data[0]; + const frame = response.data[0]; if (!frame) { return emptyDataQueryResponse; @@ -495,7 +500,7 @@ export function transformTrace(response: DataQueryResponse, nodeGraph = false): let data = [...response.data]; if (nodeGraph) { - data.push(...createGraphFrames(frame)); + data.push(...createGraphFrames(toDataFrame(frame))); } return { @@ -512,6 +517,7 @@ export function createTableFrameFromSearch(data: TraceSearchMetadata[], instance { name: 'traceID', type: FieldType.string, + values: [], config: { unit: 'string', displayNameFromDS: 'Trace ID', @@ -531,10 +537,15 @@ export function createTableFrameFromSearch(data: TraceSearchMetadata[], instance ], }, }, - { name: 'traceService', type: FieldType.string, config: { displayNameFromDS: 'Trace service' } }, - { name: 'traceName', type: FieldType.string, config: { displayNameFromDS: 'Trace name' } }, - { name: 'startTime', type: FieldType.time, config: { displayNameFromDS: 'Start time' } }, - { name: 'traceDuration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ms' } }, + { name: 'traceService', type: FieldType.string, config: { displayNameFromDS: 'Trace service' }, values: [] }, + { name: 'traceName', type: FieldType.string, config: { displayNameFromDS: 'Trace name' }, values: [] }, + { name: 'startTime', type: FieldType.time, config: { displayNameFromDS: 'Start time' }, values: [] }, + { + name: 'traceDuration', + type: FieldType.number, + config: { displayNameFromDS: 'Duration', unit: 'ms' }, + values: [], + }, ], meta: { preferredVisualisationType: 'table', @@ -826,6 +837,7 @@ const traceSubFrame = ( name: attr.key, type: FieldType.string, config: { displayNameFromDS: attr.key }, + values: [], }; }); spanSet.spans.forEach((span) => { @@ -837,6 +849,7 @@ const traceSubFrame = ( name: attr.key, type: FieldType.string, config: { displayNameFromDS: attr.key }, + values: [], }; }); }); @@ -849,10 +862,12 @@ const traceSubFrame = ( config: { custom: { hidden: true }, }, + values: [], }, { name: 'spanID', type: FieldType.string, + values: [], config: { unit: 'string', displayNameFromDS: 'Span ID', @@ -893,12 +908,14 @@ const traceSubFrame = ( { name: 'name', type: FieldType.string, + values: [], config: { displayNameFromDS: 'Name', custom: { hidden: !hasNameAttribute } }, }, ...Object.values(spanDynamicAttrs).sort((a, b) => a.name.localeCompare(b.name)), { name: 'duration', type: FieldType.number, + values: [], config: { displayNameFromDS: 'Duration', unit: 'ns', diff --git a/public/app/plugins/datasource/tempo/variables.ts b/public/app/plugins/datasource/tempo/variables.ts index afcfe6d0487..29c76129ca7 100644 --- a/public/app/plugins/datasource/tempo/variables.ts +++ b/public/app/plugins/datasource/tempo/variables.ts @@ -1,7 +1,7 @@ import { from, Observable } from 'rxjs'; import { map } from 'rxjs/operators'; -import { DataQueryRequest, DataQueryResponse, CustomVariableSupport } from '@grafana/data'; +import { DataQueryRequest, CustomVariableSupport, MetricFindValue } from '@grafana/data'; import { TempoVariableQuery, TempoVariableQueryEditor } from './VariableQueryEditor'; import { TempoDatasource } from './datasource'; @@ -13,7 +13,7 @@ export class TempoVariableSupport extends CustomVariableSupport): Observable { + query(request: DataQueryRequest): Observable<{ data: MetricFindValue[] }> { if (!this.datasource) { throw new Error('Datasource not initialized'); } diff --git a/public/app/plugins/datasource/zipkin/utils/transforms.ts b/public/app/plugins/datasource/zipkin/utils/transforms.ts index 0ecc391a39f..e3c8c0a2cdf 100644 --- a/public/app/plugins/datasource/zipkin/utils/transforms.ts +++ b/public/app/plugins/datasource/zipkin/utils/transforms.ts @@ -9,16 +9,16 @@ export function transformResponse(zSpans: ZipkinSpan[]): DataFrame { const spanRows = zSpans.map(transformSpan); const frame = new MutableDataFrame({ fields: [ - { name: 'traceID', type: FieldType.string }, - { name: 'spanID', type: FieldType.string }, - { name: 'parentSpanID', type: FieldType.string }, - { name: 'operationName', type: FieldType.string }, - { name: 'serviceName', type: FieldType.string }, - { name: 'serviceTags', type: FieldType.other }, - { name: 'startTime', type: FieldType.number }, - { name: 'duration', type: FieldType.number }, - { name: 'logs', type: FieldType.other }, - { name: 'tags', type: FieldType.other }, + { name: 'traceID', type: FieldType.string, values: [] }, + { name: 'spanID', type: FieldType.string, values: [] }, + { name: 'parentSpanID', type: FieldType.string, values: [] }, + { name: 'operationName', type: FieldType.string, values: [] }, + { name: 'serviceName', type: FieldType.string, values: [] }, + { name: 'serviceTags', type: FieldType.other, values: [] }, + { name: 'startTime', type: FieldType.number, values: [] }, + { name: 'duration', type: FieldType.number, values: [] }, + { name: 'logs', type: FieldType.other, values: [] }, + { name: 'tags', type: FieldType.other, values: [] }, ], meta: { preferredVisualisationType: 'trace', From 6983af3a70b815f5bb825ee179c4692d928a23b0 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 11 Oct 2023 09:19:13 -0700 Subject: [PATCH 065/102] Playlist: Add an api wrapper for playlist requests (#76308) --- .../playlist/PlaylistEditPage.test.tsx | 1 + .../features/playlist/PlaylistEditPage.tsx | 9 +- .../features/playlist/PlaylistForm.test.tsx | 1 + public/app/features/playlist/PlaylistForm.tsx | 2 +- .../playlist/PlaylistNewPage.test.tsx | 1 + .../app/features/playlist/PlaylistNewPage.tsx | 4 +- public/app/features/playlist/PlaylistPage.tsx | 9 +- .../app/features/playlist/PlaylistSrv.test.ts | 5 +- public/app/features/playlist/PlaylistSrv.ts | 9 +- public/app/features/playlist/api.ts | 161 ++++++++++++------ public/app/features/playlist/types.ts | 68 +++++--- 11 files changed, 183 insertions(+), 87 deletions(-) diff --git a/public/app/features/playlist/PlaylistEditPage.test.tsx b/public/app/features/playlist/PlaylistEditPage.test.tsx index 8807cdf88eb..48e85023995 100644 --- a/public/app/features/playlist/PlaylistEditPage.test.tsx +++ b/public/app/features/playlist/PlaylistEditPage.test.tsx @@ -76,6 +76,7 @@ describe('PlaylistEditPage', () => { fireEvent.submit(screen.getByRole('button', { name: /save/i })); await waitFor(() => expect(putMock).toHaveBeenCalledTimes(1)); expect(putMock).toHaveBeenCalledWith('/api/playlists/foo', { + uid: 'foo', name: 'A Name', interval: '10s', items: [{ title: 'First item', type: 'dashboard_by_uid', order: 1, value: '1' }], diff --git a/public/app/features/playlist/PlaylistEditPage.tsx b/public/app/features/playlist/PlaylistEditPage.tsx index e5d26cb12cf..7ae0f8328ed 100644 --- a/public/app/features/playlist/PlaylistEditPage.tsx +++ b/public/app/features/playlist/PlaylistEditPage.tsx @@ -8,11 +8,9 @@ import { t, Trans } from 'app/core/internationalization'; import { GrafanaRouteComponentProps } from 'app/core/navigation/types'; import { PlaylistForm } from './PlaylistForm'; -import { playlistAPI, updatePlaylist } from './api'; +import { getPlaylistAPI } from './api'; import { Playlist } from './types'; -const { getPlaylist } = playlistAPI; - export interface RouteParams { uid: string; } @@ -20,10 +18,11 @@ export interface RouteParams { interface Props extends GrafanaRouteComponentProps {} export const PlaylistEditPage = ({ match }: Props) => { - const playlist = useAsync(() => getPlaylist(match.params.uid), [match.params]); + const api = getPlaylistAPI(); + const playlist = useAsync(() => api.getPlaylist(match.params.uid), [match.params]); const onSubmit = async (playlist: Playlist) => { - await updatePlaylist(match.params.uid, playlist); + await api.updatePlaylist(playlist); locationService.push('/playlists'); }; diff --git a/public/app/features/playlist/PlaylistForm.test.tsx b/public/app/features/playlist/PlaylistForm.test.tsx index 3ab028ee755..366c730ed60 100644 --- a/public/app/features/playlist/PlaylistForm.test.tsx +++ b/public/app/features/playlist/PlaylistForm.test.tsx @@ -108,6 +108,7 @@ describe('PlaylistForm', () => { await userEvent.click(screen.getByRole('button', { name: /save/i })); expect(onSubmitMock).toHaveBeenCalledTimes(1); expect(onSubmitMock).toHaveBeenCalledWith({ + uid: 'foo', name: 'A test playlist', interval: '10m', items: [ diff --git a/public/app/features/playlist/PlaylistForm.tsx b/public/app/features/playlist/PlaylistForm.tsx index 7ea244db5bf..aa6c468cc9b 100644 --- a/public/app/features/playlist/PlaylistForm.tsx +++ b/public/app/features/playlist/PlaylistForm.tsx @@ -29,7 +29,7 @@ export const PlaylistForm = ({ onSubmit, playlist }: Props) => { const doSubmit = (list: Playlist) => { setSaving(true); - onSubmit({ ...list, items }); + onSubmit({ ...list, items, uid: playlist.uid }); }; return ( diff --git a/public/app/features/playlist/PlaylistNewPage.test.tsx b/public/app/features/playlist/PlaylistNewPage.test.tsx index 9034ca2059f..a85004cb8c0 100644 --- a/public/app/features/playlist/PlaylistNewPage.test.tsx +++ b/public/app/features/playlist/PlaylistNewPage.test.tsx @@ -60,6 +60,7 @@ describe('PlaylistNewPage', () => { await waitFor(() => expect(backendSrvMock).toHaveBeenCalledTimes(1)); expect(backendSrvMock).toHaveBeenCalledWith('/api/playlists', { name: 'A new name', + uid: '', interval: '5m', items: [], }); diff --git a/public/app/features/playlist/PlaylistNewPage.tsx b/public/app/features/playlist/PlaylistNewPage.tsx index 2c6a8eb5ef1..eb219d23e02 100644 --- a/public/app/features/playlist/PlaylistNewPage.tsx +++ b/public/app/features/playlist/PlaylistNewPage.tsx @@ -5,14 +5,14 @@ import { locationService } from '@grafana/runtime'; import { Page } from 'app/core/components/Page/Page'; import { PlaylistForm } from './PlaylistForm'; -import { createPlaylist, getDefaultPlaylist } from './api'; +import { getPlaylistAPI, getDefaultPlaylist } from './api'; import { Playlist } from './types'; export const PlaylistNewPage = () => { const [playlist] = useState(getDefaultPlaylist()); const onSubmit = async (playlist: Playlist) => { - await createPlaylist(playlist); + await getPlaylistAPI().createPlaylist(playlist); locationService.push('/playlists'); }; diff --git a/public/app/features/playlist/PlaylistPage.tsx b/public/app/features/playlist/PlaylistPage.tsx index 8a76495fb32..6c01ac2dcc8 100644 --- a/public/app/features/playlist/PlaylistPage.tsx +++ b/public/app/features/playlist/PlaylistPage.tsx @@ -11,15 +11,14 @@ import { contextSrv } from 'app/core/services/context_srv'; import { EmptyQueryListBanner } from './EmptyQueryListBanner'; import { PlaylistPageList } from './PlaylistPageList'; import { StartModal } from './StartModal'; -import { deletePlaylist, searchPlaylists, playlistAPI } from './api'; +import { getPlaylistAPI, searchPlaylists } from './api'; import { Playlist } from './types'; -const { getAllPlaylist } = playlistAPI; - export const PlaylistPage = () => { + const api = getPlaylistAPI(); const [forcePlaylistsFetch, setForcePlaylistsFetch] = useState(0); const [searchQuery, setSearchQuery] = useState(''); - const allPlaylists = useAsync(() => getAllPlaylist(), [forcePlaylistsFetch]); + const allPlaylists = useAsync(() => api.getAllPlaylist(), [forcePlaylistsFetch]); const playlists = useMemo(() => searchPlaylists(allPlaylists.value ?? [], searchQuery), [searchQuery, allPlaylists]); const [startPlaylist, setStartPlaylist] = useState(); @@ -31,7 +30,7 @@ export const PlaylistPage = () => { if (!playlistToDelete) { return; } - deletePlaylist(playlistToDelete.uid).finally(() => { + api.deletePlaylist(playlistToDelete.uid).finally(() => { setForcePlaylistsFetch(forcePlaylistsFetch + 1); setPlaylistToDelete(undefined); }); diff --git a/public/app/features/playlist/PlaylistSrv.test.ts b/public/app/features/playlist/PlaylistSrv.test.ts index 0c4910caae0..b5ccdc6d411 100644 --- a/public/app/features/playlist/PlaylistSrv.test.ts +++ b/public/app/features/playlist/PlaylistSrv.test.ts @@ -11,16 +11,17 @@ import { PlaylistSrv } from './PlaylistSrv'; import { Playlist, PlaylistItem } from './types'; jest.mock('./api', () => ({ - playlistAPI: { + getPlaylistAPI: () => ({ getPlaylist: jest.fn().mockReturnValue({ interval: '1s', uid: 'xyz', + name: 'The display', items: [ { type: 'dashboard_by_uid', value: 'aaa' }, { type: 'dashboard_by_uid', value: 'bbb' }, ], } as Playlist), - }, + }), loadDashboards: (items: PlaylistItem[]) => { return Promise.resolve( items.map((v) => ({ diff --git a/public/app/features/playlist/PlaylistSrv.ts b/public/app/features/playlist/PlaylistSrv.ts index 409469c9e5a..b74b620ab3d 100644 --- a/public/app/features/playlist/PlaylistSrv.ts +++ b/public/app/features/playlist/PlaylistSrv.ts @@ -4,9 +4,8 @@ import { pickBy } from 'lodash'; import { locationUtil, urlUtil, rangeUtil } from '@grafana/data'; import { locationService } from '@grafana/runtime'; -import { playlistAPI, loadDashboards } from './api'; - -const { getPlaylist } = playlistAPI; +import { getPlaylistAPI, loadDashboards } from './api'; +import { PlaylistAPI } from './types'; export const queryParamsToPreserve: { [key: string]: boolean } = { kiosk: true, @@ -23,11 +22,13 @@ export class PlaylistSrv { private numberOfLoops = 0; private declare validPlaylistUrl: string; private locationListenerUnsub?: () => void; + private api: PlaylistAPI; isPlaying = false; constructor() { this.locationUpdated = this.locationUpdated.bind(this); + this.api = getPlaylistAPI(); } next() { @@ -81,7 +82,7 @@ export class PlaylistSrv { this.locationListenerUnsub = locationService.getHistory().listen(this.locationUpdated); const urls: string[] = []; - let playlist = await getPlaylist(playlistUid); + let playlist = await this.api.getPlaylist(playlistUid); if (!playlist.items?.length) { // alert return; diff --git a/public/app/features/playlist/api.ts b/public/app/features/playlist/api.ts index 85b76ab99c4..f5987ad5bad 100644 --- a/public/app/features/playlist/api.ts +++ b/public/app/features/playlist/api.ts @@ -11,71 +11,132 @@ import { dispatch } from 'app/store/store'; import { DashboardQueryResult, getGrafanaSearcher, SearchQuery } from '../search/service'; -import { Playlist, PlaylistItem, KubernetesPlaylist, KubernetesPlaylistList, PlaylistAPI } from './types'; +import { Playlist, PlaylistItem, PlaylistAPI } from './types'; -export async function createPlaylist(playlist: Playlist) { - await withErrorHandling(() => getBackendSrv().post('/api/playlists', playlist)); +class LegacyAPI implements PlaylistAPI { + async getAllPlaylist(): Promise { + return getBackendSrv().get('/api/playlists/'); + } + + async getPlaylist(uid: string): Promise { + const p = await getBackendSrv().get(`/api/playlists/${uid}`); + await migrateInternalIDs(p); + return p; + } + + async createPlaylist(playlist: Playlist): Promise { + await withErrorHandling(() => getBackendSrv().post('/api/playlists', playlist)); + } + + async updatePlaylist(playlist: Playlist): Promise { + await withErrorHandling(() => getBackendSrv().put(`/api/playlists/${playlist.uid}`, playlist)); + } + + async deletePlaylist(uid: string): Promise { + await withErrorHandling(() => getBackendSrv().delete(`/api/playlists/${uid}`), 'Playlist deleted'); + } } -export async function updatePlaylist(uid: string, playlist: Playlist) { - await withErrorHandling(() => getBackendSrv().put(`/api/playlists/${uid}`, playlist)); +interface K8sPlaylistList { + playlists: K8sPlaylist[]; } -export async function deletePlaylist(uid: string) { - await withErrorHandling(() => getBackendSrv().delete(`/api/playlists/${uid}`), 'Playlist deleted'); +interface K8sPlaylist { + metadata: { + name: string; + }; + spec: { + name: string; + interval: string; + items: PlaylistItem[]; + }; } -export const playlistAPI: PlaylistAPI = { - getPlaylist: config.featureToggles.kubernetesPlaylists ? k8sGetPlaylist : legacyGetPlaylist, - getAllPlaylist: config.featureToggles.kubernetesPlaylists ? k8sGetAllPlaylist : legacyGetAllPlaylist, -}; +class K8sAPI implements PlaylistAPI { + readonly url = `/apis/playlist.x.grafana.com/v0alpha1/namespaces/org-${contextSrv.user.orgId}/playlists`; + readonly legacy = new LegacyAPI(); // set to null for full CRUD -/** This returns a playlist where all ids are replaced with UIDs */ -export async function k8sGetPlaylist(uid: string): Promise { - const k8splaylist = await getBackendSrv().get( - `/apis/playlist.x.grafana.com/v0alpha1/namespaces/org-${contextSrv.user.orgId}/playlists/${uid}` - ); - const playlist = k8splaylist.spec; - if (playlist.items) { + async getAllPlaylist(): Promise { + const result = await getBackendSrv().get(this.url); + console.log('getAllPlaylist', result); + const v = result.playlists.map(k8sResourceAsPlaylist); + console.log('after', v); + return v; + } + + async getPlaylist(uid: string): Promise { + const r = await getBackendSrv().get(this.url + '/' + uid); + const p = k8sResourceAsPlaylist(r); + await migrateInternalIDs(p); + return p; + } + + async createPlaylist(playlist: Playlist): Promise { + if (this.legacy) { + return this.legacy.createPlaylist(playlist); + } + await withErrorHandling(() => + getBackendSrv().post(this.url, { + apiVersion: 'playlists.grafana.com/v0alpha1', + kind: 'Playlist', + metadata: { + name: playlist.uid, + }, + spec: playlist, + }) + ); + } + + async updatePlaylist(playlist: Playlist): Promise { + if (this.legacy) { + return this.legacy.updatePlaylist(playlist); + } + await withErrorHandling(() => + getBackendSrv().put(`${this.url}/${playlist.uid}`, { + apiVersion: 'playlists.grafana.com/v0alpha1', + kind: 'Playlist', + metadata: { + name: playlist.uid, + }, + spec: { + ...playlist, + title: playlist.name, + }, + }) + ); + } + + async deletePlaylist(uid: string): Promise { + if (this.legacy) { + return this.legacy.deletePlaylist(uid); + } + await withErrorHandling(() => getBackendSrv().delete(`${this.url}/${uid}`), 'Playlist deleted'); + } +} + +// This converts a saved k8s resource into a playlist object +// the main difference is that k8s uses metdata.name as the uid +// to avoid future confusion, the display name is now called "title" +function k8sResourceAsPlaylist(r: K8sPlaylist): Playlist { + return { + ...r.spec, + uid: r.metadata.name, // replace the uid from the k8s name + }; +} + +/** @deprecated -- this migrates playlists saved with internal ids to uid */ +async function migrateInternalIDs(playlist: Playlist) { + if (playlist?.items) { for (const item of playlist.items) { if (item.type === 'dashboard_by_id') { item.type = 'dashboard_by_uid'; const uids = await getBackendSrv().get(`/api/dashboards/ids/${item.value}`); - if (uids.length) { + if (uids?.length) { item.value = uids[0]; } } } } - return playlist; -} - -export async function k8sGetAllPlaylist(): Promise { - const k8splaylists = await getBackendSrv().get( - `/apis/playlist.x.grafana.com/v0alpha1/namespaces/org-${contextSrv.user.orgId}/playlists` - ); - return k8splaylists.playlists.map((p) => p.spec); -} - -/** This returns a playlist where all ids are replaced with UIDs */ -export async function legacyGetPlaylist(uid: string): Promise { - const playlist = await getBackendSrv().get(`/api/playlists/${uid}`); - if (playlist.items) { - for (const item of playlist.items) { - if (item.type === 'dashboard_by_id') { - item.type = 'dashboard_by_uid'; - const uids = await getBackendSrv().get(`/api/dashboards/ids/${item.value}`); - if (uids.length) { - item.value = uids[0]; - } - } - } - } - return playlist; -} - -export async function legacyGetAllPlaylist(): Promise { - return getBackendSrv().get('/api/playlists/'); } async function withErrorHandling(apiCall: () => Promise, message = 'Playlist saved') { @@ -158,3 +219,7 @@ export function searchPlaylists(playlists: Playlist[], query?: string): Playlist query = query.toLowerCase(); return playlists.filter((v) => v.name.toLowerCase().includes(query!)); } + +export function getPlaylistAPI() { + return config.featureToggles.kubernetesPlaylists ? new K8sAPI() : new LegacyAPI(); +} diff --git a/public/app/features/playlist/types.ts b/public/app/features/playlist/types.ts index 19cf25f5fbf..61263cc0492 100644 --- a/public/app/features/playlist/types.ts +++ b/public/app/features/playlist/types.ts @@ -1,37 +1,65 @@ -import { PlaylistItem as PlaylistItemFromSchema } from '@grafana/schema'; - import { DashboardQueryResult } from '../search/service'; export type PlaylistMode = boolean | 'tv'; -export interface PlayListItemDTO { - id: number; - title: string; - playlistid: string; - type: 'dashboard' | 'tag'; -} - export interface PlaylistAPI { - getPlaylist(uid: string): Promise; getAllPlaylist(): Promise; -} - -export interface KubernetesPlaylistList { - playlists: KubernetesPlaylist[]; -} - -export interface KubernetesPlaylist { - spec: Playlist; + getPlaylist(uid: string): Promise; + createPlaylist(playlist: Playlist): Promise; + updatePlaylist(playlist: Playlist): Promise; + deletePlaylist(uid: string): Promise; } export interface Playlist { + /** + * Unique playlist identifier. Generated on creation, either by the + * creator of the playlist of by the application. + */ uid: string; + + /** + * Name of the playlist. + */ name: string; + + /** + * Interval sets the time between switching views in a playlist. + */ interval: string; + + /** + * The ordered list of items that the playlist will iterate over. + */ items?: PlaylistItem[]; } -export interface PlaylistItem extends PlaylistItemFromSchema { - // Loaded in the frontend +export interface PlaylistItem { + /** + * Type of the item. + */ + type: // Use an explicit dashboard + | 'dashboard_by_uid' + // find all dashboards with a given tag + | 'dashboard_by_tag' + // @deprecated use a dashboard with a given internal id + | 'dashboard_by_id'; + + /** + * Value depends on type and describes the playlist item. + * + * - dashboard_by_id: The value is an internal numerical identifier set by Grafana. This + * is not portable as the numerical identifier is non-deterministic between different instances. + * Will be replaced by dashboard_by_uid in the future. (deprecated) + * - dashboard_by_tag: The value is a tag which is set on any number of dashboards. All + * dashboards behind the tag will be added to the playlist. + * - dashboard_by_uid: The value is the dashboard UID + */ + value: string; + + /** + * Loaded at runtime by the frontend. + * + * The values are not stored in the backend database. + */ dashboards?: DashboardQueryResult[]; } From 046e9b76722453b3183e45a5798444107e25b035 Mon Sep 17 00:00:00 2001 From: Drew Slobodnjak <60050885+drew08t@users.noreply.github.com> Date: Wed, 11 Oct 2023 09:21:02 -0700 Subject: [PATCH 066/102] Canvas: Button API - Block Calls to Grafana (#76309) * Canvas: Button API - Block Calls to Grafana * Move origin check inside of api logic * Change grafana url source from href to origin --------- Co-authored-by: nmarrs --- .../app/plugins/panel/canvas/editor/element/utils.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/public/app/plugins/panel/canvas/editor/element/utils.ts b/public/app/plugins/panel/canvas/editor/element/utils.ts index b26544b8610..46201539f15 100644 --- a/public/app/plugins/panel/canvas/editor/element/utils.ts +++ b/public/app/plugins/panel/canvas/editor/element/utils.ts @@ -9,6 +9,11 @@ import { APIEditorConfig } from './APIEditor'; export const callApi = (api: APIEditorConfig, isTest = false) => { if (api && api.endpoint) { + // If API endpoint origin matches Grafana origin, don't call it. + if (requestMatchesGrafanaOrigin(api.endpoint)) { + appEvents.emit(AppEvents.alertError, ['Cannot call API at Grafana origin.']); + return; + } const request = getRequest(api); getBackendSrv() @@ -77,3 +82,9 @@ const getData = (api: APIEditorConfig) => { return data; }; + +const requestMatchesGrafanaOrigin = (requestEndpoint: string) => { + const requestURL = new URL(requestEndpoint); + const grafanaURL = new URL(window.location.origin); + return requestURL.origin === grafanaURL.origin; +}; From 6a8649d54420bb3e649d956350922a067564ec8b Mon Sep 17 00:00:00 2001 From: Matthew Jacobson Date: Wed, 11 Oct 2023 17:22:09 +0100 Subject: [PATCH 067/102] Alerting: Remove vendored models in migration service (#74503) This PR replaces the vendored models in the migration with their equivalent ngalert models. It also replaces the raw SQL selects and inserts with service calls. It also fills in some gaps in the testing suite around: - Migration of alert rules: verifying that the actual data model (queries, conditions) are correct 9a7cfa9 - Secure settings migration: verifying that secure fields remain encrypted for all available notifiers and certain fields migrate from plain text to encrypted secure settings correctly e7d3993 Replacing the checks for custom dashboard ACLs will be replaced in a separate targeted PR as it will be complex enough alone. --- .../backgroundsvcs/background_services.go | 4 +- pkg/server/wire.go | 4 + .../ngalert/api/api_alertmanager_test.go | 3 +- .../migration}/alert_rule.go | 227 +-- .../migration}/alert_rule_test.go | 119 +- pkg/services/ngalert/migration/channel.go | 440 ++++++ .../ngalert/migration/channel_test.go | 591 ++++++++ .../migration}/cond_trans.go | 50 +- .../ngalert/migration/migration_test.go | 1326 +++++++++++++++++ pkg/services/ngalert/migration/models.go | 121 ++ .../ngalert/migration/models/state.go | 7 + pkg/services/ngalert/migration/permissions.go | 153 ++ .../ngalert/migration/securejsondata.go | 31 + pkg/services/ngalert/migration/service.go | 137 ++ .../ngalert/migration/service_test.go | 190 +++ .../ualert => ngalert/migration}/silences.go | 51 +- .../ngalert/migration/store/database.go | 511 +++++++ .../ngalert/migration/store/models.go | 91 ++ .../ngalert/migration/store/testing.go | 57 + .../ualert => ngalert/migration}/template.go | 2 +- .../migration}/template_test.go | 5 +- pkg/services/ngalert/migration/testing.go | 29 + pkg/services/ngalert/migration/ualert.go | 115 ++ .../migration}/ualert_test.go | 108 +- pkg/services/ngalert/ngalert.go | 1 + .../ngalert/notifier/alertmanager_test.go | 3 +- .../channels_config/available_channels.go | 18 + .../ngalert/notifier/file_store_test.go | 14 +- .../notifier/multiorg_alertmanager_test.go | 9 +- pkg/services/ngalert/notifier/testing.go | 95 -- .../ngalert/provisioning/contactpoints.go | 21 +- pkg/services/ngalert/sender/router_test.go | 3 +- pkg/services/ngalert/tests/fakes/kvstore.go | 102 ++ .../migrations/external_alertmanagers.go | 27 +- .../sqlstore/migrations/migrations.go | 7 +- .../sqlstore/migrations/ualert/channel.go | 512 ------- .../migrations/ualert/channel_test.go | 471 ------ .../migrations/ualert/created_folders_mig.go | 83 ++ .../sqlstore/migrations/ualert/dash_alert.go | 130 -- .../sqlstore/migrations/ualert/dashboard.go | 108 -- .../sqlstore/migrations/ualert/datasource.go | 31 - .../ualert/migration_service_migration.go | 69 + .../migrations/ualert/migration_test.go | 847 ----------- .../sqlstore/migrations/ualert/permissions.go | 284 ---- .../ualert/rule_group_index_migration.go | 209 +++ .../migrations/ualert/securejsondata.go | 60 - .../sqlstore/migrations/ualert/tables.go | 3 - .../sqlstore/migrations/ualert/testing.go | 25 - .../sqlstore/migrations/ualert/ualert.go | 707 +-------- 49 files changed, 4564 insertions(+), 3647 deletions(-) rename pkg/services/{sqlstore/migrations/ualert => ngalert/migration}/alert_rule.go (56%) rename pkg/services/{sqlstore/migrations/ualert => ngalert/migration}/alert_rule_test.go (54%) create mode 100644 pkg/services/ngalert/migration/channel.go create mode 100644 pkg/services/ngalert/migration/channel_test.go rename pkg/services/{sqlstore/migrations/ualert => ngalert/migration}/cond_trans.go (87%) create mode 100644 pkg/services/ngalert/migration/migration_test.go create mode 100644 pkg/services/ngalert/migration/models.go create mode 100644 pkg/services/ngalert/migration/models/state.go create mode 100644 pkg/services/ngalert/migration/permissions.go create mode 100644 pkg/services/ngalert/migration/securejsondata.go create mode 100644 pkg/services/ngalert/migration/service.go create mode 100644 pkg/services/ngalert/migration/service_test.go rename pkg/services/{sqlstore/migrations/ualert => ngalert/migration}/silences.go (70%) create mode 100644 pkg/services/ngalert/migration/store/database.go create mode 100644 pkg/services/ngalert/migration/store/models.go create mode 100644 pkg/services/ngalert/migration/store/testing.go rename pkg/services/{sqlstore/migrations/ualert => ngalert/migration}/template.go (99%) rename pkg/services/{sqlstore/migrations/ualert => ngalert/migration}/template_test.go (99%) create mode 100644 pkg/services/ngalert/migration/testing.go create mode 100644 pkg/services/ngalert/migration/ualert.go rename pkg/services/{sqlstore/migrations/ualert => ngalert/migration}/ualert_test.go (53%) create mode 100644 pkg/services/ngalert/tests/fakes/kvstore.go delete mode 100644 pkg/services/sqlstore/migrations/ualert/channel.go delete mode 100644 pkg/services/sqlstore/migrations/ualert/channel_test.go create mode 100644 pkg/services/sqlstore/migrations/ualert/created_folders_mig.go delete mode 100644 pkg/services/sqlstore/migrations/ualert/dash_alert.go delete mode 100644 pkg/services/sqlstore/migrations/ualert/dashboard.go delete mode 100644 pkg/services/sqlstore/migrations/ualert/datasource.go create mode 100644 pkg/services/sqlstore/migrations/ualert/migration_service_migration.go delete mode 100644 pkg/services/sqlstore/migrations/ualert/migration_test.go delete mode 100644 pkg/services/sqlstore/migrations/ualert/permissions.go create mode 100644 pkg/services/sqlstore/migrations/ualert/rule_group_index_migration.go delete mode 100644 pkg/services/sqlstore/migrations/ualert/securejsondata.go delete mode 100644 pkg/services/sqlstore/migrations/ualert/testing.go diff --git a/pkg/registry/backgroundsvcs/background_services.go b/pkg/registry/backgroundsvcs/background_services.go index 19f8cfc0d26..3d633e66467 100644 --- a/pkg/registry/backgroundsvcs/background_services.go +++ b/pkg/registry/backgroundsvcs/background_services.go @@ -23,6 +23,7 @@ import ( "github.com/grafana/grafana/pkg/services/login/authinfoservice" "github.com/grafana/grafana/pkg/services/loginattempt/loginattemptimpl" "github.com/grafana/grafana/pkg/services/ngalert" + "github.com/grafana/grafana/pkg/services/ngalert/migration" "github.com/grafana/grafana/pkg/services/notifications" plugindashboardsservice "github.com/grafana/grafana/pkg/services/plugindashboards/service" "github.com/grafana/grafana/pkg/services/pluginsintegration/angulardetectorsprovider" @@ -44,7 +45,7 @@ import ( ) func ProvideBackgroundServiceRegistry( - httpServer *api.HTTPServer, ng *ngalert.AlertNG, cleanup *cleanup.CleanUpService, live *live.GrafanaLive, + httpServer *api.HTTPServer, alertingMigrationService *migration.MigrationService, ng *ngalert.AlertNG, cleanup *cleanup.CleanUpService, live *live.GrafanaLive, pushGateway *pushhttp.Gateway, notifications *notifications.NotificationService, pluginStore *pluginStore.Service, rendering *rendering.RenderingService, tokenService auth.UserTokenBackgroundService, tracing *tracing.TracingService, provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, usageStats *uss.UsageStats, @@ -66,6 +67,7 @@ func ProvideBackgroundServiceRegistry( ) *BackgroundServiceRegistry { return NewBackgroundServiceRegistry( httpServer, + alertingMigrationService, ng, cleanup, live, diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 44755d416b4..6b80a5aab15 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -92,6 +92,8 @@ import ( "github.com/grafana/grafana/pkg/services/ngalert" ngimage "github.com/grafana/grafana/pkg/services/ngalert/image" ngmetrics "github.com/grafana/grafana/pkg/services/ngalert/metrics" + ngmigration "github.com/grafana/grafana/pkg/services/ngalert/migration" + migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" ngstore "github.com/grafana/grafana/pkg/services/ngalert/store" "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/services/oauthtoken" @@ -239,6 +241,8 @@ var wireBasicSet = wire.NewSet( wire.Bind(new(jwt.JWTService), new(*jwt.AuthService)), ngstore.ProvideDBStore, ngimage.ProvideDeleteExpiredService, + ngmigration.ProvideService, + migrationStore.ProvideMigrationStore, ngalert.ProvideService, librarypanels.ProvideService, wire.Bind(new(librarypanels.Service), new(*librarypanels.LibraryPanelService)), diff --git a/pkg/services/ngalert/api/api_alertmanager_test.go b/pkg/services/ngalert/api/api_alertmanager_test.go index cef372a4dc9..90a9a3a0060 100644 --- a/pkg/services/ngalert/api/api_alertmanager_test.go +++ b/pkg/services/ngalert/api/api_alertmanager_test.go @@ -24,6 +24,7 @@ import ( ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/notifier" "github.com/grafana/grafana/pkg/services/ngalert/provisioning" + ngfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" @@ -663,7 +664,7 @@ func createMultiOrgAlertmanager(t *testing.T) *notifier.MultiOrgAlertmanager { orgStore := notifier.NewFakeOrgStore(t, []int64{1, 2, 3}) provStore := provisioning.NewFakeProvisioningStore() tmpDir := t.TempDir() - kvStore := notifier.NewFakeKVStore(t) + kvStore := ngfakes.NewFakeKVStore(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) reg := prometheus.NewPedanticRegistry() m := metrics.NewNGAlert(reg) diff --git a/pkg/services/sqlstore/migrations/ualert/alert_rule.go b/pkg/services/ngalert/migration/alert_rule.go similarity index 56% rename from pkg/services/sqlstore/migrations/ualert/alert_rule.go rename to pkg/services/ngalert/migration/alert_rule.go index 2039c7e1922..2ef2ec03fe9 100644 --- a/pkg/services/sqlstore/migrations/ualert/alert_rule.go +++ b/pkg/services/ngalert/migration/alert_rule.go @@ -1,6 +1,7 @@ -package ualert +package migration import ( + "context" "encoding/json" "fmt" "time" @@ -8,8 +9,11 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" + migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/store" "github.com/grafana/grafana/pkg/tsdb/graphite" + "github.com/grafana/grafana/pkg/util" ) const ( @@ -19,78 +23,7 @@ const ( ContactLabel = "__contacts__" ) -type alertRule struct { - ID int64 `xorm:"pk autoincr 'id'"` - OrgID int64 `xorm:"org_id"` - Title string - Condition string - Data []alertQuery - IntervalSeconds int64 - Version int64 - UID string `xorm:"uid"` - NamespaceUID string `xorm:"namespace_uid"` - RuleGroup string - RuleGroupIndex int `xorm:"rule_group_idx"` - NoDataState string - ExecErrState string - For duration - Updated time.Time - Annotations map[string]string - Labels map[string]string - IsPaused bool -} - -type alertRuleVersion struct { - RuleOrgID int64 `xorm:"rule_org_id"` - RuleUID string `xorm:"rule_uid"` - RuleNamespaceUID string `xorm:"rule_namespace_uid"` - RuleGroup string - RuleGroupIndex int `xorm:"rule_group_idx"` - ParentVersion int64 - RestoredFrom int64 - Version int64 - - Created time.Time - Title string - Condition string - Data []alertQuery - IntervalSeconds int64 - NoDataState string - ExecErrState string - // ideally this field should have been apimodels.ApiDuration - // but this is currently not possible because of circular dependencies - For duration - Annotations map[string]string - Labels map[string]string - IsPaused bool -} - -func (a *alertRule) makeVersion() *alertRuleVersion { - return &alertRuleVersion{ - RuleOrgID: a.OrgID, - RuleUID: a.UID, - RuleNamespaceUID: a.NamespaceUID, - RuleGroup: a.RuleGroup, - RuleGroupIndex: a.RuleGroupIndex, - ParentVersion: 0, - RestoredFrom: 0, - Version: 1, - - Created: time.Now().UTC(), - Title: a.Title, - Condition: a.Condition, - Data: a.Data, - IntervalSeconds: a.IntervalSeconds, - NoDataState: a.NoDataState, - ExecErrState: a.ExecErrState, - For: a.For, - Annotations: a.Annotations, - Labels: map[string]string{}, - IsPaused: a.IsPaused, - } -} - -func addMigrationInfo(da *dashAlert) (map[string]string, map[string]string) { +func addMigrationInfo(da *migrationStore.DashAlert, dashboardUID string) (map[string]string, map[string]string) { tagsMap := simplejson.NewFromAny(da.ParsedSettings.AlertRuleTags).MustMap() lbls := make(map[string]string, len(tagsMap)) @@ -99,15 +32,22 @@ func addMigrationInfo(da *dashAlert) (map[string]string, map[string]string) { } annotations := make(map[string]string, 3) - annotations[ngmodels.DashboardUIDAnnotation] = da.DashboardUID - annotations[ngmodels.PanelIDAnnotation] = fmt.Sprintf("%v", da.PanelId) - annotations["__alertId__"] = fmt.Sprintf("%v", da.Id) + annotations[ngmodels.DashboardUIDAnnotation] = dashboardUID + annotations[ngmodels.PanelIDAnnotation] = fmt.Sprintf("%v", da.PanelID) + annotations["__alertId__"] = fmt.Sprintf("%v", da.ID) return lbls, annotations } -func (m *migration) makeAlertRule(l log.Logger, cond condition, da dashAlert, folderUID string) (*alertRule, error) { - lbls, annotations := addMigrationInfo(&da) +// MigrateAlert migrates a single dashboard alert from legacy alerting to unified alerting. +func (om *OrgMigration) migrateAlert(ctx context.Context, l log.Logger, da *migrationStore.DashAlert, dashboardUID string, folderUID string) (*ngmodels.AlertRule, error) { + l.Debug("Migrating alert rule to Unified Alerting") + cond, err := transConditions(ctx, da, om.migrationStore) + if err != nil { + return nil, fmt.Errorf("transform conditions: %w", err) + } + + lbls, annotations := addMigrationInfo(da, dashboardUID) message := MigrateTmpl(l.New("field", "message"), da.Message) annotations["message"] = message @@ -117,33 +57,38 @@ func (m *migration) makeAlertRule(l log.Logger, cond condition, da dashAlert, fo return nil, fmt.Errorf("failed to migrate alert rule queries: %w", err) } - uid, err := m.seenUIDs.generateUid() - if err != nil { - return nil, fmt.Errorf("failed to migrate alert rule: %w", err) - } - - name := normalizeRuleName(da.Name, uid) - isPaused := false if da.State == "paused" { isPaused = true } - ar := &alertRule{ - OrgID: da.OrgId, - Title: name, // TODO: Make sure all names are unique, make new name on constraint insert error. - UID: uid, + // Here we ensure that the alert rule title is unique within the folder. + dedupSet := om.AlertTitleDeduplicator(folderUID) + name := truncateRuleName(da.Name) + if dedupSet.contains(name) { + dedupedName := dedupSet.deduplicate(name) + l.Debug("Duplicate alert rule name detected, renaming", "old_name", name, "new_name", dedupedName) + name = dedupedName + } + dedupSet.add(name) + + ar := &ngmodels.AlertRule{ + OrgID: da.OrgID, + Title: name, + UID: util.GenerateShortUID(), Condition: cond.Condition, Data: data, IntervalSeconds: ruleAdjustInterval(da.Frequency), Version: 1, NamespaceUID: folderUID, // Folder already created, comes from env var. + DashboardUID: &dashboardUID, + PanelID: &da.PanelID, RuleGroup: name, - For: duration(da.For), + For: da.For, Updated: time.Now().UTC(), Annotations: annotations, Labels: lbls, - RuleGroupIndex: 1, + RuleGroupIndex: 1, // Every rule is in its own group. IsPaused: isPaused, NoDataState: transNoData(l, da.ParsedSettings.NoDataState), ExecErrState: transExecErr(l, da.ParsedSettings.ExecutionErrorState), @@ -153,20 +98,24 @@ func (m *migration) makeAlertRule(l log.Logger, cond condition, da dashAlert, fo n, v := getLabelForSilenceMatching(ar.UID) ar.Labels[n] = v - if err := m.addErrorSilence(da, ar); err != nil { - m.mg.Logger.Error("Alert migration error: failed to create silence for Error", "rule_name", ar.Title, "err", err) + if da.ParsedSettings.ExecutionErrorState == string(legacymodels.ExecutionErrorKeepState) { + if err := om.addErrorSilence(ar); err != nil { + om.log.Error("Alert migration error: failed to create silence for Error", "rule_name", ar.Title, "err", err) + } } - if err := m.addNoDataSilence(da, ar); err != nil { - m.mg.Logger.Error("Alert migration error: failed to create silence for NoData", "rule_name", ar.Title, "err", err) + if da.ParsedSettings.NoDataState == string(legacymodels.NoDataKeepState) { + if err := om.addNoDataSilence(ar); err != nil { + om.log.Error("Alert migration error: failed to create silence for NoData", "rule_name", ar.Title, "err", err) + } } return ar, nil } // migrateAlertRuleQueries attempts to fix alert rule queries so they can work in unified alerting. Queries of some data sources are not compatible with unified alerting. -func migrateAlertRuleQueries(l log.Logger, data []alertQuery) ([]alertQuery, error) { - result := make([]alertQuery, 0, len(data)) +func migrateAlertRuleQueries(l log.Logger, data []ngmodels.AlertQuery) ([]ngmodels.AlertQuery, error) { + result := make([]ngmodels.AlertQuery, 0, len(data)) for _, d := range data { // queries that are expression are not relevant, skip them. if d.DatasourceUID == expressionDatasourceUID { @@ -274,55 +223,6 @@ func isPrometheusQuery(queryData map[string]json.RawMessage) (bool, error) { return datasource.Type == "prometheus", nil } -type alertQuery struct { - // RefID is the unique identifier of the query, set by the frontend call. - RefID string `json:"refId"` - - // QueryType is an optional identifier for the type of query. - // It can be used to distinguish different types of queries. - QueryType string `json:"queryType"` - - // RelativeTimeRange is the relative Start and End of the query as sent by the frontend. - RelativeTimeRange relativeTimeRange `json:"relativeTimeRange"` - - DatasourceUID string `json:"datasourceUid"` - - // JSON is the raw JSON query and includes the above properties as well as custom properties. - Model json.RawMessage `json:"model"` -} - -// RelativeTimeRange is the per query start and end time -// for requests. -type relativeTimeRange struct { - From duration `json:"from"` - To duration `json:"to"` -} - -// duration is a type used for marshalling durations. -type duration time.Duration - -func (d duration) String() string { - return time.Duration(d).String() -} - -func (d duration) MarshalJSON() ([]byte, error) { - return json.Marshal(time.Duration(d).Seconds()) -} - -func (d *duration) UnmarshalJSON(b []byte) error { - var v any - if err := json.Unmarshal(b, &v); err != nil { - return err - } - switch value := v.(type) { - case float64: - *d = duration(time.Duration(value) * time.Second) - return nil - default: - return fmt.Errorf("invalid duration %v", v) - } -} - func ruleAdjustInterval(freq int64) int64 { // 10 corresponds to the SchedulerCfg, but TODO not worrying about fetching for now. var baseFreq int64 = 10 @@ -332,50 +232,47 @@ func ruleAdjustInterval(freq int64) int64 { return freq - (freq % baseFreq) } -func transNoData(l log.Logger, s string) string { +func transNoData(l log.Logger, s string) ngmodels.NoDataState { switch legacymodels.NoDataOption(s) { case legacymodels.NoDataSetOK: - return string(ngmodels.OK) // values from ngalert/models/rule + return ngmodels.OK // values from ngalert/models/rule case "", legacymodels.NoDataSetNoData: - return string(ngmodels.NoData) + return ngmodels.NoData case legacymodels.NoDataSetAlerting: - return string(ngmodels.Alerting) + return ngmodels.Alerting case legacymodels.NoDataKeepState: - return string(ngmodels.NoData) // "keep last state" translates to no data because we now emit a special alert when the state is "noData". The result is that the evaluation will not return firing and instead we'll raise the special alert. + return ngmodels.NoData // "keep last state" translates to no data because we now emit a special alert when the state is "noData". The result is that the evaluation will not return firing and instead we'll raise the special alert. default: l.Warn("Unable to translate execution of NoData state. Using default execution", "old", s, "new", ngmodels.NoData) - return string(ngmodels.NoData) + return ngmodels.NoData } } -func transExecErr(l log.Logger, s string) string { +func transExecErr(l log.Logger, s string) ngmodels.ExecutionErrorState { switch legacymodels.ExecutionErrorOption(s) { case "", legacymodels.ExecutionErrorSetAlerting: - return string(ngmodels.AlertingErrState) + return ngmodels.AlertingErrState case legacymodels.ExecutionErrorKeepState: // Keep last state is translated to error as we now emit a // DatasourceError alert when the state is error - return string(ngmodels.ErrorErrState) + return ngmodels.ErrorErrState case legacymodels.ExecutionErrorSetOk: - return string(ngmodels.OkErrState) + return ngmodels.OkErrState default: l.Warn("Unable to translate execution of Error state. Using default execution", "old", s, "new", ngmodels.ErrorErrState) - return string(ngmodels.ErrorErrState) + return ngmodels.ErrorErrState } } -func normalizeRuleName(daName string, uid string) string { - // If we have to truncate, we're losing data and so there is higher risk of uniqueness conflicts. - // Append the UID to the suffix to forcibly break any collisions. - if len(daName) > DefaultFieldMaxLength { - trunc := DefaultFieldMaxLength - 1 - len(uid) - daName = daName[:trunc] + "_" + uid +// truncateRuleName truncates the rule name to the maximum allowed length. +func truncateRuleName(daName string) string { + if len(daName) > store.AlertDefinitionMaxTitleLength { + return daName[:store.AlertDefinitionMaxTitleLength] } - return daName } -func extractChannelIDs(d dashAlert) (channelUids []uidOrID) { +func extractChannelIDs(d *migrationStore.DashAlert) (channelUids []migrationStore.UidOrID) { // Extracting channel UID/ID. for _, ui := range d.ParsedSettings.Notifications { if ui.UID != "" { diff --git a/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go b/pkg/services/ngalert/migration/alert_rule_test.go similarity index 54% rename from pkg/services/sqlstore/migrations/ualert/alert_rule_test.go rename to pkg/services/ngalert/migration/alert_rule_test.go index c8f2aff7f4a..8ffa233d79a 100644 --- a/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go +++ b/pkg/services/ngalert/migration/alert_rule_test.go @@ -1,7 +1,7 @@ -package ualert +package migration import ( - "encoding/json" + "context" "strings" "testing" @@ -9,8 +9,12 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log/logtest" + legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" + migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/store" ) func TestMigrateAlertRuleQueries(t *testing.T) { @@ -73,7 +77,7 @@ func TestMigrateAlertRuleQueries(t *testing.T) { t.Run(tt.name, func(t *testing.T) { model, err := tt.input.Encode() require.NoError(t, err) - queries, err := migrateAlertRuleQueries(&logtest.Fake{}, []alertQuery{{Model: model}}) + queries, err := migrateAlertRuleQueries(&logtest.Fake{}, []models.AlertQuery{{Model: model}}) if tt.err != nil { require.Error(t, err) require.EqualError(t, err, tt.err.Error()) @@ -91,30 +95,30 @@ func TestMigrateAlertRuleQueries(t *testing.T) { func TestAddMigrationInfo(t *testing.T) { tt := []struct { name string - tagsJSON string + alert *migrationStore.DashAlert + dashboard string expectedLabels map[string]string expectedAnnotations map[string]string }{ { name: "when alert rule tags are a JSON array, they're ignored.", - tagsJSON: `{ "alertRuleTags": ["one", "two", "three", "four"] }`, + alert: &migrationStore.DashAlert{Alert: &legacymodels.Alert{ID: 43, PanelID: 42}, ParsedSettings: &migrationStore.DashAlertSettings{AlertRuleTags: []string{"one", "two", "three", "four"}}}, + dashboard: "dashboard", expectedLabels: map[string]string{}, - expectedAnnotations: map[string]string{"__alertId__": "0", "__dashboardUid__": "", "__panelId__": "0"}, + expectedAnnotations: map[string]string{"__alertId__": "43", "__dashboardUid__": "dashboard", "__panelId__": "42"}, }, { name: "when alert rule tags are a JSON object", - tagsJSON: `{ "alertRuleTags": { "key": "value", "key2": "value2" } }`, + alert: &migrationStore.DashAlert{Alert: &legacymodels.Alert{ID: 43, PanelID: 42}, ParsedSettings: &migrationStore.DashAlertSettings{AlertRuleTags: map[string]any{"key": "value", "key2": "value2"}}}, + dashboard: "dashboard", expectedLabels: map[string]string{"key": "value", "key2": "value2"}, - expectedAnnotations: map[string]string{"__alertId__": "0", "__dashboardUid__": "", "__panelId__": "0"}, + expectedAnnotations: map[string]string{"__alertId__": "43", "__dashboardUid__": "dashboard", "__panelId__": "42"}, }, } for _, tc := range tt { t.Run(tc.name, func(t *testing.T) { - var settings dashAlertSettings - require.NoError(t, json.Unmarshal([]byte(tc.tagsJSON), &settings)) - - labels, annotations := addMigrationInfo(&dashAlert{ParsedSettings: &settings}) + labels, annotations := addMigrationInfo(tc.alert, tc.dashboard) require.Equal(t, tc.expectedLabels, labels) require.Equal(t, tc.expectedAnnotations, annotations) }) @@ -122,13 +126,14 @@ func TestAddMigrationInfo(t *testing.T) { } func TestMakeAlertRule(t *testing.T) { + sqlStore := db.InitTestDB(t) t.Run("when mapping rule names", func(t *testing.T) { t.Run("leaves basic names untouched", func(t *testing.T) { - m := newTestMigration(t) + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) da := createTestDashAlert() - cnd := createTestDashAlertCondition() - ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") + ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") require.NoError(t, err) require.Equal(t, da.Name, ar.Title) @@ -136,73 +141,93 @@ func TestMakeAlertRule(t *testing.T) { }) t.Run("truncates very long names to max length", func(t *testing.T) { - m := newTestMigration(t) + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) da := createTestDashAlert() - da.Name = strings.Repeat("a", DefaultFieldMaxLength+1) - cnd := createTestDashAlertCondition() + da.Name = strings.Repeat("a", store.AlertDefinitionMaxTitleLength+1) - ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") + ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") require.NoError(t, err) - require.Len(t, ar.Title, DefaultFieldMaxLength) + require.Len(t, ar.Title, store.AlertDefinitionMaxTitleLength) + }) + + t.Run("deduplicate names in same org and folder", func(t *testing.T) { + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) + da := createTestDashAlert() + da.Name = strings.Repeat("a", store.AlertDefinitionMaxTitleLength+1) + + ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") + + require.NoError(t, err) + require.Len(t, ar.Title, store.AlertDefinitionMaxTitleLength) + + da = createTestDashAlert() + da.Name = strings.Repeat("a", store.AlertDefinitionMaxTitleLength+1) + + ar, err = m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") + + require.NoError(t, err) + require.Len(t, ar.Title, store.AlertDefinitionMaxTitleLength) parts := strings.SplitN(ar.Title, "_", 2) require.Len(t, parts, 2) require.Greater(t, len(parts[1]), 8, "unique identifier should be longer than 9 characters") - require.Equal(t, DefaultFieldMaxLength-1, len(parts[0])+len(parts[1]), "truncated name + underscore + unique identifier should together be DefaultFieldMaxLength") + require.Equal(t, store.AlertDefinitionMaxTitleLength-1, len(parts[0])+len(parts[1]), "truncated name + underscore + unique identifier should together be DefaultFieldMaxLength") require.Equal(t, ar.Title, ar.RuleGroup) }) }) t.Run("alert is not paused", func(t *testing.T) { - m := newTestMigration(t) + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) da := createTestDashAlert() - cnd := createTestDashAlertCondition() - ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") + ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") require.NoError(t, err) require.False(t, ar.IsPaused) }) t.Run("paused dash alert is paused", func(t *testing.T) { - m := newTestMigration(t) + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) da := createTestDashAlert() da.State = "paused" - cnd := createTestDashAlertCondition() - ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") + ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") require.NoError(t, err) require.True(t, ar.IsPaused) }) t.Run("use default if execution of NoData is not known", func(t *testing.T) { - m := newTestMigration(t) + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) da := createTestDashAlert() da.ParsedSettings.NoDataState = uuid.NewString() - cnd := createTestDashAlertCondition() - ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") + ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") require.Nil(t, err) - require.Equal(t, string(models.NoData), ar.NoDataState) + require.Equal(t, models.NoData, ar.NoDataState) }) t.Run("use default if execution of Error is not known", func(t *testing.T) { - m := newTestMigration(t) + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) da := createTestDashAlert() da.ParsedSettings.ExecutionErrorState = uuid.NewString() - cnd := createTestDashAlertCondition() - ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") + ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") require.Nil(t, err) - require.Equal(t, string(models.ErrorErrState), ar.ExecErrState) + require.Equal(t, models.ErrorErrState, ar.ExecErrState) }) t.Run("migrate message template", func(t *testing.T) { - m := newTestMigration(t) + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) da := createTestDashAlert() da.Message = "Instance ${instance} is down" - cnd := createTestDashAlertCondition() - ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") + ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") require.Nil(t, err) expected := "{{- $mergedLabels := mergeLabelValues $values -}}\n" + @@ -211,16 +236,12 @@ func TestMakeAlertRule(t *testing.T) { }) } -func createTestDashAlert() dashAlert { - return dashAlert{ - Id: 1, - Name: "test", - ParsedSettings: &dashAlertSettings{}, - } -} - -func createTestDashAlertCondition() condition { - return condition{ - Condition: "A", +func createTestDashAlert() migrationStore.DashAlert { + return migrationStore.DashAlert{ + Alert: &legacymodels.Alert{ + ID: 1, + Name: "test", + }, + ParsedSettings: &migrationStore.DashAlertSettings{}, } } diff --git a/pkg/services/ngalert/migration/channel.go b/pkg/services/ngalert/migration/channel.go new file mode 100644 index 00000000000..35455f7172a --- /dev/null +++ b/pkg/services/ngalert/migration/channel.go @@ -0,0 +1,440 @@ +package migration + +import ( + "context" + "crypto/md5" + "encoding/base64" + "fmt" + "regexp" + "sort" + "strings" + "time" + + alertingNotify "github.com/grafana/alerting/notify" + "github.com/prometheus/alertmanager/config" + "github.com/prometheus/alertmanager/pkg/labels" + "github.com/prometheus/common/model" + + "github.com/grafana/grafana/pkg/components/simplejson" + legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" + apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" + ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/util" +) + +const ( + // DisabledRepeatInterval is a large duration that will be used as a pseudo-disable in case a legacy channel doesn't have SendReminders enabled. + DisabledRepeatInterval = model.Duration(time.Duration(8736) * time.Hour) // 1y +) + +// channelReceiver is a convenience struct that contains a notificationChannel and its corresponding migrated PostableApiReceiver. +type channelReceiver struct { + channel *legacymodels.AlertNotification + receiver *apimodels.PostableApiReceiver +} + +// setupAlertmanagerConfigs creates Alertmanager configs with migrated receivers and routes. +func (om *OrgMigration) migrateChannels(allChannels []*legacymodels.AlertNotification, pairs []*AlertPair) (*apimodels.PostableUserConfig, error) { + var defaultChannels []*legacymodels.AlertNotification + var channels []*legacymodels.AlertNotification + for _, c := range allChannels { + if c.Type == "hipchat" || c.Type == "sensu" { + om.log.Error("Alert migration error: discontinued notification channel found", "type", c.Type, "name", c.Name, "uid", c.UID) + continue + } + + if c.IsDefault { + defaultChannels = append(defaultChannels, c) + } + channels = append(channels, c) + } + + amConfig := &apimodels.PostableUserConfig{ + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Receivers: make([]*apimodels.PostableApiReceiver, 0), + }, + } + + // Create all newly migrated receivers from legacy notification channels. + receiversMap, receivers, err := om.createReceivers(channels) + if err != nil { + return nil, fmt.Errorf("create receiver: %w", err) + } + + // No need to create an Alertmanager configuration if there are no receivers left that aren't obsolete. + if len(receivers) == 0 { + om.log.Warn("No available receivers") + return nil, nil + } + + for _, cr := range receivers { + amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, cr.receiver) + } + + defaultReceivers := make(map[string]struct{}) + // If the organization has default channels build a map of default receivers, used to create alert-specific routes later. + for _, c := range defaultChannels { + defaultReceivers[c.Name] = struct{}{} + } + defaultReceiver, defaultRoute, err := om.createDefaultRouteAndReceiver(defaultChannels) + if err != nil { + return nil, fmt.Errorf("failed to create default route & receiver in orgId %d: %w", om.orgID, err) + } + amConfig.AlertmanagerConfig.Route = defaultRoute + if defaultReceiver != nil { + amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, defaultReceiver) + } + + for _, cr := range receivers { + route, err := createRoute(cr) + if err != nil { + return nil, fmt.Errorf("failed to create route for receiver %s in orgId %d: %w", cr.receiver.Name, om.orgID, err) + } + + amConfig.AlertmanagerConfig.Route.Routes = append(amConfig.AlertmanagerConfig.Route.Routes, route) + } + + for _, pair := range pairs { + channelUids := extractChannelIDs(pair.DashAlert) + filteredReceiverNames := om.filterReceiversForAlert(pair.AlertRule.Title, channelUids, receiversMap, defaultReceivers) + + if len(filteredReceiverNames) != 0 { + // Only create a contact label if there are specific receivers, otherwise it defaults to the root-level route. + pair.AlertRule.Labels[ContactLabel] = contactListToString(filteredReceiverNames) + } + } + + // Validate the alertmanager configuration produced, this gives a chance to catch bad configuration at migration time. + // Validation between legacy and unified alerting can be different (e.g. due to bug fixes) so this would fail the migration in that case. + if err := om.validateAlertmanagerConfig(amConfig); err != nil { + return nil, fmt.Errorf("failed to validate AlertmanagerConfig in orgId %d: %w", om.orgID, err) + } + + return amConfig, nil +} + +// validateAlertmanagerConfig validates the alertmanager configuration produced by the migration against the receivers. +func (om *OrgMigration) validateAlertmanagerConfig(config *apimodels.PostableUserConfig) error { + for _, r := range config.AlertmanagerConfig.Receivers { + for _, gr := range r.GrafanaManagedReceivers { + data, err := gr.Settings.MarshalJSON() + if err != nil { + return err + } + var ( + cfg = &alertingNotify.GrafanaIntegrationConfig{ + UID: gr.UID, + Name: gr.Name, + Type: gr.Type, + DisableResolveMessage: gr.DisableResolveMessage, + Settings: data, + SecureSettings: gr.SecureSettings, + } + ) + + _, err = alertingNotify.BuildReceiverConfiguration(context.Background(), &alertingNotify.APIReceiver{ + GrafanaIntegrations: alertingNotify.GrafanaIntegrations{Integrations: []*alertingNotify.GrafanaIntegrationConfig{cfg}}, + }, om.encryptionService.GetDecryptedValue) + if err != nil { + return err + } + } + } + + return nil +} + +// contactListToString creates a sorted string representation of a given map (set) of receiver names. Each name will be comma-separated and double-quoted. Names should not contain double quotes. +func contactListToString(m map[string]any) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, quote(k)) + } + sort.Strings(keys) + + return strings.Join(keys, ",") +} + +// quote will surround the given string in double quotes. +func quote(s string) string { + return `"` + s + `"` +} + +// Create a notifier (PostableGrafanaReceiver) from a legacy notification channel +func (om *OrgMigration) createNotifier(c *legacymodels.AlertNotification) (*apimodels.PostableGrafanaReceiver, error) { + uid, err := om.determineChannelUid(c) + if err != nil { + return nil, err + } + + settings, secureSettings, err := om.migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings) + if err != nil { + return nil, err + } + + data, err := settings.MarshalJSON() + if err != nil { + return nil, err + } + + return &apimodels.PostableGrafanaReceiver{ + UID: uid, + Name: c.Name, + Type: c.Type, + DisableResolveMessage: c.DisableResolveMessage, + Settings: data, + SecureSettings: secureSettings, + }, nil +} + +// Create one receiver for every unique notification channel. +func (om *OrgMigration) createReceivers(allChannels []*legacymodels.AlertNotification) (map[migrationStore.UidOrID]*apimodels.PostableApiReceiver, []channelReceiver, error) { + receivers := make([]channelReceiver, 0, len(allChannels)) + receiversMap := make(map[migrationStore.UidOrID]*apimodels.PostableApiReceiver) + + set := make(map[string]struct{}) // Used to deduplicate sanitized names. + for _, c := range allChannels { + notifier, err := om.createNotifier(c) + if err != nil { + return nil, nil, err + } + + // We remove double quotes because this character will be used as the separator in the ContactLabel. To prevent partial matches in the Route Matcher we choose to sanitize them early on instead of complicating the Matcher regex. + sanitizedName := strings.ReplaceAll(c.Name, `"`, `_`) + // There can be name collisions after we sanitize. We check for this and attempt to make the name unique again using a short hash of the original name. + if _, ok := set[sanitizedName]; ok { + sanitizedName = sanitizedName + fmt.Sprintf("_%.3x", md5.Sum([]byte(c.Name))) + om.log.Warn("Alert contains duplicate contact name after sanitization, appending unique suffix", "type", c.Type, "name", c.Name, "new_name", sanitizedName, "uid", c.UID) + } + notifier.Name = sanitizedName + + set[sanitizedName] = struct{}{} + + cr := channelReceiver{ + channel: c, + receiver: &apimodels.PostableApiReceiver{ + Receiver: config.Receiver{ + Name: sanitizedName, // Channel name is unique within an Org. + }, + PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{ + GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{notifier}, + }, + }, + } + + receivers = append(receivers, cr) + + // Store receivers for creating routes from alert rules later. + if c.UID != "" { + receiversMap[c.UID] = cr.receiver + } + if c.ID != 0 { + // In certain circumstances, the alert rule uses ID instead of uid. So, we add this to be able to lookup by ID in case. + receiversMap[c.ID] = cr.receiver + } + } + + return receiversMap, receivers, nil +} + +// Create the root-level route with the default receiver. If no new receiver is created specifically for the root-level route, the returned receiver will be nil. +func (om *OrgMigration) createDefaultRouteAndReceiver(defaultChannels []*legacymodels.AlertNotification) (*apimodels.PostableApiReceiver, *apimodels.Route, error) { + defaultReceiverName := "autogen-contact-point-default" + defaultRoute := &apimodels.Route{ + Receiver: defaultReceiverName, + Routes: make([]*apimodels.Route, 0), + GroupByStr: []string{ngmodels.FolderTitleLabel, model.AlertNameLabel}, // To keep parity with pre-migration notifications. + RepeatInterval: nil, + } + newDefaultReceiver := &apimodels.PostableApiReceiver{ + Receiver: config.Receiver{ + Name: defaultReceiverName, + }, + PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{ + GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{}, + }, + } + + // Return early if there are no default channels + if len(defaultChannels) == 0 { + return newDefaultReceiver, defaultRoute, nil + } + + repeatInterval := DisabledRepeatInterval // If no channels have SendReminders enabled, we will use this large value as a pseudo-disable. + if len(defaultChannels) > 1 { + // If there are more than one default channels we create a separate contact group that is used only in the root policy. This is to simplify the migrated notification policy structure. + // If we ever allow more than one receiver per route this won't be necessary. + for _, c := range defaultChannels { + // Need to create a new notifier to prevent uid conflict. + defaultNotifier, err := om.createNotifier(c) + if err != nil { + return nil, nil, err + } + + newDefaultReceiver.GrafanaManagedReceivers = append(newDefaultReceiver.GrafanaManagedReceivers, defaultNotifier) + + // Choose the lowest send reminder duration from all the notifiers to use for default route. + if c.SendReminder && c.Frequency < time.Duration(repeatInterval) { + repeatInterval = model.Duration(c.Frequency) + } + } + } else { + // If there is only a single default channel, we don't need a separate receiver to hold it. We can reuse the existing receiver for that single notifier. + defaultRoute.Receiver = defaultChannels[0].Name + if defaultChannels[0].SendReminder { + repeatInterval = model.Duration(defaultChannels[0].Frequency) + } + + // No need to create a new receiver. + newDefaultReceiver = nil + } + defaultRoute.RepeatInterval = &repeatInterval + + return newDefaultReceiver, defaultRoute, nil +} + +// Create one route per contact point, matching based on ContactLabel. +func createRoute(cr channelReceiver) (*apimodels.Route, error) { + // We create a regex matcher so that each alert rule need only have a single ContactLabel entry for all contact points it sends to. + // For example, if an alert needs to send to contact1 and contact2 it will have ContactLabel=`"contact1","contact2"` and will match both routes looking + // for `.*"contact1".*` and `.*"contact2".*`. + + // We quote and escape here to ensure the regex will correctly match the ContactLabel on the alerts. + name := fmt.Sprintf(`.*%s.*`, regexp.QuoteMeta(quote(cr.receiver.Name))) + mat, err := labels.NewMatcher(labels.MatchRegexp, ContactLabel, name) + if err != nil { + return nil, err + } + + repeatInterval := DisabledRepeatInterval + if cr.channel.SendReminder { + repeatInterval = model.Duration(cr.channel.Frequency) + } + + return &apimodels.Route{ + Receiver: cr.receiver.Name, + ObjectMatchers: apimodels.ObjectMatchers{mat}, + Continue: true, // We continue so that each sibling contact point route can separately match. + RepeatInterval: &repeatInterval, + }, nil +} + +// Filter receivers to select those that were associated to the given rule as channels. +func (om *OrgMigration) filterReceiversForAlert(name string, channelIDs []migrationStore.UidOrID, receivers map[migrationStore.UidOrID]*apimodels.PostableApiReceiver, defaultReceivers map[string]struct{}) map[string]any { + if len(channelIDs) == 0 { + // If there are no channels associated, we use the default route. + return nil + } + + // Filter receiver names. + filteredReceiverNames := make(map[string]any) + for _, uidOrId := range channelIDs { + recv, ok := receivers[uidOrId] + if ok { + filteredReceiverNames[recv.Name] = struct{}{} // Deduplicate on contact point name. + } else { + om.log.Warn("Alert linked to obsolete notification channel, ignoring", "alert", name, "uid", uidOrId) + } + } + + coveredByDefault := func(names map[string]any) bool { + // Check if all receivers are also default ones and if so, just use the default route. + for n := range names { + if _, ok := defaultReceivers[n]; !ok { + return false + } + } + return true + } + + if len(filteredReceiverNames) == 0 || coveredByDefault(filteredReceiverNames) { + // Use the default route instead. + return nil + } + + // Add default receivers alongside rule-specific ones. + for n := range defaultReceivers { + filteredReceiverNames[n] = struct{}{} + } + + return filteredReceiverNames +} + +func (om *OrgMigration) determineChannelUid(c *legacymodels.AlertNotification) (string, error) { + legacyUid := c.UID + if legacyUid == "" { + newUid := util.GenerateShortUID() + om.seenUIDs.add(newUid) + om.log.Info("Legacy notification had an empty uid, generating a new one", "id", c.ID, "uid", newUid) + return newUid, nil + } + + if om.seenUIDs.contains(legacyUid) { + newUid := util.GenerateShortUID() + om.seenUIDs.add(newUid) + om.log.Warn("Legacy notification had a UID that collides with a migrated record, generating a new one", "id", c.ID, "old", legacyUid, "new", newUid) + return newUid, nil + } + + om.seenUIDs.add(legacyUid) + return legacyUid, nil +} + +var secureKeysToMigrate = map[string][]string{ + "slack": {"url", "token"}, + "pagerduty": {"integrationKey"}, + "webhook": {"password"}, + "prometheus-alertmanager": {"basicAuthPassword"}, + "opsgenie": {"apiKey"}, + "telegram": {"bottoken"}, + "line": {"token"}, + "pushover": {"apiToken", "userKey"}, + "threema": {"api_secret"}, +} + +// Some settings were migrated from settings to secure settings in between. +// See https://grafana.com/docs/grafana/latest/installation/upgrading/#ensure-encryption-of-existing-alert-notification-channel-secrets. +// migrateSettingsToSecureSettings takes care of that. +func (om *OrgMigration) migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json, secureSettings SecureJsonData) (*simplejson.Json, map[string]string, error) { + keys := secureKeysToMigrate[chanType] + newSecureSettings := secureSettings.Decrypt() + cloneSettings := simplejson.New() + settingsMap, err := settings.Map() + if err != nil { + return nil, nil, err + } + for k, v := range settingsMap { + cloneSettings.Set(k, v) + } + for _, k := range keys { + if v, ok := newSecureSettings[k]; ok && v != "" { + continue + } + + sv := cloneSettings.Get(k).MustString() + if sv != "" { + newSecureSettings[k] = sv + cloneSettings.Del(k) + } + } + + err = om.encryptSecureSettings(newSecureSettings) + if err != nil { + return nil, nil, err + } + + return cloneSettings, newSecureSettings, nil +} + +func (om *OrgMigration) encryptSecureSettings(secureSettings map[string]string) error { + for key, value := range secureSettings { + encryptedData, err := om.encryptionService.Encrypt(context.Background(), []byte(value), secrets.WithoutScope()) + if err != nil { + return fmt.Errorf("failed to encrypt secure settings: %w", err) + } + secureSettings[key] = base64.StdEncoding.EncodeToString(encryptedData) + } + return nil +} diff --git a/pkg/services/ngalert/migration/channel_test.go b/pkg/services/ngalert/migration/channel_test.go new file mode 100644 index 00000000000..88f4ddbbc45 --- /dev/null +++ b/pkg/services/ngalert/migration/channel_test.go @@ -0,0 +1,591 @@ +package migration + +import ( + "context" + "encoding/base64" + "encoding/json" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/prometheus/alertmanager/config" + "github.com/prometheus/alertmanager/pkg/labels" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/db" + legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" + apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" + ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/notifier/channels_config" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" +) + +func TestFilterReceiversForAlert(t *testing.T) { + tc := []struct { + name string + channelIds []migrationStore.UidOrID + receivers map[migrationStore.UidOrID]*apimodels.PostableApiReceiver + defaultReceivers map[string]struct{} + expected map[string]any + }{ + { + name: "when an alert has multiple channels, each should filter for the correct receiver", + channelIds: []migrationStore.UidOrID{"uid1", "uid2"}, + receivers: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ + "uid1": createPostableApiReceiver("recv1", nil), + "uid2": createPostableApiReceiver("recv2", nil), + "uid3": createPostableApiReceiver("recv3", nil), + }, + defaultReceivers: map[string]struct{}{}, + expected: map[string]any{ + "recv1": struct{}{}, + "recv2": struct{}{}, + }, + }, + { + name: "when default receivers exist, they should be added to an alert's filtered receivers", + channelIds: []migrationStore.UidOrID{"uid1"}, + receivers: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ + "uid1": createPostableApiReceiver("recv1", nil), + "uid2": createPostableApiReceiver("recv2", nil), + "uid3": createPostableApiReceiver("recv3", nil), + }, + defaultReceivers: map[string]struct{}{ + "recv2": {}, + }, + expected: map[string]any{ + "recv1": struct{}{}, // From alert + "recv2": struct{}{}, // From default + }, + }, + { + name: "when an alert has a channels associated by ID instead of UID, it should be included", + channelIds: []migrationStore.UidOrID{int64(42)}, + receivers: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ + int64(42): createPostableApiReceiver("recv1", nil), + }, + defaultReceivers: map[string]struct{}{}, + expected: map[string]any{ + "recv1": struct{}{}, + }, + }, + { + name: "when an alert's receivers are covered by the defaults, return nil to use default receiver downstream", + channelIds: []migrationStore.UidOrID{"uid1"}, + receivers: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ + "uid1": createPostableApiReceiver("recv1", nil), + "uid2": createPostableApiReceiver("recv2", nil), + "uid3": createPostableApiReceiver("recv3", nil), + }, + defaultReceivers: map[string]struct{}{ + "recv1": {}, + "recv2": {}, + }, + expected: nil, // recv1 is already a default + }, + } + + sqlStore := db.InitTestDB(t) + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) + res := m.filterReceiversForAlert("", tt.channelIds, tt.receivers, tt.defaultReceivers) + + require.Equal(t, tt.expected, res) + }) + } +} + +func TestCreateRoute(t *testing.T) { + tc := []struct { + name string + channel *legacymodels.AlertNotification + recv *apimodels.PostableApiReceiver + expected *apimodels.Route + }{ + { + name: "when a receiver is passed in, the route should regex match based on quoted name with continue=true", + channel: &legacymodels.AlertNotification{}, + recv: createPostableApiReceiver("recv1", nil), + expected: &apimodels.Route{ + Receiver: "recv1", + ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, + Routes: nil, + Continue: true, + GroupByStr: nil, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }, + }, + { + name: "notification channel should be escaped for regex in the matcher", + channel: &legacymodels.AlertNotification{}, + recv: createPostableApiReceiver(`. ^ $ * + - ? ( ) [ ] { } \ |`, nil), + expected: &apimodels.Route{ + Receiver: `. ^ $ * + - ? ( ) [ ] { } \ |`, + ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"\. \^ \$ \* \+ - \? \( \) \[ \] \{ \} \\ \|".*`}}, + Routes: nil, + Continue: true, + GroupByStr: nil, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }, + }, + { + name: "when a channel has sendReminder=true, the route should use the frequency in repeat interval", + channel: &legacymodels.AlertNotification{SendReminder: true, Frequency: time.Duration(42) * time.Hour}, + recv: createPostableApiReceiver("recv1", nil), + expected: &apimodels.Route{ + Receiver: "recv1", + ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, + Routes: nil, + Continue: true, + GroupByStr: nil, + RepeatInterval: durationPointer(model.Duration(time.Duration(42) * time.Hour)), + }, + }, + { + name: "when a channel has sendReminder=false, the route should ignore the frequency in repeat interval and use DisabledRepeatInterval", + channel: &legacymodels.AlertNotification{SendReminder: false, Frequency: time.Duration(42) * time.Hour}, + recv: createPostableApiReceiver("recv1", nil), + expected: &apimodels.Route{ + Receiver: "recv1", + ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, + Routes: nil, + Continue: true, + GroupByStr: nil, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + res, err := createRoute(channelReceiver{ + channel: tt.channel, + receiver: tt.recv, + }) + require.NoError(t, err) + + // Order of nested routes is not guaranteed. + cOpt := []cmp.Option{ + cmpopts.SortSlices(func(a, b *apimodels.Route) bool { + if a.Receiver != b.Receiver { + return a.Receiver < b.Receiver + } + return a.ObjectMatchers[0].Value < b.ObjectMatchers[0].Value + }), + cmpopts.IgnoreUnexported(apimodels.Route{}, labels.Matcher{}), + } + + if !cmp.Equal(tt.expected, res, cOpt...) { + t.Errorf("Unexpected Route: %v", cmp.Diff(tt.expected, res, cOpt...)) + } + }) + } +} + +func createNotChannel(t *testing.T, uid string, id int64, name string) *legacymodels.AlertNotification { + t.Helper() + return &legacymodels.AlertNotification{UID: uid, ID: id, Name: name, Settings: simplejson.New()} +} + +func createNotChannelWithReminder(t *testing.T, uid string, id int64, name string, frequency time.Duration) *legacymodels.AlertNotification { + t.Helper() + return &legacymodels.AlertNotification{UID: uid, ID: id, Name: name, SendReminder: true, Frequency: frequency, Settings: simplejson.New()} +} + +func TestCreateReceivers(t *testing.T) { + tc := []struct { + name string + allChannels []*legacymodels.AlertNotification + defaultChannels []*legacymodels.AlertNotification + expRecvMap map[migrationStore.UidOrID]*apimodels.PostableApiReceiver + expRecv []channelReceiver + expErr error + }{ + { + name: "when given notification channels migrate them to receivers", + allChannels: []*legacymodels.AlertNotification{createNotChannel(t, "uid1", int64(1), "name1"), createNotChannel(t, "uid2", int64(2), "name2")}, + expRecvMap: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ + "uid1": createPostableApiReceiver("name1", []string{"name1"}), + "uid2": createPostableApiReceiver("name2", []string{"name2"}), + int64(1): createPostableApiReceiver("name1", []string{"name1"}), + int64(2): createPostableApiReceiver("name2", []string{"name2"}), + }, + expRecv: []channelReceiver{ + { + channel: createNotChannel(t, "uid1", int64(1), "name1"), + receiver: createPostableApiReceiver("name1", []string{"name1"}), + }, + { + channel: createNotChannel(t, "uid2", int64(2), "name2"), + receiver: createPostableApiReceiver("name2", []string{"name2"}), + }, + }, + }, + { + name: "when given notification channel contains double quote sanitize with underscore", + allChannels: []*legacymodels.AlertNotification{createNotChannel(t, "uid1", int64(1), "name\"1")}, + expRecvMap: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ + "uid1": createPostableApiReceiver("name_1", []string{"name_1"}), + int64(1): createPostableApiReceiver("name_1", []string{"name_1"}), + }, + expRecv: []channelReceiver{ + { + channel: createNotChannel(t, "uid1", int64(1), "name\"1"), + receiver: createPostableApiReceiver("name_1", []string{"name_1"}), + }, + }, + }, + { + name: "when given notification channels collide after sanitization add short hash to end", + allChannels: []*legacymodels.AlertNotification{createNotChannel(t, "uid1", int64(1), "name\"1"), createNotChannel(t, "uid2", int64(2), "name_1")}, + expRecvMap: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ + "uid1": createPostableApiReceiver("name_1", []string{"name_1"}), + "uid2": createPostableApiReceiver("name_1_dba13d", []string{"name_1_dba13d"}), + int64(1): createPostableApiReceiver("name_1", []string{"name_1"}), + int64(2): createPostableApiReceiver("name_1_dba13d", []string{"name_1_dba13d"}), + }, + expRecv: []channelReceiver{ + { + channel: createNotChannel(t, "uid1", int64(1), "name\"1"), + receiver: createPostableApiReceiver("name_1", []string{"name_1"}), + }, + { + channel: createNotChannel(t, "uid2", int64(2), "name_1"), + receiver: createPostableApiReceiver("name_1_dba13d", []string{"name_1_dba13d"}), + }, + }, + }, + } + + sqlStore := db.InitTestDB(t) + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) + recvMap, recvs, err := m.createReceivers(tt.allChannels) + if tt.expErr != nil { + require.Error(t, err) + require.EqualError(t, err, tt.expErr.Error()) + return + } + + require.NoError(t, err) + + // We ignore certain fields for the purposes of this test + for _, recv := range recvs { + for _, not := range recv.receiver.GrafanaManagedReceivers { + not.UID = "" + not.Settings = nil + not.SecureSettings = nil + } + } + + require.Equal(t, tt.expRecvMap, recvMap) + require.ElementsMatch(t, tt.expRecv, recvs) + }) + } +} + +func TestMigrateNotificationChannelSecureSettings(t *testing.T) { + legacyEncryptFn := func(data string) string { + raw, err := util.Encrypt([]byte(data), setting.SecretKey) + require.NoError(t, err) + return string(raw) + } + decryptFn := func(data string, m *OrgMigration) string { + decoded, err := base64.StdEncoding.DecodeString(data) + require.NoError(t, err) + raw, err := m.encryptionService.Decrypt(context.Background(), decoded) + require.NoError(t, err) + return string(raw) + } + gen := func(nType string, fn func(channel *legacymodels.AlertNotification)) *legacymodels.AlertNotification { + not := &legacymodels.AlertNotification{ + UID: "uid", + ID: 1, + Name: "channel name", + Type: nType, + Settings: simplejson.NewFromAny(map[string]any{ + "something": "some value", + }), + SecureSettings: map[string][]byte{}, + } + if fn != nil { + fn(not) + } + return not + } + genExpSlack := func(fn func(channel *apimodels.PostableGrafanaReceiver)) *apimodels.PostableGrafanaReceiver { + rawSettings, err := json.Marshal(map[string]string{ + "something": "some value", + }) + require.NoError(t, err) + + recv := &apimodels.PostableGrafanaReceiver{ + UID: "uid", + Name: "channel name", + Type: "slack", + Settings: rawSettings, + SecureSettings: map[string]string{ + "token": "secure token", + "url": "secure url", + }, + } + + if fn != nil { + fn(recv) + } + return recv + } + + tc := []struct { + name string + channel *legacymodels.AlertNotification + expRecv *apimodels.PostableGrafanaReceiver + expErr error + }{ + { + name: "when secure settings exist, migrate them to receiver secure settings", + channel: gen("slack", func(channel *legacymodels.AlertNotification) { + channel.SecureSettings = map[string][]byte{ + "token": []byte(legacyEncryptFn("secure token")), + "url": []byte(legacyEncryptFn("secure url")), + } + }), + expRecv: genExpSlack(nil), + }, + { + name: "when no secure settings are encrypted, do nothing", + channel: gen("slack", nil), + expRecv: genExpSlack(func(recv *apimodels.PostableGrafanaReceiver) { + delete(recv.SecureSettings, "token") + delete(recv.SecureSettings, "url") + }), + }, + { + name: "when some secure settings are available unencrypted in settings, migrate them to secureSettings and encrypt", + channel: gen("slack", func(channel *legacymodels.AlertNotification) { + channel.SecureSettings = map[string][]byte{ + "url": []byte(legacyEncryptFn("secure url")), + } + channel.Settings.Set("token", "secure token") + }), + expRecv: genExpSlack(nil), + }, + } + sqlStore := db.InitTestDB(t) + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) + recv, err := m.createNotifier(tt.channel) + if tt.expErr != nil { + require.Error(t, err) + require.EqualError(t, err, tt.expErr.Error()) + return + } + require.NoError(t, err) + + if len(tt.expRecv.SecureSettings) > 0 { + require.NotEqual(t, tt.expRecv, recv) // Make sure they were actually encrypted at first. + } + for k, v := range recv.SecureSettings { + recv.SecureSettings[k] = decryptFn(v, m) + } + require.Equal(t, tt.expRecv, recv) + }) + } + + // Generate tests for each notification channel type. + t.Run("secure settings migrations for each notifier type", func(t *testing.T) { + notifiers := channels_config.GetAvailableNotifiers() + t.Run("migrate notification channel secure settings to receiver secure settings", func(t *testing.T) { + for _, notifier := range notifiers { + nType := notifier.Type + secureSettings, err := channels_config.GetSecretKeysForContactPointType(nType) + require.NoError(t, err) + t.Run(nType, func(t *testing.T) { + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) + channel := gen(nType, func(channel *legacymodels.AlertNotification) { + for _, key := range secureSettings { + channel.SecureSettings[key] = []byte(legacyEncryptFn("secure " + key)) + } + }) + recv, err := m.createNotifier(channel) + require.NoError(t, err) + + require.Equal(t, nType, recv.Type) + if len(secureSettings) > 0 { + for _, key := range secureSettings { + require.NotEqual(t, "secure "+key, recv.SecureSettings[key]) // Make sure they were actually encrypted at first. + } + } + require.Len(t, recv.SecureSettings, len(secureSettings)) + for _, key := range secureSettings { + require.Equal(t, "secure "+key, decryptFn(recv.SecureSettings[key], m)) + } + }) + } + }) + + t.Run("for certain legacy channel types, migrate secure fields stored in settings to secure settings", func(t *testing.T) { + for _, notifier := range notifiers { + nType := notifier.Type + secureSettings, ok := secureKeysToMigrate[nType] + if !ok { + continue + } + t.Run(nType, func(t *testing.T) { + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) + + channel := gen(nType, func(channel *legacymodels.AlertNotification) { + for _, key := range secureSettings { + // Key difference to above. We store the secure settings in the settings field and expect + // them to be migrated to secureSettings. + channel.Settings.Set(key, "secure "+key) + } + }) + recv, err := m.createNotifier(channel) + require.NoError(t, err) + + require.Equal(t, nType, recv.Type) + if len(secureSettings) > 0 { + for _, key := range secureSettings { + require.NotEqual(t, "secure "+key, recv.SecureSettings[key]) // Make sure they were actually encrypted at first. + } + } + require.Len(t, recv.SecureSettings, len(secureSettings)) + for _, key := range secureSettings { + require.Equal(t, "secure "+key, decryptFn(recv.SecureSettings[key], m)) + } + }) + } + }) + }) +} + +func TestCreateDefaultRouteAndReceiver(t *testing.T) { + tc := []struct { + name string + amConfig *apimodels.PostableUserConfig + defaultChannels []*legacymodels.AlertNotification + expRecv *apimodels.PostableApiReceiver + expRoute *apimodels.Route + expErr error + }{ + { + name: "when given multiple default notification channels migrate them to a single receiver", + defaultChannels: []*legacymodels.AlertNotification{createNotChannel(t, "uid1", int64(1), "name1"), createNotChannel(t, "uid2", int64(2), "name2")}, + expRecv: createPostableApiReceiver("autogen-contact-point-default", []string{"name1", "name2"}), + expRoute: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + Routes: make([]*apimodels.Route, 0), + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }, + }, + { + name: "when given multiple default notification channels migrate them to a single receiver with RepeatInterval set to be the minimum of all channel frequencies", + defaultChannels: []*legacymodels.AlertNotification{ + createNotChannelWithReminder(t, "uid1", int64(1), "name1", time.Duration(42)), + createNotChannelWithReminder(t, "uid2", int64(2), "name2", time.Duration(100000)), + }, + expRecv: createPostableApiReceiver("autogen-contact-point-default", []string{"name1", "name2"}), + expRoute: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + Routes: make([]*apimodels.Route, 0), + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + RepeatInterval: durationPointer(model.Duration(42)), + }, + }, + { + name: "when given no default notification channels create a single empty receiver for default", + defaultChannels: []*legacymodels.AlertNotification{}, + expRecv: createPostableApiReceiver("autogen-contact-point-default", nil), + expRoute: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + Routes: make([]*apimodels.Route, 0), + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + RepeatInterval: nil, + }, + }, + { + name: "when given a single default notification channels don't create a new default receiver", + defaultChannels: []*legacymodels.AlertNotification{createNotChannel(t, "uid1", int64(1), "name1")}, + expRecv: nil, + expRoute: &apimodels.Route{ + Receiver: "name1", + Routes: make([]*apimodels.Route, 0), + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }, + }, + { + name: "when given a single default notification channel with SendReminder=true, use the channels Frequency as the RepeatInterval", + defaultChannels: []*legacymodels.AlertNotification{createNotChannelWithReminder(t, "uid1", int64(1), "name1", time.Duration(42))}, + expRecv: nil, + expRoute: &apimodels.Route{ + Receiver: "name1", + Routes: make([]*apimodels.Route, 0), + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + RepeatInterval: durationPointer(model.Duration(42)), + }, + }, + } + + sqlStore := db.InitTestDB(t) + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + service := NewTestMigrationService(t, sqlStore, nil) + m := service.newOrgMigration(1) + recv, route, err := m.createDefaultRouteAndReceiver(tt.defaultChannels) + if tt.expErr != nil { + require.Error(t, err) + require.EqualError(t, err, tt.expErr.Error()) + return + } + + require.NoError(t, err) + + // We ignore certain fields for the purposes of this test + if recv != nil { + for _, not := range recv.GrafanaManagedReceivers { + not.UID = "" + not.Settings = nil + not.SecureSettings = nil + } + } + + require.Equal(t, tt.expRecv, recv) + require.Equal(t, tt.expRoute, route) + }) + } +} + +func createPostableApiReceiver(name string, integrationNames []string) *apimodels.PostableApiReceiver { + integrations := make([]*apimodels.PostableGrafanaReceiver, 0, len(integrationNames)) + for _, integrationName := range integrationNames { + integrations = append(integrations, &apimodels.PostableGrafanaReceiver{Name: integrationName}) + } + return &apimodels.PostableApiReceiver{ + Receiver: config.Receiver{ + Name: name, + }, + PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{ + GrafanaManagedReceivers: integrations, + }, + } +} + +func durationPointer(d model.Duration) *model.Duration { + return &d +} diff --git a/pkg/services/sqlstore/migrations/ualert/cond_trans.go b/pkg/services/ngalert/migration/cond_trans.go similarity index 87% rename from pkg/services/sqlstore/migrations/ualert/cond_trans.go rename to pkg/services/ngalert/migration/cond_trans.go index e67af621210..a2a05681d3d 100644 --- a/pkg/services/sqlstore/migrations/ualert/cond_trans.go +++ b/pkg/services/ngalert/migration/cond_trans.go @@ -1,7 +1,9 @@ -package ualert +package migration import ( + "context" "encoding/json" + "errors" "fmt" "sort" "strings" @@ -9,12 +11,22 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/services/datasources" + migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" + ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/tsdb/legacydata" "github.com/grafana/grafana/pkg/tsdb/legacydata/interval" "github.com/grafana/grafana/pkg/util" ) -func transConditions(set dashAlertSettings, orgID int64, dsUIDMap dsUIDLookup) (*condition, error) { +// It is defined in pkg/expr/service.go as "DatasourceType" +const expressionDatasourceUID = "__expr__" + +//nolint:gocyclo +func transConditions(ctx context.Context, alert *migrationStore.DashAlert, store migrationStore.Store) (*condition, error) { + // TODO: needs a significant refactor to reduce complexity. + usr := getMigrationUser(alert.OrgID) + set := alert.ParsedSettings + refIDtoCondIdx := make(map[string][]int) // a map of original refIds to their corresponding condition index for i, cond := range set.Conditions { if len(cond.Query.Params) != 3 { @@ -123,8 +135,16 @@ func transConditions(set dashAlertSettings, orgID int64, dsUIDMap dsUIDLookup) ( } } - // one could have an alert saved but datasource deleted, so can not require match. - dsUID := dsUIDMap.GetUID(orgID, set.Conditions[condIdx].Query.DatasourceID) + // Could have an alert saved but datasource deleted, so can not require match. + dsUid := "" + if ds, err := store.GetDatasource(ctx, set.Conditions[condIdx].Query.DatasourceID, usr); err == nil { + dsUid = ds.UID + } else { + if !errors.Is(err, datasources.ErrDataSourceNotFound) { + return nil, err + } + } + queryObj["refId"] = refID // See services/alerting/conditions/query.go's newQueryCondition @@ -153,11 +173,11 @@ func transConditions(set dashAlertSettings, orgID int64, dsUIDMap dsUIDLookup) ( return nil, err } - alertQuery := alertQuery{ + alertQuery := ngmodels.AlertQuery{ RefID: refID, Model: encodedObj, RelativeTimeRange: *rTR, - DatasourceUID: dsUID, + DatasourceUID: dsUid, QueryType: queryType, } newCond.Data = append(newCond.Data, alertQuery) @@ -168,7 +188,7 @@ func transConditions(set dashAlertSettings, orgID int64, dsUIDMap dsUIDLookup) ( conditions := make([]classicConditionJSON, len(set.Conditions)) for i, cond := range set.Conditions { newCond := classicConditionJSON{} - newCond.Evaluator = conditionEvalJSON{ + newCond.Evaluator = migrationStore.ConditionEvalJSON{ Type: cond.Evaluator.Type, Params: cond.Evaluator.Params, } @@ -184,7 +204,7 @@ func transConditions(set dashAlertSettings, orgID int64, dsUIDMap dsUIDLookup) ( return nil, err } newCond.Condition = ccRefID // set the alert condition to point to the classic condition - newCond.OrgID = orgID + newCond.OrgID = alert.OrgID exprModel := struct { Type string `json:"type"` @@ -201,7 +221,7 @@ func transConditions(set dashAlertSettings, orgID int64, dsUIDMap dsUIDLookup) ( return nil, err } - ccAlertQuery := alertQuery{ + ccAlertQuery := ngmodels.AlertQuery{ RefID: ccRefID, Model: exprModelJSON, DatasourceUID: expressionDatasourceUID, @@ -223,7 +243,7 @@ type condition struct { OrgID int64 `json:"-"` // Data is an array of data source queries and/or server side expressions. - Data []alertQuery `json:"data"` + Data []ngmodels.AlertQuery `json:"data"` } const alpha = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" @@ -250,7 +270,7 @@ func getNewRefID(refIDs map[string][]int) (string, error) { // getRelativeDuration turns the alerting durations for dashboard conditions // into a relative time range. -func getRelativeDuration(rawFrom, rawTo string) (*relativeTimeRange, error) { +func getRelativeDuration(rawFrom, rawTo string) (*ngmodels.RelativeTimeRange, error) { fromD, err := getFrom(rawFrom) if err != nil { return nil, err @@ -260,9 +280,9 @@ func getRelativeDuration(rawFrom, rawTo string) (*relativeTimeRange, error) { if err != nil { return nil, err } - return &relativeTimeRange{ - From: duration(fromD), - To: duration(toD), + return &ngmodels.RelativeTimeRange{ + From: ngmodels.Duration(fromD), + To: ngmodels.Duration(toD), }, nil } @@ -297,7 +317,7 @@ func getTo(to string) (time.Duration, error) { } type classicConditionJSON struct { - Evaluator conditionEvalJSON `json:"evaluator"` + Evaluator migrationStore.ConditionEvalJSON `json:"evaluator"` Operator struct { Type string `json:"type"` diff --git a/pkg/services/ngalert/migration/migration_test.go b/pkg/services/ngalert/migration/migration_test.go new file mode 100644 index 00000000000..d5edee7b274 --- /dev/null +++ b/pkg/services/ngalert/migration/migration_test.go @@ -0,0 +1,1326 @@ +package migration + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/prometheus/alertmanager/config" + "github.com/prometheus/alertmanager/pkg/labels" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + "xorm.io/xorm" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/services/alerting/models" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/datasources" + apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" + ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/tsdb/legacydata" +) + +// TestServiceStart tests the wrapper method that decides when to run the migration based on migration status and settings. +func TestServiceStart(t *testing.T) { + tc := []struct { + name string + config *setting.Cfg + isMigrationRun bool + expectedErr bool + expected bool + }{ + { + name: "when unified alerting enabled and migration not already run, then run migration", + config: &setting.Cfg{ + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: pointer(true), + }, + }, + isMigrationRun: false, + expected: true, + }, + { + name: "when unified alerting disabled, migration is already run and force migration is enabled, then revert migration", + config: &setting.Cfg{ + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: pointer(false), + }, + ForceMigration: true, + }, + isMigrationRun: true, + expected: false, + }, + { + name: "when unified alerting disabled, migration is already run and force migration is disabled, then the migration should panic", + config: &setting.Cfg{ + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: pointer(false), + }, + ForceMigration: false, + }, + isMigrationRun: true, + expected: true, + expectedErr: true, + }, + { + name: "when unified alerting enabled and migration is already run, then do nothing", + config: &setting.Cfg{ + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: pointer(true), + }, + }, + isMigrationRun: true, + expected: true, + }, + { + name: "when unified alerting disabled and migration is not already run, then do nothing", + config: &setting.Cfg{ + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: pointer(false), + }, + }, + isMigrationRun: false, + expected: false, + }, + } + + sqlStore := db.InitTestDB(t) + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + service := NewTestMigrationService(t, sqlStore, tt.config) + + err := service.migrationStore.SetMigrated(ctx, tt.isMigrationRun) + require.NoError(t, err) + + err = service.Run(ctx) + if tt.expectedErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + migrated, err := service.migrationStore.IsMigrated(ctx) + require.NoError(t, err) + require.Equal(t, tt.expected, migrated) + }) + } +} + +// TestAMConfigMigration tests the execution of the migration specifically for migrations of channels and routes. +func TestAMConfigMigration(t *testing.T) { + sqlStore := db.InitTestDB(t) + x := sqlStore.GetEngine() + service := NewTestMigrationService(t, sqlStore, &setting.Cfg{}) + tc := []struct { + name string + legacyChannels []*models.AlertNotification + alerts []*models.Alert + + expected map[int64]*apimodels.PostableUserConfig + expErr error + }{ + { + name: "general multi-org, multi-alert, multi-channel migration", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), + createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, false), + createAlertNotification(t, int64(2), "notifier4", "email", emailSettings, false), + createAlertNotification(t, int64(2), "notifier5", "slack", slackSettings, false), + createAlertNotification(t, int64(2), "notifier6", "opsgenie", opsgenieSettings, true), // default + }, + alerts: []*models.Alert{ + createAlert(t, 1, 1, 1, "alert1", []string{"notifier1"}), + createAlert(t, 1, 1, 2, "alert2", []string{"notifier2", "notifier3"}), + createAlert(t, 1, 2, 3, "alert3", []string{"notifier3"}), + createAlert(t, 2, 3, 1, "alert4", []string{"notifier4"}), + createAlert(t, 2, 3, 2, "alert5", []string{"notifier4", "notifier5", "notifier6"}), + createAlert(t, 2, 4, 3, "alert6", []string{}), + }, + expected: map[int64]*apimodels.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + {Receiver: "notifier2", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + {Receiver: "notifier3", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier3".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + }, + RepeatInterval: nil, + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, + {Receiver: config.Receiver{Name: "notifier2"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}}, + {Receiver: config.Receiver{Name: "notifier3"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}}, + {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, // empty default + + }, + }, + }, + int64(2): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "notifier6", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier4", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier4".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + {Receiver: "notifier5", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier5".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + {Receiver: "notifier6", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier6".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + }, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier4"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier4", Type: "email"}}}}, + {Receiver: config.Receiver{Name: "notifier5"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier5", Type: "slack"}}}}, + {Receiver: config.Receiver{Name: "notifier6"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier6", Type: "opsgenie"}}}}, + }, + }, + }, + }, + }, + { + name: "when no default channel, create empty autogen-contact-point-default", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + }, + alerts: []*models.Alert{}, + expected: map[int64]*apimodels.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + }, + RepeatInterval: nil, + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, + {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, + }, + }, + }, + }, + }, + { + name: "when single default channel, don't create autogen-contact-point-default", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), + }, + alerts: []*models.Alert{}, + expected: map[int64]*apimodels.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "notifier1", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + }, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, + }, + }, + }, + }, + }, + { + name: "when single default channel with SendReminder, use channel Frequency as RepeatInterval", + legacyChannels: []*models.AlertNotification{ + createAlertNotificationWithReminder(t, int64(1), "notifier1", "email", emailSettings, true, true, time.Duration(1)*time.Hour), + }, + alerts: []*models.Alert{}, + expected: map[int64]*apimodels.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "notifier1", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour))}, + }, + RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour)), + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, + }, + }, + }, + }, + }, + { + name: "when multiple default channels, add them to autogen-contact-point-default as well", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), + createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, true), + }, + alerts: []*models.Alert{}, + expected: map[int64]*apimodels.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + {Receiver: "notifier2", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + }, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, + {Receiver: config.Receiver{Name: "notifier2"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}}, + {Receiver: config.Receiver{Name: "autogen-contact-point-default"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}}, + }, + }, + }, + }, + }, + { + name: "when multiple default channels with SendReminder, use minimum channel frequency as RepeatInterval", + legacyChannels: []*models.AlertNotification{ + createAlertNotificationWithReminder(t, int64(1), "notifier1", "email", emailSettings, true, true, time.Duration(1)*time.Hour), + createAlertNotificationWithReminder(t, int64(1), "notifier2", "slack", slackSettings, true, true, time.Duration(30)*time.Minute), + }, + alerts: []*models.Alert{}, + expected: map[int64]*apimodels.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour))}, + {Receiver: "notifier2", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(30) * time.Minute))}, + }, + RepeatInterval: durationPointer(model.Duration(time.Duration(30) * time.Minute)), + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, + {Receiver: config.Receiver{Name: "notifier2"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}}, + {Receiver: config.Receiver{Name: "autogen-contact-point-default"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}}, + }, + }, + }, + }, + }, + { + name: "when default channels exist alongside non-default, add only defaults to autogen-contact-point-default", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), // default + createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), + createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, true), // default + }, + alerts: []*models.Alert{}, + expected: map[int64]*apimodels.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + {Receiver: "notifier2", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + {Receiver: "notifier3", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier3".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + }, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, + {Receiver: config.Receiver{Name: "notifier2"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}}, + {Receiver: config.Receiver{Name: "notifier3"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}}, + {Receiver: config.Receiver{Name: "autogen-contact-point-default"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier3", Type: "opsgenie"}}}}}, + }, + }, + }, + }, + { + name: "when alerts share channels, only create one receiver per legacy channel", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), + }, + alerts: []*models.Alert{ + createAlert(t, 1, 1, 1, "alert1", []string{"notifier1"}), + createAlert(t, 1, 1, 1, "alert2", []string{"notifier1", "notifier2"}), + }, + expected: map[int64]*apimodels.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + {Receiver: "notifier2", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + }, + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, + {Receiver: config.Receiver{Name: "notifier2"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}}, + {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, + }, + }, + }, + }, + }, + { + name: "when channel not linked to any alerts, still create a receiver for it", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + }, + alerts: []*models.Alert{}, + expected: map[int64]*apimodels.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + }, + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, + {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, + }, + }, + }, + }, + }, + { + name: "when unsupported channels, do not migrate them", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + createAlertNotification(t, int64(1), "notifier2", "hipchat", "", false), + createAlertNotification(t, int64(1), "notifier3", "sensu", "", false), + }, + alerts: []*models.Alert{}, + expected: map[int64]*apimodels.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + }, + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, + {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, + }, + }, + }, + }, + }, + { + name: "when unsupported channel linked to alert, do not migrate only that channel", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + createAlertNotification(t, int64(1), "notifier2", "sensu", "", false), + }, + alerts: []*models.Alert{ + createAlert(t, 1, 1, 1, "alert1", []string{"notifier1", "notifier2"}), + }, + expected: map[int64]*apimodels.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Config: apimodels.Config{Route: &apimodels.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*apimodels.Route{ + {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, + }, + }}, + Receivers: []*apimodels.PostableApiReceiver{ + {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, + {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, + }, + }, + }, + }, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + defer teardown(t, x, service) + setupLegacyAlertsTables(t, x, tt.legacyChannels, tt.alerts, nil, nil) + + err := service.Run(context.Background()) + require.NoError(t, err) + + for orgId := range tt.expected { + amConfig := getAlertmanagerConfig(t, x, orgId) + + // Order of nested GrafanaManagedReceivers is not guaranteed. + cOpt := []cmp.Option{ + cmpopts.IgnoreUnexported(apimodels.PostableApiReceiver{}), + cmpopts.IgnoreFields(apimodels.PostableGrafanaReceiver{}, "UID", "Settings", "SecureSettings"), + cmpopts.SortSlices(func(a, b *apimodels.PostableGrafanaReceiver) bool { return a.Name < b.Name }), + cmpopts.SortSlices(func(a, b *apimodels.PostableApiReceiver) bool { return a.Name < b.Name }), + } + if !cmp.Equal(tt.expected[orgId].AlertmanagerConfig.Receivers, amConfig.AlertmanagerConfig.Receivers, cOpt...) { + t.Errorf("Unexpected Receivers: %v", cmp.Diff(tt.expected[orgId].AlertmanagerConfig.Receivers, amConfig.AlertmanagerConfig.Receivers, cOpt...)) + } + + // Order of routes is not guaranteed. + cOpt = []cmp.Option{ + cmpopts.SortSlices(func(a, b *apimodels.Route) bool { + if a.Receiver != b.Receiver { + return a.Receiver < b.Receiver + } + return a.ObjectMatchers[0].Value < b.ObjectMatchers[0].Value + }), + cmpopts.IgnoreUnexported(apimodels.Route{}, labels.Matcher{}), + cmpopts.IgnoreFields(apimodels.Route{}, "GroupBy", "GroupByAll"), + } + if !cmp.Equal(tt.expected[orgId].AlertmanagerConfig.Route, amConfig.AlertmanagerConfig.Route, cOpt...) { + t.Errorf("Unexpected Route: %v", cmp.Diff(tt.expected[orgId].AlertmanagerConfig.Route, amConfig.AlertmanagerConfig.Route, cOpt...)) + } + } + }) + } +} + +// TestDashAlertMigration tests the execution of the migration specifically for alert rules. +func TestDashAlertMigration(t *testing.T) { + sqlStore := db.InitTestDB(t) + x := sqlStore.GetEngine() + service := NewTestMigrationService(t, sqlStore, &setting.Cfg{}) + + t.Run("when DashAlertMigration create ContactLabel on migrated AlertRules", func(t *testing.T) { + defer teardown(t, x, service) + legacyChannels := []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), + createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, false), + createAlertNotification(t, int64(2), "notifier4", "email", emailSettings, false), + createAlertNotification(t, int64(2), "notifier5", "slack", slackSettings, false), + createAlertNotification(t, int64(2), "notifier6", "opsgenie", opsgenieSettings, true), // default + } + alerts := []*models.Alert{ + createAlert(t, 1, 1, 1, "alert1", []string{"notifier1"}), + createAlert(t, 1, 1, 2, "alert2", []string{"notifier2", "notifier3"}), + createAlert(t, 1, 2, 3, "alert3", []string{"notifier3"}), + createAlert(t, 2, 3, 1, "alert4", []string{"notifier4"}), + createAlert(t, 2, 3, 2, "alert5", []string{"notifier4", "notifier5", "notifier6"}), + createAlert(t, 2, 4, 3, "alert6", []string{}), + } + expected := map[int64]map[string]*ngModels.AlertRule{ + int64(1): { + "alert1": {Labels: map[string]string{ContactLabel: `"notifier1"`}}, + "alert2": {Labels: map[string]string{ContactLabel: `"notifier2","notifier3"`}}, + "alert3": {Labels: map[string]string{ContactLabel: `"notifier3"`}}, + }, + int64(2): { + "alert4": {Labels: map[string]string{ContactLabel: `"notifier4","notifier6"`}}, + "alert5": {Labels: map[string]string{ContactLabel: `"notifier4","notifier5","notifier6"`}}, + "alert6": {Labels: map[string]string{}}, + }, + } + dashes := []*dashboards.Dashboard{ + createDashboard(t, 1, 1, "dash1-1", 5, nil), + createDashboard(t, 2, 1, "dash2-1", 5, nil), + createDashboard(t, 3, 2, "dash3-2", 6, nil), + createDashboard(t, 4, 2, "dash4-2", 6, nil), + } + folders := []*dashboards.Dashboard{ + createFolder(t, 5, 1, "folder5-1"), + createFolder(t, 6, 2, "folder6-2"), + } + setupLegacyAlertsTables(t, x, legacyChannels, alerts, folders, dashes) + err := service.Run(context.Background()) + require.NoError(t, err) + + for orgId := range expected { + rules := getAlertRules(t, x, orgId) + expectedRulesMap := expected[orgId] + require.Len(t, rules, len(expectedRulesMap)) + for _, r := range rules { + require.Equal(t, expectedRulesMap[r.Title].Labels[ContactLabel], r.Labels[ContactLabel]) + } + } + }) + + t.Run("when DashAlertMigration create ContactLabel with sanitized name if name contains double quote", func(t *testing.T) { + defer teardown(t, x, service) + legacyChannels := []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notif\"ier1", "email", emailSettings, false), + } + alerts := []*models.Alert{ + createAlert(t, 1, 1, 1, "alert1", []string{"notif\"ier1"}), + } + expected := map[int64]map[string]*ngModels.AlertRule{ + int64(1): { + "alert1": {Labels: map[string]string{ContactLabel: `"notif_ier1"`}}, + }, + } + dashes := []*dashboards.Dashboard{ + createDashboard(t, 1, 1, "dash1-1", 5, nil), + } + folders := []*dashboards.Dashboard{ + createFolder(t, 5, 1, "folder5-1"), + } + setupLegacyAlertsTables(t, x, legacyChannels, alerts, folders, dashes) + err := service.Run(context.Background()) + require.NoError(t, err) + + for orgId := range expected { + rules := getAlertRules(t, x, orgId) + expectedRulesMap := expected[orgId] + require.Len(t, rules, len(expectedRulesMap)) + for _, r := range rules { + require.Equal(t, expectedRulesMap[r.Title].Labels[ContactLabel], r.Labels[ContactLabel]) + } + } + }) + + t.Run("when folder is missing put alert in General folder", func(t *testing.T) { + defer teardown(t, x, service) + o := createOrg(t, 1) + folder1 := createFolder(t, 1, o.ID, "folder-1") + dash1 := createDashboard(t, 3, o.ID, "dash1", folder1.ID, nil) + dash2 := createDashboard(t, 4, o.ID, "dash2", 22, nil) // missing folder + + a1 := createAlert(t, int(o.ID), int(dash1.ID), 1, "alert-1", []string{}) + a2 := createAlert(t, int(o.ID), int(dash2.ID), 1, "alert-2", []string{}) + + _, err := x.Insert(o, folder1, dash1, dash2, a1, a2) + require.NoError(t, err) + + err = service.Run(context.Background()) + require.NoError(t, err) + + rules := getAlertRules(t, x, o.ID) + require.Len(t, rules, 2) + + var generalFolder dashboards.Dashboard + _, err = x.Table(&dashboards.Dashboard{}).Where("title = ? AND org_id = ?", generalAlertingFolderTitle, o.ID).Get(&generalFolder) + require.NoError(t, err) + + require.NotNil(t, generalFolder) + + for _, rule := range rules { + var expectedFolder dashboards.Dashboard + if rule.Title == a1.Name { + expectedFolder = *folder1 + } else { + expectedFolder = generalFolder + } + require.Equal(t, expectedFolder.UID, rule.NamespaceUID) + } + }) +} + +// TestDashAlertQueryMigration tests the execution of the migration specifically for alert rule queries. +func TestDashAlertQueryMigration(t *testing.T) { + sqlStore := db.InitTestDB(t) + x := sqlStore.GetEngine() + service := NewTestMigrationService(t, sqlStore, &setting.Cfg{}) + + createAlertQuery := func(refId string, ds string, from string, to string) ngModels.AlertQuery { + dur, _ := calculateInterval(legacydata.NewDataTimeRange(from, to), simplejson.New(), nil) + intervalMs := strconv.FormatInt(dur.Milliseconds(), 10) + rel, _ := getRelativeDuration(from, to) + return ngModels.AlertQuery{ + RefID: refId, + RelativeTimeRange: ngModels.RelativeTimeRange{From: rel.From, To: rel.To}, + DatasourceUID: ds, + Model: []byte(fmt.Sprintf(`{"datasource":{"type":"prometheus","uid":"gdev-prometheus"},"expr":"up{job=\"fake-data-gen\"}","instant":false,"intervalMs":%s,"maxDataPoints":1500,"refId":"%s"}`, intervalMs, refId)), + } + } + + createClassicConditionQuery := func(refId string, conditions []classicConditionJSON) ngModels.AlertQuery { + exprModel := struct { + Type string `json:"type"` + RefID string `json:"refId"` + Conditions []classicConditionJSON `json:"conditions"` + }{ + "classic_conditions", + refId, + conditions, + } + exprModelJSON, _ := json.Marshal(&exprModel) + + q := ngModels.AlertQuery{ + RefID: refId, + DatasourceUID: expressionDatasourceUID, + Model: exprModelJSON, + } + // IntervalMS and MaxDataPoints are created PreSave by AlertQuery. They don't appear to be necessary for expressions, + // but run PreSave here to match the expected model. + _ = q.PreSave() + return q + } + + cond := func(refId string, reducer string, evalType string, thresh float64) classicConditionJSON { + return classicConditionJSON{ + Evaluator: migrationStore.ConditionEvalJSON{Params: []float64{thresh}, Type: evalType}, + Operator: struct { + Type string `json:"type"` + }{Type: "and"}, + Query: struct { + Params []string `json:"params"` + }{Params: []string{refId}}, + Reducer: struct { + Type string `json:"type"` + }{Type: reducer}, + } + } + + genAlert := func(mutators ...ngModels.AlertRuleMutator) *ngModels.AlertRule { + rule := &ngModels.AlertRule{ + ID: 1, + OrgID: 1, + Title: "alert1", + Condition: "B", + Data: []ngModels.AlertQuery{}, + IntervalSeconds: 60, + Version: 1, + NamespaceUID: "folder5-1", + DashboardUID: pointer("dash1-1"), + PanelID: pointer(int64(1)), + RuleGroup: "alert1", + RuleGroupIndex: 1, + NoDataState: ngModels.NoData, + ExecErrState: ngModels.AlertingErrState, + For: 60 * time.Second, + Annotations: map[string]string{ + "message": "message", + }, + Labels: map[string]string{}, + IsPaused: false, + } + + for _, mutator := range mutators { + mutator(rule) + } + + rule.Annotations["__dashboardUid__"] = *rule.DashboardUID + rule.Annotations["__panelId__"] = strconv.FormatInt(*rule.PanelID, 10) + return rule + } + + type testcase struct { + name string + alerts []*models.Alert + + expectedFolder *dashboards.Dashboard + expected map[int64][]*ngModels.AlertRule + } + + tc := []testcase{ + { + name: "simple query and condition", + alerts: []*models.Alert{ + createAlertWithCond(t, 1, 1, 1, "alert1", nil, + []migrationStore.DashAlertCondition{createCondition("A", "max", "gt", 42, 1, "5m", "now")}), + createAlertWithCond(t, 2, 3, 1, "alert1", nil, + []migrationStore.DashAlertCondition{createCondition("A", "max", "gt", 42, 3, "5m", "now")}), + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(1): { + genAlert(func(rule *ngModels.AlertRule) { + rule.Data = append(rule.Data, createAlertQuery("A", "ds1-1", "5m", "now")) + rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ + cond("A", "max", "gt", 42), + })) + }), + }, + int64(2): { + genAlert(func(rule *ngModels.AlertRule) { + rule.OrgID = 2 + rule.DashboardUID = pointer("dash3-2") + rule.NamespaceUID = "folder6-2" + rule.Data = append(rule.Data, createAlertQuery("A", "ds3-2", "5m", "now")) + rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ + cond("A", "max", "gt", 42), + })) + }), + }, + }, + }, + { + name: "multiple conditions", + alerts: []*models.Alert{ + createAlertWithCond(t, 1, 1, 1, "alert1", nil, + []migrationStore.DashAlertCondition{ + createCondition("A", "avg", "gt", 42, 1, "5m", "now"), + createCondition("B", "max", "gt", 43, 2, "3m", "now"), + createCondition("C", "min", "lt", 20, 2, "3m", "now"), + }), + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(1): { + genAlert(func(rule *ngModels.AlertRule) { + rule.Condition = "D" + rule.Data = append(rule.Data, createAlertQuery("A", "ds1-1", "5m", "now")) + rule.Data = append(rule.Data, createAlertQuery("B", "ds2-1", "3m", "now")) + rule.Data = append(rule.Data, createAlertQuery("C", "ds2-1", "3m", "now")) + rule.Data = append(rule.Data, createClassicConditionQuery("D", []classicConditionJSON{ + cond("A", "avg", "gt", 42), + cond("B", "max", "gt", 43), + cond("C", "min", "lt", 20), + })) + }), + }, + }, + }, + { + name: "multiple conditions on same query with same timerange should not create multiple queries", + alerts: []*models.Alert{ + createAlertWithCond(t, 1, 1, 1, "alert1", nil, + []migrationStore.DashAlertCondition{ + createCondition("A", "max", "gt", 42, 1, "5m", "now"), + createCondition("A", "avg", "gt", 20, 1, "5m", "now"), + }), + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(1): { + genAlert(func(rule *ngModels.AlertRule) { + rule.Condition = "B" + rule.Data = append(rule.Data, createAlertQuery("A", "ds1-1", "5m", "now")) + rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ + cond("A", "max", "gt", 42), + cond("A", "avg", "gt", 20), + })) + }), + }, + }, + }, + { + name: "multiple conditions on same query with different timeranges should create multiple queries", + alerts: []*models.Alert{ + createAlertWithCond(t, 1, 1, 1, "alert1", nil, + []migrationStore.DashAlertCondition{ + createCondition("A", "max", "gt", 42, 1, "5m", "now"), + createCondition("A", "avg", "gt", 20, 1, "3m", "now"), + }), + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(1): { + genAlert(func(rule *ngModels.AlertRule) { + rule.Condition = "C" + rule.Data = append(rule.Data, createAlertQuery("A", "ds1-1", "3m", "now")) // Ordered by time range. + rule.Data = append(rule.Data, createAlertQuery("B", "ds1-1", "5m", "now")) + rule.Data = append(rule.Data, createClassicConditionQuery("C", []classicConditionJSON{ + cond("B", "max", "gt", 42), + cond("A", "avg", "gt", 20), + })) + }), + }, + }, + }, + { + name: "multiple conditions custom refIds", + alerts: []*models.Alert{ + createAlertWithCond(t, 1, 1, 1, "alert1", nil, + []migrationStore.DashAlertCondition{ + createCondition("Q1", "avg", "gt", 42, 1, "5m", "now"), + createCondition("Q2", "max", "gt", 43, 2, "3m", "now"), + createCondition("Q3", "min", "lt", 20, 2, "3m", "now"), + }), + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(1): { + genAlert(func(rule *ngModels.AlertRule) { + rule.Condition = "A" + rule.Data = append(rule.Data, createClassicConditionQuery("A", []classicConditionJSON{ + cond("Q1", "avg", "gt", 42), + cond("Q2", "max", "gt", 43), + cond("Q3", "min", "lt", 20), + })) + rule.Data = append(rule.Data, createAlertQuery("Q1", "ds1-1", "5m", "now")) + rule.Data = append(rule.Data, createAlertQuery("Q2", "ds2-1", "3m", "now")) + rule.Data = append(rule.Data, createAlertQuery("Q3", "ds2-1", "3m", "now")) + }), + }, + }, + }, + { + name: "multiple conditions out of order refIds, queries should be sorted by refId and conditions should be in original order", + alerts: []*models.Alert{ + createAlertWithCond(t, 1, 1, 1, "alert1", nil, + []migrationStore.DashAlertCondition{ + createCondition("B", "avg", "gt", 42, 1, "5m", "now"), + createCondition("C", "max", "gt", 43, 2, "3m", "now"), + createCondition("A", "min", "lt", 20, 2, "3m", "now"), + }), + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(1): { + genAlert(func(rule *ngModels.AlertRule) { + rule.Condition = "D" + rule.Data = append(rule.Data, createAlertQuery("A", "ds2-1", "3m", "now")) + rule.Data = append(rule.Data, createAlertQuery("B", "ds1-1", "5m", "now")) + rule.Data = append(rule.Data, createAlertQuery("C", "ds2-1", "3m", "now")) + rule.Data = append(rule.Data, createClassicConditionQuery("D", []classicConditionJSON{ + cond("B", "avg", "gt", 42), + cond("C", "max", "gt", 43), + cond("A", "min", "lt", 20), + })) + }), + }, + }, + }, + { + name: "multiple conditions out of order with duplicate refIds", + alerts: []*models.Alert{ + createAlertWithCond(t, 1, 1, 1, "alert1", nil, + []migrationStore.DashAlertCondition{ + createCondition("C", "avg", "gt", 42, 1, "5m", "now"), + createCondition("C", "max", "gt", 43, 1, "3m", "now"), + createCondition("B", "min", "lt", 20, 2, "5m", "now"), + createCondition("B", "min", "lt", 21, 2, "3m", "now"), + }), + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(1): { + genAlert(func(rule *ngModels.AlertRule) { + rule.Condition = "E" + rule.Data = append(rule.Data, createAlertQuery("A", "ds2-1", "3m", "now")) + rule.Data = append(rule.Data, createAlertQuery("B", "ds2-1", "5m", "now")) + rule.Data = append(rule.Data, createAlertQuery("C", "ds1-1", "3m", "now")) + rule.Data = append(rule.Data, createAlertQuery("D", "ds1-1", "5m", "now")) + rule.Data = append(rule.Data, createClassicConditionQuery("E", []classicConditionJSON{ + cond("D", "avg", "gt", 42), + cond("C", "max", "gt", 43), + cond("B", "min", "lt", 20), + cond("A", "min", "lt", 21), + })) + }), + }, + }, + }, + { + name: "alerts with unknown datasource id migrates with empty datasource uid", + alerts: []*models.Alert{ + createAlertWithCond(t, 1, 1, 1, "alert1", nil, + []migrationStore.DashAlertCondition{createCondition("A", "max", "gt", 42, 123, "5m", "now")}), // Unknown datasource id. + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(1): { + genAlert(func(rule *ngModels.AlertRule) { + rule.Data = append(rule.Data, createAlertQuery("A", "", "5m", "now")) // Empty datasource UID. + rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ + cond("A", "max", "gt", 42), + })) + }), + }, + }, + }, + { + name: "alerts with unknown dashboard do not migrate", + alerts: []*models.Alert{ + createAlertWithCond(t, 1, 22, 1, "alert1", nil, + []migrationStore.DashAlertCondition{ + createCondition("A", "avg", "gt", 42, 1, "5m", "now"), + }), + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(1): {}, + }, + }, + { + name: "alerts with unknown org do not migrate", + alerts: []*models.Alert{ + createAlertWithCond(t, 22, 1, 1, "alert1", nil, + []migrationStore.DashAlertCondition{ + createCondition("A", "avg", "gt", 42, 1, "5m", "now"), + }), + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(22): {}, + }, + }, + { + name: "alerts in general folder migrate to existing general alerting", + alerts: []*models.Alert{ + createAlertWithCond(t, 1, 8, 1, "alert1", nil, + []migrationStore.DashAlertCondition{ + createCondition("A", "avg", "gt", 42, 1, "5m", "now"), + }), + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(1): { + genAlert(func(rule *ngModels.AlertRule) { + rule.NamespaceUID = "General Alerting" + rule.DashboardUID = pointer("dash-in-general-1") + rule.Data = append(rule.Data, createAlertQuery("A", "ds1-1", "5m", "now")) + rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ + cond("A", "avg", "gt", 42), + })) + }), + }, + }, + }, + { + name: "alerts in general folder migrate to newly created general alerting if one doesn't exist", + alerts: []*models.Alert{ + createAlertWithCond(t, 2, 9, 1, "alert1", nil, // Org 2 doesn't have general alerting folder. + []migrationStore.DashAlertCondition{ + createCondition("A", "avg", "gt", 42, 3, "5m", "now"), + }), + }, + expectedFolder: &dashboards.Dashboard{ + OrgID: 2, + Title: "General Alerting", + FolderID: 0, + Slug: "general-alerting", + }, + expected: map[int64][]*ngModels.AlertRule{ + int64(2): { + genAlert(func(rule *ngModels.AlertRule) { + rule.OrgID = 2 + rule.DashboardUID = pointer("dash-in-general-2") + rule.Data = append(rule.Data, createAlertQuery("A", "ds3-2", "5m", "now")) + rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ + cond("A", "avg", "gt", 42), + })) + }), + }, + }, + }, + } + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + defer teardown(t, x, service) + dashes := []*dashboards.Dashboard{ + createDashboard(t, 1, 1, "dash1-1", 5, nil), + createDashboard(t, 2, 1, "dash2-1", 5, nil), + createDashboard(t, 3, 2, "dash3-2", 6, nil), + createDashboard(t, 4, 2, "dash4-2", 6, nil), + createDashboard(t, 8, 1, "dash-in-general-1", 0, nil), + createDashboard(t, 9, 2, "dash-in-general-2", 0, nil), + createDashboard(t, 10, 1, "dash-with-acl-1", 5, func(d *dashboards.Dashboard) { + d.Title = "Dashboard With ACL 1" + d.HasACL = true + }), + } + folders := []*dashboards.Dashboard{ + createFolder(t, 5, 1, "folder5-1"), + createFolder(t, 6, 2, "folder6-2"), + createFolder(t, 7, 1, "General Alerting"), + } + setupLegacyAlertsTables(t, x, nil, tt.alerts, folders, dashes) + + err := service.Run(context.Background()) + require.NoError(t, err) + + for orgId, expected := range tt.expected { + rules := getAlertRules(t, x, orgId) + + for _, r := range rules { + // Remove generated fields. + require.NotEqual(t, r.Labels["rule_uid"], "") + delete(r.Labels, "rule_uid") + require.NotEqual(t, r.Annotations["__alertId__"], "") + delete(r.Annotations, "__alertId__") + + // If folder is created, we check if separately + if tt.expectedFolder != nil { + folder := getDashboard(t, x, orgId, r.NamespaceUID) + require.Equal(t, tt.expectedFolder.Title, folder.Title) + require.Equal(t, tt.expectedFolder.OrgID, folder.OrgID) + require.Equal(t, tt.expectedFolder.FolderID, folder.FolderID) + } + } + + cOpt := []cmp.Option{ + cmpopts.SortSlices(func(a, b *ngModels.AlertRule) bool { + return a.ID < b.ID + }), + cmpopts.IgnoreUnexported(ngModels.AlertRule{}, ngModels.AlertQuery{}), + cmpopts.IgnoreFields(ngModels.AlertRule{}, "Updated", "UID", "ID"), + } + if tt.expectedFolder != nil { + cOpt = append(cOpt, cmpopts.IgnoreFields(ngModels.AlertRule{}, "NamespaceUID")) + } + if !cmp.Equal(expected, rules, cOpt...) { + t.Errorf("Unexpected Rule: %v", cmp.Diff(expected, rules, cOpt...)) + } + } + }) + } +} + +const ( + emailSettings = `{"addresses": "test"}` + slackSettings = `{"recipient": "test", "token": "test"}` + opsgenieSettings = `{"apiKey": "test"}` +) + +var ( + now = time.Now() +) + +// createAlertNotificationWithReminder creates a legacy alert notification channel for inserting into the test database. +func createAlertNotificationWithReminder(t *testing.T, orgId int64, uid string, channelType string, settings string, defaultChannel bool, sendReminder bool, frequency time.Duration) *models.AlertNotification { + t.Helper() + settingsJson := simplejson.New() + if settings != "" { + s, err := simplejson.NewJson([]byte(settings)) + if err != nil { + t.Fatalf("Failed to unmarshal alert notification json: %v", err) + } + settingsJson = s + } + + return &models.AlertNotification{ + OrgID: orgId, + UID: uid, + Name: uid, // Same as uid to make testing easier. + Type: channelType, + DisableResolveMessage: false, + IsDefault: defaultChannel, + Settings: settingsJson, + SecureSettings: make(map[string][]byte), + Created: now, + Updated: now, + SendReminder: sendReminder, + Frequency: frequency, + } +} + +// createAlertNotification creates a legacy alert notification channel for inserting into the test database. +func createAlertNotification(t *testing.T, orgId int64, uid string, channelType string, settings string, defaultChannel bool) *models.AlertNotification { + return createAlertNotificationWithReminder(t, orgId, uid, channelType, settings, defaultChannel, false, time.Duration(0)) +} + +var queryModel = `{"datasource":{"type":"prometheus","uid":"gdev-prometheus"},"expr":"up{job=\"fake-data-gen\"}","instant":false,"refId":"%s"}` + +func createCondition(refId string, reducer string, evalType string, thresh float64, datasourceId int64, from string, to string) migrationStore.DashAlertCondition { + return migrationStore.DashAlertCondition{ + Evaluator: migrationStore.ConditionEvalJSON{ + Params: []float64{thresh}, + Type: evalType, + }, + Operator: struct { + Type string `json:"type"` + }{ + Type: "and", + }, + Query: struct { + Params []string `json:"params"` + DatasourceID int64 `json:"datasourceId"` + Model json.RawMessage + }{ + Params: []string{refId, from, to}, + DatasourceID: datasourceId, + Model: []byte(fmt.Sprintf(queryModel, refId)), + }, + Reducer: struct { + Type string `json:"type"` + }{ + Type: reducer, + }, + } +} + +// createAlert creates a legacy alert rule for inserting into the test database. +func createAlert(t *testing.T, orgId int, dashboardId int, panelsId int, name string, notifierUids []string) *models.Alert { + return createAlertWithCond(t, orgId, dashboardId, panelsId, name, notifierUids, []migrationStore.DashAlertCondition{}) +} + +// createAlert creates a legacy alert rule for inserting into the test database. +func createAlertWithCond(t *testing.T, orgId int, dashboardId int, panelsId int, name string, notifierUids []string, cond []migrationStore.DashAlertCondition) *models.Alert { + t.Helper() + + var settings = simplejson.New() + if len(notifierUids) != 0 { + notifiers := make([]any, 0) + for _, n := range notifierUids { + notifiers = append(notifiers, struct { + Uid string + }{Uid: n}) + } + + settings.Set("notifications", notifiers) + } + settings.Set("conditions", cond) + + return &models.Alert{ + OrgID: int64(orgId), + DashboardID: int64(dashboardId), + PanelID: int64(panelsId), + Name: name, + Message: "message", + Frequency: int64(60), + For: 60 * time.Second, + State: models.AlertStateOK, + Settings: settings, + NewStateDate: now, + Created: now, + Updated: now, + } +} + +// createDashboard creates a folder for inserting into the test database. +func createFolder(t *testing.T, id int64, orgId int64, uid string) *dashboards.Dashboard { + f := createDashboard(t, id, orgId, uid, 0, nil) + f.IsFolder = true + return f +} + +// createDashboard creates a dashboard for inserting into the test database. +func createDashboard(t *testing.T, id int64, orgId int64, uid string, folderId int64, mut func(*dashboards.Dashboard)) *dashboards.Dashboard { + t.Helper() + d := &dashboards.Dashboard{ + ID: id, + OrgID: orgId, + UID: uid, + Created: now, + Updated: now, + Title: uid, // Not tested, needed to satisfy constraint. + FolderID: folderId, + Data: simplejson.New(), + Version: 1, + } + if mut != nil { + mut(d) + } + return d +} + +// createDatasource creates a datasource for inserting into the test database. +func createDatasource(t *testing.T, id int64, orgId int64, uid string) *datasources.DataSource { + t.Helper() + return &datasources.DataSource{ + ID: id, + OrgID: orgId, + UID: uid, + Created: now, + Updated: now, + Name: uid, // Not tested, needed to satisfy constraint. + } +} + +func createOrg(t *testing.T, id int64) *org.Org { + t.Helper() + return &org.Org{ + ID: id, + Version: 1, + Name: fmt.Sprintf("org_%d", id), + Created: time.Now(), + Updated: time.Now(), + } +} + +// teardown cleans the input tables between test cases. +func teardown(t *testing.T, x *xorm.Engine, service *MigrationService) { + _, err := x.Exec("DELETE from org") + require.NoError(t, err) + _, err = x.Exec("DELETE from alert") + require.NoError(t, err) + _, err = x.Exec("DELETE from alert_notification") + require.NoError(t, err) + _, err = x.Exec("DELETE from dashboard") + require.NoError(t, err) + _, err = x.Exec("DELETE from data_source") + require.NoError(t, err) + err = service.migrationStore.RevertAllOrgs(context.Background()) + require.NoError(t, err) +} + +// setupLegacyAlertsTables inserts data into the legacy alerting tables that is needed for testing the +func setupLegacyAlertsTables(t *testing.T, x *xorm.Engine, legacyChannels []*models.AlertNotification, alerts []*models.Alert, folders []*dashboards.Dashboard, dashes []*dashboards.Dashboard) { + t.Helper() + + orgs := []org.Org{ + *createOrg(t, 1), + *createOrg(t, 2), + } + + // Setup folders. + if len(folders) > 0 { + _, err := x.Insert(folders) + require.NoError(t, err) + } + + // Setup dashboards. + if len(dashes) > 0 { + _, err := x.Insert(dashes) + require.NoError(t, err) + } + + // Setup data_sources. + dataSources := []datasources.DataSource{ + *createDatasource(t, 1, 1, "ds1-1"), + *createDatasource(t, 2, 1, "ds2-1"), + *createDatasource(t, 3, 2, "ds3-2"), + *createDatasource(t, 4, 2, "ds4-2"), + } + + _, errOrgs := x.Insert(orgs) + require.NoError(t, errOrgs) + + _, errDataSourcess := x.Insert(dataSources) + require.NoError(t, errDataSourcess) + + if len(legacyChannels) > 0 { + _, channelErr := x.Insert(legacyChannels) + require.NoError(t, channelErr) + } + + if len(alerts) > 0 { + _, alertErr := x.Insert(alerts) + require.NoError(t, alertErr) + } +} + +// getAlertmanagerConfig retreives the Alertmanager Config from the database for a given orgId. +func getAlertmanagerConfig(t *testing.T, x *xorm.Engine, orgId int64) *apimodels.PostableUserConfig { + amConfig := "" + _, err := x.Table("alert_configuration").Where("org_id = ?", orgId).Cols("alertmanager_configuration").Get(&amConfig) + require.NoError(t, err) + + config := apimodels.PostableUserConfig{} + err = json.Unmarshal([]byte(amConfig), &config) + require.NoError(t, err) + return &config +} + +// getAlertmanagerConfig retreives the Alertmanager Config from the database for a given orgId. +func getAlertRules(t *testing.T, x *xorm.Engine, orgId int64) []*ngModels.AlertRule { + rules := make([]*ngModels.AlertRule, 0) + err := x.Table("alert_rule").Where("org_id = ?", orgId).Find(&rules) + require.NoError(t, err) + + return rules +} + +// getDashboard retrieves a dashboard from the database for a given org, uid. +func getDashboard(t *testing.T, x *xorm.Engine, orgId int64, uid string) *dashboards.Dashboard { + dashes := make([]*dashboards.Dashboard, 0) + err := x.Table("dashboard").Where("org_id = ? AND uid = ?", orgId, uid).Find(&dashes) + require.NoError(t, err) + if len(dashes) > 1 { + t.Error("Expected only one dashboard to be returned") + } + if len(dashes) == 0 { + return nil + } + + return dashes[0] +} + +func pointer[T any](b T) *T { + return &b +} diff --git a/pkg/services/ngalert/migration/models.go b/pkg/services/ngalert/migration/models.go new file mode 100644 index 00000000000..90553f88b70 --- /dev/null +++ b/pkg/services/ngalert/migration/models.go @@ -0,0 +1,121 @@ +package migration + +import ( + "strings" + + pb "github.com/prometheus/alertmanager/silence/silencepb" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/folder" + migmodels "github.com/grafana/grafana/pkg/services/ngalert/migration/models" + migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" + "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/store" + "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" +) + +// OrgMigration is a helper struct for migrating alerts for a single org. It contains state, services, and caches. +type OrgMigration struct { + cfg *setting.Cfg + log log.Logger + + migrationStore migrationStore.Store + encryptionService secrets.Service + + orgID int64 + seenUIDs Deduplicator + silences []*pb.MeshSilence + alertRuleTitleDedup map[string]Deduplicator // Folder -> Deduplicator (Title). + + // cache for folders created for dashboards that have custom permissions + folderCache map[string]*folder.Folder + generalAlertingFolder *folder.Folder + + state *migmodels.OrgMigrationState +} + +// newOrgMigration creates a new OrgMigration for the given orgID. +func (ms *MigrationService) newOrgMigration(orgID int64) *OrgMigration { + return &OrgMigration{ + cfg: ms.cfg, + log: ms.log.New("orgID", orgID), + + migrationStore: ms.migrationStore, + encryptionService: ms.encryptionService, + + orgID: orgID, + // We deduplicate for case-insensitive matching in MySQL-compatible backend flavours because they use case-insensitive collation. + seenUIDs: Deduplicator{set: make(map[string]struct{}), caseInsensitive: ms.migrationStore.CaseInsensitive()}, + silences: make([]*pb.MeshSilence, 0), + alertRuleTitleDedup: make(map[string]Deduplicator), + + folderCache: make(map[string]*folder.Folder), + + state: &migmodels.OrgMigrationState{ + OrgID: orgID, + CreatedFolders: make([]string, 0), + }, + } +} + +func (om *OrgMigration) AlertTitleDeduplicator(folderUID string) Deduplicator { + if _, ok := om.alertRuleTitleDedup[folderUID]; !ok { + om.alertRuleTitleDedup[folderUID] = Deduplicator{ + set: make(map[string]struct{}), + caseInsensitive: om.migrationStore.CaseInsensitive(), + maxLen: store.AlertDefinitionMaxTitleLength, + } + } + return om.alertRuleTitleDedup[folderUID] +} + +type AlertPair struct { + AlertRule *models.AlertRule + DashAlert *migrationStore.DashAlert +} + +// Deduplicator is a wrapper around map[string]struct{} and util.GenerateShortUID() which aims help maintain and generate +// unique strings (such as uids or titles). if caseInsensitive is true, all uniqueness is determined in a +// case-insensitive manner. if maxLen is greater than 0, all strings will be truncated to maxLen before being checked in +// contains and dedup will always return a string of length maxLen or less. +type Deduplicator struct { + set map[string]struct{} + caseInsensitive bool + maxLen int +} + +// contains checks whether the given string has already been seen by this Deduplicator. +func (s *Deduplicator) contains(u string) bool { + dedup := u + if s.caseInsensitive { + dedup = strings.ToLower(dedup) + } + if s.maxLen > 0 && len(dedup) > s.maxLen { + dedup = dedup[:s.maxLen] + } + _, seen := s.set[dedup] + return seen +} + +// deduplicate returns a unique string based on the given string by appending a uuid to it. Will truncate the given string if +// the resulting string would be longer than maxLen. +func (s *Deduplicator) deduplicate(dedup string) string { + uid := util.GenerateShortUID() + if s.maxLen > 0 && len(dedup)+1+len(uid) > s.maxLen { + trunc := s.maxLen - 1 - len(uid) + dedup = dedup[:trunc] + } + + return dedup + "_" + uid +} + +// add adds the given string to the Deduplicator. +func (s *Deduplicator) add(uid string) { + dedup := uid + if s.caseInsensitive { + dedup = strings.ToLower(dedup) + } + s.set[dedup] = struct{}{} +} diff --git a/pkg/services/ngalert/migration/models/state.go b/pkg/services/ngalert/migration/models/state.go new file mode 100644 index 00000000000..e7c5b78120b --- /dev/null +++ b/pkg/services/ngalert/migration/models/state.go @@ -0,0 +1,7 @@ +package models + +// OrgMigrationState contains information about the state of an org migration. +type OrgMigrationState struct { + OrgID int64 `json:"orgId"` + CreatedFolders []string `json:"createdFolders"` +} diff --git a/pkg/services/ngalert/migration/permissions.go b/pkg/services/ngalert/migration/permissions.go new file mode 100644 index 00000000000..ce6f139b880 --- /dev/null +++ b/pkg/services/ngalert/migration/permissions.go @@ -0,0 +1,153 @@ +package migration + +import ( + "context" + "errors" + "fmt" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/auth/identity" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/folder" + "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/services/user" +) + +const DASHBOARD_FOLDER = "%s Alerts - %s" + +// MaxFolderName is the maximum length of the folder name generated using DASHBOARD_FOLDER format +const MaxFolderName = 255 + +var ( + migratorPermissions = []accesscontrol.Permission{ + {Action: dashboards.ActionFoldersRead, Scope: dashboards.ScopeFoldersAll}, + {Action: dashboards.ActionDashboardsRead, Scope: dashboards.ScopeDashboardsAll}, + {Action: dashboards.ActionFoldersPermissionsRead, Scope: dashboards.ScopeFoldersAll}, + {Action: dashboards.ActionDashboardsPermissionsRead, Scope: dashboards.ScopeDashboardsAll}, + {Action: dashboards.ActionFoldersCreate}, + {Action: dashboards.ActionDashboardsCreate, Scope: dashboards.ScopeFoldersAll}, + {Action: datasources.ActionRead, Scope: datasources.ScopeAll}, + } + generalAlertingFolderTitle = "General Alerting" +) + +// getMigrationUser returns a background user for the given orgID with permissions to execute migration-related tasks. +func getMigrationUser(orgID int64) identity.Requester { + return accesscontrol.BackgroundUser("ngalert_migration", orgID, org.RoleAdmin, migratorPermissions) +} + +// getAlertFolderNameFromDashboard generates a folder name for alerts that belong to a dashboard. Formats the string according to DASHBOARD_FOLDER format. +// If the resulting string exceeds the migrations.MaxTitleLength, the dashboard title is stripped to be at the maximum length +func getAlertFolderNameFromDashboard(dash *dashboards.Dashboard) string { + maxLen := MaxFolderName - len(fmt.Sprintf(DASHBOARD_FOLDER, "", dash.UID)) + title := dash.Title + if len(title) > maxLen { + title = title[:maxLen] + } + return fmt.Sprintf(DASHBOARD_FOLDER, title, dash.UID) // include UID to the name to avoid collision +} + +func (om *OrgMigration) getOrCreateMigratedFolder(ctx context.Context, log log.Logger, dashID int64) (*dashboards.Dashboard, *folder.Folder, error) { + dash, err := om.migrationStore.GetDashboard(ctx, om.orgID, dashID) + if err != nil { + if errors.Is(err, dashboards.ErrFolderNotFound) { + return nil, nil, fmt.Errorf("dashboard with ID %v under organisation %d not found: %w", dashID, om.orgID, err) + } + return nil, nil, fmt.Errorf("failed to get dashboard with ID %v under organisation %d: %w", dashID, om.orgID, err) + } + l := log.New( + "dashboardTitle", dash.Title, + "dashboardUID", dash.UID, + ) + + var migratedFolder *folder.Folder + switch { + case dash.HasACL: + folderName := getAlertFolderNameFromDashboard(dash) + f, ok := om.folderCache[folderName] + if !ok { + l.Info("create a new folder for alerts that belongs to dashboard because it has custom permissions", "folder", folderName) + // create folder and assign the permissions of the dashboard (included default and inherited) + f, err = om.createFolder(ctx, om.orgID, folderName) + if err != nil { + return nil, nil, fmt.Errorf("create new folder: %w", err) + } + permissions, err := om.migrationStore.GetACL(ctx, dash.OrgID, dash.ID) + if err != nil { + return nil, nil, fmt.Errorf("failed to get dashboard %d under organisation %d permissions: %w", dash.ID, dash.OrgID, err) + } + err = om.migrationStore.SetACL(ctx, f.OrgID, f.ID, permissions) + if err != nil { + return nil, nil, fmt.Errorf("failed to set folder %d under organisation %d permissions: %w", f.ID, f.OrgID, err) + } + om.folderCache[folderName] = f + } + migratedFolder = f + case dash.FolderID > 0: + // get folder if exists + f, err := om.migrationStore.GetFolder(ctx, &folder.GetFolderQuery{ID: &dash.FolderID, OrgID: dash.OrgID, SignedInUser: getMigrationUser(dash.OrgID)}) + if err != nil { + // If folder does not exist then the dashboard is an orphan and we migrate the alert to the general folder. + l.Warn("Failed to find folder for dashboard. Migrate rule to the default folder", "missing_folder_id", dash.FolderID, "error", err) + migratedFolder, err = om.getOrCreateGeneralFolder(ctx, dash.OrgID) + if err != nil { + return nil, nil, err + } + } else { + migratedFolder = f + } + default: + migratedFolder, err = om.getOrCreateGeneralFolder(ctx, dash.OrgID) + if err != nil { + return nil, nil, err + } + } + + if migratedFolder.UID == "" { + return nil, nil, fmt.Errorf("empty folder identifier") + } + + return dash, migratedFolder, nil +} + +// getOrCreateGeneralFolder returns the general folder under the specific organisation +// If the general folder does not exist it creates it. +func (om *OrgMigration) getOrCreateGeneralFolder(ctx context.Context, orgID int64) (*folder.Folder, error) { + if om.generalAlertingFolder != nil { + return om.generalAlertingFolder, nil + } + f, err := om.migrationStore.GetFolder(ctx, &folder.GetFolderQuery{OrgID: orgID, Title: &generalAlertingFolderTitle, SignedInUser: getMigrationUser(orgID)}) + if err != nil { + if errors.Is(err, dashboards.ErrFolderNotFound) { + // create folder + generalAlertingFolder, err := om.createFolder(ctx, orgID, generalAlertingFolderTitle) + if err != nil { + return nil, fmt.Errorf("create general alerting folder '%s': %w", generalAlertingFolderTitle, err) + } + om.generalAlertingFolder = generalAlertingFolder + return om.generalAlertingFolder, nil + } + return nil, fmt.Errorf("get general alerting folder '%s': %w", generalAlertingFolderTitle, err) + } + om.generalAlertingFolder = f + + return om.generalAlertingFolder, nil +} + +// createFolder creates a new folder with given permissions. +func (om *OrgMigration) createFolder(ctx context.Context, orgID int64, title string) (*folder.Folder, error) { + f, err := om.migrationStore.CreateFolder(ctx, &folder.CreateFolderCommand{ + OrgID: orgID, + Title: title, + SignedInUser: getMigrationUser(orgID).(*user.SignedInUser), + }) + if err != nil { + return nil, err + } + + om.state.CreatedFolders = append(om.state.CreatedFolders, f.UID) + + return f, nil +} diff --git a/pkg/services/ngalert/migration/securejsondata.go b/pkg/services/ngalert/migration/securejsondata.go new file mode 100644 index 00000000000..9de357bb00a --- /dev/null +++ b/pkg/services/ngalert/migration/securejsondata.go @@ -0,0 +1,31 @@ +package migration + +import ( + "os" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" +) + +// SecureJsonData is used to store encrypted data (for example in data_source table). Only values are separately +// encrypted. +type SecureJsonData map[string][]byte + +var seclogger = log.New("securejsondata") + +// Decrypt returns map of the same type but where the all the values are decrypted. Opposite of what +// GetEncryptedJsonData is doing. +func (s SecureJsonData) Decrypt() map[string]string { + decrypted := make(map[string]string) + for key, data := range s { + decryptedData, err := util.Decrypt(data, setting.SecretKey) + if err != nil { + seclogger.Error(err.Error()) + os.Exit(1) + } + + decrypted[key] = string(decryptedData) + } + return decrypted +} diff --git a/pkg/services/ngalert/migration/service.go b/pkg/services/ngalert/migration/service.go new file mode 100644 index 00000000000..876a0b2532a --- /dev/null +++ b/pkg/services/ngalert/migration/service.go @@ -0,0 +1,137 @@ +package migration + +import ( + "context" + "fmt" + "time" + + "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/serverlock" + migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" + "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/setting" +) + +// actionName is the unique row-level lock name for serverlock.ServerLockService. +const actionName = "alerting migration" + +//nolint:stylecheck +var ForceMigrationError = fmt.Errorf("Grafana has already been migrated to Unified Alerting. Any alert rules created while using Unified Alerting will be deleted by rolling back. Set force_migration=true in your grafana.ini and restart Grafana to roll back and delete Unified Alerting configuration data.") + +type MigrationService struct { + lock *serverlock.ServerLockService + cfg *setting.Cfg + log log.Logger + store db.DB + migrationStore migrationStore.Store + + encryptionService secrets.Service +} + +func ProvideService( + lock *serverlock.ServerLockService, + cfg *setting.Cfg, + store db.DB, + migrationStore migrationStore.Store, + encryptionService secrets.Service, +) (*MigrationService, error) { + return &MigrationService{ + lock: lock, + log: log.New("ngalert.migration"), + cfg: cfg, + store: store, + migrationStore: migrationStore, + encryptionService: encryptionService, + }, nil +} + +// Run starts the migration. This will either migrate from legacy alerting to unified alerting or revert the migration. +// If the migration status in the kvstore is not set and unified alerting is enabled, the migration will be executed. +// If the migration status in the kvstore is set and both unified alerting is disabled and ForceMigration is set to true, the migration will be reverted. +func (ms *MigrationService) Run(ctx context.Context) error { + var errMigration error + errLock := ms.lock.LockExecuteAndRelease(ctx, actionName, time.Minute*10, func(context.Context) { + ms.log.Info("Starting") + errMigration = ms.store.InTransaction(ctx, func(ctx context.Context) error { + migrated, err := ms.migrationStore.IsMigrated(ctx) + if err != nil { + return fmt.Errorf("getting migration status: %w", err) + } + if migrated == ms.cfg.UnifiedAlerting.IsEnabled() { + // Nothing to do. + ms.log.Info("No migrations to run") + return nil + } + + if migrated { + // If legacy alerting is also disabled, there is nothing to do + if setting.AlertingEnabled != nil && !*setting.AlertingEnabled { + return nil + } + + // Safeguard to prevent data loss when reverting from UA to LA. + if !ms.cfg.ForceMigration { + return ForceMigrationError + } + + // Revert migration + ms.log.Info("Reverting legacy migration") + err := ms.migrationStore.RevertAllOrgs(ctx) + if err != nil { + return fmt.Errorf("reverting migration: %w", err) + } + ms.log.Info("Legacy migration reverted") + return nil + } + + ms.log.Info("Starting legacy migration") + err = ms.migrateAllOrgs(ctx) + if err != nil { + return fmt.Errorf("executing migration: %w", err) + } + + err = ms.migrationStore.SetMigrated(ctx, true) + if err != nil { + return fmt.Errorf("setting migration status: %w", err) + } + + ms.log.Info("Completed legacy migration") + return nil + }) + }) + if errLock != nil { + ms.log.Warn("Server lock for alerting migration already exists") + return nil + } + if errMigration != nil { + return fmt.Errorf("migration failed: %w", errMigration) + } + return nil +} + +// IsDisabled returns true if the cfg is nil. +func (ms *MigrationService) IsDisabled() bool { + return ms.cfg == nil +} + +// migrateAllOrgs executes the migration for all orgs. +func (ms *MigrationService) migrateAllOrgs(ctx context.Context) error { + orgs, err := ms.migrationStore.GetAllOrgs(ctx) + if err != nil { + return fmt.Errorf("get orgs: %w", err) + } + + for _, o := range orgs { + om := ms.newOrgMigration(o.ID) + if err := om.migrateOrg(ctx); err != nil { + return fmt.Errorf("migrate org %d: %w", o.ID, err) + } + + err = om.migrationStore.SetOrgMigrationState(ctx, o.ID, om.state) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/services/ngalert/migration/service_test.go b/pkg/services/ngalert/migration/service_test.go new file mode 100644 index 00000000000..4bc386a5fe5 --- /dev/null +++ b/pkg/services/ngalert/migration/service_test.go @@ -0,0 +1,190 @@ +package migration + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/infra/db" + legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/setting" +) + +// TestServiceRevert tests migration revert. +func TestServiceRevert(t *testing.T) { + alerts := []*legacymodels.Alert{ + createAlert(t, 1, 1, 1, "alert1", []string{"notifier1"}), + } + channels := []*legacymodels.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + } + dashes := []*dashboards.Dashboard{ + createDashboard(t, 1, 1, "dash1-1", 5, nil), + createDashboard(t, 2, 1, "dash2-1", 5, nil), + createDashboard(t, 8, 1, "dash-in-general-1", 0, nil), + } + folders := []*dashboards.Dashboard{ + createFolder(t, 5, 1, "folder5-1"), + } + + t.Run("revert deletes UA resources", func(t *testing.T) { + sqlStore := db.InitTestDB(t) + x := sqlStore.GetEngine() + + setupLegacyAlertsTables(t, x, channels, alerts, folders, dashes) + + dashCount, err := x.Table("dashboard").Count(&dashboards.Dashboard{}) + require.NoError(t, err) + require.Equal(t, int64(4), dashCount) + + // Run migration. + ctx := context.Background() + cfg := &setting.Cfg{ + ForceMigration: true, + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: pointer(true), + }, + } + service := NewTestMigrationService(t, sqlStore, cfg) + + err = service.migrationStore.SetMigrated(ctx, false) + require.NoError(t, err) + + err = service.Run(ctx) + require.NoError(t, err) + + // Verify migration was run. + migrated, err := service.migrationStore.IsMigrated(ctx) + require.NoError(t, err) + require.Equal(t, true, migrated) + + // Currently, we fill in some random data for tables that aren't populated during migration. + _, err = x.Table("ngalert_configuration").Insert(models.AdminConfiguration{}) + require.NoError(t, err) + _, err = x.Table("alert_instance").Insert(models.AlertInstance{ + AlertInstanceKey: models.AlertInstanceKey{ + RuleOrgID: 1, + RuleUID: "alert1", + LabelsHash: "", + }, + CurrentState: models.InstanceStateNormal, + CurrentStateSince: time.Now(), + CurrentStateEnd: time.Now(), + LastEvalTime: time.Now(), + }) + require.NoError(t, err) + + // Verify various UA resources exist + tables := []string{ + "alert_rule", + "alert_rule_version", + "alert_configuration", + "ngalert_configuration", + "alert_instance", + } + for _, table := range tables { + count, err := x.Table(table).Count() + require.NoError(t, err) + require.True(t, count > 0, "table %s should have at least one row", table) + } + + // Revert migration. + service.cfg.UnifiedAlerting.Enabled = pointer(false) + err = service.Run(context.Background()) + require.NoError(t, err) + + // Verify revert was run. + migrated, err = service.migrationStore.IsMigrated(ctx) + require.NoError(t, err) + require.Equal(t, false, migrated) + + // Verify various UA resources are gone + for _, table := range tables { + count, err := x.Table(table).Count() + require.NoError(t, err) + require.Equal(t, int64(0), count, "table %s should have no rows", table) + } + }) + + t.Run("revert deletes folders created during migration", func(t *testing.T) { + sqlStore := db.InitTestDB(t) + x := sqlStore.GetEngine() + alerts = []*legacymodels.Alert{ + createAlert(t, 1, 8, 1, "alert1", []string{"notifier1"}), + } + setupLegacyAlertsTables(t, x, channels, alerts, folders, dashes) + + dashCount, err := x.Table("dashboard").Count(&dashboards.Dashboard{}) + require.NoError(t, err) + require.Equal(t, int64(4), dashCount) + + // Run migration. + ctx := context.Background() + cfg := &setting.Cfg{ + ForceMigration: true, + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: pointer(true), + }, + } + service := NewTestMigrationService(t, sqlStore, cfg) + + err = service.migrationStore.SetMigrated(ctx, false) + require.NoError(t, err) + + err = service.Run(ctx) + require.NoError(t, err) + + // Verify migration was run. + migrated, err := service.migrationStore.IsMigrated(ctx) + require.NoError(t, err) + require.Equal(t, true, migrated) + + // Verify we created some folders. + newDashCount, err := x.Table("dashboard").Count(&dashboards.Dashboard{}) + require.NoError(t, err) + require.Truef(t, newDashCount > dashCount, "newDashCount: %d should be greater than dashCount: %d", newDashCount, dashCount) + + // Check that dashboards and folders from before migration still exist. + require.NotNil(t, getDashboard(t, x, 1, "dash1-1")) + require.NotNil(t, getDashboard(t, x, 1, "dash2-1")) + require.NotNil(t, getDashboard(t, x, 1, "dash-in-general-1")) + + state, err := service.migrationStore.GetOrgMigrationState(ctx, 1) + require.NoError(t, err) + + // Verify list of created folders. + require.NotEmpty(t, state.CreatedFolders) + for _, uid := range state.CreatedFolders { + require.NotNil(t, getDashboard(t, x, 1, uid)) + } + + // Revert migration. + service.cfg.UnifiedAlerting.Enabled = pointer(false) + err = service.Run(context.Background()) + require.NoError(t, err) + + // Verify revert was run. + migrated, err = service.migrationStore.IsMigrated(ctx) + require.NoError(t, err) + require.Equal(t, false, migrated) + + // Verify we are back to the original count. + newDashCount, err = x.Table("dashboard").Count(&dashboards.Dashboard{}) + require.NoError(t, err) + require.Equalf(t, dashCount, newDashCount, "newDashCount: %d should be equal to dashCount: %d after revert", newDashCount, dashCount) + + // Check that dashboards and folders from before migration still exist. + require.NotNil(t, getDashboard(t, x, 1, "dash1-1")) + require.NotNil(t, getDashboard(t, x, 1, "dash2-1")) + require.NotNil(t, getDashboard(t, x, 1, "dash-in-general-1")) + + // Check that folders created during migration are gone. + for _, uid := range state.CreatedFolders { + require.Nil(t, getDashboard(t, x, 1, uid)) + } + }) +} diff --git a/pkg/services/sqlstore/migrations/ualert/silences.go b/pkg/services/ngalert/migration/silences.go similarity index 70% rename from pkg/services/sqlstore/migrations/ualert/silences.go rename to pkg/services/ngalert/migration/silences.go index 69d1a2b320c..0263605c0a8 100644 --- a/pkg/services/sqlstore/migrations/ualert/silences.go +++ b/pkg/services/ngalert/migration/silences.go @@ -1,4 +1,4 @@ -package ualert +package migration import ( "bytes" @@ -16,7 +16,7 @@ import ( pb "github.com/prometheus/alertmanager/silence/silencepb" "github.com/prometheus/common/model" - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/services/ngalert/models" ) const ( @@ -26,14 +26,10 @@ const ( ErrorAlertName = "DatasourceError" ) -func (m *migration) addErrorSilence(da dashAlert, rule *alertRule) error { - if da.ParsedSettings.ExecutionErrorState != "keep_state" { - return nil - } - +func (om *OrgMigration) addErrorSilence(rule *models.AlertRule) error { uid, err := uuid.NewRandom() if err != nil { - return errors.New("failed to create uuid for silence") + return errors.New("create uuid for silence") } s := &pb.MeshSilence{ @@ -58,21 +54,14 @@ func (m *migration) addErrorSilence(da dashAlert, rule *alertRule) error { }, ExpiresAt: time.Now().AddDate(1, 0, 0), // 1 year } - if _, ok := m.silences[da.OrgId]; !ok { - m.silences[da.OrgId] = make([]*pb.MeshSilence, 0) - } - m.silences[da.OrgId] = append(m.silences[da.OrgId], s) + om.silences = append(om.silences, s) return nil } -func (m *migration) addNoDataSilence(da dashAlert, rule *alertRule) error { - if da.ParsedSettings.NoDataState != "keep_state" { - return nil - } - +func (om *OrgMigration) addNoDataSilence(rule *models.AlertRule) error { uid, err := uuid.NewRandom() if err != nil { - return errors.New("failed to create uuid for silence") + return errors.New("create uuid for silence") } s := &pb.MeshSilence{ @@ -97,28 +86,20 @@ func (m *migration) addNoDataSilence(da dashAlert, rule *alertRule) error { }, ExpiresAt: time.Now().AddDate(1, 0, 0), // 1 year. } - _, ok := m.silences[da.OrgId] - if !ok { - m.silences[da.OrgId] = make([]*pb.MeshSilence, 0) - } - m.silences[da.OrgId] = append(m.silences[da.OrgId], s) + om.silences = append(om.silences, s) return nil } -func (m *migration) writeSilencesFile(orgID int64) error { +func (om *OrgMigration) writeSilencesFile() error { var buf bytes.Buffer - orgSilences, ok := m.silences[orgID] - if !ok { - return nil - } - - for _, e := range orgSilences { + om.log.Debug("Writing silences file", "silences", len(om.silences)) + for _, e := range om.silences { if _, err := pbutil.WriteDelimited(&buf, e); err != nil { return err } } - f, err := openReplace(silencesFileNameForOrg(m.mg, orgID)) + f, err := openReplace(silencesFileNameForOrg(om.cfg.DataPath, om.orgID)) if err != nil { return err } @@ -130,12 +111,8 @@ func (m *migration) writeSilencesFile(orgID int64) error { return f.Close() } -func getSilenceFileNamesForAllOrgs(mg *migrator.Migrator) ([]string, error) { - return filepath.Glob(filepath.Join(mg.Cfg.DataPath, "alerting", "*", "silences")) -} - -func silencesFileNameForOrg(mg *migrator.Migrator, orgID int64) string { - return filepath.Join(mg.Cfg.DataPath, "alerting", strconv.Itoa(int(orgID)), "silences") +func silencesFileNameForOrg(dataPath string, orgID int64) string { + return filepath.Join(dataPath, "alerting", strconv.Itoa(int(orgID)), "silences") } // replaceFile wraps a file that is moved to another filename on closing. diff --git a/pkg/services/ngalert/migration/store/database.go b/pkg/services/ngalert/migration/store/database.go new file mode 100644 index 00000000000..99589b57f49 --- /dev/null +++ b/pkg/services/ngalert/migration/store/database.go @@ -0,0 +1,511 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/infra/kvstore" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/accesscontrol" + legacyalerting "github.com/grafana/grafana/pkg/services/alerting" + legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" + "github.com/grafana/grafana/pkg/services/auth/identity" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/folder" + apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + migmodels "github.com/grafana/grafana/pkg/services/ngalert/migration/models" + "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/notifier" + "github.com/grafana/grafana/pkg/services/ngalert/store" + "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/services/user" + "github.com/grafana/grafana/pkg/setting" +) + +// Store is the database abstraction for migration persistence. +type Store interface { + InsertAlertRules(ctx context.Context, rules ...models.AlertRule) error + + SaveAlertmanagerConfiguration(ctx context.Context, orgID int64, amConfig *apimodels.PostableUserConfig) error + + GetAllOrgs(ctx context.Context) ([]*org.OrgDTO, error) + + GetDatasource(ctx context.Context, datasourceID int64, user identity.Requester) (*datasources.DataSource, error) + + GetNotificationChannels(ctx context.Context, orgID int64) ([]*legacymodels.AlertNotification, error) + + GetOrgDashboardAlerts(ctx context.Context, orgID int64) (map[int64][]*DashAlert, int, error) + + GetACL(ctx context.Context, orgID int64, dashID int64) ([]*DashboardACL, error) + SetACL(ctx context.Context, orgID int64, dashboardID int64, items []*DashboardACL) error + + GetDashboard(ctx context.Context, orgID int64, id int64) (*dashboards.Dashboard, error) + GetFolder(ctx context.Context, cmd *folder.GetFolderQuery) (*folder.Folder, error) + CreateFolder(ctx context.Context, cmd *folder.CreateFolderCommand) (*folder.Folder, error) + + IsMigrated(ctx context.Context) (bool, error) + SetMigrated(ctx context.Context, migrated bool) error + GetOrgMigrationState(ctx context.Context, orgID int64) (*migmodels.OrgMigrationState, error) + SetOrgMigrationState(ctx context.Context, orgID int64, summary *migmodels.OrgMigrationState) error + + RevertAllOrgs(ctx context.Context) error + + CaseInsensitive() bool +} + +type migrationStore struct { + store db.DB + cfg *setting.Cfg + log log.Logger + kv kvstore.KVStore + alertingStore *store.DBstore + dashboardService dashboards.DashboardService + folderService folder.Service + dataSourceCache datasources.CacheService + orgService org.Service + legacyAlertNotificationService *legacyalerting.AlertNotificationService +} + +// MigrationStore implements the Store interface. +var _ Store = (*migrationStore)(nil) + +func ProvideMigrationStore( + cfg *setting.Cfg, + sqlStore db.DB, + kv kvstore.KVStore, + alertingStore *store.DBstore, + dashboardService dashboards.DashboardService, + folderService folder.Service, + dataSourceCache datasources.CacheService, + orgService org.Service, + legacyAlertNotificationService *legacyalerting.AlertNotificationService, +) (Store, error) { + return &migrationStore{ + log: log.New("ngalert.migration-store"), + cfg: cfg, + store: sqlStore, + kv: kv, + alertingStore: alertingStore, + dashboardService: dashboardService, + folderService: folderService, + dataSourceCache: dataSourceCache, + orgService: orgService, + legacyAlertNotificationService: legacyAlertNotificationService, + }, nil +} + +// KVNamespace is the kvstore namespace used for the migration status. +const KVNamespace = "ngalert.migration" + +// migratedKey is the kvstore key used for the migration status. +const migratedKey = "migrated" + +// stateKey is the kvstore key used for the OrgMigrationState. +const stateKey = "stateKey" + +const anyOrg = 0 + +// IsMigrated returns the migration status from the kvstore. +func (ms *migrationStore) IsMigrated(ctx context.Context) (bool, error) { + kv := kvstore.WithNamespace(ms.kv, anyOrg, KVNamespace) + content, exists, err := kv.Get(ctx, migratedKey) + if err != nil { + return false, err + } + + if !exists { + return false, nil + } + + return strconv.ParseBool(content) +} + +// SetMigrated sets the migration status in the kvstore. +func (ms *migrationStore) SetMigrated(ctx context.Context, migrated bool) error { + kv := kvstore.WithNamespace(ms.kv, anyOrg, KVNamespace) + return kv.Set(ctx, migratedKey, strconv.FormatBool(migrated)) +} + +// GetOrgMigrationState returns a summary of a previous migration. +func (ms *migrationStore) GetOrgMigrationState(ctx context.Context, orgID int64) (*migmodels.OrgMigrationState, error) { + kv := kvstore.WithNamespace(ms.kv, orgID, KVNamespace) + content, exists, err := kv.Get(ctx, stateKey) + if err != nil { + return nil, err + } + + if !exists { + return &migmodels.OrgMigrationState{OrgID: orgID}, nil + } + + var summary migmodels.OrgMigrationState + err = json.Unmarshal([]byte(content), &summary) + if err != nil { + return nil, err + } + + return &summary, nil +} + +// SetOrgMigrationState sets the summary of a previous migration. +func (ms *migrationStore) SetOrgMigrationState(ctx context.Context, orgID int64, summary *migmodels.OrgMigrationState) error { + kv := kvstore.WithNamespace(ms.kv, orgID, KVNamespace) + raw, err := json.Marshal(summary) + if err != nil { + return err + } + + return kv.Set(ctx, stateKey, string(raw)) +} + +func (ms *migrationStore) InsertAlertRules(ctx context.Context, rules ...models.AlertRule) error { + if ms.store.GetDialect().DriverName() == migrator.Postgres { + // Postgresql which will automatically rollback the whole transaction on constraint violation. + // So, for postgresql, insertions will execute in a subtransaction. + err := ms.store.InTransaction(ctx, func(subCtx context.Context) error { + _, err := ms.alertingStore.InsertAlertRules(subCtx, rules) + if err != nil { + return err + } + return nil + }) + if err != nil { + return err + } + } else { + _, err := ms.alertingStore.InsertAlertRules(ctx, rules) + if err != nil { + return err + } + } + + return nil +} + +func (ms *migrationStore) SaveAlertmanagerConfiguration(ctx context.Context, orgID int64, amConfig *apimodels.PostableUserConfig) error { + rawAmConfig, err := json.Marshal(amConfig) + if err != nil { + return err + } + + cmd := models.SaveAlertmanagerConfigurationCmd{ + AlertmanagerConfiguration: string(rawAmConfig), + ConfigurationVersion: fmt.Sprintf("v%d", models.AlertConfigurationVersion), + Default: false, + OrgID: orgID, + LastApplied: 0, + } + return ms.alertingStore.SaveAlertmanagerConfiguration(ctx, &cmd) +} + +// revertPermissions are the permissions required for the background user to revert the migration. +var revertPermissions = []accesscontrol.Permission{ + {Action: dashboards.ActionFoldersDelete, Scope: dashboards.ScopeFoldersAll}, + {Action: dashboards.ActionFoldersRead, Scope: dashboards.ScopeFoldersAll}, +} + +// RevertAllOrgs reverts the migration, deleting all unified alerting resources such as alert rules, alertmanager configurations, and silence files. +// In addition, it will delete all folders and permissions originally created by this migration, these are stored in the kvstore. +func (ms *migrationStore) RevertAllOrgs(ctx context.Context) error { + return ms.store.WithTransactionalDbSession(ctx, func(sess *db.Session) error { + if _, err := sess.Exec("DELETE FROM alert_rule"); err != nil { + return err + } + + if _, err := sess.Exec("DELETE FROM alert_rule_version"); err != nil { + return err + } + + orgs, err := ms.GetAllOrgs(ctx) + if err != nil { + return fmt.Errorf("get orgs: %w", err) + } + for _, o := range orgs { + if err := ms.DeleteMigratedFolders(ctx, o.ID); err != nil { + return err + } + } + + if _, err := sess.Exec("DELETE FROM alert_configuration"); err != nil { + return err + } + + if _, err := sess.Exec("DELETE FROM ngalert_configuration"); err != nil { + return err + } + + if _, err := sess.Exec("DELETE FROM alert_instance"); err != nil { + return err + } + + if _, err := sess.Exec("DELETE FROM kv_store WHERE namespace = ?", notifier.KVNamespace); err != nil { + return err + } + + if _, err := sess.Exec("DELETE FROM kv_store WHERE namespace = ?", KVNamespace); err != nil { + return err + } + + files, err := filepath.Glob(filepath.Join(ms.cfg.DataPath, "alerting", "*", "silences")) + if err != nil { + return err + } + for _, f := range files { + if err := os.Remove(f); err != nil { + ms.log.Error("Failed to remove silence file", "file", f, "err", err) + } + } + + err = ms.SetMigrated(ctx, false) + if err != nil { + return fmt.Errorf("setting migration status: %w", err) + } + + return nil + }) +} + +// DeleteMigratedFolders deletes all folders created by the previous migration run for the given org. This includes all folder permissions. +// If the folder is not empty of all descendants the operation will fail and return an error. +func (ms *migrationStore) DeleteMigratedFolders(ctx context.Context, orgID int64) error { + summary, err := ms.GetOrgMigrationState(ctx, orgID) + if err != nil { + return err + } + return ms.DeleteFolders(ctx, orgID, summary.CreatedFolders...) +} + +// DeleteFolders deletes the folders from the given orgs with the given UIDs. This includes all folder permissions. +// If the folder is not empty of all descendants the operation will fail and return an error. +func (ms *migrationStore) DeleteFolders(ctx context.Context, orgID int64, uids ...string) error { + if len(uids) == 0 { + return nil + } + + usr := accesscontrol.BackgroundUser("ngalert_migration_revert", orgID, org.RoleAdmin, revertPermissions) + for _, folderUID := range uids { + cmd := folder.DeleteFolderCommand{ + UID: folderUID, + OrgID: orgID, + SignedInUser: usr.(*user.SignedInUser), + } + err := ms.folderService.Delete(ctx, &cmd) // Also handles permissions and other related entities. + if err != nil { + return err + } + } + return nil +} + +func (ms *migrationStore) GetDashboard(ctx context.Context, orgID int64, id int64) (*dashboards.Dashboard, error) { + return ms.dashboardService.GetDashboard(ctx, &dashboards.GetDashboardQuery{ID: id, OrgID: orgID}) +} + +func (ms *migrationStore) GetAllOrgs(ctx context.Context) ([]*org.OrgDTO, error) { + orgQuery := &org.SearchOrgsQuery{} + return ms.orgService.Search(ctx, orgQuery) +} + +func (ms *migrationStore) GetDatasource(ctx context.Context, datasourceID int64, user identity.Requester) (*datasources.DataSource, error) { + return ms.dataSourceCache.GetDatasource(ctx, datasourceID, user, false) +} + +// GetNotificationChannels returns all channels for this org. +func (ms *migrationStore) GetNotificationChannels(ctx context.Context, orgID int64) ([]*legacymodels.AlertNotification, error) { + return ms.legacyAlertNotificationService.GetAllAlertNotifications(ctx, &legacymodels.GetAllAlertNotificationsQuery{ + OrgID: orgID, + }) +} + +func (ms *migrationStore) GetFolder(ctx context.Context, cmd *folder.GetFolderQuery) (*folder.Folder, error) { + return ms.folderService.Get(ctx, cmd) +} + +func (ms *migrationStore) CreateFolder(ctx context.Context, cmd *folder.CreateFolderCommand) (*folder.Folder, error) { + return ms.folderService.Create(ctx, cmd) +} + +// based on SQLStore.GetDashboardACLInfoList() +func (ms *migrationStore) GetACL(ctx context.Context, orgID, dashboardID int64) ([]*DashboardACL, error) { + var err error + + falseStr := ms.store.GetDialect().BooleanStr(false) + + result := make([]*DashboardACL, 0) + rawSQL := ` + -- get distinct permissions for the dashboard and its parent folder + SELECT DISTINCT + da.id, + da.user_id, + da.team_id, + da.permission, + da.role + FROM dashboard as d + LEFT JOIN dashboard folder on folder.id = d.folder_id + LEFT JOIN dashboard_acl AS da ON + da.dashboard_id = d.id OR + da.dashboard_id = d.folder_id OR + ( + -- include default permissions -- + da.org_id = -1 AND ( + (folder.id IS NOT NULL AND folder.has_acl = ` + falseStr + `) OR + (folder.id IS NULL AND d.has_acl = ` + falseStr + `) + ) + ) + WHERE d.org_id = ? AND d.id = ? AND da.id IS NOT NULL + ORDER BY da.id ASC + ` + err = ms.store.WithDbSession(ctx, func(sess *db.Session) error { + return sess.SQL(rawSQL, orgID, dashboardID).Find(&result) + }) + if err != nil { + return nil, err + } + return result, err +} + +// based on SQLStore.UpdateDashboardACL() +// it should be called from inside a transaction +func (ms *migrationStore) SetACL(ctx context.Context, orgID int64, dashboardID int64, items []*DashboardACL) error { + if dashboardID <= 0 { + return fmt.Errorf("folder id must be greater than zero for a folder permission") + } + return ms.store.WithDbSession(ctx, func(sess *db.Session) error { + // userPermissionsMap is a map keeping the highest permission per user + // for handling conficting inherited (folder) and non-inherited (dashboard) user permissions + userPermissionsMap := make(map[int64]*DashboardACL, len(items)) + // teamPermissionsMap is a map keeping the highest permission per team + // for handling conficting inherited (folder) and non-inherited (dashboard) team permissions + teamPermissionsMap := make(map[int64]*DashboardACL, len(items)) + for _, item := range items { + if item.UserID != 0 { + acl, ok := userPermissionsMap[item.UserID] + if !ok { + userPermissionsMap[item.UserID] = item + } else { + if item.Permission > acl.Permission { + // the higher permission wins + userPermissionsMap[item.UserID] = item + } + } + } + + if item.TeamID != 0 { + acl, ok := teamPermissionsMap[item.TeamID] + if !ok { + teamPermissionsMap[item.TeamID] = item + } else { + if item.Permission > acl.Permission { + // the higher permission wins + teamPermissionsMap[item.TeamID] = item + } + } + } + } + + type keyType struct { + UserID int64 `xorm:"user_id"` + TeamID int64 `xorm:"team_id"` + Role RoleType + Permission permissionType + } + // seen keeps track of inserted perrmissions to avoid duplicates (due to inheritance) + seen := make(map[keyType]struct{}, len(items)) + for _, item := range items { + if item.UserID == 0 && item.TeamID == 0 && (item.Role == nil || !item.Role.IsValid()) { + return dashboards.ErrDashboardACLInfoMissing + } + + // ignore duplicate user permissions + if item.UserID != 0 { + acl, ok := userPermissionsMap[item.UserID] + if ok { + if acl.Id != item.Id { + continue + } + } + } + + // ignore duplicate team permissions + if item.TeamID != 0 { + acl, ok := teamPermissionsMap[item.TeamID] + if ok { + if acl.Id != item.Id { + continue + } + } + } + + key := keyType{UserID: item.UserID, TeamID: item.TeamID, Role: "", Permission: item.Permission} + if item.Role != nil { + key.Role = *item.Role + } + if _, ok := seen[key]; ok { + continue + } + + // unset Id so that the new record will get a different one + item.Id = 0 + item.OrgID = orgID + item.DashboardID = dashboardID + item.Created = time.Now() + item.Updated = time.Now() + + sess.Nullable("user_id", "team_id") + if _, err := sess.Insert(item); err != nil { + return err + } + seen[key] = struct{}{} + } + + // Update dashboard HasACL flag + dashboard := dashboards.Dashboard{HasACL: true} + _, err := sess.Cols("has_acl").Where("id=?", dashboardID).Update(&dashboard) + + return err + }) +} + +// GetOrgDashboardAlerts loads all legacy dashboard alerts for the given org mapped by dashboard id. +func (ms *migrationStore) GetOrgDashboardAlerts(ctx context.Context, orgID int64) (map[int64][]*DashAlert, int, error) { + var alerts []legacymodels.Alert + err := ms.store.WithDbSession(ctx, func(sess *db.Session) error { + return sess.SQL("select * from alert WHERE org_id = ? AND dashboard_id IN (SELECT id from dashboard)", orgID).Find(&alerts) + }) + if err != nil { + return nil, 0, err + } + + mappedAlerts := make(map[int64][]*DashAlert) + for i := range alerts { + alert := alerts[i] + + rawSettings, err := json.Marshal(alert.Settings) + if err != nil { + return nil, 0, fmt.Errorf("get settings for alert rule ID:%d, name:'%s', orgID:%d: %w", alert.ID, alert.Name, alert.OrgID, err) + } + var parsedSettings DashAlertSettings + err = json.Unmarshal(rawSettings, &parsedSettings) + if err != nil { + return nil, 0, fmt.Errorf("parse settings for alert rule ID:%d, name:'%s', orgID:%d: %w", alert.ID, alert.Name, alert.OrgID, err) + } + + mappedAlerts[alert.DashboardID] = append(mappedAlerts[alert.DashboardID], &DashAlert{ + Alert: &alerts[i], + ParsedSettings: &parsedSettings, + }) + } + return mappedAlerts, len(alerts), nil +} + +func (ms *migrationStore) CaseInsensitive() bool { + return ms.store.GetDialect().SupportEngine() +} diff --git a/pkg/services/ngalert/migration/store/models.go b/pkg/services/ngalert/migration/store/models.go new file mode 100644 index 00000000000..7053e4199ed --- /dev/null +++ b/pkg/services/ngalert/migration/store/models.go @@ -0,0 +1,91 @@ +package store + +import ( + "encoding/json" + "time" + + legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" +) + +type RoleType string + +const ( + RoleNone RoleType = "None" + RoleViewer RoleType = "Viewer" + RoleEditor RoleType = "Editor" + RoleAdmin RoleType = "Admin" +) + +func (r RoleType) IsValid() bool { + return r == RoleViewer || r == RoleAdmin || r == RoleEditor || r == RoleNone +} + +type permissionType int + +type DashboardACL struct { + // nolint:stylecheck + Id int64 + OrgID int64 `xorm:"org_id"` + DashboardID int64 `xorm:"dashboard_id"` + + UserID int64 `xorm:"user_id"` + TeamID int64 `xorm:"team_id"` + Role *RoleType // pointer to be nullable + Permission permissionType + + Created time.Time + Updated time.Time +} + +func (p DashboardACL) TableName() string { return "dashboard_acl" } + +// uidOrID for both uid and ID, primarily used for mapping legacy channel to migrated receiver. +type UidOrID any + +type DashAlert struct { + *legacymodels.Alert + ParsedSettings *DashAlertSettings +} + +// dashAlertSettings is a type for the JSON that is in the settings field of +// the alert table. +type DashAlertSettings struct { + NoDataState string `json:"noDataState"` + ExecutionErrorState string `json:"executionErrorState"` + Conditions []DashAlertCondition `json:"conditions"` + AlertRuleTags any `json:"alertRuleTags"` + Notifications []DashAlertNot `json:"notifications"` +} + +// dashAlertNot is the object that represents the Notifications array in +// dashAlertSettings +type DashAlertNot struct { + UID string `json:"uid,omitempty"` + ID int64 `json:"id,omitempty"` +} + +// dashAlertingConditionJSON is like classic.ClassicConditionJSON except that it +// includes the model property with the query. +type DashAlertCondition struct { + Evaluator ConditionEvalJSON `json:"evaluator"` + + Operator struct { + Type string `json:"type"` + } `json:"operator"` + + Query struct { + Params []string `json:"params"` + DatasourceID int64 `json:"datasourceId"` + Model json.RawMessage + } `json:"query"` + + Reducer struct { + // Params []any `json:"params"` (Unused) + Type string `json:"type"` + } +} + +type ConditionEvalJSON struct { + Params []float64 `json:"params"` + Type string `json:"type"` // e.g. "gt" +} diff --git a/pkg/services/ngalert/migration/store/testing.go b/pkg/services/ngalert/migration/store/testing.go new file mode 100644 index 00000000000..7f912000f92 --- /dev/null +++ b/pkg/services/ngalert/migration/store/testing.go @@ -0,0 +1,57 @@ +package store + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/localcache" + "github.com/grafana/grafana/pkg/infra/log/logtest" + "github.com/grafana/grafana/pkg/infra/tracing" + legacyalerting "github.com/grafana/grafana/pkg/services/alerting" + "github.com/grafana/grafana/pkg/services/datasources/guardian" + datasourceService "github.com/grafana/grafana/pkg/services/datasources/service" + encryptionservice "github.com/grafana/grafana/pkg/services/encryption/service" + "github.com/grafana/grafana/pkg/services/folder/folderimpl" + "github.com/grafana/grafana/pkg/services/ngalert/store" + "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" + "github.com/grafana/grafana/pkg/services/ngalert/testutil" + "github.com/grafana/grafana/pkg/services/org/orgimpl" + "github.com/grafana/grafana/pkg/services/quota/quotatest" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/setting" +) + +func NewTestMigrationStore(t *testing.T, sqlStore *sqlstore.SQLStore, cfg *setting.Cfg) *migrationStore { + if cfg.UnifiedAlerting.BaseInterval == 0 { + cfg.UnifiedAlerting.BaseInterval = time.Second * 10 + } + alertingStore := store.DBstore{ + SQLStore: sqlStore, + Cfg: cfg.UnifiedAlerting, + } + bus := bus.ProvideBus(tracing.InitializeTracerForTest()) + folderStore := folderimpl.ProvideDashboardFolderStore(sqlStore) + dashboardService, dashboardStore := testutil.SetupDashboardService(t, sqlStore, folderStore, cfg) + folderService := testutil.SetupFolderService(t, cfg, sqlStore, dashboardStore, folderStore, bus) + + quotaService := "atest.FakeQuotaService{} + orgService, err := orgimpl.ProvideService(sqlStore, cfg, quotaService) + require.NoError(t, err) + + cache := localcache.ProvideService() + return &migrationStore{ + log: &logtest.Fake{}, + cfg: cfg, + store: sqlStore, + kv: fakes.NewFakeKVStore(t), + alertingStore: &alertingStore, + dashboardService: dashboardService, + folderService: folderService, + dataSourceCache: datasourceService.ProvideCacheService(cache, sqlStore, guardian.ProvideGuardian()), + orgService: orgService, + legacyAlertNotificationService: legacyalerting.ProvideService(sqlStore, encryptionservice.SetupTestService(t), nil), + } +} diff --git a/pkg/services/sqlstore/migrations/ualert/template.go b/pkg/services/ngalert/migration/template.go similarity index 99% rename from pkg/services/sqlstore/migrations/ualert/template.go rename to pkg/services/ngalert/migration/template.go index 08eb3dfbc43..fb218d582c9 100644 --- a/pkg/services/sqlstore/migrations/ualert/template.go +++ b/pkg/services/ngalert/migration/template.go @@ -1,7 +1,7 @@ // This file contains code that parses templates from old alerting into a sequence // of tokens. Each token can be either a string literal or a variable. -package ualert +package migration import ( "bytes" diff --git a/pkg/services/sqlstore/migrations/ualert/template_test.go b/pkg/services/ngalert/migration/template_test.go similarity index 99% rename from pkg/services/sqlstore/migrations/ualert/template_test.go rename to pkg/services/ngalert/migration/template_test.go index f95fbc757f3..88786af5e46 100644 --- a/pkg/services/sqlstore/migrations/ualert/template_test.go +++ b/pkg/services/ngalert/migration/template_test.go @@ -1,11 +1,12 @@ -package ualert +package migration import ( "fmt" "testing" - "github.com/grafana/grafana/pkg/infra/log" "github.com/stretchr/testify/assert" + + "github.com/grafana/grafana/pkg/infra/log" ) func TestTokenString(t *testing.T) { diff --git a/pkg/services/ngalert/migration/testing.go b/pkg/services/ngalert/migration/testing.go new file mode 100644 index 00000000000..4f5fcaf7769 --- /dev/null +++ b/pkg/services/ngalert/migration/testing.go @@ -0,0 +1,29 @@ +package migration + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/infra/serverlock" + "github.com/grafana/grafana/pkg/infra/tracing" + migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" + fake_secrets "github.com/grafana/grafana/pkg/services/secrets/fakes" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/setting" +) + +func NewTestMigrationService(t *testing.T, sqlStore *sqlstore.SQLStore, cfg *setting.Cfg) *MigrationService { + if cfg == nil { + cfg = setting.NewCfg() + } + ms, err := ProvideService( + serverlock.ProvideService(sqlStore, tracing.InitializeTracerForTest()), + cfg, + sqlStore, + migrationStore.NewTestMigrationStore(t, sqlStore, cfg), + fake_secrets.NewFakeSecretsService(), + ) + require.NoError(t, err) + return ms +} diff --git a/pkg/services/ngalert/migration/ualert.go b/pkg/services/ngalert/migration/ualert.go new file mode 100644 index 00000000000..085960d14de --- /dev/null +++ b/pkg/services/ngalert/migration/ualert.go @@ -0,0 +1,115 @@ +package migration + +import ( + "context" + "fmt" + + apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" + "github.com/grafana/grafana/pkg/services/ngalert/models" +) + +func (om *OrgMigration) migrateAlerts(ctx context.Context, alerts []*migrationStore.DashAlert, dashboardUID string, folderUID string) ([]*AlertPair, error) { + log := om.log.New( + "dashboardUID", dashboardUID, + "newFolderUID", folderUID, + ) + + pairs := make([]*AlertPair, 0, len(alerts)) + for _, da := range alerts { + al := log.New("ruleID", da.ID, "ruleName", da.Name) + alertRule, err := om.migrateAlert(ctx, al, da, dashboardUID, folderUID) + if err != nil { + return nil, fmt.Errorf("migrate alert: %w", err) + } + pairs = append(pairs, &AlertPair{AlertRule: alertRule, DashAlert: da}) + } + + return pairs, nil +} + +func (om *OrgMigration) migrateDashboard(ctx context.Context, dashID int64, alerts []*migrationStore.DashAlert) ([]*AlertPair, error) { + dash, newFolder, err := om.getOrCreateMigratedFolder(ctx, om.log, dashID) + if err != nil { + return nil, fmt.Errorf("get or create migrated folder: %w", err) + } + + pairs, err := om.migrateAlerts(ctx, alerts, dash.UID, newFolder.UID) + if err != nil { + return nil, fmt.Errorf("migrate and save alerts: %w", err) + } + + return pairs, nil +} + +func (om *OrgMigration) migrateOrgAlerts(ctx context.Context) ([]*AlertPair, error) { + mappedAlerts, cnt, err := om.migrationStore.GetOrgDashboardAlerts(ctx, om.orgID) + if err != nil { + return nil, fmt.Errorf("load alerts: %w", err) + } + om.log.Info("Alerts found to migrate", "alerts", cnt) + + pairs := make([]*AlertPair, 0, cnt) + for dashID, alerts := range mappedAlerts { + dashPairs, err := om.migrateDashboard(ctx, dashID, alerts) + if err != nil { + return nil, fmt.Errorf("migrate and save dashboard '%d': %w", dashID, err) + } + pairs = append(pairs, dashPairs...) + } + return pairs, nil +} + +func (om *OrgMigration) migrateOrgChannels(ctx context.Context, pairs []*AlertPair) (*apimodels.PostableUserConfig, error) { + channels, err := om.migrationStore.GetNotificationChannels(ctx, om.orgID) + if err != nil { + return nil, fmt.Errorf("load notification channels: %w", err) + } + + amConfig, err := om.migrateChannels(channels, pairs) + if err != nil { + return nil, err + } + + return amConfig, nil +} + +func (om *OrgMigration) migrateOrg(ctx context.Context) error { + om.log.Info("Migrating alerts for organisation") + + pairs, err := om.migrateOrgAlerts(ctx) + if err != nil { + return fmt.Errorf("migrate alerts: %w", err) + } + + // This must happen before we insert the rules into the database because it modifies the alert labels. This will + // be changed in the future when we improve how notification policies are created. + amConfig, err := om.migrateOrgChannels(ctx, pairs) + if err != nil { + return fmt.Errorf("migrate channels: %w", err) + } + + if err := om.writeSilencesFile(); err != nil { + return fmt.Errorf("write silence file for org %d: %w", om.orgID, err) + } + + if len(pairs) > 0 { + om.log.Debug("Inserting migrated alert rules", "count", len(pairs)) + rules := make([]models.AlertRule, 0, len(pairs)) + for _, p := range pairs { + rules = append(rules, *p.AlertRule) + } + err := om.migrationStore.InsertAlertRules(ctx, rules...) + if err != nil { + return fmt.Errorf("insert alert rules: %w", err) + } + } + + if amConfig != nil { + if err := om.migrationStore.SaveAlertmanagerConfiguration(ctx, om.orgID, amConfig); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/services/sqlstore/migrations/ualert/ualert_test.go b/pkg/services/ngalert/migration/ualert_test.go similarity index 53% rename from pkg/services/sqlstore/migrations/ualert/ualert_test.go rename to pkg/services/ngalert/migration/ualert_test.go index 98a56c07c6f..2f4cd49f90b 100644 --- a/pkg/services/sqlstore/migrations/ualert/ualert_test.go +++ b/pkg/services/ngalert/migration/ualert_test.go @@ -1,74 +1,33 @@ -package ualert +package migration import ( - "encoding/base64" "encoding/json" "fmt" - "sort" "strings" "testing" - "github.com/prometheus/alertmanager/pkg/labels" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/services/dashboards" + apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/util" ) -var MigTitle = migTitle -var RmMigTitle = rmMigTitle -var ClearMigrationEntryTitle = clearMigrationEntryTitle - -type RmMigration = rmMigration - -// UnmarshalJSON implements the json.Unmarshaler interface for Matchers. Vendored from definitions.ObjectMatchers. -func (m *ObjectMatchers) UnmarshalJSON(data []byte) error { - var rawMatchers [][3]string - if err := json.Unmarshal(data, &rawMatchers); err != nil { - return err - } - for _, rawMatcher := range rawMatchers { - var matchType labels.MatchType - switch rawMatcher[1] { - case "=": - matchType = labels.MatchEqual - case "!=": - matchType = labels.MatchNotEqual - case "=~": - matchType = labels.MatchRegexp - case "!~": - matchType = labels.MatchNotRegexp - default: - return fmt.Errorf("unsupported match type %q in matcher", rawMatcher[1]) - } - - rawMatcher[2] = strings.TrimPrefix(rawMatcher[2], "\"") - rawMatcher[2] = strings.TrimSuffix(rawMatcher[2], "\"") - - matcher, err := labels.NewMatcher(matchType, rawMatcher[0], rawMatcher[2]) - if err != nil { - return err - } - *m = append(*m, matcher) - } - sort.Sort(labels.Matchers(*m)) - return nil -} - func Test_validateAlertmanagerConfig(t *testing.T) { tc := []struct { name string - receivers []*PostableGrafanaReceiver + receivers []*apimodels.PostableGrafanaReceiver err error }{ { name: "when a slack receiver does not have a valid URL - it should error", - receivers: []*PostableGrafanaReceiver{ + receivers: []*apimodels.PostableGrafanaReceiver{ { UID: "test-uid", Name: "SlackWithBadURL", Type: "slack", - Settings: simplejson.NewFromAny(map[string]interface{}{}), + Settings: mustRawMessage(map[string]any{}), SecureSettings: map[string]string{"url": invalidUri}, }, }, @@ -76,36 +35,38 @@ func Test_validateAlertmanagerConfig(t *testing.T) { }, { name: "when a slack receiver has an invalid recipient - it should not error", - receivers: []*PostableGrafanaReceiver{ + receivers: []*apimodels.PostableGrafanaReceiver{ { UID: util.GenerateShortUID(), Name: "SlackWithBadRecipient", Type: "slack", - Settings: simplejson.NewFromAny(map[string]interface{}{"recipient": "this passes"}), + Settings: mustRawMessage(map[string]any{"recipient": "this passes"}), SecureSettings: map[string]string{"url": "http://webhook.slack.com/myuser"}, }, }, }, { name: "when the configuration is valid - it should not error", - receivers: []*PostableGrafanaReceiver{ + receivers: []*apimodels.PostableGrafanaReceiver{ { UID: util.GenerateShortUID(), Name: "SlackWithBadURL", Type: "slack", - Settings: simplejson.NewFromAny(map[string]interface{}{"recipient": "#a-good-channel"}), + Settings: mustRawMessage(map[string]interface{}{"recipient": "#a-good-channel"}), SecureSettings: map[string]string{"url": "http://webhook.slack.com/myuser"}, }, }, }, } + sqlStore := db.InitTestDB(t) for _, tt := range tc { t.Run(tt.name, func(t *testing.T) { - mg := newTestMigration(t) + service := NewTestMigrationService(t, sqlStore, nil) + mg := service.newOrgMigration(1) config := configFromReceivers(t, tt.receivers) - require.NoError(t, config.EncryptSecureSettings()) // make sure we encrypt the settings + require.NoError(t, encryptSecureSettings(config, mg)) // make sure we encrypt the settings err := mg.validateAlertmanagerConfig(config) if tt.err != nil { require.Error(t, err) @@ -117,24 +78,24 @@ func Test_validateAlertmanagerConfig(t *testing.T) { } } -func configFromReceivers(t *testing.T, receivers []*PostableGrafanaReceiver) *PostableUserConfig { +func configFromReceivers(t *testing.T, receivers []*apimodels.PostableGrafanaReceiver) *apimodels.PostableUserConfig { t.Helper() - return &PostableUserConfig{ - AlertmanagerConfig: PostableApiAlertingConfig{ - Receivers: []*PostableApiReceiver{ - {GrafanaManagedReceivers: receivers}, + return &apimodels.PostableUserConfig{ + AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ + Receivers: []*apimodels.PostableApiReceiver{ + {PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: receivers}}, }, }, } } -func (c *PostableUserConfig) EncryptSecureSettings() error { +func encryptSecureSettings(c *apimodels.PostableUserConfig, m *OrgMigration) error { for _, r := range c.AlertmanagerConfig.Receivers { for _, gr := range r.GrafanaManagedReceivers { - encryptedData := GetEncryptedJsonData(gr.SecureSettings) - for k, v := range encryptedData { - gr.SecureSettings[k] = base64.StdEncoding.EncodeToString(v) + err := m.encryptSecureSettings(gr.SecureSettings) + if err != nil { + return err } } } @@ -145,13 +106,13 @@ const invalidUri = "�6�M��)uk譹1(�h`$�o�N>mĕ����cS2�dh func Test_getAlertFolderNameFromDashboard(t *testing.T) { t.Run("should include full title", func(t *testing.T) { - dash := &dashboard{ - Uid: util.GenerateShortUID(), + dash := &dashboards.Dashboard{ + UID: util.GenerateShortUID(), Title: "TEST", } folder := getAlertFolderNameFromDashboard(dash) require.Contains(t, folder, dash.Title) - require.Contains(t, folder, dash.Uid) + require.Contains(t, folder, dash.UID) }) t.Run("should cut title to the length", func(t *testing.T) { title := "" @@ -163,25 +124,25 @@ func Test_getAlertFolderNameFromDashboard(t *testing.T) { } } - dash := &dashboard{ - Uid: util.GenerateShortUID(), + dash := &dashboards.Dashboard{ + UID: util.GenerateShortUID(), Title: title, } folder := getAlertFolderNameFromDashboard(dash) require.Len(t, folder, MaxFolderName) - require.Contains(t, folder, dash.Uid) + require.Contains(t, folder, dash.UID) }) } func Test_shortUIDCaseInsensitiveConflicts(t *testing.T) { - s := uidSet{ + s := Deduplicator{ set: make(map[string]struct{}), caseInsensitive: true, } // 10000 uids seems to be enough to cause a collision in almost every run if using util.GenerateShortUID directly. for i := 0; i < 10000; i++ { - _, _ = s.generateUid() + s.add(util.GenerateShortUID()) } // check if any are case-insensitive duplicates. @@ -192,3 +153,8 @@ func Test_shortUIDCaseInsensitiveConflicts(t *testing.T) { require.Equal(t, len(s.set), len(deduped)) } + +func mustRawMessage[T any](s T) apimodels.RawMessage { + js, _ := json.Marshal(s) + return js +} diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 521106afd3a..f73ddbfbc8a 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -319,6 +319,7 @@ func subscribeToFolderChanges(logger log.Logger, bus bus.Bus, dbStore api.RuleSt // Run starts the scheduler and Alertmanager. func (ng *AlertNG) Run(ctx context.Context) error { ng.Log.Debug("Starting") + ng.stateManager.Warm(ctx, ng.store) children, subCtx := errgroup.WithContext(ctx) diff --git a/pkg/services/ngalert/notifier/alertmanager_test.go b/pkg/services/ngalert/notifier/alertmanager_test.go index 6b9569c99c5..e0a14bda177 100644 --- a/pkg/services/ngalert/notifier/alertmanager_test.go +++ b/pkg/services/ngalert/notifier/alertmanager_test.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/ngalert/metrics" "github.com/grafana/grafana/pkg/services/ngalert/store" + "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" "github.com/grafana/grafana/pkg/services/secrets/database" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/setting" @@ -37,7 +38,7 @@ func setupAMTest(t *testing.T) *alertmanager { DashboardService: dashboards.NewFakeDashboardService(t), } - kvStore := NewFakeKVStore(t) + kvStore := fakes.NewFakeKVStore(t) secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(sqlStore)) decryptFn := secretsService.GetDecryptedValue am, err := newAlertmanager(context.Background(), 1, cfg, s, kvStore, &NilPeer{}, decryptFn, nil, m) diff --git a/pkg/services/ngalert/notifier/channels_config/available_channels.go b/pkg/services/ngalert/notifier/channels_config/available_channels.go index 2af9a0cc00e..45072be5009 100644 --- a/pkg/services/ngalert/notifier/channels_config/available_channels.go +++ b/pkg/services/ngalert/notifier/channels_config/available_channels.go @@ -1,6 +1,7 @@ package channels_config import ( + "fmt" "os" alertingOpsgenie "github.com/grafana/alerting/receivers/opsgenie" @@ -1340,3 +1341,20 @@ func GetAvailableNotifiers() []*NotifierPlugin { }, } } + +// GetSecretKeysForContactPointType returns settings keys of contact point of the given type that are expected to be secrets. Returns error is contact point type is not known. +func GetSecretKeysForContactPointType(contactPointType string) ([]string, error) { + notifiers := GetAvailableNotifiers() + for _, n := range notifiers { + if n.Type == contactPointType { + var secureFields []string + for _, field := range n.Options { + if field.Secure { + secureFields = append(secureFields, field.PropertyName) + } + } + return secureFields, nil + } + } + return nil, fmt.Errorf("no secrets configured for type '%s'", contactPointType) +} diff --git a/pkg/services/ngalert/notifier/file_store_test.go b/pkg/services/ngalert/notifier/file_store_test.go index 255a7e65716..92182396fa0 100644 --- a/pkg/services/ngalert/notifier/file_store_test.go +++ b/pkg/services/ngalert/notifier/file_store_test.go @@ -7,10 +7,12 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" ) func TestFileStore_FilepathFor_DirectoryNotExist(t *testing.T) { - store := NewFakeKVStore(t) + store := fakes.NewFakeKVStore(t) workingDir := filepath.Join(t.TempDir(), "notexistdir") fs := NewFileStore(1, store, workingDir) filekey := "silences" @@ -31,7 +33,7 @@ func TestFileStore_FilepathFor_DirectoryNotExist(t *testing.T) { } } func TestFileStore_FilepathFor(t *testing.T) { - store := NewFakeKVStore(t) + store := fakes.NewFakeKVStore(t) workingDir := t.TempDir() fs := NewFileStore(1, store, workingDir) filekey := "silences" @@ -73,7 +75,7 @@ func TestFileStore_FilepathFor(t *testing.T) { } func TestFileStore_Persist(t *testing.T) { - store := NewFakeKVStore(t) + store := fakes.NewFakeKVStore(t) state := &fakeState{data: "something to marshal"} workingDir := t.TempDir() fs := NewFileStore(1, store, workingDir) @@ -82,9 +84,9 @@ func TestFileStore_Persist(t *testing.T) { size, err := fs.Persist(context.Background(), filekey, state) require.NoError(t, err) require.Equal(t, int64(20), size) - store.mtx.Lock() - require.Len(t, store.store, 1) - store.mtx.Unlock() + store.Mtx.Lock() + require.Len(t, store.Store, 1) + store.Mtx.Unlock() v, ok, err := store.Get(context.Background(), 1, KVNamespace, filekey) require.NoError(t, err) require.True(t, ok) diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go index 93b6ac1a715..b92678a833e 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go @@ -19,6 +19,7 @@ import ( "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/provisioning" "github.com/grafana/grafana/pkg/services/ngalert/store" + ngfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/setting" @@ -31,7 +32,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) { } tmpDir := t.TempDir() - kvStore := NewFakeKVStore(t) + kvStore := ngfakes.NewFakeKVStore(t) provStore := provisioning.NewFakeProvisioningStore() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue @@ -165,7 +166,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T) } tmpDir := t.TempDir() - kvStore := NewFakeKVStore(t) + kvStore := ngfakes.NewFakeKVStore(t) provStore := provisioning.NewFakeProvisioningStore() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue @@ -259,7 +260,7 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) { DataPath: tmpDir, UnifiedAlerting: setting.UnifiedAlertingSettings{AlertmanagerConfigPollInterval: 3 * time.Minute, DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration()}, // do not poll in tests. } - kvStore := NewFakeKVStore(t) + kvStore := ngfakes.NewFakeKVStore(t) provStore := provisioning.NewFakeProvisioningStore() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue @@ -310,7 +311,7 @@ func TestMultiOrgAlertmanager_ActivateHistoricalConfiguration(t *testing.T) { DataPath: tmpDir, UnifiedAlerting: setting.UnifiedAlertingSettings{AlertmanagerConfigPollInterval: 3 * time.Minute, DefaultConfiguration: defaultConfig}, // do not poll in tests. } - kvStore := NewFakeKVStore(t) + kvStore := ngfakes.NewFakeKVStore(t) provStore := provisioning.NewFakeProvisioningStore() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue diff --git a/pkg/services/ngalert/notifier/testing.go b/pkg/services/ngalert/notifier/testing.go index cab6386561d..ba57d944574 100644 --- a/pkg/services/ngalert/notifier/testing.go +++ b/pkg/services/ngalert/notifier/testing.go @@ -5,12 +5,9 @@ import ( "crypto/md5" "errors" "fmt" - "strings" - "sync" "testing" "time" - "github.com/grafana/grafana/pkg/infra/kvstore" "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/store" @@ -195,98 +192,6 @@ func (f *FakeOrgStore) GetOrgs(_ context.Context) ([]int64, error) { return f.orgs, nil } -type FakeKVStore struct { - mtx sync.Mutex - store map[int64]map[string]map[string]string -} - -func NewFakeKVStore(t *testing.T) *FakeKVStore { - t.Helper() - - return &FakeKVStore{ - store: map[int64]map[string]map[string]string{}, - } -} - -func (fkv *FakeKVStore) Get(_ context.Context, orgId int64, namespace string, key string) (string, bool, error) { - fkv.mtx.Lock() - defer fkv.mtx.Unlock() - org, ok := fkv.store[orgId] - if !ok { - return "", false, nil - } - k, ok := org[namespace] - if !ok { - return "", false, nil - } - - v, ok := k[key] - if !ok { - return "", false, nil - } - - return v, true, nil -} -func (fkv *FakeKVStore) Set(_ context.Context, orgId int64, namespace string, key string, value string) error { - fkv.mtx.Lock() - defer fkv.mtx.Unlock() - org, ok := fkv.store[orgId] - if !ok { - fkv.store[orgId] = map[string]map[string]string{} - } - _, ok = org[namespace] - if !ok { - fkv.store[orgId][namespace] = map[string]string{} - } - - fkv.store[orgId][namespace][key] = value - - return nil -} -func (fkv *FakeKVStore) Del(_ context.Context, orgId int64, namespace string, key string) error { - fkv.mtx.Lock() - defer fkv.mtx.Unlock() - org, ok := fkv.store[orgId] - if !ok { - return nil - } - _, ok = org[namespace] - if !ok { - return nil - } - - delete(fkv.store[orgId][namespace], key) - - return nil -} - -func (fkv *FakeKVStore) Keys(ctx context.Context, orgID int64, namespace string, keyPrefix string) ([]kvstore.Key, error) { - fkv.mtx.Lock() - defer fkv.mtx.Unlock() - var keys []kvstore.Key - for orgIDFromStore, namespaceMap := range fkv.store { - if orgID != kvstore.AllOrganizations && orgID != orgIDFromStore { - continue - } - if keyMap, exists := namespaceMap[namespace]; exists { - for k := range keyMap { - if strings.HasPrefix(k, keyPrefix) { - keys = append(keys, kvstore.Key{ - OrgId: orgIDFromStore, - Namespace: namespace, - Key: keyPrefix, - }) - } - } - } - } - return keys, nil -} - -func (fkv *FakeKVStore) GetAll(ctx context.Context, orgId int64, namespace string) (map[int64]map[string]string, error) { - return nil, nil -} - type fakeState struct { data string } diff --git a/pkg/services/ngalert/provisioning/contactpoints.go b/pkg/services/ngalert/provisioning/contactpoints.go index d20415fc634..7695c6df58d 100644 --- a/pkg/services/ngalert/provisioning/contactpoints.go +++ b/pkg/services/ngalert/provisioning/contactpoints.go @@ -242,7 +242,7 @@ func (ecp *ContactPointService) UpdateContactPoint(ctx context.Context, orgID in if err != nil { return err } - secretKeys, err := GetSecretKeysForContactPointType(contactPoint.Type) + secretKeys, err := channels_config.GetSecretKeysForContactPointType(contactPoint.Type) if err != nil { return fmt.Errorf("%w: %s", ErrValidation, err.Error()) } @@ -531,27 +531,10 @@ func ValidateContactPoint(ctx context.Context, e apimodels.EmbeddedContactPoint, return nil } -// GetSecretKeysForContactPointType returns settings keys of contact point of the given type that are expected to be secrets. Returns error is contact point type is not known. -func GetSecretKeysForContactPointType(contactPointType string) ([]string, error) { - notifiers := channels_config.GetAvailableNotifiers() - for _, n := range notifiers { - if n.Type == contactPointType { - var secureFields []string - for _, field := range n.Options { - if field.Secure { - secureFields = append(secureFields, field.PropertyName) - } - } - return secureFields, nil - } - } - return nil, fmt.Errorf("no secrets configured for type '%s'", contactPointType) -} - // RemoveSecretsForContactPoint removes all secrets from the contact point's settings and returns them as a map. Returns error if contact point type is not known. func RemoveSecretsForContactPoint(e *apimodels.EmbeddedContactPoint) (map[string]string, error) { s := map[string]string{} - secretKeys, err := GetSecretKeysForContactPointType(e.Type) + secretKeys, err := channels_config.GetSecretKeysForContactPointType(e.Type) if err != nil { return nil, err } diff --git a/pkg/services/ngalert/sender/router_test.go b/pkg/services/ngalert/sender/router_test.go index 1414ebf5d26..77bc7b168c4 100644 --- a/pkg/services/ngalert/sender/router_test.go +++ b/pkg/services/ngalert/sender/router_test.go @@ -25,6 +25,7 @@ import ( "github.com/grafana/grafana/pkg/services/ngalert/notifier" "github.com/grafana/grafana/pkg/services/ngalert/provisioning" "github.com/grafana/grafana/pkg/services/ngalert/store" + "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" fake_secrets "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/setting" @@ -405,7 +406,7 @@ func createMultiOrgAlertmanager(t *testing.T, orgs []int64) *notifier.MultiOrgAl } cfgStore := notifier.NewFakeConfigStore(t, make(map[int64]*models.AlertConfiguration)) - kvStore := notifier.NewFakeKVStore(t) + kvStore := fakes.NewFakeKVStore(t) registry := prometheus.NewPedanticRegistry() m := metrics.NewNGAlert(registry) secretsService := secretsManager.SetupTestService(t, fake_secrets.NewFakeSecretsStore()) diff --git a/pkg/services/ngalert/tests/fakes/kvstore.go b/pkg/services/ngalert/tests/fakes/kvstore.go new file mode 100644 index 00000000000..7ef60184b83 --- /dev/null +++ b/pkg/services/ngalert/tests/fakes/kvstore.go @@ -0,0 +1,102 @@ +package fakes + +import ( + "context" + "strings" + "sync" + "testing" + + "github.com/grafana/grafana/pkg/infra/kvstore" +) + +type FakeKVStore struct { + Mtx sync.Mutex + Store map[int64]map[string]map[string]string +} + +func NewFakeKVStore(t *testing.T) *FakeKVStore { + t.Helper() + + return &FakeKVStore{ + Store: map[int64]map[string]map[string]string{}, + } +} + +func (fkv *FakeKVStore) Get(_ context.Context, orgId int64, namespace string, key string) (string, bool, error) { + fkv.Mtx.Lock() + defer fkv.Mtx.Unlock() + org, ok := fkv.Store[orgId] + if !ok { + return "", false, nil + } + k, ok := org[namespace] + if !ok { + return "", false, nil + } + + v, ok := k[key] + if !ok { + return "", false, nil + } + + return v, true, nil +} +func (fkv *FakeKVStore) Set(_ context.Context, orgId int64, namespace string, key string, value string) error { + fkv.Mtx.Lock() + defer fkv.Mtx.Unlock() + org, ok := fkv.Store[orgId] + if !ok { + fkv.Store[orgId] = map[string]map[string]string{} + } + _, ok = org[namespace] + if !ok { + fkv.Store[orgId][namespace] = map[string]string{} + } + + fkv.Store[orgId][namespace][key] = value + + return nil +} +func (fkv *FakeKVStore) Del(_ context.Context, orgId int64, namespace string, key string) error { + fkv.Mtx.Lock() + defer fkv.Mtx.Unlock() + org, ok := fkv.Store[orgId] + if !ok { + return nil + } + _, ok = org[namespace] + if !ok { + return nil + } + + delete(fkv.Store[orgId][namespace], key) + + return nil +} + +func (fkv *FakeKVStore) Keys(ctx context.Context, orgID int64, namespace string, keyPrefix string) ([]kvstore.Key, error) { + fkv.Mtx.Lock() + defer fkv.Mtx.Unlock() + var keys []kvstore.Key + for orgIDFromStore, namespaceMap := range fkv.Store { + if orgID != kvstore.AllOrganizations && orgID != orgIDFromStore { + continue + } + if keyMap, exists := namespaceMap[namespace]; exists { + for k := range keyMap { + if strings.HasPrefix(k, keyPrefix) { + keys = append(keys, kvstore.Key{ + OrgId: orgIDFromStore, + Namespace: namespace, + Key: keyPrefix, + }) + } + } + } + } + return keys, nil +} + +func (fkv *FakeKVStore) GetAll(ctx context.Context, orgId int64, namespace string) (map[int64]map[string]string, error) { + return nil, nil +} diff --git a/pkg/services/sqlstore/migrations/external_alertmanagers.go b/pkg/services/sqlstore/migrations/external_alertmanagers.go index ceea68ead7a..78d91192a9d 100644 --- a/pkg/services/sqlstore/migrations/external_alertmanagers.go +++ b/pkg/services/sqlstore/migrations/external_alertmanagers.go @@ -3,14 +3,16 @@ package migrations import ( "fmt" "net/url" + "os" "time" "xorm.io/xorm" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/datasources" - "github.com/grafana/grafana/pkg/services/sqlstore/migrations/ualert" "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" ) @@ -75,9 +77,9 @@ func (e externalAlertmanagerToDatasources) Exec(sess *xorm.Session, mg *migrator ds.BasicAuth = true ds.BasicAuthUser = u.User.Username() if password, ok := u.User.Password(); ok { - ds.SecureJsonData = ualert.GetEncryptedJsonData(map[string]string{ + ds.SecureJsonData = getEncryptedJsonData(map[string]string{ "basicAuthPassword": password, - }) + }, log.New("securejsondata")) } } @@ -124,3 +126,22 @@ func generateNewDatasourceUid(sess *xorm.Session, orgId int64) (string, error) { return "", datasources.ErrDataSourceFailedGenerateUniqueUid } + +// SecureJsonData is used to store encrypted data (for example in data_source table). Only values are separately +// encrypted. +type secureJsonData map[string][]byte + +// getEncryptedJsonData returns map where all keys are encrypted. +func getEncryptedJsonData(sjd map[string]string, log log.Logger) secureJsonData { + encrypted := make(secureJsonData) + for key, data := range sjd { + encryptedData, err := util.Encrypt([]byte(data), setting.SecretKey) + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } + + encrypted[key] = encryptedData + } + return encrypted +} diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index 56df4323cc9..60b2def385d 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -54,10 +54,9 @@ func (*OSSMigrations) AddMigration(mg *Migrator) { addCacheMigration(mg) addShortURLMigrations(mg) ualert.AddTablesMigrations(mg) - ualert.AddDashAlertMigration(mg) addLibraryElementsMigrations(mg) - ualert.RerunDashAlertMigration(mg) + ualert.FixEarlyMigration(mg) addSecretsMigration(mg) addKVStoreMigrations(mg) ualert.AddDashboardUIDPanelIDMigration(mg) @@ -76,7 +75,6 @@ func (*OSSMigrations) AddMigration(mg *Migrator) { addEntityEventsTableMigration(mg) addPublicDashboardMigration(mg) - ualert.CreateDefaultFoldersForAlertingMigration(mg) addDbFileStorageMigration(mg) accesscontrol.AddManagedPermissionsMigration(mg, accesscontrol.ManagedPermissionsMigrationID) @@ -101,6 +99,9 @@ func (*OSSMigrations) AddMigration(mg *Migrator) { anonservice.AddMigration(mg) signingkeys.AddMigration(mg) + + ualert.MigrationServiceMigration(mg) + ualert.CreatedFoldersMigration(mg) } func addStarMigrations(mg *Migrator) { diff --git a/pkg/services/sqlstore/migrations/ualert/channel.go b/pkg/services/sqlstore/migrations/ualert/channel.go deleted file mode 100644 index 1268ec92e59..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/channel.go +++ /dev/null @@ -1,512 +0,0 @@ -package ualert - -import ( - "crypto/md5" - "encoding/base64" - "encoding/json" - "fmt" - "regexp" - "sort" - "strings" - "time" - - "github.com/prometheus/alertmanager/pkg/labels" - "github.com/prometheus/common/model" - - "github.com/grafana/grafana/pkg/components/simplejson" - ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" -) - -const ( - // DisabledRepeatInterval is a large duration that will be used as a pseudo-disable in case a legacy channel doesn't have SendReminders enabled. - DisabledRepeatInterval = model.Duration(time.Duration(8736) * time.Hour) // 1y -) - -type notificationChannel struct { - ID int64 `xorm:"id"` - OrgID int64 `xorm:"org_id"` - Uid string `xorm:"uid"` - Name string `xorm:"name"` - Type string `xorm:"type"` - DisableResolveMessage bool `xorm:"disable_resolve_message"` - IsDefault bool `xorm:"is_default"` - Settings *simplejson.Json `xorm:"settings"` - SecureSettings SecureJsonData `xorm:"secure_settings"` - SendReminder bool `xorm:"send_reminder"` - Frequency model.Duration `xorm:"frequency"` -} - -// channelsPerOrg maps notification channels per organisation -type channelsPerOrg map[int64][]*notificationChannel - -// channelMap maps notification channels per organisation -type defaultChannelsPerOrg map[int64][]*notificationChannel - -// uidOrID for both uid and ID, primarily used for mapping legacy channel to migrated receiver. -type uidOrID any - -// channelReceiver is a convenience struct that contains a notificationChannel and its corresponding migrated PostableApiReceiver. -type channelReceiver struct { - channel *notificationChannel - receiver *PostableApiReceiver -} - -// setupAlertmanagerConfigs creates Alertmanager configs with migrated receivers and routes. -func (m *migration) setupAlertmanagerConfigs(rulesPerOrg map[int64]map[*alertRule][]uidOrID) (amConfigsPerOrg, error) { - // allChannels: channelUID -> channelConfig - allChannelsPerOrg, defaultChannelsPerOrg, err := m.getNotificationChannelMap() - if err != nil { - return nil, fmt.Errorf("failed to load notification channels: %w", err) - } - - amConfigPerOrg := make(amConfigsPerOrg, len(allChannelsPerOrg)) - for orgID, channels := range allChannelsPerOrg { - amConfig := &PostableUserConfig{ - AlertmanagerConfig: PostableApiAlertingConfig{ - Receivers: make([]*PostableApiReceiver, 0), - }, - } - amConfigPerOrg[orgID] = amConfig - - // Create all newly migrated receivers from legacy notification channels. - receiversMap, receivers, err := m.createReceivers(channels) - if err != nil { - return nil, fmt.Errorf("failed to create receiver in orgId %d: %w", orgID, err) - } - - // No need to create an Alertmanager configuration if there are no receivers left that aren't obsolete. - if len(receivers) == 0 { - m.mg.Logger.Warn("No available receivers", "orgId", orgID) - continue - } - - for _, cr := range receivers { - amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, cr.receiver) - } - - defaultReceivers := make(map[string]struct{}) - defaultChannels, ok := defaultChannelsPerOrg[orgID] - if ok { - // If the organization has default channels build a map of default receivers, used to create alert-specific routes later. - for _, c := range defaultChannels { - defaultReceivers[c.Name] = struct{}{} - } - } - defaultReceiver, defaultRoute, err := m.createDefaultRouteAndReceiver(defaultChannels) - if err != nil { - return nil, fmt.Errorf("failed to create default route & receiver in orgId %d: %w", orgID, err) - } - amConfig.AlertmanagerConfig.Route = defaultRoute - if defaultReceiver != nil { - amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, defaultReceiver) - } - - for _, cr := range receivers { - route, err := createRoute(cr) - if err != nil { - return nil, fmt.Errorf("failed to create route for receiver %s in orgId %d: %w", cr.receiver.Name, orgID, err) - } - - amConfigPerOrg[orgID].AlertmanagerConfig.Route.Routes = append(amConfigPerOrg[orgID].AlertmanagerConfig.Route.Routes, route) - } - - for ar, channelUids := range rulesPerOrg[orgID] { - filteredReceiverNames := m.filterReceiversForAlert(ar.Title, channelUids, receiversMap, defaultReceivers) - - if len(filteredReceiverNames) != 0 { - // Only create a contact label if there are specific receivers, otherwise it defaults to the root-level route. - ar.Labels[ContactLabel] = contactListToString(filteredReceiverNames) - } - } - - // Validate the alertmanager configuration produced, this gives a chance to catch bad configuration at migration time. - // Validation between legacy and unified alerting can be different (e.g. due to bug fixes) so this would fail the migration in that case. - if err := m.validateAlertmanagerConfig(amConfig); err != nil { - return nil, fmt.Errorf("failed to validate AlertmanagerConfig in orgId %d: %w", orgID, err) - } - } - - return amConfigPerOrg, nil -} - -// contactListToString creates a sorted string representation of a given map (set) of receiver names. Each name will be comma-separated and double-quoted. Names should not contain double quotes. -func contactListToString(m map[string]any) string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, quote(k)) - } - sort.Strings(keys) - - return strings.Join(keys, ",") -} - -// quote will surround the given string in double quotes. -func quote(s string) string { - return `"` + s + `"` -} - -// getNotificationChannelMap returns a map of all channelUIDs to channel config as well as a separate map for just those channels that are default. -// For any given Organization, all channels in defaultChannelsPerOrg should also exist in channelsPerOrg. -func (m *migration) getNotificationChannelMap() (channelsPerOrg, defaultChannelsPerOrg, error) { - q := ` - SELECT id, - org_id, - uid, - name, - type, - disable_resolve_message, - is_default, - settings, - secure_settings, - send_reminder, - frequency - FROM - alert_notification - ` - allChannels := []notificationChannel{} - err := m.sess.SQL(q).Find(&allChannels) - if err != nil { - return nil, nil, err - } - - if len(allChannels) == 0 { - return nil, nil, nil - } - - allChannelsMap := make(channelsPerOrg) - defaultChannelsMap := make(defaultChannelsPerOrg) - for i, c := range allChannels { - if c.Type == "hipchat" || c.Type == "sensu" { - m.mg.Logger.Error("Alert migration error: discontinued notification channel found", "type", c.Type, "name", c.Name, "uid", c.Uid) - continue - } - - allChannelsMap[c.OrgID] = append(allChannelsMap[c.OrgID], &allChannels[i]) - - if c.IsDefault { - defaultChannelsMap[c.OrgID] = append(defaultChannelsMap[c.OrgID], &allChannels[i]) - } - } - - return allChannelsMap, defaultChannelsMap, nil -} - -// Create a notifier (PostableGrafanaReceiver) from a legacy notification channel -func (m *migration) createNotifier(c *notificationChannel) (*PostableGrafanaReceiver, error) { - uid, err := m.determineChannelUid(c) - if err != nil { - return nil, err - } - - settings, secureSettings, err := migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings) - if err != nil { - return nil, err - } - - return &PostableGrafanaReceiver{ - UID: uid, - Name: c.Name, - Type: c.Type, - DisableResolveMessage: c.DisableResolveMessage, - Settings: settings, - SecureSettings: secureSettings, - }, nil -} - -// Create one receiver for every unique notification channel. -func (m *migration) createReceivers(allChannels []*notificationChannel) (map[uidOrID]*PostableApiReceiver, []channelReceiver, error) { - receivers := make([]channelReceiver, 0, len(allChannels)) - receiversMap := make(map[uidOrID]*PostableApiReceiver) - - set := make(map[string]struct{}) // Used to deduplicate sanitized names. - for _, c := range allChannels { - notifier, err := m.createNotifier(c) - if err != nil { - return nil, nil, err - } - - // We remove double quotes because this character will be used as the separator in the ContactLabel. To prevent partial matches in the Route Matcher we choose to sanitize them early on instead of complicating the Matcher regex. - sanitizedName := strings.ReplaceAll(c.Name, `"`, `_`) - // There can be name collisions after we sanitize. We check for this and attempt to make the name unique again using a short hash of the original name. - if _, ok := set[sanitizedName]; ok { - sanitizedName = sanitizedName + fmt.Sprintf("_%.3x", md5.Sum([]byte(c.Name))) - m.mg.Logger.Warn("Alert contains duplicate contact name after sanitization, appending unique suffix", "type", c.Type, "name", c.Name, "new_name", sanitizedName, "uid", c.Uid) - } - notifier.Name = sanitizedName - - set[sanitizedName] = struct{}{} - - cr := channelReceiver{ - channel: c, - receiver: &PostableApiReceiver{ - Name: sanitizedName, // Channel name is unique within an Org. - GrafanaManagedReceivers: []*PostableGrafanaReceiver{notifier}, - }, - } - - receivers = append(receivers, cr) - - // Store receivers for creating routes from alert rules later. - if c.Uid != "" { - receiversMap[c.Uid] = cr.receiver - } - if c.ID != 0 { - // In certain circumstances, the alert rule uses ID instead of uid. So, we add this to be able to lookup by ID in case. - receiversMap[c.ID] = cr.receiver - } - } - - return receiversMap, receivers, nil -} - -// Create the root-level route with the default receiver. If no new receiver is created specifically for the root-level route, the returned receiver will be nil. -func (m *migration) createDefaultRouteAndReceiver(defaultChannels []*notificationChannel) (*PostableApiReceiver, *Route, error) { - defaultReceiverName := "autogen-contact-point-default" - defaultRoute := &Route{ - Receiver: defaultReceiverName, - Routes: make([]*Route, 0), - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, // To keep parity with pre-migration notifications. - RepeatInterval: nil, - } - newDefaultReceiver := &PostableApiReceiver{ - Name: defaultReceiverName, - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - } - - // Return early if there are no default channels - if len(defaultChannels) == 0 { - return newDefaultReceiver, defaultRoute, nil - } - - repeatInterval := DisabledRepeatInterval // If no channels have SendReminders enabled, we will use this large value as a pseudo-disable. - if len(defaultChannels) > 1 { - // If there are more than one default channels we create a separate contact group that is used only in the root policy. This is to simplify the migrated notification policy structure. - // If we ever allow more than one receiver per route this won't be necessary. - for _, c := range defaultChannels { - // Need to create a new notifier to prevent uid conflict. - defaultNotifier, err := m.createNotifier(c) - if err != nil { - return nil, nil, err - } - - newDefaultReceiver.GrafanaManagedReceivers = append(newDefaultReceiver.GrafanaManagedReceivers, defaultNotifier) - - // Choose the lowest send reminder duration from all the notifiers to use for default route. - if c.SendReminder && c.Frequency < repeatInterval { - repeatInterval = c.Frequency - } - } - } else { - // If there is only a single default channel, we don't need a separate receiver to hold it. We can reuse the existing receiver for that single notifier. - defaultRoute.Receiver = defaultChannels[0].Name - if defaultChannels[0].SendReminder { - repeatInterval = defaultChannels[0].Frequency - } - - // No need to create a new receiver. - newDefaultReceiver = nil - } - defaultRoute.RepeatInterval = &repeatInterval - - return newDefaultReceiver, defaultRoute, nil -} - -// Create one route per contact point, matching based on ContactLabel. -func createRoute(cr channelReceiver) (*Route, error) { - // We create a regex matcher so that each alert rule need only have a single ContactLabel entry for all contact points it sends to. - // For example, if an alert needs to send to contact1 and contact2 it will have ContactLabel=`"contact1","contact2"` and will match both routes looking - // for `.*"contact1".*` and `.*"contact2".*`. - - // We quote and escape here to ensure the regex will correctly match the ContactLabel on the alerts. - name := fmt.Sprintf(`.*%s.*`, regexp.QuoteMeta(quote(cr.receiver.Name))) - mat, err := labels.NewMatcher(labels.MatchRegexp, ContactLabel, name) - if err != nil { - return nil, err - } - - repeatInterval := DisabledRepeatInterval - if cr.channel.SendReminder { - repeatInterval = cr.channel.Frequency - } - - return &Route{ - Receiver: cr.receiver.Name, - ObjectMatchers: ObjectMatchers{mat}, - Continue: true, // We continue so that each sibling contact point route can separately match. - RepeatInterval: &repeatInterval, - }, nil -} - -// Filter receivers to select those that were associated to the given rule as channels. -func (m *migration) filterReceiversForAlert(name string, channelIDs []uidOrID, receivers map[uidOrID]*PostableApiReceiver, defaultReceivers map[string]struct{}) map[string]any { - if len(channelIDs) == 0 { - // If there are no channels associated, we use the default route. - return nil - } - - // Filter receiver names. - filteredReceiverNames := make(map[string]any) - for _, uidOrId := range channelIDs { - recv, ok := receivers[uidOrId] - if ok { - filteredReceiverNames[recv.Name] = struct{}{} // Deduplicate on contact point name. - } else { - m.mg.Logger.Warn("Alert linked to obsolete notification channel, ignoring", "alert", name, "uid", uidOrId) - } - } - - coveredByDefault := func(names map[string]any) bool { - // Check if all receivers are also default ones and if so, just use the default route. - for n := range names { - if _, ok := defaultReceivers[n]; !ok { - return false - } - } - return true - } - - if len(filteredReceiverNames) == 0 || coveredByDefault(filteredReceiverNames) { - // Use the default route instead. - return nil - } - - // Add default receivers alongside rule-specific ones. - for n := range defaultReceivers { - filteredReceiverNames[n] = struct{}{} - } - - return filteredReceiverNames -} - -func (m *migration) determineChannelUid(c *notificationChannel) (string, error) { - legacyUid := c.Uid - if legacyUid == "" { - newUid, err := m.seenUIDs.generateUid() - if err != nil { - return "", err - } - m.mg.Logger.Info("Legacy notification had an empty uid, generating a new one", "id", c.ID, "uid", newUid) - return newUid, nil - } - - if m.seenUIDs.contains(legacyUid) { - newUid, err := m.seenUIDs.generateUid() - if err != nil { - return "", err - } - m.mg.Logger.Warn("Legacy notification had a UID that collides with a migrated record, generating a new one", "id", c.ID, "old", legacyUid, "new", newUid) - return newUid, nil - } - - return legacyUid, nil -} - -// Some settings were migrated from settings to secure settings in between. -// See https://grafana.com/docs/grafana/latest/installation/upgrading/#ensure-encryption-of-existing-alert-notification-channel-secrets. -// migrateSettingsToSecureSettings takes care of that. -func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json, secureSettings SecureJsonData) (*simplejson.Json, map[string]string, error) { - keys := []string{} - switch chanType { - case "slack": - keys = []string{"url", "token"} - case "pagerduty": - keys = []string{"integrationKey"} - case "webhook": - keys = []string{"password"} - case "prometheus-alertmanager": - keys = []string{"basicAuthPassword"} - case "opsgenie": - keys = []string{"apiKey"} - case "telegram": - keys = []string{"bottoken"} - case "line": - keys = []string{"token"} - case "pushover": - keys = []string{"apiToken", "userKey"} - case "threema": - keys = []string{"api_secret"} - } - - newSecureSettings := secureSettings.Decrypt() - cloneSettings := simplejson.New() - settingsMap, err := settings.Map() - if err != nil { - return nil, nil, err - } - for k, v := range settingsMap { - cloneSettings.Set(k, v) - } - for _, k := range keys { - if v, ok := newSecureSettings[k]; ok && v != "" { - continue - } - - sv := cloneSettings.Get(k).MustString() - if sv != "" { - newSecureSettings[k] = sv - cloneSettings.Del(k) - } - } - - encryptedData := GetEncryptedJsonData(newSecureSettings) - for k, v := range encryptedData { - newSecureSettings[k] = base64.StdEncoding.EncodeToString(v) - } - - return cloneSettings, newSecureSettings, nil -} - -// Below is a snapshot of all the config and supporting functions imported -// to avoid vendoring those packages. - -type PostableUserConfig struct { - TemplateFiles map[string]string `yaml:"template_files" json:"template_files"` - AlertmanagerConfig PostableApiAlertingConfig `yaml:"alertmanager_config" json:"alertmanager_config"` -} - -type amConfigsPerOrg = map[int64]*PostableUserConfig - -type PostableApiAlertingConfig struct { - Route *Route `yaml:"route,omitempty" json:"route,omitempty"` - Templates []string `yaml:"templates" json:"templates"` - Receivers []*PostableApiReceiver `yaml:"receivers,omitempty" json:"receivers,omitempty"` -} - -type Route struct { - Receiver string `yaml:"receiver,omitempty" json:"receiver,omitempty"` - ObjectMatchers ObjectMatchers `yaml:"object_matchers,omitempty" json:"object_matchers,omitempty"` - Routes []*Route `yaml:"routes,omitempty" json:"routes,omitempty"` - Continue bool `yaml:"continue,omitempty" json:"continue,omitempty"` - GroupByStr []string `yaml:"group_by,omitempty" json:"group_by,omitempty"` - RepeatInterval *model.Duration `yaml:"repeat_interval,omitempty" json:"repeat_interval,omitempty"` -} - -type ObjectMatchers labels.Matchers - -// MarshalJSON implements the json.Marshaler interface for Matchers. Vendored from definitions.ObjectMatchers. -func (m ObjectMatchers) MarshalJSON() ([]byte, error) { - if len(m) == 0 { - return nil, nil - } - result := make([][3]string, len(m)) - for i, matcher := range m { - result[i] = [3]string{matcher.Name, matcher.Type.String(), matcher.Value} - } - return json.Marshal(result) -} - -type PostableApiReceiver struct { - Name string `yaml:"name" json:"name"` - GrafanaManagedReceivers []*PostableGrafanaReceiver `yaml:"grafana_managed_receiver_configs,omitempty" json:"grafana_managed_receiver_configs,omitempty"` -} - -type PostableGrafanaReceiver CreateAlertNotificationCommand - -type CreateAlertNotificationCommand struct { - UID string `json:"uid"` - Name string `json:"name"` - Type string `json:"type"` - DisableResolveMessage bool `json:"disableResolveMessage"` - Settings *simplejson.Json `json:"settings"` - SecureSettings map[string]string `json:"secureSettings"` -} diff --git a/pkg/services/sqlstore/migrations/ualert/channel_test.go b/pkg/services/sqlstore/migrations/ualert/channel_test.go deleted file mode 100644 index a31405bd94b..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/channel_test.go +++ /dev/null @@ -1,471 +0,0 @@ -package ualert - -import ( - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/prometheus/alertmanager/pkg/labels" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/components/simplejson" - ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" -) - -func TestFilterReceiversForAlert(t *testing.T) { - tc := []struct { - name string - channelIds []uidOrID - receivers map[uidOrID]*PostableApiReceiver - defaultReceivers map[string]struct{} - expected map[string]any - }{ - { - name: "when an alert has multiple channels, each should filter for the correct receiver", - channelIds: []uidOrID{"uid1", "uid2"}, - receivers: map[uidOrID]*PostableApiReceiver{ - "uid1": { - Name: "recv1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - }, - "uid2": { - Name: "recv2", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - }, - "uid3": { - Name: "recv3", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - }, - }, - defaultReceivers: map[string]struct{}{}, - expected: map[string]any{ - "recv1": struct{}{}, - "recv2": struct{}{}, - }, - }, - { - name: "when default receivers exist, they should be added to an alert's filtered receivers", - channelIds: []uidOrID{"uid1"}, - receivers: map[uidOrID]*PostableApiReceiver{ - "uid1": { - Name: "recv1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - }, - "uid2": { - Name: "recv2", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - }, - "uid3": { - Name: "recv3", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - }, - }, - defaultReceivers: map[string]struct{}{ - "recv2": {}, - }, - expected: map[string]any{ - "recv1": struct{}{}, // From alert - "recv2": struct{}{}, // From default - }, - }, - { - name: "when an alert has a channels associated by ID instead of UID, it should be included", - channelIds: []uidOrID{int64(42)}, - receivers: map[uidOrID]*PostableApiReceiver{ - int64(42): { - Name: "recv1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - }, - }, - defaultReceivers: map[string]struct{}{}, - expected: map[string]any{ - "recv1": struct{}{}, - }, - }, - { - name: "when an alert's receivers are covered by the defaults, return nil to use default receiver downstream", - channelIds: []uidOrID{"uid1"}, - receivers: map[uidOrID]*PostableApiReceiver{ - "uid1": { - Name: "recv1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - }, - "uid2": { - Name: "recv2", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - }, - "uid3": { - Name: "recv3", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - }, - }, - defaultReceivers: map[string]struct{}{ - "recv1": {}, - "recv2": {}, - }, - expected: nil, // recv1 is already a default - }, - } - - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - m := newTestMigration(t) - res := m.filterReceiversForAlert("", tt.channelIds, tt.receivers, tt.defaultReceivers) - - require.Equal(t, tt.expected, res) - }) - } -} - -func TestCreateRoute(t *testing.T) { - tc := []struct { - name string - channel *notificationChannel - recv *PostableApiReceiver - expected *Route - }{ - { - name: "when a receiver is passed in, the route should regex match based on quoted name with continue=true", - channel: ¬ificationChannel{}, - recv: &PostableApiReceiver{ - Name: "recv1", - }, - expected: &Route{ - Receiver: "recv1", - ObjectMatchers: ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, - Routes: nil, - Continue: true, - GroupByStr: nil, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }, - }, - { - name: "notification channel should be escaped for regex in the matcher", - channel: ¬ificationChannel{}, - recv: &PostableApiReceiver{ - Name: `. ^ $ * + - ? ( ) [ ] { } \ |`, - }, - expected: &Route{ - Receiver: `. ^ $ * + - ? ( ) [ ] { } \ |`, - ObjectMatchers: ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"\. \^ \$ \* \+ - \? \( \) \[ \] \{ \} \\ \|".*`}}, - Routes: nil, - Continue: true, - GroupByStr: nil, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }, - }, - { - name: "when a channel has sendReminder=true, the route should use the frequency in repeat interval", - channel: ¬ificationChannel{SendReminder: true, Frequency: model.Duration(time.Duration(42) * time.Hour)}, - recv: &PostableApiReceiver{ - Name: "recv1", - }, - expected: &Route{ - Receiver: "recv1", - ObjectMatchers: ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, - Routes: nil, - Continue: true, - GroupByStr: nil, - RepeatInterval: durationPointer(model.Duration(time.Duration(42) * time.Hour)), - }, - }, - { - name: "when a channel has sendReminder=false, the route should ignore the frequency in repeat interval and use DisabledRepeatInterval", - channel: ¬ificationChannel{SendReminder: false, Frequency: model.Duration(time.Duration(42) * time.Hour)}, - recv: &PostableApiReceiver{ - Name: "recv1", - }, - expected: &Route{ - Receiver: "recv1", - ObjectMatchers: ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, - Routes: nil, - Continue: true, - GroupByStr: nil, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }, - }, - } - - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - res, err := createRoute(channelReceiver{ - channel: tt.channel, - receiver: tt.recv, - }) - require.NoError(t, err) - - // Order of nested routes is not guaranteed. - cOpt := []cmp.Option{ - cmpopts.SortSlices(func(a, b *Route) bool { - if a.Receiver != b.Receiver { - return a.Receiver < b.Receiver - } - return a.ObjectMatchers[0].Value < b.ObjectMatchers[0].Value - }), - cmpopts.IgnoreUnexported(Route{}, labels.Matcher{}), - } - - if !cmp.Equal(tt.expected, res, cOpt...) { - t.Errorf("Unexpected Route: %v", cmp.Diff(tt.expected, res, cOpt...)) - } - }) - } -} - -func createNotChannel(t *testing.T, uid string, id int64, name string) *notificationChannel { - t.Helper() - return ¬ificationChannel{Uid: uid, ID: id, Name: name, Settings: simplejson.New()} -} - -func createNotChannelWithReminder(t *testing.T, uid string, id int64, name string, frequency model.Duration) *notificationChannel { - t.Helper() - return ¬ificationChannel{Uid: uid, ID: id, Name: name, SendReminder: true, Frequency: frequency, Settings: simplejson.New()} -} - -func TestCreateReceivers(t *testing.T) { - tc := []struct { - name string - allChannels []*notificationChannel - defaultChannels []*notificationChannel - expRecvMap map[uidOrID]*PostableApiReceiver - expRecv []channelReceiver - expErr error - }{ - { - name: "when given notification channels migrate them to receivers", - allChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name1"), createNotChannel(t, "uid2", int64(2), "name2")}, - expRecvMap: map[uidOrID]*PostableApiReceiver{ - "uid1": { - Name: "name1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}}, - }, - "uid2": { - Name: "name2", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name2"}}, - }, - int64(1): { - Name: "name1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}}, - }, - int64(2): { - Name: "name2", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name2"}}, - }, - }, - expRecv: []channelReceiver{ - { - channel: createNotChannel(t, "uid1", int64(1), "name1"), - receiver: &PostableApiReceiver{ - Name: "name1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}}, - }, - }, - { - channel: createNotChannel(t, "uid2", int64(2), "name2"), - receiver: &PostableApiReceiver{ - Name: "name2", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name2"}}, - }, - }, - }, - }, - { - name: "when given notification channel contains double quote sanitize with underscore", - allChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name\"1")}, - expRecvMap: map[uidOrID]*PostableApiReceiver{ - "uid1": { - Name: "name_1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, - }, - int64(1): { - Name: "name_1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, - }, - }, - expRecv: []channelReceiver{ - { - channel: createNotChannel(t, "uid1", int64(1), "name\"1"), - receiver: &PostableApiReceiver{ - Name: "name_1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, - }, - }, - }, - }, - { - name: "when given notification channels collide after sanitization add short hash to end", - allChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name\"1"), createNotChannel(t, "uid2", int64(2), "name_1")}, - expRecvMap: map[uidOrID]*PostableApiReceiver{ - "uid1": { - Name: "name_1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, - }, - "uid2": { - Name: "name_1_dba13d", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1_dba13d"}}, - }, - int64(1): { - Name: "name_1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, - }, - int64(2): { - Name: "name_1_dba13d", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1_dba13d"}}, - }, - }, - expRecv: []channelReceiver{ - { - channel: createNotChannel(t, "uid1", int64(1), "name\"1"), - receiver: &PostableApiReceiver{ - Name: "name_1", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, - }, - }, - { - channel: createNotChannel(t, "uid2", int64(2), "name_1"), - receiver: &PostableApiReceiver{ - Name: "name_1_dba13d", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1_dba13d"}}, - }, - }, - }, - }, - } - - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - m := newTestMigration(t) - recvMap, recvs, err := m.createReceivers(tt.allChannels) - if tt.expErr != nil { - require.Error(t, err) - require.EqualError(t, err, tt.expErr.Error()) - return - } - - require.NoError(t, err) - - // We ignore certain fields for the purposes of this test - for _, recv := range recvs { - for _, not := range recv.receiver.GrafanaManagedReceivers { - not.UID = "" - not.Settings = nil - not.SecureSettings = nil - } - } - - require.Equal(t, tt.expRecvMap, recvMap) - require.ElementsMatch(t, tt.expRecv, recvs) - }) - } -} - -func TestCreateDefaultRouteAndReceiver(t *testing.T) { - tc := []struct { - name string - amConfig *PostableUserConfig - defaultChannels []*notificationChannel - expRecv *PostableApiReceiver - expRoute *Route - expErr error - }{ - { - name: "when given multiple default notification channels migrate them to a single receiver", - defaultChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name1"), createNotChannel(t, "uid2", int64(2), "name2")}, - expRecv: &PostableApiReceiver{ - Name: "autogen-contact-point-default", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}, {Name: "name2"}}, - }, - expRoute: &Route{ - Receiver: "autogen-contact-point-default", - Routes: make([]*Route, 0), - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }, - }, - { - name: "when given multiple default notification channels migrate them to a single receiver with RepeatInterval set to be the minimum of all channel frequencies", - defaultChannels: []*notificationChannel{ - createNotChannelWithReminder(t, "uid1", int64(1), "name1", model.Duration(42)), - createNotChannelWithReminder(t, "uid2", int64(2), "name2", model.Duration(100000)), - }, - expRecv: &PostableApiReceiver{ - Name: "autogen-contact-point-default", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}, {Name: "name2"}}, - }, - expRoute: &Route{ - Receiver: "autogen-contact-point-default", - Routes: make([]*Route, 0), - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - RepeatInterval: durationPointer(model.Duration(42)), - }, - }, - { - name: "when given no default notification channels create a single empty receiver for default", - defaultChannels: []*notificationChannel{}, - expRecv: &PostableApiReceiver{ - Name: "autogen-contact-point-default", - GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, - }, - expRoute: &Route{ - Receiver: "autogen-contact-point-default", - Routes: make([]*Route, 0), - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - RepeatInterval: nil, - }, - }, - { - name: "when given a single default notification channels don't create a new default receiver", - defaultChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name1")}, - expRecv: nil, - expRoute: &Route{ - Receiver: "name1", - Routes: make([]*Route, 0), - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }, - }, - { - name: "when given a single default notification channel with SendReminder=true, use the channels Frequency as the RepeatInterval", - defaultChannels: []*notificationChannel{createNotChannelWithReminder(t, "uid1", int64(1), "name1", model.Duration(42))}, - expRecv: nil, - expRoute: &Route{ - Receiver: "name1", - Routes: make([]*Route, 0), - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - RepeatInterval: durationPointer(model.Duration(42)), - }, - }, - } - - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - m := newTestMigration(t) - recv, route, err := m.createDefaultRouteAndReceiver(tt.defaultChannels) - if tt.expErr != nil { - require.Error(t, err) - require.EqualError(t, err, tt.expErr.Error()) - return - } - - require.NoError(t, err) - - // We ignore certain fields for the purposes of this test - if recv != nil { - for _, not := range recv.GrafanaManagedReceivers { - not.UID = "" - not.Settings = nil - not.SecureSettings = nil - } - } - - require.Equal(t, tt.expRecv, recv) - require.Equal(t, tt.expRoute, route) - }) - } -} - -func durationPointer(d model.Duration) *model.Duration { - return &d -} diff --git a/pkg/services/sqlstore/migrations/ualert/created_folders_mig.go b/pkg/services/sqlstore/migrations/ualert/created_folders_mig.go new file mode 100644 index 00000000000..1d4f1da80c0 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/created_folders_mig.go @@ -0,0 +1,83 @@ +package ualert + +import ( + "encoding/json" + "fmt" + "time" + + "xorm.io/xorm" + + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" +) + +// stateKey is a vendored migrationStore.stateKey. +var stateKey = "stateKey" + +// CreatedFoldersMigration moves the record of created folders during legacy migration from Dashboard created_by=-8 +// to the kvstore. If there are no dashboards with created_by=-.8, then nothing needs to be done. +func CreatedFoldersMigration(mg *migrator.Migrator) { + mg.AddMigration("migrate record of created folders during legacy migration to kvstore", &createdFoldersToKVStore{}) +} + +type createdFoldersToKVStore struct { + migrator.MigrationBase +} + +func (c createdFoldersToKVStore) SQL(migrator.Dialect) string { + return codeMigration +} + +func (c createdFoldersToKVStore) Exec(sess *xorm.Session, mg *migrator.Migrator) error { + var results []struct { + UID string `xorm:"uid"` + OrgID int64 `xorm:"org_id"` + } + folderCreatedBy := -8 + if err := sess.SQL("select * from dashboard where created_by = ?", folderCreatedBy).Find(&results); err != nil { + return err + } + + if len(results) == 0 { + mg.Logger.Debug("no dashboards with created_by=-8, nothing to set in kvstore") + return nil + } + + type orgMigrationState struct { + OrgID int64 `json:"orgId"` + CreatedFolders []string `json:"createdFolders"` + } + states := make(map[int64]*orgMigrationState) + for _, r := range results { + if _, ok := states[r.OrgID]; !ok { + states[r.OrgID] = &orgMigrationState{ + OrgID: r.OrgID, + CreatedFolders: []string{}, + } + } + states[r.OrgID].CreatedFolders = append(states[r.OrgID].CreatedFolders, r.UID) + } + + now := time.Now() + for _, state := range states { + raw, err := json.Marshal(state) + if err != nil { + return err + } + + orgId := state.OrgID + entry := kvStoreV1Entry{ + OrgID: &orgId, + Namespace: &KVNamespace, + Key: &stateKey, + Value: string(raw), + Created: now, + Updated: now, + } + if _, errCreate := sess.Table("kv_store").Insert(&entry); errCreate != nil { + mg.Logger.Error("failed to insert record of created folders to kvstore", "err", errCreate) + return fmt.Errorf("failed to insert record of created folders to kvstore: %w", errCreate) + } + } + + return nil +} diff --git a/pkg/services/sqlstore/migrations/ualert/dash_alert.go b/pkg/services/sqlstore/migrations/ualert/dash_alert.go deleted file mode 100644 index cf385464999..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/dash_alert.go +++ /dev/null @@ -1,130 +0,0 @@ -package ualert - -import ( - "encoding/json" - "fmt" - "time" -) - -type dashAlert struct { - Id int64 - OrgId int64 - DashboardId int64 - PanelId int64 - Name string - Message string - Frequency int64 - For time.Duration - State string - - Settings json.RawMessage - ParsedSettings *dashAlertSettings - DashboardUID string // Set from separate call -} - -var slurpDashSQL = ` -SELECT id, - org_id, - dashboard_id, - panel_id, - org_id, - name, - message, - frequency, - %s, - state, - settings -FROM - alert -WHERE org_id IN (SELECT id from org) - AND dashboard_id IN (SELECT id from dashboard) -` - -// slurpDashAlerts loads all alerts from the alert database table into -// the dashAlert type. If there are alerts that belong to either organization or dashboard that does not exist, those alerts will not be returned/ -// Additionally it unmarshals the json settings for the alert into the -// ParsedSettings property of the dash alert. -func (m *migration) slurpDashAlerts() ([]dashAlert, error) { - dashAlerts := []dashAlert{} - err := m.sess.SQL(fmt.Sprintf(slurpDashSQL, m.mg.Dialect.Quote("for"))).Find(&dashAlerts) - - if err != nil { - return nil, err - } - - for i := range dashAlerts { - err = json.Unmarshal(dashAlerts[i].Settings, &dashAlerts[i].ParsedSettings) - if err != nil { - da := dashAlerts[i] - return nil, fmt.Errorf("failed to parse alert rule ID:%d, name:'%s', orgID:%d: %w", da.Id, da.Name, da.OrgId, err) - } - } - - return dashAlerts, nil -} - -// dashAlertSettings is a type for the JSON that is in the settings field of -// the alert table. -type dashAlertSettings struct { - NoDataState string `json:"noDataState"` - ExecutionErrorState string `json:"executionErrorState"` - Conditions []dashAlertCondition `json:"conditions"` - AlertRuleTags any `json:"alertRuleTags"` - Notifications []dashAlertNot `json:"notifications"` -} - -// dashAlertNot is the object that represents the Notifications array in -// dashAlertSettings -type dashAlertNot struct { - UID string `json:"uid,omitempty"` - ID int64 `json:"id,omitempty"` -} - -// dashAlertingConditionJSON is like classic.ClassicConditionJSON except that it -// includes the model property with the query. -type dashAlertCondition struct { - Evaluator conditionEvalJSON `json:"evaluator"` - - Operator struct { - Type string `json:"type"` - } `json:"operator"` - - Query struct { - Params []string `json:"params"` - DatasourceID int64 `json:"datasourceId"` - Model json.RawMessage - } `json:"query"` - - Reducer struct { - // Params []any `json:"params"` (Unused) - Type string `json:"type"` - } -} - -type conditionEvalJSON struct { - Params []float64 `json:"params"` - Type string `json:"type"` // e.g. "gt" -} - -// slurpDashUIDs returns a map of [orgID, dashboardId] -> dashUID. -func (m *migration) slurpDashUIDs() (map[[2]int64]string, error) { - dashIDs := []struct { - OrgID int64 `xorm:"org_id"` - ID int64 `xorm:"id"` - UID string `xorm:"uid"` - }{} - - err := m.sess.SQL(`SELECT org_id, id, uid FROM dashboard`).Find(&dashIDs) - - if err != nil { - return nil, err - } - - idToUID := make(map[[2]int64]string, len(dashIDs)) - - for _, ds := range dashIDs { - idToUID[[2]int64{ds.OrgID, ds.ID}] = ds.UID - } - - return idToUID, nil -} diff --git a/pkg/services/sqlstore/migrations/ualert/dashboard.go b/pkg/services/sqlstore/migrations/ualert/dashboard.go deleted file mode 100644 index d7440f60b02..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/dashboard.go +++ /dev/null @@ -1,108 +0,0 @@ -package ualert - -import ( - "time" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/infra/slugify" -) - -type dashboard struct { - Id int64 - Uid string - Slug string - OrgId int64 - GnetId int64 - Version int - PluginId string - - Created time.Time - Updated time.Time - - UpdatedBy int64 - CreatedBy int64 - FolderId int64 - IsFolder bool - HasACL bool `xorm:"has_acl"` - - Title string - Data *simplejson.Json -} - -func (d *dashboard) setUid(uid string) { - d.Uid = uid - d.Data.Set("uid", uid) -} - -func (d *dashboard) setVersion(version int) { - d.Version = version - d.Data.Set("version", version) -} - -// UpdateSlug updates the slug -func (d *dashboard) updateSlug() { - title := d.Data.Get("title").MustString() - d.Slug = slugify.Slugify(title) -} - -func newDashboardFromJson(data *simplejson.Json) *dashboard { - dash := &dashboard{} - dash.Data = data - dash.Title = dash.Data.Get("title").MustString() - dash.updateSlug() - update := false - - if id, err := dash.Data.Get("id").Float64(); err == nil { - dash.Id = int64(id) - update = true - } - - if uid, err := dash.Data.Get("uid").String(); err == nil { - dash.Uid = uid - update = true - } - - if version, err := dash.Data.Get("version").Float64(); err == nil && update { - dash.Version = int(version) - dash.Updated = time.Now() - } else { - dash.Data.Set("version", 0) - dash.Created = time.Now() - dash.Updated = time.Now() - } - - if gnetId, err := dash.Data.Get("gnetId").Float64(); err == nil { - dash.GnetId = int64(gnetId) - } - - return dash -} - -type saveFolderCommand struct { - Dashboard *simplejson.Json `json:"dashboard" binding:"Required"` - UserId int64 `json:"userId"` - Message string `json:"message"` - OrgId int64 `json:"-"` - RestoredFrom int `json:"-"` - PluginId string `json:"-"` - FolderId int64 `json:"folderId"` - IsFolder bool `json:"isFolder"` -} - -// GetDashboardModel turns the command into the saveable model -func (cmd *saveFolderCommand) getDashboardModel() *dashboard { - dash := newDashboardFromJson(cmd.Dashboard) - userId := cmd.UserId - - if userId == 0 { - userId = -1 - } - - dash.UpdatedBy = userId - dash.OrgId = cmd.OrgId - dash.PluginId = cmd.PluginId - dash.IsFolder = cmd.IsFolder - dash.FolderId = cmd.FolderId - dash.updateSlug() - return dash -} diff --git a/pkg/services/sqlstore/migrations/ualert/datasource.go b/pkg/services/sqlstore/migrations/ualert/datasource.go deleted file mode 100644 index 03027a4d747..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/datasource.go +++ /dev/null @@ -1,31 +0,0 @@ -package ualert - -type dsUIDLookup map[[2]int64]string - -// GetUID fetch thes datasource UID based on orgID+datasourceID -func (d dsUIDLookup) GetUID(orgID, datasourceID int64) string { - return d[[2]int64{orgID, datasourceID}] -} - -// slurpDSIDs returns a map of [orgID, dataSourceId] -> UID. -func (m *migration) slurpDSIDs() (dsUIDLookup, error) { - dsIDs := []struct { - OrgID int64 `xorm:"org_id"` - ID int64 `xorm:"id"` - UID string `xorm:"uid"` - }{} - - err := m.sess.SQL(`SELECT org_id, id, uid FROM data_source`).Find(&dsIDs) - - if err != nil { - return nil, err - } - - idToUID := make(dsUIDLookup, len(dsIDs)) - - for _, ds := range dsIDs { - idToUID[[2]int64{ds.OrgID, ds.ID}] = ds.UID - } - - return idToUID, nil -} diff --git a/pkg/services/sqlstore/migrations/ualert/migration_service_migration.go b/pkg/services/sqlstore/migrations/ualert/migration_service_migration.go new file mode 100644 index 00000000000..ad491b794fc --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/migration_service_migration.go @@ -0,0 +1,69 @@ +package ualert + +import ( + "fmt" + "os" + "strconv" + "time" + + "xorm.io/xorm" + + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" +) + +// KVNamespace is a vendored migration.KVNamespace. +var KVNamespace = "ngalert.migration" + +// migratedKey is a vendored migration.migratedKey. +var migratedKey = "migrated" + +// MigrationServiceMigration moves the legacy alert migration status from the migration log to kvstore. +func MigrationServiceMigration(mg *migrator.Migrator) { + mg.AddMigration("set legacy alert migration status in kvstore", &migrationLogToKVStore{}) +} + +type migrationLogToKVStore struct { + migrator.MigrationBase +} + +func (c migrationLogToKVStore) SQL(migrator.Dialect) string { + return codeMigration +} + +func (c migrationLogToKVStore) Exec(sess *xorm.Session, mg *migrator.Migrator) error { + logs, err := mg.GetMigrationLog() + if err != nil { + mg.Logger.Error("alert migration failure: could not get migration log", "error", err) + os.Exit(1) + } + + _, migrationRun := logs[migTitle] + + var anyOrg int64 = 0 + now := time.Now() + entry := kvStoreV1Entry{ + OrgID: &anyOrg, + Namespace: &KVNamespace, + Key: &migratedKey, + Value: strconv.FormatBool(migrationRun), + Created: now, + Updated: now, + } + if _, errCreate := sess.Table("kv_store").Insert(&entry); errCreate != nil { + mg.Logger.Error("failed to insert migration status to kvstore", "err", errCreate) + return fmt.Errorf("failed to insert migration status to kvstore: %w", errCreate) + } + return nil +} + +// kvStoreV1Entry is a vendored kvstore.Item. +type kvStoreV1Entry struct { + ID int64 `xorm:"pk autoincr 'id'"` + OrgID *int64 `xorm:"org_id"` + Namespace *string `xorm:"namespace"` + Key *string `xorm:"key"` + Value string `xorm:"value"` + + Created time.Time `xorm:"created"` + Updated time.Time `xorm:"updated"` +} diff --git a/pkg/services/sqlstore/migrations/ualert/migration_test.go b/pkg/services/sqlstore/migrations/ualert/migration_test.go deleted file mode 100644 index 0c76f6321bf..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/migration_test.go +++ /dev/null @@ -1,847 +0,0 @@ -package ualert_test - -import ( - "encoding/json" - "fmt" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/prometheus/alertmanager/pkg/labels" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - "gopkg.in/ini.v1" - "xorm.io/xorm" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/services/alerting/models" - "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/datasources" - ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/services/org" - "github.com/grafana/grafana/pkg/services/sqlstore/migrations" - "github.com/grafana/grafana/pkg/services/sqlstore/migrations/ualert" - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" - "github.com/grafana/grafana/pkg/services/sqlstore/sqlutil" - "github.com/grafana/grafana/pkg/setting" -) - -// TestAddDashAlertMigration tests the AddDashAlertMigration wrapper method that decides when to run the migration based on migration status and settings. -func TestAddDashAlertMigration(t *testing.T) { - x := setupTestDB(t) - - tc := []struct { - name string - config *setting.Cfg - isMigrationRun bool - shouldPanic bool - expected []string // set of migration titles - }{ - { - name: "when unified alerting enabled and migration not already run, then add main migration and clear rmMigration log entry", - config: &setting.Cfg{ - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: boolPointer(true), - }, - }, - isMigrationRun: false, - expected: []string{fmt.Sprintf(ualert.ClearMigrationEntryTitle, ualert.RmMigTitle), ualert.MigTitle}, - }, - { - name: "when unified alerting disabled and migration is already run, then add rmMigration and clear main migration log entry", - config: &setting.Cfg{ - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: boolPointer(false), - }, - ForceMigration: true, - }, - isMigrationRun: true, - expected: []string{fmt.Sprintf(ualert.ClearMigrationEntryTitle, ualert.MigTitle), ualert.RmMigTitle}, - }, - { - name: "when unified alerting disabled, migration is already run and force migration is disabled, then the migration should panic", - config: &setting.Cfg{ - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: boolPointer(false), - }, - ForceMigration: false, - }, - isMigrationRun: true, - expected: []string{fmt.Sprintf(ualert.ClearMigrationEntryTitle, ualert.MigTitle), ualert.RmMigTitle}, - }, - { - name: "when unified alerting enabled and migration is already run, then do nothing", - config: &setting.Cfg{ - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: boolPointer(true), - }, - }, - isMigrationRun: true, - expected: []string{}, - }, - { - name: "when unified alerting disabled and migration is not already run, then do nothing", - config: &setting.Cfg{ - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: boolPointer(false), - }, - }, - isMigrationRun: false, - expected: []string{}, - }, - } - - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - defer func() { - // if the code should panic, make sure it has - if r := recover(); r == nil && tt.shouldPanic { - t.Errorf("The code did not panic") - } - }() - if tt.isMigrationRun { - log := migrator.MigrationLog{ - MigrationID: ualert.MigTitle, - SQL: "", - Timestamp: time.Now(), - Success: true, - } - _, err := x.Insert(log) - require.NoError(t, err) - } else { - _, err := x.Exec("DELETE FROM migration_log WHERE migration_id = ?", ualert.MigTitle) - require.NoError(t, err) - } - - mg := migrator.NewMigrator(x, tt.config) - - ualert.AddDashAlertMigration(mg) - require.Equal(t, tt.expected, mg.GetMigrationIDs(false)) - }) - } -} - -// TestAMConfigMigration tests the execution of the main DashAlertMigration specifically for migrations of channels and routes. -func TestAMConfigMigration(t *testing.T) { - // Run initial migration to have a working DB. - x := setupTestDB(t) - - tc := []struct { - name string - legacyChannels []*models.AlertNotification - alerts []*models.Alert - - expected map[int64]*ualert.PostableUserConfig - expErr error - }{ - { - name: "general multi-org, multi-alert, multi-channel migration", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), - createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, false), - createAlertNotification(t, int64(2), "notifier4", "email", emailSettings, false), - createAlertNotification(t, int64(2), "notifier5", "slack", slackSettings, false), - createAlertNotification(t, int64(2), "notifier6", "opsgenie", opsgenieSettings, true), // default - }, - alerts: []*models.Alert{ - createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1"}), - createAlert(t, int64(1), int64(1), int64(2), "alert2", []string{"notifier2", "notifier3"}), - createAlert(t, int64(1), int64(2), int64(3), "alert3", []string{"notifier3"}), - createAlert(t, int64(2), int64(3), int64(1), "alert4", []string{"notifier4"}), - createAlert(t, int64(2), int64(3), int64(2), "alert5", []string{"notifier4", "notifier5", "notifier6"}), - createAlert(t, int64(2), int64(4), int64(3), "alert6", []string{}), - }, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - {Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - {Receiver: "notifier3", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier3".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - }, - RepeatInterval: nil, - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, - {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, - {Name: "notifier3", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}, - {Name: "autogen-contact-point-default"}, // empty default - }, - }, - }, - int64(2): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "notifier6", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier4", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier4".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - {Receiver: "notifier5", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier5".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - {Receiver: "notifier6", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier6".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - }, - RepeatInterval: durationPointer(ualert.DisabledRepeatInterval), - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier4", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier4", Type: "email"}}}, - {Name: "notifier5", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier5", Type: "slack"}}}, - {Name: "notifier6", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier6", Type: "opsgenie"}}}, - }, - }, - }, - }, - }, - { - name: "when no default channel, create empty autogen-contact-point-default", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - }, - alerts: []*models.Alert{}, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - }, - RepeatInterval: nil, - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, - {Name: "autogen-contact-point-default"}, - }, - }, - }, - }, - }, - { - name: "when single default channel, don't create autogen-contact-point-default", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), - }, - alerts: []*models.Alert{}, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "notifier1", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - }, - RepeatInterval: durationPointer(ualert.DisabledRepeatInterval), - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, - }, - }, - }, - }, - }, - { - name: "when single default channel with SendReminder, use channel Frequency as RepeatInterval", - legacyChannels: []*models.AlertNotification{ - createAlertNotificationWithReminder(t, int64(1), "notifier1", "email", emailSettings, true, true, time.Duration(1)*time.Hour), - }, - alerts: []*models.Alert{}, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "notifier1", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour))}, - }, - RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour)), - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, - }, - }, - }, - }, - }, - { - name: "when multiple default channels, add them to autogen-contact-point-default as well", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), - createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, true), - }, - alerts: []*models.Alert{}, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - {Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - }, - RepeatInterval: durationPointer(ualert.DisabledRepeatInterval), - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, - {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, - {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}, - }, - }, - }, - }, - }, - { - name: "when multiple default channels with SendReminder, use minimum channel frequency as RepeatInterval", - legacyChannels: []*models.AlertNotification{ - createAlertNotificationWithReminder(t, int64(1), "notifier1", "email", emailSettings, true, true, time.Duration(1)*time.Hour), - createAlertNotificationWithReminder(t, int64(1), "notifier2", "slack", slackSettings, true, true, time.Duration(30)*time.Minute), - }, - alerts: []*models.Alert{}, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour))}, - {Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(30) * time.Minute))}, - }, - RepeatInterval: durationPointer(model.Duration(time.Duration(30) * time.Minute)), - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, - {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, - {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}, - }, - }, - }, - }, - }, - { - name: "when default channels exist alongside non-default, add only defaults to autogen-contact-point-default", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), // default - createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), - createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, true), // default - }, - alerts: []*models.Alert{}, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - {Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - {Receiver: "notifier3", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier3".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - }, - RepeatInterval: durationPointer(ualert.DisabledRepeatInterval), - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, - {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, - {Name: "notifier3", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}, - {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier3", Type: "opsgenie"}}}}, - }, - }, - }, - }, - { - name: "when alerts share channels, only create one receiver per legacy channel", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), - }, - alerts: []*models.Alert{ - createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1"}), - createAlert(t, int64(1), int64(1), int64(1), "alert2", []string{"notifier1", "notifier2"}), - }, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - {Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - }, - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, - {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, - {Name: "autogen-contact-point-default"}, - }, - }, - }, - }, - }, - { - name: "when channel not linked to any alerts, still create a receiver for it", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - }, - alerts: []*models.Alert{}, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - }, - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, - {Name: "autogen-contact-point-default"}, - }, - }, - }, - }, - }, - { - name: "when unsupported channels, do not migrate them", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - createAlertNotification(t, int64(1), "notifier2", "hipchat", "", false), - createAlertNotification(t, int64(1), "notifier3", "sensu", "", false), - }, - alerts: []*models.Alert{}, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - }, - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, - {Name: "autogen-contact-point-default"}, - }, - }, - }, - }, - }, - { - name: "when unsupported channel linked to alert, do not migrate only that channel", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - createAlertNotification(t, int64(1), "notifier2", "sensu", "", false), - }, - alerts: []*models.Alert{ - createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1", "notifier2"}), - }, - expected: map[int64]*ualert.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: ualert.PostableApiAlertingConfig{ - Route: &ualert.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*ualert.Route{ - {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, - }, - }, - Receivers: []*ualert.PostableApiReceiver{ - {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, - {Name: "autogen-contact-point-default"}, - }, - }, - }, - }, - }, - } - - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - defer teardown(t, x) - setupLegacyAlertsTables(t, x, tt.legacyChannels, tt.alerts) - runDashAlertMigrationTestRun(t, x) - - for orgId := range tt.expected { - amConfig := getAlertmanagerConfig(t, x, orgId) - - // Order of nested GrafanaManagedReceivers is not guaranteed. - cOpt := []cmp.Option{ - cmpopts.IgnoreUnexported(ualert.PostableApiReceiver{}), - cmpopts.IgnoreFields(ualert.PostableGrafanaReceiver{}, "UID", "Settings", "SecureSettings"), - cmpopts.SortSlices(func(a, b *ualert.PostableGrafanaReceiver) bool { return a.Name < b.Name }), - cmpopts.SortSlices(func(a, b *ualert.PostableApiReceiver) bool { return a.Name < b.Name }), - } - if !cmp.Equal(tt.expected[orgId].AlertmanagerConfig.Receivers, amConfig.AlertmanagerConfig.Receivers, cOpt...) { - t.Errorf("Unexpected Receivers: %v", cmp.Diff(tt.expected[orgId].AlertmanagerConfig.Receivers, amConfig.AlertmanagerConfig.Receivers, cOpt...)) - } - - // Order of routes is not guaranteed. - cOpt = []cmp.Option{ - cmpopts.SortSlices(func(a, b *ualert.Route) bool { - if a.Receiver != b.Receiver { - return a.Receiver < b.Receiver - } - return a.ObjectMatchers[0].Value < b.ObjectMatchers[0].Value - }), - cmpopts.IgnoreUnexported(ualert.Route{}, labels.Matcher{}), - } - if !cmp.Equal(tt.expected[orgId].AlertmanagerConfig.Route, amConfig.AlertmanagerConfig.Route, cOpt...) { - t.Errorf("Unexpected Route: %v", cmp.Diff(tt.expected[orgId].AlertmanagerConfig.Route, amConfig.AlertmanagerConfig.Route, cOpt...)) - } - } - }) - } -} - -// TestDashAlertMigration tests the execution of the main DashAlertMigration specifically for migrations of models. -func TestDashAlertMigration(t *testing.T) { - // Run initial migration to have a working DB. - x := setupTestDB(t) - - t.Run("when DashAlertMigration create ContactLabel on migrated AlertRules", func(t *testing.T) { - defer teardown(t, x) - legacyChannels := []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), - createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, false), - createAlertNotification(t, int64(2), "notifier4", "email", emailSettings, false), - createAlertNotification(t, int64(2), "notifier5", "slack", slackSettings, false), - createAlertNotification(t, int64(2), "notifier6", "opsgenie", opsgenieSettings, true), // default - } - alerts := []*models.Alert{ - createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1"}), - createAlert(t, int64(1), int64(1), int64(2), "alert2", []string{"notifier2", "notifier3"}), - createAlert(t, int64(1), int64(2), int64(3), "alert3", []string{"notifier3"}), - createAlert(t, int64(2), int64(3), int64(1), "alert4", []string{"notifier4"}), - createAlert(t, int64(2), int64(3), int64(2), "alert5", []string{"notifier4", "notifier5", "notifier6"}), - createAlert(t, int64(2), int64(4), int64(3), "alert6", []string{}), - } - expected := map[int64]map[string]*ngModels.AlertRule{ - int64(1): { - "alert1": {Labels: map[string]string{ualert.ContactLabel: `"notifier1"`}}, - "alert2": {Labels: map[string]string{ualert.ContactLabel: `"notifier2","notifier3"`}}, - "alert3": {Labels: map[string]string{ualert.ContactLabel: `"notifier3"`}}, - }, - int64(2): { - "alert4": {Labels: map[string]string{ualert.ContactLabel: `"notifier4","notifier6"`}}, - "alert5": {Labels: map[string]string{ualert.ContactLabel: `"notifier4","notifier5","notifier6"`}}, - "alert6": {Labels: map[string]string{}}, - }, - } - setupLegacyAlertsTables(t, x, legacyChannels, alerts) - runDashAlertMigrationTestRun(t, x) - - for orgId := range expected { - rules := getAlertRules(t, x, orgId) - expectedRulesMap := expected[orgId] - require.Len(t, rules, len(expectedRulesMap)) - for _, r := range rules { - require.Equal(t, expectedRulesMap[r.Title].Labels[ualert.ContactLabel], r.Labels[ualert.ContactLabel]) - } - } - }) - - t.Run("when DashAlertMigration create ContactLabel with sanitized name if name contains double quote", func(t *testing.T) { - defer teardown(t, x) - legacyChannels := []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notif\"ier1", "email", emailSettings, false), - } - alerts := []*models.Alert{ - createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notif\"ier1"}), - } - expected := map[int64]map[string]*ngModels.AlertRule{ - int64(1): { - "alert1": {Labels: map[string]string{ualert.ContactLabel: `"notif_ier1"`}}, - }, - } - setupLegacyAlertsTables(t, x, legacyChannels, alerts) - runDashAlertMigrationTestRun(t, x) - - for orgId := range expected { - rules := getAlertRules(t, x, orgId) - expectedRulesMap := expected[orgId] - require.Len(t, rules, len(expectedRulesMap)) - for _, r := range rules { - require.Equal(t, expectedRulesMap[r.Title].Labels[ualert.ContactLabel], r.Labels[ualert.ContactLabel]) - } - } - }) - - t.Run("when folder is missing put alert in General folder", func(t *testing.T) { - o := createOrg(t, 1) - folder1 := createDashboard(t, 1, o.ID, "folder-1") - folder1.IsFolder = true - dash1 := createDashboard(t, 3, o.ID, "dash1") - dash1.FolderID = folder1.ID - dash2 := createDashboard(t, 4, o.ID, "dash2") - dash2.FolderID = 22 // missing folder - - a1 := createAlert(t, o.ID, dash1.ID, int64(1), "alert-1", []string{}) - a2 := createAlert(t, o.ID, dash2.ID, int64(1), "alert-2", []string{}) - - _, err := x.Insert(o, folder1, dash1, dash2, a1, a2) - require.NoError(t, err) - - runDashAlertMigrationTestRun(t, x) - - rules := getAlertRules(t, x, o.ID) - require.Len(t, rules, 2) - - var generalFolder dashboards.Dashboard - _, err = x.Table(&dashboards.Dashboard{}).Where("title = ? AND org_id = ?", ualert.GENERAL_FOLDER, o.ID).Get(&generalFolder) - require.NoError(t, err) - - require.NotNil(t, generalFolder) - - for _, rule := range rules { - var expectedFolder dashboards.Dashboard - if rule.Title == a1.Name { - expectedFolder = *folder1 - } else { - expectedFolder = generalFolder - } - require.Equal(t, expectedFolder.UID, rule.NamespaceUID) - } - }) -} - -const ( - emailSettings = `{"addresses": "test"}` - slackSettings = `{"recipient": "test", "token": "test"}` - opsgenieSettings = `{"apiKey": "test"}` -) - -// setupTestDB prepares the sqlite database and runs OSS migrations to initialize the schemas. -func setupTestDB(t *testing.T) *xorm.Engine { - t.Helper() - testDB := sqlutil.SQLite3TestDB() - - x, err := xorm.NewEngine(testDB.DriverName, testDB.ConnStr) - require.NoError(t, err) - - err = migrator.NewDialect(x.DriverName()).CleanDB(x) - require.NoError(t, err) - - mg := migrator.NewMigrator(x, &setting.Cfg{Raw: ini.Empty()}) - migrations := &migrations.OSSMigrations{} - migrations.AddMigration(mg) - - err = mg.Start(false, 0) - require.NoError(t, err) - - return x -} - -var ( - now = time.Now() -) - -// createAlertNotificationWithReminder creates a legacy alert notification channel for inserting into the test database. -func createAlertNotificationWithReminder(t *testing.T, orgId int64, uid string, channelType string, settings string, defaultChannel bool, sendReminder bool, frequency time.Duration) *models.AlertNotification { - t.Helper() - settingsJson := simplejson.New() - if settings != "" { - s, err := simplejson.NewJson([]byte(settings)) - if err != nil { - t.Fatalf("Failed to unmarshal alert notification json: %v", err) - } - settingsJson = s - } - - return &models.AlertNotification{ - OrgID: orgId, - UID: uid, - Name: uid, // Same as uid to make testing easier. - Type: channelType, - DisableResolveMessage: false, - IsDefault: defaultChannel, - Settings: settingsJson, - SecureSettings: make(map[string][]byte), - Created: now, - Updated: now, - SendReminder: sendReminder, - Frequency: frequency, - } -} - -// createAlertNotification creates a legacy alert notification channel for inserting into the test database. -func createAlertNotification(t *testing.T, orgId int64, uid string, channelType string, settings string, defaultChannel bool) *models.AlertNotification { - return createAlertNotificationWithReminder(t, orgId, uid, channelType, settings, defaultChannel, false, time.Duration(0)) -} - -// createAlert creates a legacy alert rule for inserting into the test database. -func createAlert(t *testing.T, orgId int64, dashboardId int64, panelsId int64, name string, notifierUids []string) *models.Alert { - t.Helper() - - var settings = simplejson.New() - if len(notifierUids) != 0 { - notifiers := make([]interface{}, 0) - for _, n := range notifierUids { - notifiers = append(notifiers, struct { - Uid string - }{Uid: n}) - } - - settings.Set("notifications", notifiers) - } - - return &models.Alert{ - OrgID: orgId, - DashboardID: dashboardId, - PanelID: panelsId, - Name: name, - Message: "message", - Frequency: int64(60), - For: time.Duration(time.Duration(60).Seconds()), - State: models.AlertStateOK, - Settings: settings, - NewStateDate: now, - Created: now, - Updated: now, - } -} - -// createDashboard creates a dashboard for inserting into the test database. -func createDashboard(t *testing.T, id int64, orgId int64, uid string) *dashboards.Dashboard { - t.Helper() - return &dashboards.Dashboard{ - ID: id, - OrgID: orgId, - UID: uid, - Created: now, - Updated: now, - Title: uid, // Not tested, needed to satisfy contraint. - } -} - -// createDatasource creates a ddatasource for inserting into the test database. -func createDatasource(t *testing.T, id int64, orgId int64, uid string) *datasources.DataSource { - t.Helper() - return &datasources.DataSource{ - ID: id, - OrgID: orgId, - UID: uid, - Created: now, - Updated: now, - Name: uid, // Not tested, needed to satisfy contraint. - } -} - -func createOrg(t *testing.T, id int64) *org.Org { - t.Helper() - return &org.Org{ - ID: id, - Version: 1, - Name: fmt.Sprintf("org_%d", id), - Created: time.Now(), - Updated: time.Now(), - } -} - -// teardown cleans the input tables between test cases. -func teardown(t *testing.T, x *xorm.Engine) { - _, err := x.Exec("DELETE from org") - require.NoError(t, err) - _, err = x.Exec("DELETE from alert") - require.NoError(t, err) - _, err = x.Exec("DELETE from alert_notification") - require.NoError(t, err) - _, err = x.Exec("DELETE from dashboard") - require.NoError(t, err) - _, err = x.Exec("DELETE from data_source") - require.NoError(t, err) -} - -// setupDashAlertMigrationTestRun runs DashAlertMigration for a new test run. -func runDashAlertMigrationTestRun(t *testing.T, x *xorm.Engine) { - _, errDeleteMig := x.Exec("DELETE FROM migration_log WHERE migration_id = ?", ualert.MigTitle) - require.NoError(t, errDeleteMig) - - alertMigrator := migrator.NewMigrator(x, &setting.Cfg{}) - alertMigrator.AddMigration(ualert.RmMigTitle, &ualert.RmMigration{}) - ualert.AddDashAlertMigration(alertMigrator) - - errRunningMig := alertMigrator.Start(false, 0) - require.NoError(t, errRunningMig) -} - -// setupLegacyAlertsTables inserts data into the legacy alerting tables that is needed for testing the migration. -func setupLegacyAlertsTables(t *testing.T, x *xorm.Engine, legacyChannels []*models.AlertNotification, alerts []*models.Alert) { - t.Helper() - - orgs := []org.Org{ - *createOrg(t, 1), - *createOrg(t, 2), - } - - // Setup dashboards. - dashboards := []dashboards.Dashboard{ - *createDashboard(t, 1, 1, "dash1-1"), - *createDashboard(t, 2, 1, "dash2-1"), - *createDashboard(t, 3, 2, "dash3-2"), - *createDashboard(t, 4, 2, "dash4-2"), - } - _, errDashboards := x.Insert(dashboards) - require.NoError(t, errDashboards) - - // Setup data_sources. - dataSources := []datasources.DataSource{ - *createDatasource(t, 1, 1, "ds1-1"), - *createDatasource(t, 2, 1, "ds2-1"), - *createDatasource(t, 3, 2, "ds3-2"), - *createDatasource(t, 4, 2, "ds4-2"), - } - - _, errOrgs := x.Insert(orgs) - require.NoError(t, errOrgs) - - _, errDataSourcess := x.Insert(dataSources) - require.NoError(t, errDataSourcess) - - if len(legacyChannels) > 0 { - _, channelErr := x.Insert(legacyChannels) - require.NoError(t, channelErr) - } - - if len(alerts) > 0 { - _, alertErr := x.Insert(alerts) - require.NoError(t, alertErr) - } -} - -// getAlertmanagerConfig retreives the Alertmanager Config from the database for a given orgId. -func getAlertmanagerConfig(t *testing.T, x *xorm.Engine, orgId int64) *ualert.PostableUserConfig { - amConfig := "" - _, err := x.Table("alert_configuration").Where("org_id = ?", orgId).Cols("alertmanager_configuration").Get(&amConfig) - require.NoError(t, err) - - config := ualert.PostableUserConfig{} - err = json.Unmarshal([]byte(amConfig), &config) - require.NoError(t, err) - return &config -} - -// getAlertmanagerConfig retreives the Alertmanager Config from the database for a given orgId. -func getAlertRules(t *testing.T, x *xorm.Engine, orgId int64) []*ngModels.AlertRule { - rules := make([]*ngModels.AlertRule, 0) - err := x.Table("alert_rule").Where("org_id = ?", orgId).Find(&rules) - require.NoError(t, err) - - return rules -} - -func boolPointer(b bool) *bool { - return &b -} - -func durationPointer(d model.Duration) *model.Duration { - return &d -} diff --git a/pkg/services/sqlstore/migrations/ualert/permissions.go b/pkg/services/sqlstore/migrations/ualert/permissions.go deleted file mode 100644 index 49cad344c7d..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/permissions.go +++ /dev/null @@ -1,284 +0,0 @@ -package ualert - -import ( - "fmt" - "time" - - "xorm.io/xorm" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/infra/metrics" - "github.com/grafana/grafana/pkg/services/dashboards" - dashver "github.com/grafana/grafana/pkg/services/dashboardversion" - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" - "github.com/grafana/grafana/pkg/util" -) - -type roleType string - -const ( - RoleNone roleType = "None" - RoleViewer roleType = "Viewer" - RoleEditor roleType = "Editor" - RoleAdmin roleType = "Admin" -) - -func (r roleType) IsValid() bool { - return r == RoleViewer || r == RoleAdmin || r == RoleEditor || r == RoleNone -} - -type permissionType int - -type dashboardACL struct { - // nolint:stylecheck - Id int64 - OrgID int64 `xorm:"org_id"` - DashboardID int64 `xorm:"dashboard_id"` - - UserID int64 `xorm:"user_id"` - TeamID int64 `xorm:"team_id"` - Role *roleType // pointer to be nullable - Permission permissionType - - Created time.Time - Updated time.Time -} - -func (p dashboardACL) TableName() string { return "dashboard_acl" } - -type folderHelper struct { - sess *xorm.Session - mg *migrator.Migrator -} - -// getOrCreateGeneralFolder returns the general folder under the specific organisation -// If the general folder does not exist it creates it. -func (m *folderHelper) getOrCreateGeneralFolder(orgID int64) (*dashboard, error) { - // there is a unique constraint on org_id, folder_id, title - // there are no nested folders so the parent folder id is always 0 - dashboard := dashboard{OrgId: orgID, FolderId: 0, Title: GENERAL_FOLDER} - has, err := m.sess.Get(&dashboard) - if err != nil { - return nil, err - } else if !has { - // create folder - return m.createGeneralFolder(orgID) - } - return &dashboard, nil -} - -func (m *folderHelper) createGeneralFolder(orgID int64) (*dashboard, error) { - return m.createFolder(orgID, GENERAL_FOLDER) -} - -// returns the folder of the given dashboard (if exists) -func (m *folderHelper) getFolder(dash dashboard, da dashAlert) (dashboard, error) { - // get folder if exists - folder := dashboard{} - if dash.FolderId > 0 { - exists, err := m.sess.Where("id=?", dash.FolderId).Get(&folder) - if err != nil { - return folder, fmt.Errorf("failed to get folder %d: %w", dash.FolderId, err) - } - if !exists { - return folder, fmt.Errorf("folder with id %v not found", dash.FolderId) - } - if !folder.IsFolder { - return folder, fmt.Errorf("id %v is a dashboard not a folder", dash.FolderId) - } - } - return folder, nil -} - -// based on sqlstore.saveDashboard() -// it should be called from inside a transaction -func (m *folderHelper) createFolder(orgID int64, title string) (*dashboard, error) { - cmd := saveFolderCommand{ - OrgId: orgID, - FolderId: 0, - IsFolder: true, - Dashboard: simplejson.NewFromAny(map[string]any{ - "title": title, - }), - } - dash := cmd.getDashboardModel() - dash.setUid(util.GenerateShortUID()) - - parentVersion := dash.Version - dash.setVersion(1) - dash.Created = time.Now() - dash.CreatedBy = FOLDER_CREATED_BY - dash.Updated = time.Now() - dash.UpdatedBy = FOLDER_CREATED_BY - metrics.MApiDashboardInsert.Inc() - - if _, err := m.sess.Insert(dash); err != nil { - return nil, err - } - - dashVersion := &dashver.DashboardVersion{ - DashboardID: dash.Id, - ParentVersion: parentVersion, - RestoredFrom: cmd.RestoredFrom, - Version: dash.Version, - Created: time.Now(), - CreatedBy: dash.UpdatedBy, - Message: cmd.Message, - Data: dash.Data, - } - - // insert version entry - if _, err := m.sess.Insert(dashVersion); err != nil { - return nil, err - } - return dash, nil -} - -// based on SQLStore.UpdateDashboardACL() -// it should be called from inside a transaction -func (m *folderHelper) setACL(orgID int64, dashboardID int64, items []*dashboardACL) error { - if dashboardID <= 0 { - return fmt.Errorf("folder id must be greater than zero for a folder permission") - } - - // userPermissionsMap is a map keeping the highest permission per user - // for handling conficting inherited (folder) and non-inherited (dashboard) user permissions - userPermissionsMap := make(map[int64]*dashboardACL, len(items)) - // teamPermissionsMap is a map keeping the highest permission per team - // for handling conficting inherited (folder) and non-inherited (dashboard) team permissions - teamPermissionsMap := make(map[int64]*dashboardACL, len(items)) - for _, item := range items { - if item.UserID != 0 { - acl, ok := userPermissionsMap[item.UserID] - if !ok { - userPermissionsMap[item.UserID] = item - } else { - if item.Permission > acl.Permission { - // the higher permission wins - userPermissionsMap[item.UserID] = item - } - } - } - - if item.TeamID != 0 { - acl, ok := teamPermissionsMap[item.TeamID] - if !ok { - teamPermissionsMap[item.TeamID] = item - } else { - if item.Permission > acl.Permission { - // the higher permission wins - teamPermissionsMap[item.TeamID] = item - } - } - } - } - - type keyType struct { - UserID int64 `xorm:"user_id"` - TeamID int64 `xorm:"team_id"` - Role roleType - Permission permissionType - } - // seen keeps track of inserted perrmissions to avoid duplicates (due to inheritance) - seen := make(map[keyType]struct{}, len(items)) - for _, item := range items { - if item.UserID == 0 && item.TeamID == 0 && (item.Role == nil || !item.Role.IsValid()) { - return dashboards.ErrDashboardACLInfoMissing - } - - // ignore duplicate user permissions - if item.UserID != 0 { - acl, ok := userPermissionsMap[item.UserID] - if ok { - if acl.Id != item.Id { - continue - } - } - } - - // ignore duplicate team permissions - if item.TeamID != 0 { - acl, ok := teamPermissionsMap[item.TeamID] - if ok { - if acl.Id != item.Id { - continue - } - } - } - - key := keyType{UserID: item.UserID, TeamID: item.TeamID, Role: "", Permission: item.Permission} - if item.Role != nil { - key.Role = *item.Role - } - if _, ok := seen[key]; ok { - continue - } - - // unset Id so that the new record will get a different one - item.Id = 0 - item.OrgID = orgID - item.DashboardID = dashboardID - item.Created = time.Now() - item.Updated = time.Now() - - m.sess.Nullable("user_id", "team_id") - if _, err := m.sess.Insert(item); err != nil { - return err - } - seen[key] = struct{}{} - } - - // Update dashboard HasACL flag - dashboard := dashboards.Dashboard{HasACL: true} - _, err := m.sess.Cols("has_acl").Where("id=?", dashboardID).Update(&dashboard) - return err -} - -// based on SQLStore.GetDashboardACLInfoList() -func (m *folderHelper) getACL(orgID, dashboardID int64) ([]*dashboardACL, error) { - var err error - - falseStr := m.mg.Dialect.BooleanStr(false) - - result := make([]*dashboardACL, 0) - rawSQL := ` - -- get distinct permissions for the dashboard and its parent folder - SELECT DISTINCT - da.id, - da.user_id, - da.team_id, - da.permission, - da.role - FROM dashboard as d - LEFT JOIN dashboard folder on folder.id = d.folder_id - LEFT JOIN dashboard_acl AS da ON - da.dashboard_id = d.id OR - da.dashboard_id = d.folder_id OR - ( - -- include default permissions -- - da.org_id = -1 AND ( - (folder.id IS NOT NULL AND folder.has_acl = ` + falseStr + `) OR - (folder.id IS NULL AND d.has_acl = ` + falseStr + `) - ) - ) - WHERE d.org_id = ? AND d.id = ? AND da.id IS NOT NULL - ORDER BY da.id ASC - ` - err = m.sess.SQL(rawSQL, orgID, dashboardID).Find(&result) - return result, err -} - -// getOrgsThatHaveFolders returns a unique list of organization ID that have at least one folder -func (m *folderHelper) getOrgsIDThatHaveFolders() (map[int64]struct{}, error) { - // get folder if exists - var rows []int64 - err := m.sess.Table(&dashboard{}).Where("is_folder=?", true).Distinct("org_id").Find(&rows) - if err != nil { - return nil, err - } - result := make(map[int64]struct{}, len(rows)) - for _, s := range rows { - result[s] = struct{}{} - } - return result, nil -} diff --git a/pkg/services/sqlstore/migrations/ualert/rule_group_index_migration.go b/pkg/services/sqlstore/migrations/ualert/rule_group_index_migration.go new file mode 100644 index 00000000000..611bf66e290 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/rule_group_index_migration.go @@ -0,0 +1,209 @@ +package ualert + +import ( + "encoding/json" + "fmt" + "time" + + "xorm.io/xorm" + + ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" +) + +// UpdateRuleGroupIndexMigration updates a new field rule_group_index for alert rules that belong to a group with more than 1 alert. +func UpdateRuleGroupIndexMigration(mg *migrator.Migrator) { + mg.AddMigration("update group index for alert rules", &updateRulesOrderInGroup{}) +} + +type updateRulesOrderInGroup struct { + migrator.MigrationBase +} + +func (c updateRulesOrderInGroup) SQL(migrator.Dialect) string { + return codeMigration +} + +func (c updateRulesOrderInGroup) Exec(sess *xorm.Session, migrator *migrator.Migrator) error { + var rows []*alertRule + if err := sess.Table(alertRule{}).Asc("id").Find(&rows); err != nil { + return fmt.Errorf("failed to read the list of alert rules: %w", err) + } + + if len(rows) == 0 { + migrator.Logger.Debug("No rules to migrate.") + return nil + } + + groups := map[ngmodels.AlertRuleGroupKey][]*alertRule{} + + for _, row := range rows { + groupKey := ngmodels.AlertRuleGroupKey{ + OrgID: row.OrgID, + NamespaceUID: row.NamespaceUID, + RuleGroup: row.RuleGroup, + } + groups[groupKey] = append(groups[groupKey], row) + } + + toUpdate := make([]*alertRule, 0, len(rows)) + + for _, rules := range groups { + for i, rule := range rules { + if rule.RuleGroupIndex == i+1 { + continue + } + rule.RuleGroupIndex = i + 1 + toUpdate = append(toUpdate, rule) + } + } + + if len(toUpdate) == 0 { + migrator.Logger.Debug("No rules to upgrade group index") + return nil + } + + updated := time.Now() + versions := make([]interface{}, 0, len(toUpdate)) + + for _, rule := range toUpdate { + rule.Updated = updated + version := rule.makeVersion() + version.Version = rule.Version + 1 + version.ParentVersion = rule.Version + rule.Version++ + _, err := sess.ID(rule.ID).Cols("version", "updated", "rule_group_idx").Update(rule) + if err != nil { + migrator.Logger.Error("failed to update alert rule", "uid", rule.UID, "err", err) + return fmt.Errorf("unable to update alert rules with group index: %w", err) + } + migrator.Logger.Debug("updated group index for alert rule", "rule_uid", rule.UID) + versions = append(versions, version) + } + _, err := sess.Insert(versions...) + if err != nil { + migrator.Logger.Error("failed to insert changes to alert_rule_version", "err", err) + return fmt.Errorf("unable to update alert rules with group index: %w", err) + } + return nil +} + +type alertRule struct { + ID int64 `xorm:"pk autoincr 'id'"` + OrgID int64 `xorm:"org_id"` + Title string + Condition string + Data []alertQuery + IntervalSeconds int64 + Version int64 + UID string `xorm:"uid"` + NamespaceUID string `xorm:"namespace_uid"` + RuleGroup string + RuleGroupIndex int `xorm:"rule_group_idx"` + NoDataState string + ExecErrState string + For duration + Updated time.Time + Annotations map[string]string + Labels map[string]string + IsPaused bool +} + +type alertRuleVersion struct { + RuleOrgID int64 `xorm:"rule_org_id"` + RuleUID string `xorm:"rule_uid"` + RuleNamespaceUID string `xorm:"rule_namespace_uid"` + RuleGroup string + RuleGroupIndex int `xorm:"rule_group_idx"` + ParentVersion int64 + RestoredFrom int64 + Version int64 + + Created time.Time + Title string + Condition string + Data []alertQuery + IntervalSeconds int64 + NoDataState string + ExecErrState string + // ideally this field should have been apimodels.ApiDuration + // but this is currently not possible because of circular dependencies + For duration + Annotations map[string]string + Labels map[string]string + IsPaused bool +} + +func (a *alertRule) makeVersion() *alertRuleVersion { + return &alertRuleVersion{ + RuleOrgID: a.OrgID, + RuleUID: a.UID, + RuleNamespaceUID: a.NamespaceUID, + RuleGroup: a.RuleGroup, + RuleGroupIndex: a.RuleGroupIndex, + ParentVersion: 0, + RestoredFrom: 0, + Version: 1, + + Created: time.Now().UTC(), + Title: a.Title, + Condition: a.Condition, + Data: a.Data, + IntervalSeconds: a.IntervalSeconds, + NoDataState: a.NoDataState, + ExecErrState: a.ExecErrState, + For: a.For, + Annotations: a.Annotations, + Labels: map[string]string{}, + IsPaused: a.IsPaused, + } +} + +type alertQuery struct { + // RefID is the unique identifier of the query, set by the frontend call. + RefID string `json:"refId"` + + // QueryType is an optional identifier for the type of query. + // It can be used to distinguish different types of queries. + QueryType string `json:"queryType"` + + // RelativeTimeRange is the relative Start and End of the query as sent by the frontend. + RelativeTimeRange relativeTimeRange `json:"relativeTimeRange"` + + DatasourceUID string `json:"datasourceUid"` + + // JSON is the raw JSON query and includes the above properties as well as custom properties. + Model json.RawMessage `json:"model"` +} + +// RelativeTimeRange is the per query start and end time +// for requests. +type relativeTimeRange struct { + From duration `json:"from"` + To duration `json:"to"` +} + +// duration is a type used for marshalling durations. +type duration time.Duration + +func (d duration) String() string { + return time.Duration(d).String() +} + +func (d duration) MarshalJSON() ([]byte, error) { + return json.Marshal(time.Duration(d).Seconds()) +} + +func (d *duration) UnmarshalJSON(b []byte) error { + var v interface{} + if err := json.Unmarshal(b, &v); err != nil { + return err + } + switch value := v.(type) { + case float64: + *d = duration(time.Duration(value) * time.Second) + return nil + default: + return fmt.Errorf("invalid duration %v", v) + } +} diff --git a/pkg/services/sqlstore/migrations/ualert/securejsondata.go b/pkg/services/sqlstore/migrations/ualert/securejsondata.go deleted file mode 100644 index 20279f10e33..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/securejsondata.go +++ /dev/null @@ -1,60 +0,0 @@ -package ualert - -import ( - "os" - - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/util" -) - -// SecureJsonData is used to store encrypted data (for example in data_source table). Only values are separately -// encrypted. -type SecureJsonData map[string][]byte - -var seclogger = log.New("securejsondata") - -// DecryptedValue returns single decrypted value from SecureJsonData. Similar to normal map access second return value -// is true if the key exists and false if not. -func (s SecureJsonData) DecryptedValue(key string) (string, bool) { - if value, ok := s[key]; ok { - decryptedData, err := util.Decrypt(value, setting.SecretKey) - if err != nil { - seclogger.Error(err.Error()) - os.Exit(1) - } - return string(decryptedData), true - } - return "", false -} - -// Decrypt returns map of the same type but where the all the values are decrypted. Opposite of what -// GetEncryptedJsonData is doing. -func (s SecureJsonData) Decrypt() map[string]string { - decrypted := make(map[string]string) - for key, data := range s { - decryptedData, err := util.Decrypt(data, setting.SecretKey) - if err != nil { - seclogger.Error(err.Error()) - os.Exit(1) - } - - decrypted[key] = string(decryptedData) - } - return decrypted -} - -// GetEncryptedJsonData returns map where all keys are encrypted. -func GetEncryptedJsonData(sjd map[string]string) SecureJsonData { - encrypted := make(SecureJsonData) - for key, data := range sjd { - encryptedData, err := util.Encrypt([]byte(data), setting.SecretKey) - if err != nil { - seclogger.Error(err.Error()) - os.Exit(1) - } - - encrypted[key] = encryptedData - } - return encrypted -} diff --git a/pkg/services/sqlstore/migrations/ualert/tables.go b/pkg/services/sqlstore/migrations/ualert/tables.go index 78add4d1ce5..b6babe3409d 100644 --- a/pkg/services/sqlstore/migrations/ualert/tables.go +++ b/pkg/services/sqlstore/migrations/ualert/tables.go @@ -492,9 +492,6 @@ func addAlertImageMigrations(mg *migrator.Migrator) { } func extractAlertmanagerConfigurationHistoryMigration(mg *migrator.Migrator) { - if !mg.Cfg.UnifiedAlerting.IsEnabled() { - return - } // Since it's not always consistent as to what state the org ID indexes are in, just drop them all and rebuild from scratch. // This is not expensive since this table is guaranteed to have a small number of rows. mg.AddMigration("drop non-unique orgID index on alert_configuration", migrator.NewDropIndexMigration(migrator.Table{Name: "alert_configuration"}, &migrator.Index{Cols: []string{"org_id"}})) diff --git a/pkg/services/sqlstore/migrations/ualert/testing.go b/pkg/services/sqlstore/migrations/ualert/testing.go deleted file mode 100644 index 9a74c5d04d9..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/testing.go +++ /dev/null @@ -1,25 +0,0 @@ -package ualert - -import ( - "testing" - - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" - "github.com/prometheus/alertmanager/silence/silencepb" -) - -// newTestMigration generates an empty migration to use in tests. -func newTestMigration(t *testing.T) *migration { - t.Helper() - - return &migration{ - mg: &migrator.Migrator{ - - Logger: log.New("test"), - }, - seenUIDs: uidSet{ - set: make(map[string]struct{}), - }, - silences: make(map[int64][]*silencepb.MeshSilence), - } -} diff --git a/pkg/services/sqlstore/migrations/ualert/ualert.go b/pkg/services/sqlstore/migrations/ualert/ualert.go index 65080ebe3c3..d59b187ff7c 100644 --- a/pkg/services/sqlstore/migrations/ualert/ualert.go +++ b/pkg/services/sqlstore/migrations/ualert/ualert.go @@ -1,48 +1,21 @@ package ualert import ( - "context" - "encoding/json" - "errors" "fmt" "os" "path/filepath" "strconv" - "strings" - "time" - alertingNotify "github.com/grafana/alerting/notify" - pb "github.com/prometheus/alertmanager/silence/silencepb" "xorm.io/xorm" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/sqlstore/migrator" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/util" ) -const GENERAL_FOLDER = "General Alerting" -const DASHBOARD_FOLDER = "%s Alerts - %s" - -// MaxFolderName is the maximum length of the folder name generated using DASHBOARD_FOLDER format -const MaxFolderName = 255 - -// FOLDER_CREATED_BY us used to track folders created by this migration -// during alert migration cleanup. -const FOLDER_CREATED_BY = -8 - -const KV_NAMESPACE = "alertmanager" - var migTitle = "move dashboard alerts to unified alerting" -var rmMigTitle = "remove unified alerting data" - -const clearMigrationEntryTitle = "clear migration entry %q" const codeMigration = "code migration" -// It is defined in pkg/expr/service.go as "DatasourceType" -const expressionDatasourceUID = "__expr__" - type MigrationError struct { AlertId int64 Err error @@ -54,99 +27,16 @@ func (e MigrationError) Error() string { func (e *MigrationError) Unwrap() error { return e.Err } -func AddDashAlertMigration(mg *migrator.Migrator) { - logs, err := mg.GetMigrationLog() - if err != nil { - mg.Logger.Error("Alert migration failure: could not get migration log", "error", err) - os.Exit(1) - } - - _, migrationRun := logs[migTitle] - - switch { - // If unified alerting is enabled and the upgrade migration has not been run - case mg.Cfg.UnifiedAlerting.IsEnabled() && !migrationRun: - // Remove the migration entry that removes all unified alerting data. This is so when the feature - // flag is removed in future the "remove unified alerting data" migration will be run again. - mg.AddMigration(fmt.Sprintf(clearMigrationEntryTitle, rmMigTitle), &clearMigrationEntry{ - migrationID: rmMigTitle, - }) - if err != nil { - mg.Logger.Error("Alert migration error: could not clear alert migration for removing data", "error", err) - } - mg.AddMigration(migTitle, &migration{ - // We deduplicate for case-insensitive matching in MySQL-compatible backend flavours because they use case-insensitive collation. - seenUIDs: uidSet{set: make(map[string]struct{}), caseInsensitive: mg.Dialect.SupportEngine()}, - silences: make(map[int64][]*pb.MeshSilence), - }) - // If unified alerting is disabled and upgrade migration has been run - case !mg.Cfg.UnifiedAlerting.IsEnabled() && migrationRun: - // If legacy alerting is also disabled, there is nothing to do - if setting.AlertingEnabled != nil && !*setting.AlertingEnabled { - return - } - - // Safeguard to prevent data loss when migrating from UA to LA - if !mg.Cfg.ForceMigration { - panic("Grafana has already been migrated to Unified Alerting.\nAny alert rules created while using Unified Alerting will be deleted by rolling back.\n\nSet force_migration=true in your grafana.ini and restart Grafana to roll back and delete Unified Alerting configuration data.") - } - // Remove the migration entry that creates unified alerting data. This is so when the feature - // flag is enabled in the future the migration "move dashboard alerts to unified alerting" will be run again. - mg.AddMigration(fmt.Sprintf(clearMigrationEntryTitle, migTitle), &clearMigrationEntry{ - migrationID: migTitle, - }) - if err != nil { - mg.Logger.Error("Alert migration error: could not clear dashboard alert migration", "error", err) - } - mg.AddMigration(rmMigTitle, &rmMigration{}) - } -} - -// RerunDashAlertMigration force the dashboard alert migration to run -// to make sure that the Alertmanager configurations will be created for each organisation -func RerunDashAlertMigration(mg *migrator.Migrator) { - logs, err := mg.GetMigrationLog() - if err != nil { - mg.Logger.Error("Alert migration failure: could not get migration log", "error", err) - os.Exit(1) - } - +// FixEarlyMigration fixes UA configs created before 8.2 with org_id=0 and moves some files like __default__.tmpl. +// The only use of this migration is when a user enabled ng-alerting before 8.2. +func FixEarlyMigration(mg *migrator.Migrator) { cloneMigTitle := fmt.Sprintf("clone %s", migTitle) - - _, migrationRun := logs[cloneMigTitle] - ngEnabled := mg.Cfg.UnifiedAlerting.IsEnabled() - - switch { - case ngEnabled && !migrationRun: - // The only use of this migration is when a user enabled ng-alerting before 8.2. - mg.AddMigration(cloneMigTitle, &upgradeNgAlerting{}) - // if user disables the feature flag and enables it back. - // This migration does not need to be run because the original migration AddDashAlertMigration does what's needed - } + mg.AddMigration(cloneMigTitle, &upgradeNgAlerting{}) } func AddDashboardUIDPanelIDMigration(mg *migrator.Migrator) { - logs, err := mg.GetMigrationLog() - if err != nil { - mg.Logger.Error("Alert migration failure: could not get migration log", "error", err) - os.Exit(1) - } - migrationID := "update dashboard_uid and panel_id from existing annotations" - _, migrationRun := logs[migrationID] - ngEnabled := mg.Cfg.UnifiedAlerting.IsEnabled() - undoMigrationID := "undo " + migrationID - - if ngEnabled && !migrationRun { - // If ngalert is enabled and the migration has not been run then run it. - mg.AddMigration(migrationID, &updateDashboardUIDPanelIDMigration{}) - } else if !ngEnabled && migrationRun { - // If ngalert is disabled and the migration has been run then remove it - // from the migration log so it will run if ngalert is re-enabled. - mg.AddMigration(undoMigrationID, &clearMigrationEntry{ - migrationID: migrationID, - }) - } + mg.AddMigration(migrationID, &updateDashboardUIDPanelIDMigration{}) } // updateDashboardUIDPanelIDMigration sets the dashboard_uid and panel_id columns @@ -198,327 +88,6 @@ func (m *updateDashboardUIDPanelIDMigration) Exec(sess *xorm.Session, mg *migrat return nil } -// clearMigrationEntry removes an entry fromt the migration_log table. -// This migration is not recorded in the migration_log so that it can re-run several times. -type clearMigrationEntry struct { - migrator.MigrationBase - - migrationID string -} - -func (m *clearMigrationEntry) SQL(dialect migrator.Dialect) string { - return "clear migration entry code migration" -} - -func (m *clearMigrationEntry) Exec(sess *xorm.Session, mg *migrator.Migrator) error { - _, err := sess.SQL(`DELETE from migration_log where migration_id = ?`, m.migrationID).Query() - if err != nil { - return fmt.Errorf("failed to clear migration entry %v: %w", m.migrationID, err) - } - return nil -} - -func (m *clearMigrationEntry) SkipMigrationLog() bool { - return true -} - -type migration struct { - migrator.MigrationBase - // session and mg are attached for convenience. - sess *xorm.Session - mg *migrator.Migrator - - seenUIDs uidSet - silences map[int64][]*pb.MeshSilence -} - -func (m *migration) SQL(dialect migrator.Dialect) string { - return codeMigration -} - -//nolint:gocyclo -func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { - m.sess = sess - m.mg = mg - - dashAlerts, err := m.slurpDashAlerts() - if err != nil { - return err - } - mg.Logger.Info("Alerts found to migrate", "alerts", len(dashAlerts)) - - // [orgID, dataSourceId] -> UID - dsIDMap, err := m.slurpDSIDs() - if err != nil { - return err - } - - // [orgID, dashboardId] -> dashUID - dashIDMap, err := m.slurpDashUIDs() - if err != nil { - return err - } - - // cache for folders created for dashboards that have custom permissions - folderCache := make(map[string]*dashboard) - // cache for the general folders - generalFolderCache := make(map[int64]*dashboard) - - folderHelper := folderHelper{ - sess: sess, - mg: mg, - } - - gf := func(dash dashboard, da dashAlert) (*dashboard, error) { - f, ok := generalFolderCache[dash.OrgId] - if !ok { - // get or create general folder - f, err = folderHelper.getOrCreateGeneralFolder(dash.OrgId) - if err != nil { - return nil, MigrationError{ - Err: fmt.Errorf("failed to get or create general folder under organisation %d: %w", dash.OrgId, err), - AlertId: da.Id, - } - } - generalFolderCache[dash.OrgId] = f - } - // No need to assign default permissions to general folder - // because they are included to the query result if it's a folder with no permissions - // https://github.com/grafana/grafana/blob/076e2ce06a6ecf15804423fcc8dca1b620a321e5/pkg/services/sqlstore/dashboard_acl.go#L109 - return f, nil - } - - // Per org map of newly created rules to which notification channels it should send to. - rulesPerOrg := make(map[int64]map[*alertRule][]uidOrID) - - for _, da := range dashAlerts { - l := mg.Logger.New("ruleID", da.Id, "ruleName", da.Name, "dashboardUID", da.DashboardUID, "orgID", da.OrgId) - l.Debug("Migrating alert rule to Unified Alerting") - newCond, err := transConditions(*da.ParsedSettings, da.OrgId, dsIDMap) - if err != nil { - return err - } - - da.DashboardUID = dashIDMap[[2]int64{da.OrgId, da.DashboardId}] - - // get dashboard - dash := dashboard{} - exists, err := m.sess.Where("org_id=? AND uid=?", da.OrgId, da.DashboardUID).Get(&dash) - if err != nil { - return MigrationError{ - Err: fmt.Errorf("failed to get dashboard %s under organisation %d: %w", da.DashboardUID, da.OrgId, err), - AlertId: da.Id, - } - } - if !exists { - return MigrationError{ - Err: fmt.Errorf("dashboard with UID %v under organisation %d not found: %w", da.DashboardUID, da.OrgId, err), - AlertId: da.Id, - } - } - - var folder *dashboard - switch { - case dash.HasACL: - folderName := getAlertFolderNameFromDashboard(&dash) - f, ok := folderCache[folderName] - if !ok { - l.Info("Create a new folder for alerts that belongs to dashboard because it has custom permissions", "folder", folderName) - // create folder and assign the permissions of the dashboard (included default and inherited) - f, err = folderHelper.createFolder(dash.OrgId, folderName) - if err != nil { - return MigrationError{ - Err: fmt.Errorf("failed to create folder: %w", err), - AlertId: da.Id, - } - } - permissions, err := folderHelper.getACL(dash.OrgId, dash.Id) - if err != nil { - return MigrationError{ - Err: fmt.Errorf("failed to get dashboard %d under organisation %d permissions: %w", dash.Id, dash.OrgId, err), - AlertId: da.Id, - } - } - err = folderHelper.setACL(f.OrgId, f.Id, permissions) - if err != nil { - return MigrationError{ - Err: fmt.Errorf("failed to set folder %d under organisation %d permissions: %w", f.Id, f.OrgId, err), - AlertId: da.Id, - } - } - folderCache[folderName] = f - } - folder = f - case dash.FolderId > 0: - // get folder if exists - f, err := folderHelper.getFolder(dash, da) - if err != nil { - // If folder does not exist then the dashboard is an orphan and we migrate the alert to the general folder. - l.Warn("Failed to find folder for dashboard. Migrate rule to the default folder", "rule_name", da.Name, "dashboard_uid", da.DashboardUID, "missing_folder_id", dash.FolderId) - folder, err = gf(dash, da) - if err != nil { - return err - } - } else { - folder = &f - } - default: - folder, err = gf(dash, da) - if err != nil { - return err - } - } - - if folder.Uid == "" { - return MigrationError{ - Err: fmt.Errorf("empty folder identifier"), - AlertId: da.Id, - } - } - rule, err := m.makeAlertRule(l, *newCond, da, folder.Uid) - if err != nil { - return fmt.Errorf("failed to migrate alert rule '%s' [ID:%d, DashboardUID:%s, orgID:%d]: %w", da.Name, da.Id, da.DashboardUID, da.OrgId, err) - } - - if _, ok := rulesPerOrg[rule.OrgID]; !ok { - rulesPerOrg[rule.OrgID] = make(map[*alertRule][]uidOrID) - } - if _, ok := rulesPerOrg[rule.OrgID][rule]; !ok { - rulesPerOrg[rule.OrgID][rule] = extractChannelIDs(da) - } else { - return MigrationError{ - Err: fmt.Errorf("duplicate generated rule UID"), - AlertId: da.Id, - } - } - } - - for orgID := range rulesPerOrg { - if err := m.writeSilencesFile(orgID); err != nil { - m.mg.Logger.Error("Alert migration error: failed to write silence file", "err", err) - } - } - - amConfigPerOrg, err := m.setupAlertmanagerConfigs(rulesPerOrg) - if err != nil { - return err - } - - err = m.insertRules(mg, rulesPerOrg) - if err != nil { - return err - } - - for orgID, amConfig := range amConfigPerOrg { - if err := m.writeAlertmanagerConfig(orgID, amConfig); err != nil { - return err - } - } - - return nil -} - -func (m *migration) insertRules(mg *migrator.Migrator, rulesPerOrg map[int64]map[*alertRule][]uidOrID) error { - for _, rules := range rulesPerOrg { - for rule := range rules { - var err error - if strings.HasPrefix(mg.Dialect.DriverName(), migrator.Postgres) { - err = mg.InTransaction(func(sess *xorm.Session) error { - _, err := sess.Insert(rule) - return err - }) - } else { - _, err = m.sess.Insert(rule) - } - if err != nil { - // TODO better error handling, if constraint - rule.Title += fmt.Sprintf(" %v", rule.UID) - rule.RuleGroup += fmt.Sprintf(" %v", rule.UID) - - _, err = m.sess.Insert(rule) - if err != nil { - return err - } - } - - // create entry in alert_rule_version - _, err = m.sess.Insert(rule.makeVersion()) - if err != nil { - return err - } - } - } - return nil -} - -func (m *migration) writeAlertmanagerConfig(orgID int64, amConfig *PostableUserConfig) error { - rawAmConfig, err := json.Marshal(amConfig) - if err != nil { - return err - } - - // remove an existing configuration, which could have been left during switching back to legacy alerting - _, _ = m.sess.Delete(AlertConfiguration{OrgID: orgID}) - - // We don't need to apply the configuration, given the multi org alertmanager will do an initial sync before the server is ready. - _, err = m.sess.Insert(AlertConfiguration{ - AlertmanagerConfiguration: string(rawAmConfig), - // Since we are migration for a snapshot of the code, it is always going to migrate to - // the v1 config. - ConfigurationVersion: "v1", - OrgID: orgID, - }) - if err != nil { - return err - } - - return nil -} - -// validateAlertmanagerConfig validates the alertmanager configuration produced by the migration against the receivers. -func (m *migration) validateAlertmanagerConfig(config *PostableUserConfig) error { - for _, r := range config.AlertmanagerConfig.Receivers { - for _, gr := range r.GrafanaManagedReceivers { - data, err := gr.Settings.MarshalJSON() - if err != nil { - return err - } - var ( - cfg = &alertingNotify.GrafanaIntegrationConfig{ - UID: gr.UID, - Name: gr.Name, - Type: gr.Type, - DisableResolveMessage: gr.DisableResolveMessage, - Settings: data, - SecureSettings: gr.SecureSettings, - } - ) - - // decryptFunc represents the legacy way of decrypting data. Before the migration, we don't need any new way, - // given that the previous alerting will never support it. - decryptFunc := func(_ context.Context, sjd map[string][]byte, key string, fallback string) string { - if value, ok := sjd[key]; ok { - decryptedData, err := util.Decrypt(value, setting.SecretKey) - if err != nil { - m.mg.Logger.Warn("Unable to decrypt key '%s' for %s receiver with uid %s, returning fallback.", key, gr.Type, gr.UID) - return fallback - } - return string(decryptedData) - } - return fallback - } - _, err = alertingNotify.BuildReceiverConfiguration(context.Background(), &alertingNotify.APIReceiver{ - GrafanaIntegrations: alertingNotify.GrafanaIntegrations{Integrations: []*alertingNotify.GrafanaIntegrationConfig{cfg}}, - }, decryptFunc) - if err != nil { - return err - } - } - } - - return nil -} - type AlertConfiguration struct { ID int64 `xorm:"pk autoincr 'id'"` OrgID int64 `xorm:"org_id"` @@ -528,84 +97,6 @@ type AlertConfiguration struct { CreatedAt int64 `xorm:"created"` } -// rmMigration removes Grafana 8 alert data -type rmMigration struct { - migrator.MigrationBase -} - -func (m *rmMigration) SQL(dialect migrator.Dialect) string { - return codeMigration -} - -func (m *rmMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { - _, err := sess.Exec("delete from alert_rule") - if err != nil { - return err - } - - _, err = sess.Exec("delete from alert_rule_version") - if err != nil { - return err - } - - _, err = sess.Exec("delete from dashboard_acl where dashboard_id IN (select id from dashboard where created_by = ?)", FOLDER_CREATED_BY) - if err != nil { - return err - } - - _, err = sess.Exec("delete from dashboard where created_by = ?", FOLDER_CREATED_BY) - if err != nil { - return err - } - - _, err = sess.Exec("delete from alert_configuration") - if err != nil { - return err - } - - _, err = sess.Exec("delete from ngalert_configuration") - if err != nil { - return err - } - - _, err = sess.Exec("delete from alert_instance") - if err != nil { - return err - } - - exists, err := sess.IsTableExist("kv_store") - if err != nil { - return err - } - - if exists { - _, err = sess.Exec("delete from kv_store where namespace = ?", KV_NAMESPACE) - if err != nil { - return err - } - } - - files, err := getSilenceFileNamesForAllOrgs(mg) - if err != nil { - return err - } - for _, f := range files { - if err := os.Remove(f); err != nil { - mg.Logger.Error("Alert migration error: failed to remove silence file", "file", f, "err", err) - } - } - - return nil -} - -// rmMigrationWithoutLogging is similar migration to rmMigration -// but is not recorded in the migration_log table so that it can rerun in the future -type rmMigrationWithoutLogging = rmMigration - -func (m *rmMigrationWithoutLogging) SkipMigrationLog() bool { - return true -} - type upgradeNgAlerting struct { migrator.MigrationBase } @@ -735,191 +226,3 @@ func (u *upgradeNgAlerting) updateAlertmanagerFiles(orgId int64, migrator *migra func (u *upgradeNgAlerting) SQL(migrator.Dialect) string { return codeMigration } - -// getAlertFolderNameFromDashboard generates a folder name for alerts that belong to a dashboard. Formats the string according to DASHBOARD_FOLDER format. -// If the resulting string exceeds the migrations.MaxTitleLength, the dashboard title is stripped to be at the maximum length -func getAlertFolderNameFromDashboard(dash *dashboard) string { - maxLen := MaxFolderName - len(fmt.Sprintf(DASHBOARD_FOLDER, "", dash.Uid)) - title := dash.Title - if len(title) > maxLen { - title = title[:maxLen] - } - return fmt.Sprintf(DASHBOARD_FOLDER, title, dash.Uid) // include UID to the name to avoid collision -} - -// CreateDefaultFoldersForAlertingMigration creates a folder dedicated for alerting if no folders exist -func CreateDefaultFoldersForAlertingMigration(mg *migrator.Migrator) { - if !mg.Cfg.UnifiedAlerting.IsEnabled() { - return - } - mg.AddMigration("create default alerting folders", &createDefaultFoldersForAlertingMigration{}) -} - -type createDefaultFoldersForAlertingMigration struct { - migrator.MigrationBase -} - -func (c createDefaultFoldersForAlertingMigration) Exec(sess *xorm.Session, migrator *migrator.Migrator) error { - helper := folderHelper{ - sess: sess, - mg: migrator, - } - - var rows []struct { - Id int64 - Name string - } - - if err := sess.Table("org").Cols("id", "name").Find(&rows); err != nil { - return fmt.Errorf("failed to read the list of organizations: %w", err) - } - - orgsWithFolders, err := helper.getOrgsIDThatHaveFolders() - if err != nil { - return fmt.Errorf("failed to list organizations that have at least one folder: %w", err) - } - - for _, row := range rows { - // if there's at least one folder in the org or if alerting is disabled for that org, skip adding the default folder - if _, ok := orgsWithFolders[row.Id]; ok { - migrator.Logger.Debug("Skip adding default alerting folder because organization already has at least one folder", "org_id", row.Id) - continue - } - if _, ok := migrator.Cfg.UnifiedAlerting.DisabledOrgs[row.Id]; ok { - migrator.Logger.Debug("Skip adding default alerting folder because alerting is disabled for the organization ", "org_id", row.Id) - continue - } - folder, err := helper.createGeneralFolder(row.Id) - if err != nil { - return fmt.Errorf("failed to create the default alerting folder for organization %s (ID: %d): %w", row.Name, row.Id, err) - } - migrator.Logger.Info("Created the default folder for alerting", "org_id", row.Id, "folder_name", folder.Title, "folder_uid", folder.Uid) - } - return nil -} - -func (c createDefaultFoldersForAlertingMigration) SQL(migrator.Dialect) string { - return codeMigration -} - -// UpdateRuleGroupIndexMigration updates a new field rule_group_index for alert rules that belong to a group with more than 1 alert. -func UpdateRuleGroupIndexMigration(mg *migrator.Migrator) { - if !mg.Cfg.UnifiedAlerting.IsEnabled() { - return - } - mg.AddMigration("update group index for alert rules", &updateRulesOrderInGroup{}) -} - -type updateRulesOrderInGroup struct { - migrator.MigrationBase -} - -func (c updateRulesOrderInGroup) SQL(migrator.Dialect) string { - return codeMigration -} - -func (c updateRulesOrderInGroup) Exec(sess *xorm.Session, migrator *migrator.Migrator) error { - var rows []*alertRule - if err := sess.Table(alertRule{}).Asc("id").Find(&rows); err != nil { - return fmt.Errorf("failed to read the list of alert rules: %w", err) - } - - if len(rows) == 0 { - migrator.Logger.Debug("No rules to migrate.") - return nil - } - - groups := map[ngmodels.AlertRuleGroupKey][]*alertRule{} - - for _, row := range rows { - groupKey := ngmodels.AlertRuleGroupKey{ - OrgID: row.OrgID, - NamespaceUID: row.NamespaceUID, - RuleGroup: row.RuleGroup, - } - groups[groupKey] = append(groups[groupKey], row) - } - - toUpdate := make([]*alertRule, 0, len(rows)) - - for _, rules := range groups { - for i, rule := range rules { - if rule.RuleGroupIndex == i+1 { - continue - } - rule.RuleGroupIndex = i + 1 - toUpdate = append(toUpdate, rule) - } - } - - if len(toUpdate) == 0 { - migrator.Logger.Debug("No rules to upgrade group index") - return nil - } - - updated := time.Now() - versions := make([]any, 0, len(toUpdate)) - - for _, rule := range toUpdate { - rule.Updated = updated - version := rule.makeVersion() - version.Version = rule.Version + 1 - version.ParentVersion = rule.Version - rule.Version++ - _, err := sess.ID(rule.ID).Cols("version", "updated", "rule_group_idx").Update(rule) - if err != nil { - migrator.Logger.Error("Failed to update alert rule", "uid", rule.UID, "err", err) - return fmt.Errorf("unable to update alert rules with group index: %w", err) - } - migrator.Logger.Debug("Updated group index for alert rule", "rule_uid", rule.UID) - versions = append(versions, version) - } - _, err := sess.Insert(versions...) - if err != nil { - migrator.Logger.Error("Failed to insert changes to alert_rule_version", "err", err) - return fmt.Errorf("unable to update alert rules with group index: %w", err) - } - return nil -} - -// uidSet is a wrapper around map[string]struct{} and util.GenerateShortUID() which aims help generate uids in quick -// succession while taking into consideration case sensitivity requirements. if caseInsensitive is true, all generated -// uids must also be unique when compared in a case-insensitive manner. -type uidSet struct { - set map[string]struct{} - caseInsensitive bool -} - -// contains checks whether the given uid has already been generated in this uidSet. -func (s *uidSet) contains(uid string) bool { - dedup := uid - if s.caseInsensitive { - dedup = strings.ToLower(dedup) - } - _, seen := s.set[dedup] - return seen -} - -// add adds the given uid to the uidSet. -func (s *uidSet) add(uid string) { - dedup := uid - if s.caseInsensitive { - dedup = strings.ToLower(dedup) - } - s.set[dedup] = struct{}{} -} - -// generateUid will generate a new unique uid that is not already contained in the uidSet. -// If it fails to create one that has not already been generated it will make multiple, but not unlimited, attempts. -// If all attempts are exhausted an error will be returned. -func (s *uidSet) generateUid() (string, error) { - for i := 0; i < 5; i++ { - gen := util.GenerateShortUID() - if !s.contains(gen) { - s.add(gen) - return gen, nil - } - } - - return "", errors.New("failed to generate UID") -} From 54050be9f639ae50d27186234c1f5d954e3e17a7 Mon Sep 17 00:00:00 2001 From: Giordano Ricci Date: Wed, 11 Oct 2023 19:52:19 +0100 Subject: [PATCH 068/102] Explore: Use short units in graphs (#76358) --- public/app/features/explore/Graph/ExploreGraph.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/features/explore/Graph/ExploreGraph.tsx b/public/app/features/explore/Graph/ExploreGraph.tsx index ec0ccb3e8c4..652ccffd42b 100644 --- a/public/app/features/explore/Graph/ExploreGraph.tsx +++ b/public/app/features/explore/Graph/ExploreGraph.tsx @@ -99,6 +99,7 @@ export function ExploreGraph({ defaults: { min: anchorToZero ? 0 : undefined, max: yAxisMaximum || undefined, + unit: 'short', color: { mode: FieldColorModeId.PaletteClassic, }, From f6649d7a97d7660e6e83c0a91adb674e67312125 Mon Sep 17 00:00:00 2001 From: Alexander Weaver Date: Wed, 11 Oct 2023 14:21:21 -0500 Subject: [PATCH 069/102] Revert "Alerting: Remove vendored models in migration service" (#76387) Revert "Alerting: Remove vendored models in migration service (#74503)" This reverts commit 6a8649d54420bb3e649d956350922a067564ec8b. --- .../backgroundsvcs/background_services.go | 4 +- pkg/server/wire.go | 4 - .../ngalert/api/api_alertmanager_test.go | 3 +- pkg/services/ngalert/migration/channel.go | 440 ------ .../ngalert/migration/channel_test.go | 591 -------- .../ngalert/migration/migration_test.go | 1326 ----------------- pkg/services/ngalert/migration/models.go | 121 -- .../ngalert/migration/models/state.go | 7 - pkg/services/ngalert/migration/permissions.go | 153 -- .../ngalert/migration/securejsondata.go | 31 - pkg/services/ngalert/migration/service.go | 137 -- .../ngalert/migration/service_test.go | 190 --- .../ngalert/migration/store/database.go | 511 ------- .../ngalert/migration/store/models.go | 91 -- .../ngalert/migration/store/testing.go | 57 - pkg/services/ngalert/migration/testing.go | 29 - pkg/services/ngalert/migration/ualert.go | 115 -- pkg/services/ngalert/ngalert.go | 1 - .../ngalert/notifier/alertmanager_test.go | 3 +- .../channels_config/available_channels.go | 18 - .../ngalert/notifier/file_store_test.go | 14 +- .../notifier/multiorg_alertmanager_test.go | 9 +- pkg/services/ngalert/notifier/testing.go | 95 ++ .../ngalert/provisioning/contactpoints.go | 21 +- pkg/services/ngalert/sender/router_test.go | 3 +- pkg/services/ngalert/tests/fakes/kvstore.go | 102 -- .../migrations/external_alertmanagers.go | 27 +- .../sqlstore/migrations/migrations.go | 7 +- .../migrations/ualert}/alert_rule.go | 227 ++- .../migrations/ualert}/alert_rule_test.go | 119 +- .../sqlstore/migrations/ualert/channel.go | 512 +++++++ .../migrations/ualert/channel_test.go | 471 ++++++ .../migrations/ualert}/cond_trans.go | 50 +- .../migrations/ualert/created_folders_mig.go | 83 -- .../sqlstore/migrations/ualert/dash_alert.go | 130 ++ .../sqlstore/migrations/ualert/dashboard.go | 108 ++ .../sqlstore/migrations/ualert/datasource.go | 31 + .../ualert/migration_service_migration.go | 69 - .../migrations/ualert/migration_test.go | 847 +++++++++++ .../sqlstore/migrations/ualert/permissions.go | 284 ++++ .../ualert/rule_group_index_migration.go | 209 --- .../migrations/ualert/securejsondata.go | 60 + .../migrations/ualert}/silences.go | 51 +- .../sqlstore/migrations/ualert/tables.go | 3 + .../migrations/ualert}/template.go | 2 +- .../migrations/ualert}/template_test.go | 5 +- .../sqlstore/migrations/ualert/testing.go | 25 + .../sqlstore/migrations/ualert/ualert.go | 707 ++++++++- .../migrations/ualert}/ualert_test.go | 108 +- 49 files changed, 3647 insertions(+), 4564 deletions(-) delete mode 100644 pkg/services/ngalert/migration/channel.go delete mode 100644 pkg/services/ngalert/migration/channel_test.go delete mode 100644 pkg/services/ngalert/migration/migration_test.go delete mode 100644 pkg/services/ngalert/migration/models.go delete mode 100644 pkg/services/ngalert/migration/models/state.go delete mode 100644 pkg/services/ngalert/migration/permissions.go delete mode 100644 pkg/services/ngalert/migration/securejsondata.go delete mode 100644 pkg/services/ngalert/migration/service.go delete mode 100644 pkg/services/ngalert/migration/service_test.go delete mode 100644 pkg/services/ngalert/migration/store/database.go delete mode 100644 pkg/services/ngalert/migration/store/models.go delete mode 100644 pkg/services/ngalert/migration/store/testing.go delete mode 100644 pkg/services/ngalert/migration/testing.go delete mode 100644 pkg/services/ngalert/migration/ualert.go delete mode 100644 pkg/services/ngalert/tests/fakes/kvstore.go rename pkg/services/{ngalert/migration => sqlstore/migrations/ualert}/alert_rule.go (56%) rename pkg/services/{ngalert/migration => sqlstore/migrations/ualert}/alert_rule_test.go (54%) create mode 100644 pkg/services/sqlstore/migrations/ualert/channel.go create mode 100644 pkg/services/sqlstore/migrations/ualert/channel_test.go rename pkg/services/{ngalert/migration => sqlstore/migrations/ualert}/cond_trans.go (87%) delete mode 100644 pkg/services/sqlstore/migrations/ualert/created_folders_mig.go create mode 100644 pkg/services/sqlstore/migrations/ualert/dash_alert.go create mode 100644 pkg/services/sqlstore/migrations/ualert/dashboard.go create mode 100644 pkg/services/sqlstore/migrations/ualert/datasource.go delete mode 100644 pkg/services/sqlstore/migrations/ualert/migration_service_migration.go create mode 100644 pkg/services/sqlstore/migrations/ualert/migration_test.go create mode 100644 pkg/services/sqlstore/migrations/ualert/permissions.go delete mode 100644 pkg/services/sqlstore/migrations/ualert/rule_group_index_migration.go create mode 100644 pkg/services/sqlstore/migrations/ualert/securejsondata.go rename pkg/services/{ngalert/migration => sqlstore/migrations/ualert}/silences.go (70%) rename pkg/services/{ngalert/migration => sqlstore/migrations/ualert}/template.go (99%) rename pkg/services/{ngalert/migration => sqlstore/migrations/ualert}/template_test.go (99%) create mode 100644 pkg/services/sqlstore/migrations/ualert/testing.go rename pkg/services/{ngalert/migration => sqlstore/migrations/ualert}/ualert_test.go (53%) diff --git a/pkg/registry/backgroundsvcs/background_services.go b/pkg/registry/backgroundsvcs/background_services.go index 3d633e66467..19f8cfc0d26 100644 --- a/pkg/registry/backgroundsvcs/background_services.go +++ b/pkg/registry/backgroundsvcs/background_services.go @@ -23,7 +23,6 @@ import ( "github.com/grafana/grafana/pkg/services/login/authinfoservice" "github.com/grafana/grafana/pkg/services/loginattempt/loginattemptimpl" "github.com/grafana/grafana/pkg/services/ngalert" - "github.com/grafana/grafana/pkg/services/ngalert/migration" "github.com/grafana/grafana/pkg/services/notifications" plugindashboardsservice "github.com/grafana/grafana/pkg/services/plugindashboards/service" "github.com/grafana/grafana/pkg/services/pluginsintegration/angulardetectorsprovider" @@ -45,7 +44,7 @@ import ( ) func ProvideBackgroundServiceRegistry( - httpServer *api.HTTPServer, alertingMigrationService *migration.MigrationService, ng *ngalert.AlertNG, cleanup *cleanup.CleanUpService, live *live.GrafanaLive, + httpServer *api.HTTPServer, ng *ngalert.AlertNG, cleanup *cleanup.CleanUpService, live *live.GrafanaLive, pushGateway *pushhttp.Gateway, notifications *notifications.NotificationService, pluginStore *pluginStore.Service, rendering *rendering.RenderingService, tokenService auth.UserTokenBackgroundService, tracing *tracing.TracingService, provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, usageStats *uss.UsageStats, @@ -67,7 +66,6 @@ func ProvideBackgroundServiceRegistry( ) *BackgroundServiceRegistry { return NewBackgroundServiceRegistry( httpServer, - alertingMigrationService, ng, cleanup, live, diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 6b80a5aab15..44755d416b4 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -92,8 +92,6 @@ import ( "github.com/grafana/grafana/pkg/services/ngalert" ngimage "github.com/grafana/grafana/pkg/services/ngalert/image" ngmetrics "github.com/grafana/grafana/pkg/services/ngalert/metrics" - ngmigration "github.com/grafana/grafana/pkg/services/ngalert/migration" - migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" ngstore "github.com/grafana/grafana/pkg/services/ngalert/store" "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/services/oauthtoken" @@ -241,8 +239,6 @@ var wireBasicSet = wire.NewSet( wire.Bind(new(jwt.JWTService), new(*jwt.AuthService)), ngstore.ProvideDBStore, ngimage.ProvideDeleteExpiredService, - ngmigration.ProvideService, - migrationStore.ProvideMigrationStore, ngalert.ProvideService, librarypanels.ProvideService, wire.Bind(new(librarypanels.Service), new(*librarypanels.LibraryPanelService)), diff --git a/pkg/services/ngalert/api/api_alertmanager_test.go b/pkg/services/ngalert/api/api_alertmanager_test.go index 90a9a3a0060..cef372a4dc9 100644 --- a/pkg/services/ngalert/api/api_alertmanager_test.go +++ b/pkg/services/ngalert/api/api_alertmanager_test.go @@ -24,7 +24,6 @@ import ( ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/notifier" "github.com/grafana/grafana/pkg/services/ngalert/provisioning" - ngfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" @@ -664,7 +663,7 @@ func createMultiOrgAlertmanager(t *testing.T) *notifier.MultiOrgAlertmanager { orgStore := notifier.NewFakeOrgStore(t, []int64{1, 2, 3}) provStore := provisioning.NewFakeProvisioningStore() tmpDir := t.TempDir() - kvStore := ngfakes.NewFakeKVStore(t) + kvStore := notifier.NewFakeKVStore(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) reg := prometheus.NewPedanticRegistry() m := metrics.NewNGAlert(reg) diff --git a/pkg/services/ngalert/migration/channel.go b/pkg/services/ngalert/migration/channel.go deleted file mode 100644 index 35455f7172a..00000000000 --- a/pkg/services/ngalert/migration/channel.go +++ /dev/null @@ -1,440 +0,0 @@ -package migration - -import ( - "context" - "crypto/md5" - "encoding/base64" - "fmt" - "regexp" - "sort" - "strings" - "time" - - alertingNotify "github.com/grafana/alerting/notify" - "github.com/prometheus/alertmanager/config" - "github.com/prometheus/alertmanager/pkg/labels" - "github.com/prometheus/common/model" - - "github.com/grafana/grafana/pkg/components/simplejson" - legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" - apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" - migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" - ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/services/secrets" - "github.com/grafana/grafana/pkg/util" -) - -const ( - // DisabledRepeatInterval is a large duration that will be used as a pseudo-disable in case a legacy channel doesn't have SendReminders enabled. - DisabledRepeatInterval = model.Duration(time.Duration(8736) * time.Hour) // 1y -) - -// channelReceiver is a convenience struct that contains a notificationChannel and its corresponding migrated PostableApiReceiver. -type channelReceiver struct { - channel *legacymodels.AlertNotification - receiver *apimodels.PostableApiReceiver -} - -// setupAlertmanagerConfigs creates Alertmanager configs with migrated receivers and routes. -func (om *OrgMigration) migrateChannels(allChannels []*legacymodels.AlertNotification, pairs []*AlertPair) (*apimodels.PostableUserConfig, error) { - var defaultChannels []*legacymodels.AlertNotification - var channels []*legacymodels.AlertNotification - for _, c := range allChannels { - if c.Type == "hipchat" || c.Type == "sensu" { - om.log.Error("Alert migration error: discontinued notification channel found", "type", c.Type, "name", c.Name, "uid", c.UID) - continue - } - - if c.IsDefault { - defaultChannels = append(defaultChannels, c) - } - channels = append(channels, c) - } - - amConfig := &apimodels.PostableUserConfig{ - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Receivers: make([]*apimodels.PostableApiReceiver, 0), - }, - } - - // Create all newly migrated receivers from legacy notification channels. - receiversMap, receivers, err := om.createReceivers(channels) - if err != nil { - return nil, fmt.Errorf("create receiver: %w", err) - } - - // No need to create an Alertmanager configuration if there are no receivers left that aren't obsolete. - if len(receivers) == 0 { - om.log.Warn("No available receivers") - return nil, nil - } - - for _, cr := range receivers { - amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, cr.receiver) - } - - defaultReceivers := make(map[string]struct{}) - // If the organization has default channels build a map of default receivers, used to create alert-specific routes later. - for _, c := range defaultChannels { - defaultReceivers[c.Name] = struct{}{} - } - defaultReceiver, defaultRoute, err := om.createDefaultRouteAndReceiver(defaultChannels) - if err != nil { - return nil, fmt.Errorf("failed to create default route & receiver in orgId %d: %w", om.orgID, err) - } - amConfig.AlertmanagerConfig.Route = defaultRoute - if defaultReceiver != nil { - amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, defaultReceiver) - } - - for _, cr := range receivers { - route, err := createRoute(cr) - if err != nil { - return nil, fmt.Errorf("failed to create route for receiver %s in orgId %d: %w", cr.receiver.Name, om.orgID, err) - } - - amConfig.AlertmanagerConfig.Route.Routes = append(amConfig.AlertmanagerConfig.Route.Routes, route) - } - - for _, pair := range pairs { - channelUids := extractChannelIDs(pair.DashAlert) - filteredReceiverNames := om.filterReceiversForAlert(pair.AlertRule.Title, channelUids, receiversMap, defaultReceivers) - - if len(filteredReceiverNames) != 0 { - // Only create a contact label if there are specific receivers, otherwise it defaults to the root-level route. - pair.AlertRule.Labels[ContactLabel] = contactListToString(filteredReceiverNames) - } - } - - // Validate the alertmanager configuration produced, this gives a chance to catch bad configuration at migration time. - // Validation between legacy and unified alerting can be different (e.g. due to bug fixes) so this would fail the migration in that case. - if err := om.validateAlertmanagerConfig(amConfig); err != nil { - return nil, fmt.Errorf("failed to validate AlertmanagerConfig in orgId %d: %w", om.orgID, err) - } - - return amConfig, nil -} - -// validateAlertmanagerConfig validates the alertmanager configuration produced by the migration against the receivers. -func (om *OrgMigration) validateAlertmanagerConfig(config *apimodels.PostableUserConfig) error { - for _, r := range config.AlertmanagerConfig.Receivers { - for _, gr := range r.GrafanaManagedReceivers { - data, err := gr.Settings.MarshalJSON() - if err != nil { - return err - } - var ( - cfg = &alertingNotify.GrafanaIntegrationConfig{ - UID: gr.UID, - Name: gr.Name, - Type: gr.Type, - DisableResolveMessage: gr.DisableResolveMessage, - Settings: data, - SecureSettings: gr.SecureSettings, - } - ) - - _, err = alertingNotify.BuildReceiverConfiguration(context.Background(), &alertingNotify.APIReceiver{ - GrafanaIntegrations: alertingNotify.GrafanaIntegrations{Integrations: []*alertingNotify.GrafanaIntegrationConfig{cfg}}, - }, om.encryptionService.GetDecryptedValue) - if err != nil { - return err - } - } - } - - return nil -} - -// contactListToString creates a sorted string representation of a given map (set) of receiver names. Each name will be comma-separated and double-quoted. Names should not contain double quotes. -func contactListToString(m map[string]any) string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, quote(k)) - } - sort.Strings(keys) - - return strings.Join(keys, ",") -} - -// quote will surround the given string in double quotes. -func quote(s string) string { - return `"` + s + `"` -} - -// Create a notifier (PostableGrafanaReceiver) from a legacy notification channel -func (om *OrgMigration) createNotifier(c *legacymodels.AlertNotification) (*apimodels.PostableGrafanaReceiver, error) { - uid, err := om.determineChannelUid(c) - if err != nil { - return nil, err - } - - settings, secureSettings, err := om.migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings) - if err != nil { - return nil, err - } - - data, err := settings.MarshalJSON() - if err != nil { - return nil, err - } - - return &apimodels.PostableGrafanaReceiver{ - UID: uid, - Name: c.Name, - Type: c.Type, - DisableResolveMessage: c.DisableResolveMessage, - Settings: data, - SecureSettings: secureSettings, - }, nil -} - -// Create one receiver for every unique notification channel. -func (om *OrgMigration) createReceivers(allChannels []*legacymodels.AlertNotification) (map[migrationStore.UidOrID]*apimodels.PostableApiReceiver, []channelReceiver, error) { - receivers := make([]channelReceiver, 0, len(allChannels)) - receiversMap := make(map[migrationStore.UidOrID]*apimodels.PostableApiReceiver) - - set := make(map[string]struct{}) // Used to deduplicate sanitized names. - for _, c := range allChannels { - notifier, err := om.createNotifier(c) - if err != nil { - return nil, nil, err - } - - // We remove double quotes because this character will be used as the separator in the ContactLabel. To prevent partial matches in the Route Matcher we choose to sanitize them early on instead of complicating the Matcher regex. - sanitizedName := strings.ReplaceAll(c.Name, `"`, `_`) - // There can be name collisions after we sanitize. We check for this and attempt to make the name unique again using a short hash of the original name. - if _, ok := set[sanitizedName]; ok { - sanitizedName = sanitizedName + fmt.Sprintf("_%.3x", md5.Sum([]byte(c.Name))) - om.log.Warn("Alert contains duplicate contact name after sanitization, appending unique suffix", "type", c.Type, "name", c.Name, "new_name", sanitizedName, "uid", c.UID) - } - notifier.Name = sanitizedName - - set[sanitizedName] = struct{}{} - - cr := channelReceiver{ - channel: c, - receiver: &apimodels.PostableApiReceiver{ - Receiver: config.Receiver{ - Name: sanitizedName, // Channel name is unique within an Org. - }, - PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{ - GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{notifier}, - }, - }, - } - - receivers = append(receivers, cr) - - // Store receivers for creating routes from alert rules later. - if c.UID != "" { - receiversMap[c.UID] = cr.receiver - } - if c.ID != 0 { - // In certain circumstances, the alert rule uses ID instead of uid. So, we add this to be able to lookup by ID in case. - receiversMap[c.ID] = cr.receiver - } - } - - return receiversMap, receivers, nil -} - -// Create the root-level route with the default receiver. If no new receiver is created specifically for the root-level route, the returned receiver will be nil. -func (om *OrgMigration) createDefaultRouteAndReceiver(defaultChannels []*legacymodels.AlertNotification) (*apimodels.PostableApiReceiver, *apimodels.Route, error) { - defaultReceiverName := "autogen-contact-point-default" - defaultRoute := &apimodels.Route{ - Receiver: defaultReceiverName, - Routes: make([]*apimodels.Route, 0), - GroupByStr: []string{ngmodels.FolderTitleLabel, model.AlertNameLabel}, // To keep parity with pre-migration notifications. - RepeatInterval: nil, - } - newDefaultReceiver := &apimodels.PostableApiReceiver{ - Receiver: config.Receiver{ - Name: defaultReceiverName, - }, - PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{ - GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{}, - }, - } - - // Return early if there are no default channels - if len(defaultChannels) == 0 { - return newDefaultReceiver, defaultRoute, nil - } - - repeatInterval := DisabledRepeatInterval // If no channels have SendReminders enabled, we will use this large value as a pseudo-disable. - if len(defaultChannels) > 1 { - // If there are more than one default channels we create a separate contact group that is used only in the root policy. This is to simplify the migrated notification policy structure. - // If we ever allow more than one receiver per route this won't be necessary. - for _, c := range defaultChannels { - // Need to create a new notifier to prevent uid conflict. - defaultNotifier, err := om.createNotifier(c) - if err != nil { - return nil, nil, err - } - - newDefaultReceiver.GrafanaManagedReceivers = append(newDefaultReceiver.GrafanaManagedReceivers, defaultNotifier) - - // Choose the lowest send reminder duration from all the notifiers to use for default route. - if c.SendReminder && c.Frequency < time.Duration(repeatInterval) { - repeatInterval = model.Duration(c.Frequency) - } - } - } else { - // If there is only a single default channel, we don't need a separate receiver to hold it. We can reuse the existing receiver for that single notifier. - defaultRoute.Receiver = defaultChannels[0].Name - if defaultChannels[0].SendReminder { - repeatInterval = model.Duration(defaultChannels[0].Frequency) - } - - // No need to create a new receiver. - newDefaultReceiver = nil - } - defaultRoute.RepeatInterval = &repeatInterval - - return newDefaultReceiver, defaultRoute, nil -} - -// Create one route per contact point, matching based on ContactLabel. -func createRoute(cr channelReceiver) (*apimodels.Route, error) { - // We create a regex matcher so that each alert rule need only have a single ContactLabel entry for all contact points it sends to. - // For example, if an alert needs to send to contact1 and contact2 it will have ContactLabel=`"contact1","contact2"` and will match both routes looking - // for `.*"contact1".*` and `.*"contact2".*`. - - // We quote and escape here to ensure the regex will correctly match the ContactLabel on the alerts. - name := fmt.Sprintf(`.*%s.*`, regexp.QuoteMeta(quote(cr.receiver.Name))) - mat, err := labels.NewMatcher(labels.MatchRegexp, ContactLabel, name) - if err != nil { - return nil, err - } - - repeatInterval := DisabledRepeatInterval - if cr.channel.SendReminder { - repeatInterval = model.Duration(cr.channel.Frequency) - } - - return &apimodels.Route{ - Receiver: cr.receiver.Name, - ObjectMatchers: apimodels.ObjectMatchers{mat}, - Continue: true, // We continue so that each sibling contact point route can separately match. - RepeatInterval: &repeatInterval, - }, nil -} - -// Filter receivers to select those that were associated to the given rule as channels. -func (om *OrgMigration) filterReceiversForAlert(name string, channelIDs []migrationStore.UidOrID, receivers map[migrationStore.UidOrID]*apimodels.PostableApiReceiver, defaultReceivers map[string]struct{}) map[string]any { - if len(channelIDs) == 0 { - // If there are no channels associated, we use the default route. - return nil - } - - // Filter receiver names. - filteredReceiverNames := make(map[string]any) - for _, uidOrId := range channelIDs { - recv, ok := receivers[uidOrId] - if ok { - filteredReceiverNames[recv.Name] = struct{}{} // Deduplicate on contact point name. - } else { - om.log.Warn("Alert linked to obsolete notification channel, ignoring", "alert", name, "uid", uidOrId) - } - } - - coveredByDefault := func(names map[string]any) bool { - // Check if all receivers are also default ones and if so, just use the default route. - for n := range names { - if _, ok := defaultReceivers[n]; !ok { - return false - } - } - return true - } - - if len(filteredReceiverNames) == 0 || coveredByDefault(filteredReceiverNames) { - // Use the default route instead. - return nil - } - - // Add default receivers alongside rule-specific ones. - for n := range defaultReceivers { - filteredReceiverNames[n] = struct{}{} - } - - return filteredReceiverNames -} - -func (om *OrgMigration) determineChannelUid(c *legacymodels.AlertNotification) (string, error) { - legacyUid := c.UID - if legacyUid == "" { - newUid := util.GenerateShortUID() - om.seenUIDs.add(newUid) - om.log.Info("Legacy notification had an empty uid, generating a new one", "id", c.ID, "uid", newUid) - return newUid, nil - } - - if om.seenUIDs.contains(legacyUid) { - newUid := util.GenerateShortUID() - om.seenUIDs.add(newUid) - om.log.Warn("Legacy notification had a UID that collides with a migrated record, generating a new one", "id", c.ID, "old", legacyUid, "new", newUid) - return newUid, nil - } - - om.seenUIDs.add(legacyUid) - return legacyUid, nil -} - -var secureKeysToMigrate = map[string][]string{ - "slack": {"url", "token"}, - "pagerduty": {"integrationKey"}, - "webhook": {"password"}, - "prometheus-alertmanager": {"basicAuthPassword"}, - "opsgenie": {"apiKey"}, - "telegram": {"bottoken"}, - "line": {"token"}, - "pushover": {"apiToken", "userKey"}, - "threema": {"api_secret"}, -} - -// Some settings were migrated from settings to secure settings in between. -// See https://grafana.com/docs/grafana/latest/installation/upgrading/#ensure-encryption-of-existing-alert-notification-channel-secrets. -// migrateSettingsToSecureSettings takes care of that. -func (om *OrgMigration) migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json, secureSettings SecureJsonData) (*simplejson.Json, map[string]string, error) { - keys := secureKeysToMigrate[chanType] - newSecureSettings := secureSettings.Decrypt() - cloneSettings := simplejson.New() - settingsMap, err := settings.Map() - if err != nil { - return nil, nil, err - } - for k, v := range settingsMap { - cloneSettings.Set(k, v) - } - for _, k := range keys { - if v, ok := newSecureSettings[k]; ok && v != "" { - continue - } - - sv := cloneSettings.Get(k).MustString() - if sv != "" { - newSecureSettings[k] = sv - cloneSettings.Del(k) - } - } - - err = om.encryptSecureSettings(newSecureSettings) - if err != nil { - return nil, nil, err - } - - return cloneSettings, newSecureSettings, nil -} - -func (om *OrgMigration) encryptSecureSettings(secureSettings map[string]string) error { - for key, value := range secureSettings { - encryptedData, err := om.encryptionService.Encrypt(context.Background(), []byte(value), secrets.WithoutScope()) - if err != nil { - return fmt.Errorf("failed to encrypt secure settings: %w", err) - } - secureSettings[key] = base64.StdEncoding.EncodeToString(encryptedData) - } - return nil -} diff --git a/pkg/services/ngalert/migration/channel_test.go b/pkg/services/ngalert/migration/channel_test.go deleted file mode 100644 index 88f4ddbbc45..00000000000 --- a/pkg/services/ngalert/migration/channel_test.go +++ /dev/null @@ -1,591 +0,0 @@ -package migration - -import ( - "context" - "encoding/base64" - "encoding/json" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/prometheus/alertmanager/config" - "github.com/prometheus/alertmanager/pkg/labels" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/infra/db" - legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" - apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" - migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" - ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/services/ngalert/notifier/channels_config" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/util" -) - -func TestFilterReceiversForAlert(t *testing.T) { - tc := []struct { - name string - channelIds []migrationStore.UidOrID - receivers map[migrationStore.UidOrID]*apimodels.PostableApiReceiver - defaultReceivers map[string]struct{} - expected map[string]any - }{ - { - name: "when an alert has multiple channels, each should filter for the correct receiver", - channelIds: []migrationStore.UidOrID{"uid1", "uid2"}, - receivers: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ - "uid1": createPostableApiReceiver("recv1", nil), - "uid2": createPostableApiReceiver("recv2", nil), - "uid3": createPostableApiReceiver("recv3", nil), - }, - defaultReceivers: map[string]struct{}{}, - expected: map[string]any{ - "recv1": struct{}{}, - "recv2": struct{}{}, - }, - }, - { - name: "when default receivers exist, they should be added to an alert's filtered receivers", - channelIds: []migrationStore.UidOrID{"uid1"}, - receivers: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ - "uid1": createPostableApiReceiver("recv1", nil), - "uid2": createPostableApiReceiver("recv2", nil), - "uid3": createPostableApiReceiver("recv3", nil), - }, - defaultReceivers: map[string]struct{}{ - "recv2": {}, - }, - expected: map[string]any{ - "recv1": struct{}{}, // From alert - "recv2": struct{}{}, // From default - }, - }, - { - name: "when an alert has a channels associated by ID instead of UID, it should be included", - channelIds: []migrationStore.UidOrID{int64(42)}, - receivers: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ - int64(42): createPostableApiReceiver("recv1", nil), - }, - defaultReceivers: map[string]struct{}{}, - expected: map[string]any{ - "recv1": struct{}{}, - }, - }, - { - name: "when an alert's receivers are covered by the defaults, return nil to use default receiver downstream", - channelIds: []migrationStore.UidOrID{"uid1"}, - receivers: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ - "uid1": createPostableApiReceiver("recv1", nil), - "uid2": createPostableApiReceiver("recv2", nil), - "uid3": createPostableApiReceiver("recv3", nil), - }, - defaultReceivers: map[string]struct{}{ - "recv1": {}, - "recv2": {}, - }, - expected: nil, // recv1 is already a default - }, - } - - sqlStore := db.InitTestDB(t) - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) - res := m.filterReceiversForAlert("", tt.channelIds, tt.receivers, tt.defaultReceivers) - - require.Equal(t, tt.expected, res) - }) - } -} - -func TestCreateRoute(t *testing.T) { - tc := []struct { - name string - channel *legacymodels.AlertNotification - recv *apimodels.PostableApiReceiver - expected *apimodels.Route - }{ - { - name: "when a receiver is passed in, the route should regex match based on quoted name with continue=true", - channel: &legacymodels.AlertNotification{}, - recv: createPostableApiReceiver("recv1", nil), - expected: &apimodels.Route{ - Receiver: "recv1", - ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, - Routes: nil, - Continue: true, - GroupByStr: nil, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }, - }, - { - name: "notification channel should be escaped for regex in the matcher", - channel: &legacymodels.AlertNotification{}, - recv: createPostableApiReceiver(`. ^ $ * + - ? ( ) [ ] { } \ |`, nil), - expected: &apimodels.Route{ - Receiver: `. ^ $ * + - ? ( ) [ ] { } \ |`, - ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"\. \^ \$ \* \+ - \? \( \) \[ \] \{ \} \\ \|".*`}}, - Routes: nil, - Continue: true, - GroupByStr: nil, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }, - }, - { - name: "when a channel has sendReminder=true, the route should use the frequency in repeat interval", - channel: &legacymodels.AlertNotification{SendReminder: true, Frequency: time.Duration(42) * time.Hour}, - recv: createPostableApiReceiver("recv1", nil), - expected: &apimodels.Route{ - Receiver: "recv1", - ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, - Routes: nil, - Continue: true, - GroupByStr: nil, - RepeatInterval: durationPointer(model.Duration(time.Duration(42) * time.Hour)), - }, - }, - { - name: "when a channel has sendReminder=false, the route should ignore the frequency in repeat interval and use DisabledRepeatInterval", - channel: &legacymodels.AlertNotification{SendReminder: false, Frequency: time.Duration(42) * time.Hour}, - recv: createPostableApiReceiver("recv1", nil), - expected: &apimodels.Route{ - Receiver: "recv1", - ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, - Routes: nil, - Continue: true, - GroupByStr: nil, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }, - }, - } - - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - res, err := createRoute(channelReceiver{ - channel: tt.channel, - receiver: tt.recv, - }) - require.NoError(t, err) - - // Order of nested routes is not guaranteed. - cOpt := []cmp.Option{ - cmpopts.SortSlices(func(a, b *apimodels.Route) bool { - if a.Receiver != b.Receiver { - return a.Receiver < b.Receiver - } - return a.ObjectMatchers[0].Value < b.ObjectMatchers[0].Value - }), - cmpopts.IgnoreUnexported(apimodels.Route{}, labels.Matcher{}), - } - - if !cmp.Equal(tt.expected, res, cOpt...) { - t.Errorf("Unexpected Route: %v", cmp.Diff(tt.expected, res, cOpt...)) - } - }) - } -} - -func createNotChannel(t *testing.T, uid string, id int64, name string) *legacymodels.AlertNotification { - t.Helper() - return &legacymodels.AlertNotification{UID: uid, ID: id, Name: name, Settings: simplejson.New()} -} - -func createNotChannelWithReminder(t *testing.T, uid string, id int64, name string, frequency time.Duration) *legacymodels.AlertNotification { - t.Helper() - return &legacymodels.AlertNotification{UID: uid, ID: id, Name: name, SendReminder: true, Frequency: frequency, Settings: simplejson.New()} -} - -func TestCreateReceivers(t *testing.T) { - tc := []struct { - name string - allChannels []*legacymodels.AlertNotification - defaultChannels []*legacymodels.AlertNotification - expRecvMap map[migrationStore.UidOrID]*apimodels.PostableApiReceiver - expRecv []channelReceiver - expErr error - }{ - { - name: "when given notification channels migrate them to receivers", - allChannels: []*legacymodels.AlertNotification{createNotChannel(t, "uid1", int64(1), "name1"), createNotChannel(t, "uid2", int64(2), "name2")}, - expRecvMap: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ - "uid1": createPostableApiReceiver("name1", []string{"name1"}), - "uid2": createPostableApiReceiver("name2", []string{"name2"}), - int64(1): createPostableApiReceiver("name1", []string{"name1"}), - int64(2): createPostableApiReceiver("name2", []string{"name2"}), - }, - expRecv: []channelReceiver{ - { - channel: createNotChannel(t, "uid1", int64(1), "name1"), - receiver: createPostableApiReceiver("name1", []string{"name1"}), - }, - { - channel: createNotChannel(t, "uid2", int64(2), "name2"), - receiver: createPostableApiReceiver("name2", []string{"name2"}), - }, - }, - }, - { - name: "when given notification channel contains double quote sanitize with underscore", - allChannels: []*legacymodels.AlertNotification{createNotChannel(t, "uid1", int64(1), "name\"1")}, - expRecvMap: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ - "uid1": createPostableApiReceiver("name_1", []string{"name_1"}), - int64(1): createPostableApiReceiver("name_1", []string{"name_1"}), - }, - expRecv: []channelReceiver{ - { - channel: createNotChannel(t, "uid1", int64(1), "name\"1"), - receiver: createPostableApiReceiver("name_1", []string{"name_1"}), - }, - }, - }, - { - name: "when given notification channels collide after sanitization add short hash to end", - allChannels: []*legacymodels.AlertNotification{createNotChannel(t, "uid1", int64(1), "name\"1"), createNotChannel(t, "uid2", int64(2), "name_1")}, - expRecvMap: map[migrationStore.UidOrID]*apimodels.PostableApiReceiver{ - "uid1": createPostableApiReceiver("name_1", []string{"name_1"}), - "uid2": createPostableApiReceiver("name_1_dba13d", []string{"name_1_dba13d"}), - int64(1): createPostableApiReceiver("name_1", []string{"name_1"}), - int64(2): createPostableApiReceiver("name_1_dba13d", []string{"name_1_dba13d"}), - }, - expRecv: []channelReceiver{ - { - channel: createNotChannel(t, "uid1", int64(1), "name\"1"), - receiver: createPostableApiReceiver("name_1", []string{"name_1"}), - }, - { - channel: createNotChannel(t, "uid2", int64(2), "name_1"), - receiver: createPostableApiReceiver("name_1_dba13d", []string{"name_1_dba13d"}), - }, - }, - }, - } - - sqlStore := db.InitTestDB(t) - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) - recvMap, recvs, err := m.createReceivers(tt.allChannels) - if tt.expErr != nil { - require.Error(t, err) - require.EqualError(t, err, tt.expErr.Error()) - return - } - - require.NoError(t, err) - - // We ignore certain fields for the purposes of this test - for _, recv := range recvs { - for _, not := range recv.receiver.GrafanaManagedReceivers { - not.UID = "" - not.Settings = nil - not.SecureSettings = nil - } - } - - require.Equal(t, tt.expRecvMap, recvMap) - require.ElementsMatch(t, tt.expRecv, recvs) - }) - } -} - -func TestMigrateNotificationChannelSecureSettings(t *testing.T) { - legacyEncryptFn := func(data string) string { - raw, err := util.Encrypt([]byte(data), setting.SecretKey) - require.NoError(t, err) - return string(raw) - } - decryptFn := func(data string, m *OrgMigration) string { - decoded, err := base64.StdEncoding.DecodeString(data) - require.NoError(t, err) - raw, err := m.encryptionService.Decrypt(context.Background(), decoded) - require.NoError(t, err) - return string(raw) - } - gen := func(nType string, fn func(channel *legacymodels.AlertNotification)) *legacymodels.AlertNotification { - not := &legacymodels.AlertNotification{ - UID: "uid", - ID: 1, - Name: "channel name", - Type: nType, - Settings: simplejson.NewFromAny(map[string]any{ - "something": "some value", - }), - SecureSettings: map[string][]byte{}, - } - if fn != nil { - fn(not) - } - return not - } - genExpSlack := func(fn func(channel *apimodels.PostableGrafanaReceiver)) *apimodels.PostableGrafanaReceiver { - rawSettings, err := json.Marshal(map[string]string{ - "something": "some value", - }) - require.NoError(t, err) - - recv := &apimodels.PostableGrafanaReceiver{ - UID: "uid", - Name: "channel name", - Type: "slack", - Settings: rawSettings, - SecureSettings: map[string]string{ - "token": "secure token", - "url": "secure url", - }, - } - - if fn != nil { - fn(recv) - } - return recv - } - - tc := []struct { - name string - channel *legacymodels.AlertNotification - expRecv *apimodels.PostableGrafanaReceiver - expErr error - }{ - { - name: "when secure settings exist, migrate them to receiver secure settings", - channel: gen("slack", func(channel *legacymodels.AlertNotification) { - channel.SecureSettings = map[string][]byte{ - "token": []byte(legacyEncryptFn("secure token")), - "url": []byte(legacyEncryptFn("secure url")), - } - }), - expRecv: genExpSlack(nil), - }, - { - name: "when no secure settings are encrypted, do nothing", - channel: gen("slack", nil), - expRecv: genExpSlack(func(recv *apimodels.PostableGrafanaReceiver) { - delete(recv.SecureSettings, "token") - delete(recv.SecureSettings, "url") - }), - }, - { - name: "when some secure settings are available unencrypted in settings, migrate them to secureSettings and encrypt", - channel: gen("slack", func(channel *legacymodels.AlertNotification) { - channel.SecureSettings = map[string][]byte{ - "url": []byte(legacyEncryptFn("secure url")), - } - channel.Settings.Set("token", "secure token") - }), - expRecv: genExpSlack(nil), - }, - } - sqlStore := db.InitTestDB(t) - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) - recv, err := m.createNotifier(tt.channel) - if tt.expErr != nil { - require.Error(t, err) - require.EqualError(t, err, tt.expErr.Error()) - return - } - require.NoError(t, err) - - if len(tt.expRecv.SecureSettings) > 0 { - require.NotEqual(t, tt.expRecv, recv) // Make sure they were actually encrypted at first. - } - for k, v := range recv.SecureSettings { - recv.SecureSettings[k] = decryptFn(v, m) - } - require.Equal(t, tt.expRecv, recv) - }) - } - - // Generate tests for each notification channel type. - t.Run("secure settings migrations for each notifier type", func(t *testing.T) { - notifiers := channels_config.GetAvailableNotifiers() - t.Run("migrate notification channel secure settings to receiver secure settings", func(t *testing.T) { - for _, notifier := range notifiers { - nType := notifier.Type - secureSettings, err := channels_config.GetSecretKeysForContactPointType(nType) - require.NoError(t, err) - t.Run(nType, func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) - channel := gen(nType, func(channel *legacymodels.AlertNotification) { - for _, key := range secureSettings { - channel.SecureSettings[key] = []byte(legacyEncryptFn("secure " + key)) - } - }) - recv, err := m.createNotifier(channel) - require.NoError(t, err) - - require.Equal(t, nType, recv.Type) - if len(secureSettings) > 0 { - for _, key := range secureSettings { - require.NotEqual(t, "secure "+key, recv.SecureSettings[key]) // Make sure they were actually encrypted at first. - } - } - require.Len(t, recv.SecureSettings, len(secureSettings)) - for _, key := range secureSettings { - require.Equal(t, "secure "+key, decryptFn(recv.SecureSettings[key], m)) - } - }) - } - }) - - t.Run("for certain legacy channel types, migrate secure fields stored in settings to secure settings", func(t *testing.T) { - for _, notifier := range notifiers { - nType := notifier.Type - secureSettings, ok := secureKeysToMigrate[nType] - if !ok { - continue - } - t.Run(nType, func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) - - channel := gen(nType, func(channel *legacymodels.AlertNotification) { - for _, key := range secureSettings { - // Key difference to above. We store the secure settings in the settings field and expect - // them to be migrated to secureSettings. - channel.Settings.Set(key, "secure "+key) - } - }) - recv, err := m.createNotifier(channel) - require.NoError(t, err) - - require.Equal(t, nType, recv.Type) - if len(secureSettings) > 0 { - for _, key := range secureSettings { - require.NotEqual(t, "secure "+key, recv.SecureSettings[key]) // Make sure they were actually encrypted at first. - } - } - require.Len(t, recv.SecureSettings, len(secureSettings)) - for _, key := range secureSettings { - require.Equal(t, "secure "+key, decryptFn(recv.SecureSettings[key], m)) - } - }) - } - }) - }) -} - -func TestCreateDefaultRouteAndReceiver(t *testing.T) { - tc := []struct { - name string - amConfig *apimodels.PostableUserConfig - defaultChannels []*legacymodels.AlertNotification - expRecv *apimodels.PostableApiReceiver - expRoute *apimodels.Route - expErr error - }{ - { - name: "when given multiple default notification channels migrate them to a single receiver", - defaultChannels: []*legacymodels.AlertNotification{createNotChannel(t, "uid1", int64(1), "name1"), createNotChannel(t, "uid2", int64(2), "name2")}, - expRecv: createPostableApiReceiver("autogen-contact-point-default", []string{"name1", "name2"}), - expRoute: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - Routes: make([]*apimodels.Route, 0), - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }, - }, - { - name: "when given multiple default notification channels migrate them to a single receiver with RepeatInterval set to be the minimum of all channel frequencies", - defaultChannels: []*legacymodels.AlertNotification{ - createNotChannelWithReminder(t, "uid1", int64(1), "name1", time.Duration(42)), - createNotChannelWithReminder(t, "uid2", int64(2), "name2", time.Duration(100000)), - }, - expRecv: createPostableApiReceiver("autogen-contact-point-default", []string{"name1", "name2"}), - expRoute: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - Routes: make([]*apimodels.Route, 0), - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - RepeatInterval: durationPointer(model.Duration(42)), - }, - }, - { - name: "when given no default notification channels create a single empty receiver for default", - defaultChannels: []*legacymodels.AlertNotification{}, - expRecv: createPostableApiReceiver("autogen-contact-point-default", nil), - expRoute: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - Routes: make([]*apimodels.Route, 0), - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - RepeatInterval: nil, - }, - }, - { - name: "when given a single default notification channels don't create a new default receiver", - defaultChannels: []*legacymodels.AlertNotification{createNotChannel(t, "uid1", int64(1), "name1")}, - expRecv: nil, - expRoute: &apimodels.Route{ - Receiver: "name1", - Routes: make([]*apimodels.Route, 0), - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }, - }, - { - name: "when given a single default notification channel with SendReminder=true, use the channels Frequency as the RepeatInterval", - defaultChannels: []*legacymodels.AlertNotification{createNotChannelWithReminder(t, "uid1", int64(1), "name1", time.Duration(42))}, - expRecv: nil, - expRoute: &apimodels.Route{ - Receiver: "name1", - Routes: make([]*apimodels.Route, 0), - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - RepeatInterval: durationPointer(model.Duration(42)), - }, - }, - } - - sqlStore := db.InitTestDB(t) - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) - recv, route, err := m.createDefaultRouteAndReceiver(tt.defaultChannels) - if tt.expErr != nil { - require.Error(t, err) - require.EqualError(t, err, tt.expErr.Error()) - return - } - - require.NoError(t, err) - - // We ignore certain fields for the purposes of this test - if recv != nil { - for _, not := range recv.GrafanaManagedReceivers { - not.UID = "" - not.Settings = nil - not.SecureSettings = nil - } - } - - require.Equal(t, tt.expRecv, recv) - require.Equal(t, tt.expRoute, route) - }) - } -} - -func createPostableApiReceiver(name string, integrationNames []string) *apimodels.PostableApiReceiver { - integrations := make([]*apimodels.PostableGrafanaReceiver, 0, len(integrationNames)) - for _, integrationName := range integrationNames { - integrations = append(integrations, &apimodels.PostableGrafanaReceiver{Name: integrationName}) - } - return &apimodels.PostableApiReceiver{ - Receiver: config.Receiver{ - Name: name, - }, - PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{ - GrafanaManagedReceivers: integrations, - }, - } -} - -func durationPointer(d model.Duration) *model.Duration { - return &d -} diff --git a/pkg/services/ngalert/migration/migration_test.go b/pkg/services/ngalert/migration/migration_test.go deleted file mode 100644 index d5edee7b274..00000000000 --- a/pkg/services/ngalert/migration/migration_test.go +++ /dev/null @@ -1,1326 +0,0 @@ -package migration - -import ( - "context" - "encoding/json" - "fmt" - "strconv" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/prometheus/alertmanager/config" - "github.com/prometheus/alertmanager/pkg/labels" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - "xorm.io/xorm" - - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/infra/db" - "github.com/grafana/grafana/pkg/services/alerting/models" - "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/datasources" - apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" - migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" - ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/services/org" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/tsdb/legacydata" -) - -// TestServiceStart tests the wrapper method that decides when to run the migration based on migration status and settings. -func TestServiceStart(t *testing.T) { - tc := []struct { - name string - config *setting.Cfg - isMigrationRun bool - expectedErr bool - expected bool - }{ - { - name: "when unified alerting enabled and migration not already run, then run migration", - config: &setting.Cfg{ - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: pointer(true), - }, - }, - isMigrationRun: false, - expected: true, - }, - { - name: "when unified alerting disabled, migration is already run and force migration is enabled, then revert migration", - config: &setting.Cfg{ - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: pointer(false), - }, - ForceMigration: true, - }, - isMigrationRun: true, - expected: false, - }, - { - name: "when unified alerting disabled, migration is already run and force migration is disabled, then the migration should panic", - config: &setting.Cfg{ - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: pointer(false), - }, - ForceMigration: false, - }, - isMigrationRun: true, - expected: true, - expectedErr: true, - }, - { - name: "when unified alerting enabled and migration is already run, then do nothing", - config: &setting.Cfg{ - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: pointer(true), - }, - }, - isMigrationRun: true, - expected: true, - }, - { - name: "when unified alerting disabled and migration is not already run, then do nothing", - config: &setting.Cfg{ - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: pointer(false), - }, - }, - isMigrationRun: false, - expected: false, - }, - } - - sqlStore := db.InitTestDB(t) - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - service := NewTestMigrationService(t, sqlStore, tt.config) - - err := service.migrationStore.SetMigrated(ctx, tt.isMigrationRun) - require.NoError(t, err) - - err = service.Run(ctx) - if tt.expectedErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } - - migrated, err := service.migrationStore.IsMigrated(ctx) - require.NoError(t, err) - require.Equal(t, tt.expected, migrated) - }) - } -} - -// TestAMConfigMigration tests the execution of the migration specifically for migrations of channels and routes. -func TestAMConfigMigration(t *testing.T) { - sqlStore := db.InitTestDB(t) - x := sqlStore.GetEngine() - service := NewTestMigrationService(t, sqlStore, &setting.Cfg{}) - tc := []struct { - name string - legacyChannels []*models.AlertNotification - alerts []*models.Alert - - expected map[int64]*apimodels.PostableUserConfig - expErr error - }{ - { - name: "general multi-org, multi-alert, multi-channel migration", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), - createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, false), - createAlertNotification(t, int64(2), "notifier4", "email", emailSettings, false), - createAlertNotification(t, int64(2), "notifier5", "slack", slackSettings, false), - createAlertNotification(t, int64(2), "notifier6", "opsgenie", opsgenieSettings, true), // default - }, - alerts: []*models.Alert{ - createAlert(t, 1, 1, 1, "alert1", []string{"notifier1"}), - createAlert(t, 1, 1, 2, "alert2", []string{"notifier2", "notifier3"}), - createAlert(t, 1, 2, 3, "alert3", []string{"notifier3"}), - createAlert(t, 2, 3, 1, "alert4", []string{"notifier4"}), - createAlert(t, 2, 3, 2, "alert5", []string{"notifier4", "notifier5", "notifier6"}), - createAlert(t, 2, 4, 3, "alert6", []string{}), - }, - expected: map[int64]*apimodels.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - {Receiver: "notifier2", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - {Receiver: "notifier3", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier3".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - }, - RepeatInterval: nil, - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, - {Receiver: config.Receiver{Name: "notifier2"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}}, - {Receiver: config.Receiver{Name: "notifier3"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}}, - {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, // empty default - - }, - }, - }, - int64(2): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "notifier6", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier4", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier4".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - {Receiver: "notifier5", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier5".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - {Receiver: "notifier6", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier6".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - }, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier4"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier4", Type: "email"}}}}, - {Receiver: config.Receiver{Name: "notifier5"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier5", Type: "slack"}}}}, - {Receiver: config.Receiver{Name: "notifier6"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier6", Type: "opsgenie"}}}}, - }, - }, - }, - }, - }, - { - name: "when no default channel, create empty autogen-contact-point-default", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - }, - alerts: []*models.Alert{}, - expected: map[int64]*apimodels.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - }, - RepeatInterval: nil, - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, - {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, - }, - }, - }, - }, - }, - { - name: "when single default channel, don't create autogen-contact-point-default", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), - }, - alerts: []*models.Alert{}, - expected: map[int64]*apimodels.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "notifier1", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - }, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, - }, - }, - }, - }, - }, - { - name: "when single default channel with SendReminder, use channel Frequency as RepeatInterval", - legacyChannels: []*models.AlertNotification{ - createAlertNotificationWithReminder(t, int64(1), "notifier1", "email", emailSettings, true, true, time.Duration(1)*time.Hour), - }, - alerts: []*models.Alert{}, - expected: map[int64]*apimodels.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "notifier1", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour))}, - }, - RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour)), - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, - }, - }, - }, - }, - }, - { - name: "when multiple default channels, add them to autogen-contact-point-default as well", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), - createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, true), - }, - alerts: []*models.Alert{}, - expected: map[int64]*apimodels.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - {Receiver: "notifier2", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - }, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, - {Receiver: config.Receiver{Name: "notifier2"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}}, - {Receiver: config.Receiver{Name: "autogen-contact-point-default"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}}, - }, - }, - }, - }, - }, - { - name: "when multiple default channels with SendReminder, use minimum channel frequency as RepeatInterval", - legacyChannels: []*models.AlertNotification{ - createAlertNotificationWithReminder(t, int64(1), "notifier1", "email", emailSettings, true, true, time.Duration(1)*time.Hour), - createAlertNotificationWithReminder(t, int64(1), "notifier2", "slack", slackSettings, true, true, time.Duration(30)*time.Minute), - }, - alerts: []*models.Alert{}, - expected: map[int64]*apimodels.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour))}, - {Receiver: "notifier2", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(30) * time.Minute))}, - }, - RepeatInterval: durationPointer(model.Duration(time.Duration(30) * time.Minute)), - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, - {Receiver: config.Receiver{Name: "notifier2"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}}, - {Receiver: config.Receiver{Name: "autogen-contact-point-default"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}}, - }, - }, - }, - }, - }, - { - name: "when default channels exist alongside non-default, add only defaults to autogen-contact-point-default", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), // default - createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), - createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, true), // default - }, - alerts: []*models.Alert{}, - expected: map[int64]*apimodels.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - {Receiver: "notifier2", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - {Receiver: "notifier3", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier3".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - }, - RepeatInterval: durationPointer(DisabledRepeatInterval), - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, - {Receiver: config.Receiver{Name: "notifier2"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}}, - {Receiver: config.Receiver{Name: "notifier3"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}}, - {Receiver: config.Receiver{Name: "autogen-contact-point-default"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier3", Type: "opsgenie"}}}}}, - }, - }, - }, - }, - { - name: "when alerts share channels, only create one receiver per legacy channel", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), - }, - alerts: []*models.Alert{ - createAlert(t, 1, 1, 1, "alert1", []string{"notifier1"}), - createAlert(t, 1, 1, 1, "alert2", []string{"notifier1", "notifier2"}), - }, - expected: map[int64]*apimodels.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - {Receiver: "notifier2", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - }, - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, - {Receiver: config.Receiver{Name: "notifier2"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}}, - {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, - }, - }, - }, - }, - }, - { - name: "when channel not linked to any alerts, still create a receiver for it", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - }, - alerts: []*models.Alert{}, - expected: map[int64]*apimodels.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - }, - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, - {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, - }, - }, - }, - }, - }, - { - name: "when unsupported channels, do not migrate them", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - createAlertNotification(t, int64(1), "notifier2", "hipchat", "", false), - createAlertNotification(t, int64(1), "notifier3", "sensu", "", false), - }, - alerts: []*models.Alert{}, - expected: map[int64]*apimodels.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - }, - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, - {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, - }, - }, - }, - }, - }, - { - name: "when unsupported channel linked to alert, do not migrate only that channel", - legacyChannels: []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - createAlertNotification(t, int64(1), "notifier2", "sensu", "", false), - }, - alerts: []*models.Alert{ - createAlert(t, 1, 1, 1, "alert1", []string{"notifier1", "notifier2"}), - }, - expected: map[int64]*apimodels.PostableUserConfig{ - int64(1): { - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Config: apimodels.Config{Route: &apimodels.Route{ - Receiver: "autogen-contact-point-default", - GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, - Routes: []*apimodels.Route{ - {Receiver: "notifier1", ObjectMatchers: apimodels.ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(DisabledRepeatInterval)}, - }, - }}, - Receivers: []*apimodels.PostableApiReceiver{ - {Receiver: config.Receiver{Name: "notifier1"}, PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: []*apimodels.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}}, - {Receiver: config.Receiver{Name: "autogen-contact-point-default"}}, - }, - }, - }, - }, - }, - } - - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - defer teardown(t, x, service) - setupLegacyAlertsTables(t, x, tt.legacyChannels, tt.alerts, nil, nil) - - err := service.Run(context.Background()) - require.NoError(t, err) - - for orgId := range tt.expected { - amConfig := getAlertmanagerConfig(t, x, orgId) - - // Order of nested GrafanaManagedReceivers is not guaranteed. - cOpt := []cmp.Option{ - cmpopts.IgnoreUnexported(apimodels.PostableApiReceiver{}), - cmpopts.IgnoreFields(apimodels.PostableGrafanaReceiver{}, "UID", "Settings", "SecureSettings"), - cmpopts.SortSlices(func(a, b *apimodels.PostableGrafanaReceiver) bool { return a.Name < b.Name }), - cmpopts.SortSlices(func(a, b *apimodels.PostableApiReceiver) bool { return a.Name < b.Name }), - } - if !cmp.Equal(tt.expected[orgId].AlertmanagerConfig.Receivers, amConfig.AlertmanagerConfig.Receivers, cOpt...) { - t.Errorf("Unexpected Receivers: %v", cmp.Diff(tt.expected[orgId].AlertmanagerConfig.Receivers, amConfig.AlertmanagerConfig.Receivers, cOpt...)) - } - - // Order of routes is not guaranteed. - cOpt = []cmp.Option{ - cmpopts.SortSlices(func(a, b *apimodels.Route) bool { - if a.Receiver != b.Receiver { - return a.Receiver < b.Receiver - } - return a.ObjectMatchers[0].Value < b.ObjectMatchers[0].Value - }), - cmpopts.IgnoreUnexported(apimodels.Route{}, labels.Matcher{}), - cmpopts.IgnoreFields(apimodels.Route{}, "GroupBy", "GroupByAll"), - } - if !cmp.Equal(tt.expected[orgId].AlertmanagerConfig.Route, amConfig.AlertmanagerConfig.Route, cOpt...) { - t.Errorf("Unexpected Route: %v", cmp.Diff(tt.expected[orgId].AlertmanagerConfig.Route, amConfig.AlertmanagerConfig.Route, cOpt...)) - } - } - }) - } -} - -// TestDashAlertMigration tests the execution of the migration specifically for alert rules. -func TestDashAlertMigration(t *testing.T) { - sqlStore := db.InitTestDB(t) - x := sqlStore.GetEngine() - service := NewTestMigrationService(t, sqlStore, &setting.Cfg{}) - - t.Run("when DashAlertMigration create ContactLabel on migrated AlertRules", func(t *testing.T) { - defer teardown(t, x, service) - legacyChannels := []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), - createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, false), - createAlertNotification(t, int64(2), "notifier4", "email", emailSettings, false), - createAlertNotification(t, int64(2), "notifier5", "slack", slackSettings, false), - createAlertNotification(t, int64(2), "notifier6", "opsgenie", opsgenieSettings, true), // default - } - alerts := []*models.Alert{ - createAlert(t, 1, 1, 1, "alert1", []string{"notifier1"}), - createAlert(t, 1, 1, 2, "alert2", []string{"notifier2", "notifier3"}), - createAlert(t, 1, 2, 3, "alert3", []string{"notifier3"}), - createAlert(t, 2, 3, 1, "alert4", []string{"notifier4"}), - createAlert(t, 2, 3, 2, "alert5", []string{"notifier4", "notifier5", "notifier6"}), - createAlert(t, 2, 4, 3, "alert6", []string{}), - } - expected := map[int64]map[string]*ngModels.AlertRule{ - int64(1): { - "alert1": {Labels: map[string]string{ContactLabel: `"notifier1"`}}, - "alert2": {Labels: map[string]string{ContactLabel: `"notifier2","notifier3"`}}, - "alert3": {Labels: map[string]string{ContactLabel: `"notifier3"`}}, - }, - int64(2): { - "alert4": {Labels: map[string]string{ContactLabel: `"notifier4","notifier6"`}}, - "alert5": {Labels: map[string]string{ContactLabel: `"notifier4","notifier5","notifier6"`}}, - "alert6": {Labels: map[string]string{}}, - }, - } - dashes := []*dashboards.Dashboard{ - createDashboard(t, 1, 1, "dash1-1", 5, nil), - createDashboard(t, 2, 1, "dash2-1", 5, nil), - createDashboard(t, 3, 2, "dash3-2", 6, nil), - createDashboard(t, 4, 2, "dash4-2", 6, nil), - } - folders := []*dashboards.Dashboard{ - createFolder(t, 5, 1, "folder5-1"), - createFolder(t, 6, 2, "folder6-2"), - } - setupLegacyAlertsTables(t, x, legacyChannels, alerts, folders, dashes) - err := service.Run(context.Background()) - require.NoError(t, err) - - for orgId := range expected { - rules := getAlertRules(t, x, orgId) - expectedRulesMap := expected[orgId] - require.Len(t, rules, len(expectedRulesMap)) - for _, r := range rules { - require.Equal(t, expectedRulesMap[r.Title].Labels[ContactLabel], r.Labels[ContactLabel]) - } - } - }) - - t.Run("when DashAlertMigration create ContactLabel with sanitized name if name contains double quote", func(t *testing.T) { - defer teardown(t, x, service) - legacyChannels := []*models.AlertNotification{ - createAlertNotification(t, int64(1), "notif\"ier1", "email", emailSettings, false), - } - alerts := []*models.Alert{ - createAlert(t, 1, 1, 1, "alert1", []string{"notif\"ier1"}), - } - expected := map[int64]map[string]*ngModels.AlertRule{ - int64(1): { - "alert1": {Labels: map[string]string{ContactLabel: `"notif_ier1"`}}, - }, - } - dashes := []*dashboards.Dashboard{ - createDashboard(t, 1, 1, "dash1-1", 5, nil), - } - folders := []*dashboards.Dashboard{ - createFolder(t, 5, 1, "folder5-1"), - } - setupLegacyAlertsTables(t, x, legacyChannels, alerts, folders, dashes) - err := service.Run(context.Background()) - require.NoError(t, err) - - for orgId := range expected { - rules := getAlertRules(t, x, orgId) - expectedRulesMap := expected[orgId] - require.Len(t, rules, len(expectedRulesMap)) - for _, r := range rules { - require.Equal(t, expectedRulesMap[r.Title].Labels[ContactLabel], r.Labels[ContactLabel]) - } - } - }) - - t.Run("when folder is missing put alert in General folder", func(t *testing.T) { - defer teardown(t, x, service) - o := createOrg(t, 1) - folder1 := createFolder(t, 1, o.ID, "folder-1") - dash1 := createDashboard(t, 3, o.ID, "dash1", folder1.ID, nil) - dash2 := createDashboard(t, 4, o.ID, "dash2", 22, nil) // missing folder - - a1 := createAlert(t, int(o.ID), int(dash1.ID), 1, "alert-1", []string{}) - a2 := createAlert(t, int(o.ID), int(dash2.ID), 1, "alert-2", []string{}) - - _, err := x.Insert(o, folder1, dash1, dash2, a1, a2) - require.NoError(t, err) - - err = service.Run(context.Background()) - require.NoError(t, err) - - rules := getAlertRules(t, x, o.ID) - require.Len(t, rules, 2) - - var generalFolder dashboards.Dashboard - _, err = x.Table(&dashboards.Dashboard{}).Where("title = ? AND org_id = ?", generalAlertingFolderTitle, o.ID).Get(&generalFolder) - require.NoError(t, err) - - require.NotNil(t, generalFolder) - - for _, rule := range rules { - var expectedFolder dashboards.Dashboard - if rule.Title == a1.Name { - expectedFolder = *folder1 - } else { - expectedFolder = generalFolder - } - require.Equal(t, expectedFolder.UID, rule.NamespaceUID) - } - }) -} - -// TestDashAlertQueryMigration tests the execution of the migration specifically for alert rule queries. -func TestDashAlertQueryMigration(t *testing.T) { - sqlStore := db.InitTestDB(t) - x := sqlStore.GetEngine() - service := NewTestMigrationService(t, sqlStore, &setting.Cfg{}) - - createAlertQuery := func(refId string, ds string, from string, to string) ngModels.AlertQuery { - dur, _ := calculateInterval(legacydata.NewDataTimeRange(from, to), simplejson.New(), nil) - intervalMs := strconv.FormatInt(dur.Milliseconds(), 10) - rel, _ := getRelativeDuration(from, to) - return ngModels.AlertQuery{ - RefID: refId, - RelativeTimeRange: ngModels.RelativeTimeRange{From: rel.From, To: rel.To}, - DatasourceUID: ds, - Model: []byte(fmt.Sprintf(`{"datasource":{"type":"prometheus","uid":"gdev-prometheus"},"expr":"up{job=\"fake-data-gen\"}","instant":false,"intervalMs":%s,"maxDataPoints":1500,"refId":"%s"}`, intervalMs, refId)), - } - } - - createClassicConditionQuery := func(refId string, conditions []classicConditionJSON) ngModels.AlertQuery { - exprModel := struct { - Type string `json:"type"` - RefID string `json:"refId"` - Conditions []classicConditionJSON `json:"conditions"` - }{ - "classic_conditions", - refId, - conditions, - } - exprModelJSON, _ := json.Marshal(&exprModel) - - q := ngModels.AlertQuery{ - RefID: refId, - DatasourceUID: expressionDatasourceUID, - Model: exprModelJSON, - } - // IntervalMS and MaxDataPoints are created PreSave by AlertQuery. They don't appear to be necessary for expressions, - // but run PreSave here to match the expected model. - _ = q.PreSave() - return q - } - - cond := func(refId string, reducer string, evalType string, thresh float64) classicConditionJSON { - return classicConditionJSON{ - Evaluator: migrationStore.ConditionEvalJSON{Params: []float64{thresh}, Type: evalType}, - Operator: struct { - Type string `json:"type"` - }{Type: "and"}, - Query: struct { - Params []string `json:"params"` - }{Params: []string{refId}}, - Reducer: struct { - Type string `json:"type"` - }{Type: reducer}, - } - } - - genAlert := func(mutators ...ngModels.AlertRuleMutator) *ngModels.AlertRule { - rule := &ngModels.AlertRule{ - ID: 1, - OrgID: 1, - Title: "alert1", - Condition: "B", - Data: []ngModels.AlertQuery{}, - IntervalSeconds: 60, - Version: 1, - NamespaceUID: "folder5-1", - DashboardUID: pointer("dash1-1"), - PanelID: pointer(int64(1)), - RuleGroup: "alert1", - RuleGroupIndex: 1, - NoDataState: ngModels.NoData, - ExecErrState: ngModels.AlertingErrState, - For: 60 * time.Second, - Annotations: map[string]string{ - "message": "message", - }, - Labels: map[string]string{}, - IsPaused: false, - } - - for _, mutator := range mutators { - mutator(rule) - } - - rule.Annotations["__dashboardUid__"] = *rule.DashboardUID - rule.Annotations["__panelId__"] = strconv.FormatInt(*rule.PanelID, 10) - return rule - } - - type testcase struct { - name string - alerts []*models.Alert - - expectedFolder *dashboards.Dashboard - expected map[int64][]*ngModels.AlertRule - } - - tc := []testcase{ - { - name: "simple query and condition", - alerts: []*models.Alert{ - createAlertWithCond(t, 1, 1, 1, "alert1", nil, - []migrationStore.DashAlertCondition{createCondition("A", "max", "gt", 42, 1, "5m", "now")}), - createAlertWithCond(t, 2, 3, 1, "alert1", nil, - []migrationStore.DashAlertCondition{createCondition("A", "max", "gt", 42, 3, "5m", "now")}), - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(1): { - genAlert(func(rule *ngModels.AlertRule) { - rule.Data = append(rule.Data, createAlertQuery("A", "ds1-1", "5m", "now")) - rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ - cond("A", "max", "gt", 42), - })) - }), - }, - int64(2): { - genAlert(func(rule *ngModels.AlertRule) { - rule.OrgID = 2 - rule.DashboardUID = pointer("dash3-2") - rule.NamespaceUID = "folder6-2" - rule.Data = append(rule.Data, createAlertQuery("A", "ds3-2", "5m", "now")) - rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ - cond("A", "max", "gt", 42), - })) - }), - }, - }, - }, - { - name: "multiple conditions", - alerts: []*models.Alert{ - createAlertWithCond(t, 1, 1, 1, "alert1", nil, - []migrationStore.DashAlertCondition{ - createCondition("A", "avg", "gt", 42, 1, "5m", "now"), - createCondition("B", "max", "gt", 43, 2, "3m", "now"), - createCondition("C", "min", "lt", 20, 2, "3m", "now"), - }), - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(1): { - genAlert(func(rule *ngModels.AlertRule) { - rule.Condition = "D" - rule.Data = append(rule.Data, createAlertQuery("A", "ds1-1", "5m", "now")) - rule.Data = append(rule.Data, createAlertQuery("B", "ds2-1", "3m", "now")) - rule.Data = append(rule.Data, createAlertQuery("C", "ds2-1", "3m", "now")) - rule.Data = append(rule.Data, createClassicConditionQuery("D", []classicConditionJSON{ - cond("A", "avg", "gt", 42), - cond("B", "max", "gt", 43), - cond("C", "min", "lt", 20), - })) - }), - }, - }, - }, - { - name: "multiple conditions on same query with same timerange should not create multiple queries", - alerts: []*models.Alert{ - createAlertWithCond(t, 1, 1, 1, "alert1", nil, - []migrationStore.DashAlertCondition{ - createCondition("A", "max", "gt", 42, 1, "5m", "now"), - createCondition("A", "avg", "gt", 20, 1, "5m", "now"), - }), - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(1): { - genAlert(func(rule *ngModels.AlertRule) { - rule.Condition = "B" - rule.Data = append(rule.Data, createAlertQuery("A", "ds1-1", "5m", "now")) - rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ - cond("A", "max", "gt", 42), - cond("A", "avg", "gt", 20), - })) - }), - }, - }, - }, - { - name: "multiple conditions on same query with different timeranges should create multiple queries", - alerts: []*models.Alert{ - createAlertWithCond(t, 1, 1, 1, "alert1", nil, - []migrationStore.DashAlertCondition{ - createCondition("A", "max", "gt", 42, 1, "5m", "now"), - createCondition("A", "avg", "gt", 20, 1, "3m", "now"), - }), - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(1): { - genAlert(func(rule *ngModels.AlertRule) { - rule.Condition = "C" - rule.Data = append(rule.Data, createAlertQuery("A", "ds1-1", "3m", "now")) // Ordered by time range. - rule.Data = append(rule.Data, createAlertQuery("B", "ds1-1", "5m", "now")) - rule.Data = append(rule.Data, createClassicConditionQuery("C", []classicConditionJSON{ - cond("B", "max", "gt", 42), - cond("A", "avg", "gt", 20), - })) - }), - }, - }, - }, - { - name: "multiple conditions custom refIds", - alerts: []*models.Alert{ - createAlertWithCond(t, 1, 1, 1, "alert1", nil, - []migrationStore.DashAlertCondition{ - createCondition("Q1", "avg", "gt", 42, 1, "5m", "now"), - createCondition("Q2", "max", "gt", 43, 2, "3m", "now"), - createCondition("Q3", "min", "lt", 20, 2, "3m", "now"), - }), - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(1): { - genAlert(func(rule *ngModels.AlertRule) { - rule.Condition = "A" - rule.Data = append(rule.Data, createClassicConditionQuery("A", []classicConditionJSON{ - cond("Q1", "avg", "gt", 42), - cond("Q2", "max", "gt", 43), - cond("Q3", "min", "lt", 20), - })) - rule.Data = append(rule.Data, createAlertQuery("Q1", "ds1-1", "5m", "now")) - rule.Data = append(rule.Data, createAlertQuery("Q2", "ds2-1", "3m", "now")) - rule.Data = append(rule.Data, createAlertQuery("Q3", "ds2-1", "3m", "now")) - }), - }, - }, - }, - { - name: "multiple conditions out of order refIds, queries should be sorted by refId and conditions should be in original order", - alerts: []*models.Alert{ - createAlertWithCond(t, 1, 1, 1, "alert1", nil, - []migrationStore.DashAlertCondition{ - createCondition("B", "avg", "gt", 42, 1, "5m", "now"), - createCondition("C", "max", "gt", 43, 2, "3m", "now"), - createCondition("A", "min", "lt", 20, 2, "3m", "now"), - }), - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(1): { - genAlert(func(rule *ngModels.AlertRule) { - rule.Condition = "D" - rule.Data = append(rule.Data, createAlertQuery("A", "ds2-1", "3m", "now")) - rule.Data = append(rule.Data, createAlertQuery("B", "ds1-1", "5m", "now")) - rule.Data = append(rule.Data, createAlertQuery("C", "ds2-1", "3m", "now")) - rule.Data = append(rule.Data, createClassicConditionQuery("D", []classicConditionJSON{ - cond("B", "avg", "gt", 42), - cond("C", "max", "gt", 43), - cond("A", "min", "lt", 20), - })) - }), - }, - }, - }, - { - name: "multiple conditions out of order with duplicate refIds", - alerts: []*models.Alert{ - createAlertWithCond(t, 1, 1, 1, "alert1", nil, - []migrationStore.DashAlertCondition{ - createCondition("C", "avg", "gt", 42, 1, "5m", "now"), - createCondition("C", "max", "gt", 43, 1, "3m", "now"), - createCondition("B", "min", "lt", 20, 2, "5m", "now"), - createCondition("B", "min", "lt", 21, 2, "3m", "now"), - }), - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(1): { - genAlert(func(rule *ngModels.AlertRule) { - rule.Condition = "E" - rule.Data = append(rule.Data, createAlertQuery("A", "ds2-1", "3m", "now")) - rule.Data = append(rule.Data, createAlertQuery("B", "ds2-1", "5m", "now")) - rule.Data = append(rule.Data, createAlertQuery("C", "ds1-1", "3m", "now")) - rule.Data = append(rule.Data, createAlertQuery("D", "ds1-1", "5m", "now")) - rule.Data = append(rule.Data, createClassicConditionQuery("E", []classicConditionJSON{ - cond("D", "avg", "gt", 42), - cond("C", "max", "gt", 43), - cond("B", "min", "lt", 20), - cond("A", "min", "lt", 21), - })) - }), - }, - }, - }, - { - name: "alerts with unknown datasource id migrates with empty datasource uid", - alerts: []*models.Alert{ - createAlertWithCond(t, 1, 1, 1, "alert1", nil, - []migrationStore.DashAlertCondition{createCondition("A", "max", "gt", 42, 123, "5m", "now")}), // Unknown datasource id. - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(1): { - genAlert(func(rule *ngModels.AlertRule) { - rule.Data = append(rule.Data, createAlertQuery("A", "", "5m", "now")) // Empty datasource UID. - rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ - cond("A", "max", "gt", 42), - })) - }), - }, - }, - }, - { - name: "alerts with unknown dashboard do not migrate", - alerts: []*models.Alert{ - createAlertWithCond(t, 1, 22, 1, "alert1", nil, - []migrationStore.DashAlertCondition{ - createCondition("A", "avg", "gt", 42, 1, "5m", "now"), - }), - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(1): {}, - }, - }, - { - name: "alerts with unknown org do not migrate", - alerts: []*models.Alert{ - createAlertWithCond(t, 22, 1, 1, "alert1", nil, - []migrationStore.DashAlertCondition{ - createCondition("A", "avg", "gt", 42, 1, "5m", "now"), - }), - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(22): {}, - }, - }, - { - name: "alerts in general folder migrate to existing general alerting", - alerts: []*models.Alert{ - createAlertWithCond(t, 1, 8, 1, "alert1", nil, - []migrationStore.DashAlertCondition{ - createCondition("A", "avg", "gt", 42, 1, "5m", "now"), - }), - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(1): { - genAlert(func(rule *ngModels.AlertRule) { - rule.NamespaceUID = "General Alerting" - rule.DashboardUID = pointer("dash-in-general-1") - rule.Data = append(rule.Data, createAlertQuery("A", "ds1-1", "5m", "now")) - rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ - cond("A", "avg", "gt", 42), - })) - }), - }, - }, - }, - { - name: "alerts in general folder migrate to newly created general alerting if one doesn't exist", - alerts: []*models.Alert{ - createAlertWithCond(t, 2, 9, 1, "alert1", nil, // Org 2 doesn't have general alerting folder. - []migrationStore.DashAlertCondition{ - createCondition("A", "avg", "gt", 42, 3, "5m", "now"), - }), - }, - expectedFolder: &dashboards.Dashboard{ - OrgID: 2, - Title: "General Alerting", - FolderID: 0, - Slug: "general-alerting", - }, - expected: map[int64][]*ngModels.AlertRule{ - int64(2): { - genAlert(func(rule *ngModels.AlertRule) { - rule.OrgID = 2 - rule.DashboardUID = pointer("dash-in-general-2") - rule.Data = append(rule.Data, createAlertQuery("A", "ds3-2", "5m", "now")) - rule.Data = append(rule.Data, createClassicConditionQuery("B", []classicConditionJSON{ - cond("A", "avg", "gt", 42), - })) - }), - }, - }, - }, - } - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - defer teardown(t, x, service) - dashes := []*dashboards.Dashboard{ - createDashboard(t, 1, 1, "dash1-1", 5, nil), - createDashboard(t, 2, 1, "dash2-1", 5, nil), - createDashboard(t, 3, 2, "dash3-2", 6, nil), - createDashboard(t, 4, 2, "dash4-2", 6, nil), - createDashboard(t, 8, 1, "dash-in-general-1", 0, nil), - createDashboard(t, 9, 2, "dash-in-general-2", 0, nil), - createDashboard(t, 10, 1, "dash-with-acl-1", 5, func(d *dashboards.Dashboard) { - d.Title = "Dashboard With ACL 1" - d.HasACL = true - }), - } - folders := []*dashboards.Dashboard{ - createFolder(t, 5, 1, "folder5-1"), - createFolder(t, 6, 2, "folder6-2"), - createFolder(t, 7, 1, "General Alerting"), - } - setupLegacyAlertsTables(t, x, nil, tt.alerts, folders, dashes) - - err := service.Run(context.Background()) - require.NoError(t, err) - - for orgId, expected := range tt.expected { - rules := getAlertRules(t, x, orgId) - - for _, r := range rules { - // Remove generated fields. - require.NotEqual(t, r.Labels["rule_uid"], "") - delete(r.Labels, "rule_uid") - require.NotEqual(t, r.Annotations["__alertId__"], "") - delete(r.Annotations, "__alertId__") - - // If folder is created, we check if separately - if tt.expectedFolder != nil { - folder := getDashboard(t, x, orgId, r.NamespaceUID) - require.Equal(t, tt.expectedFolder.Title, folder.Title) - require.Equal(t, tt.expectedFolder.OrgID, folder.OrgID) - require.Equal(t, tt.expectedFolder.FolderID, folder.FolderID) - } - } - - cOpt := []cmp.Option{ - cmpopts.SortSlices(func(a, b *ngModels.AlertRule) bool { - return a.ID < b.ID - }), - cmpopts.IgnoreUnexported(ngModels.AlertRule{}, ngModels.AlertQuery{}), - cmpopts.IgnoreFields(ngModels.AlertRule{}, "Updated", "UID", "ID"), - } - if tt.expectedFolder != nil { - cOpt = append(cOpt, cmpopts.IgnoreFields(ngModels.AlertRule{}, "NamespaceUID")) - } - if !cmp.Equal(expected, rules, cOpt...) { - t.Errorf("Unexpected Rule: %v", cmp.Diff(expected, rules, cOpt...)) - } - } - }) - } -} - -const ( - emailSettings = `{"addresses": "test"}` - slackSettings = `{"recipient": "test", "token": "test"}` - opsgenieSettings = `{"apiKey": "test"}` -) - -var ( - now = time.Now() -) - -// createAlertNotificationWithReminder creates a legacy alert notification channel for inserting into the test database. -func createAlertNotificationWithReminder(t *testing.T, orgId int64, uid string, channelType string, settings string, defaultChannel bool, sendReminder bool, frequency time.Duration) *models.AlertNotification { - t.Helper() - settingsJson := simplejson.New() - if settings != "" { - s, err := simplejson.NewJson([]byte(settings)) - if err != nil { - t.Fatalf("Failed to unmarshal alert notification json: %v", err) - } - settingsJson = s - } - - return &models.AlertNotification{ - OrgID: orgId, - UID: uid, - Name: uid, // Same as uid to make testing easier. - Type: channelType, - DisableResolveMessage: false, - IsDefault: defaultChannel, - Settings: settingsJson, - SecureSettings: make(map[string][]byte), - Created: now, - Updated: now, - SendReminder: sendReminder, - Frequency: frequency, - } -} - -// createAlertNotification creates a legacy alert notification channel for inserting into the test database. -func createAlertNotification(t *testing.T, orgId int64, uid string, channelType string, settings string, defaultChannel bool) *models.AlertNotification { - return createAlertNotificationWithReminder(t, orgId, uid, channelType, settings, defaultChannel, false, time.Duration(0)) -} - -var queryModel = `{"datasource":{"type":"prometheus","uid":"gdev-prometheus"},"expr":"up{job=\"fake-data-gen\"}","instant":false,"refId":"%s"}` - -func createCondition(refId string, reducer string, evalType string, thresh float64, datasourceId int64, from string, to string) migrationStore.DashAlertCondition { - return migrationStore.DashAlertCondition{ - Evaluator: migrationStore.ConditionEvalJSON{ - Params: []float64{thresh}, - Type: evalType, - }, - Operator: struct { - Type string `json:"type"` - }{ - Type: "and", - }, - Query: struct { - Params []string `json:"params"` - DatasourceID int64 `json:"datasourceId"` - Model json.RawMessage - }{ - Params: []string{refId, from, to}, - DatasourceID: datasourceId, - Model: []byte(fmt.Sprintf(queryModel, refId)), - }, - Reducer: struct { - Type string `json:"type"` - }{ - Type: reducer, - }, - } -} - -// createAlert creates a legacy alert rule for inserting into the test database. -func createAlert(t *testing.T, orgId int, dashboardId int, panelsId int, name string, notifierUids []string) *models.Alert { - return createAlertWithCond(t, orgId, dashboardId, panelsId, name, notifierUids, []migrationStore.DashAlertCondition{}) -} - -// createAlert creates a legacy alert rule for inserting into the test database. -func createAlertWithCond(t *testing.T, orgId int, dashboardId int, panelsId int, name string, notifierUids []string, cond []migrationStore.DashAlertCondition) *models.Alert { - t.Helper() - - var settings = simplejson.New() - if len(notifierUids) != 0 { - notifiers := make([]any, 0) - for _, n := range notifierUids { - notifiers = append(notifiers, struct { - Uid string - }{Uid: n}) - } - - settings.Set("notifications", notifiers) - } - settings.Set("conditions", cond) - - return &models.Alert{ - OrgID: int64(orgId), - DashboardID: int64(dashboardId), - PanelID: int64(panelsId), - Name: name, - Message: "message", - Frequency: int64(60), - For: 60 * time.Second, - State: models.AlertStateOK, - Settings: settings, - NewStateDate: now, - Created: now, - Updated: now, - } -} - -// createDashboard creates a folder for inserting into the test database. -func createFolder(t *testing.T, id int64, orgId int64, uid string) *dashboards.Dashboard { - f := createDashboard(t, id, orgId, uid, 0, nil) - f.IsFolder = true - return f -} - -// createDashboard creates a dashboard for inserting into the test database. -func createDashboard(t *testing.T, id int64, orgId int64, uid string, folderId int64, mut func(*dashboards.Dashboard)) *dashboards.Dashboard { - t.Helper() - d := &dashboards.Dashboard{ - ID: id, - OrgID: orgId, - UID: uid, - Created: now, - Updated: now, - Title: uid, // Not tested, needed to satisfy constraint. - FolderID: folderId, - Data: simplejson.New(), - Version: 1, - } - if mut != nil { - mut(d) - } - return d -} - -// createDatasource creates a datasource for inserting into the test database. -func createDatasource(t *testing.T, id int64, orgId int64, uid string) *datasources.DataSource { - t.Helper() - return &datasources.DataSource{ - ID: id, - OrgID: orgId, - UID: uid, - Created: now, - Updated: now, - Name: uid, // Not tested, needed to satisfy constraint. - } -} - -func createOrg(t *testing.T, id int64) *org.Org { - t.Helper() - return &org.Org{ - ID: id, - Version: 1, - Name: fmt.Sprintf("org_%d", id), - Created: time.Now(), - Updated: time.Now(), - } -} - -// teardown cleans the input tables between test cases. -func teardown(t *testing.T, x *xorm.Engine, service *MigrationService) { - _, err := x.Exec("DELETE from org") - require.NoError(t, err) - _, err = x.Exec("DELETE from alert") - require.NoError(t, err) - _, err = x.Exec("DELETE from alert_notification") - require.NoError(t, err) - _, err = x.Exec("DELETE from dashboard") - require.NoError(t, err) - _, err = x.Exec("DELETE from data_source") - require.NoError(t, err) - err = service.migrationStore.RevertAllOrgs(context.Background()) - require.NoError(t, err) -} - -// setupLegacyAlertsTables inserts data into the legacy alerting tables that is needed for testing the -func setupLegacyAlertsTables(t *testing.T, x *xorm.Engine, legacyChannels []*models.AlertNotification, alerts []*models.Alert, folders []*dashboards.Dashboard, dashes []*dashboards.Dashboard) { - t.Helper() - - orgs := []org.Org{ - *createOrg(t, 1), - *createOrg(t, 2), - } - - // Setup folders. - if len(folders) > 0 { - _, err := x.Insert(folders) - require.NoError(t, err) - } - - // Setup dashboards. - if len(dashes) > 0 { - _, err := x.Insert(dashes) - require.NoError(t, err) - } - - // Setup data_sources. - dataSources := []datasources.DataSource{ - *createDatasource(t, 1, 1, "ds1-1"), - *createDatasource(t, 2, 1, "ds2-1"), - *createDatasource(t, 3, 2, "ds3-2"), - *createDatasource(t, 4, 2, "ds4-2"), - } - - _, errOrgs := x.Insert(orgs) - require.NoError(t, errOrgs) - - _, errDataSourcess := x.Insert(dataSources) - require.NoError(t, errDataSourcess) - - if len(legacyChannels) > 0 { - _, channelErr := x.Insert(legacyChannels) - require.NoError(t, channelErr) - } - - if len(alerts) > 0 { - _, alertErr := x.Insert(alerts) - require.NoError(t, alertErr) - } -} - -// getAlertmanagerConfig retreives the Alertmanager Config from the database for a given orgId. -func getAlertmanagerConfig(t *testing.T, x *xorm.Engine, orgId int64) *apimodels.PostableUserConfig { - amConfig := "" - _, err := x.Table("alert_configuration").Where("org_id = ?", orgId).Cols("alertmanager_configuration").Get(&amConfig) - require.NoError(t, err) - - config := apimodels.PostableUserConfig{} - err = json.Unmarshal([]byte(amConfig), &config) - require.NoError(t, err) - return &config -} - -// getAlertmanagerConfig retreives the Alertmanager Config from the database for a given orgId. -func getAlertRules(t *testing.T, x *xorm.Engine, orgId int64) []*ngModels.AlertRule { - rules := make([]*ngModels.AlertRule, 0) - err := x.Table("alert_rule").Where("org_id = ?", orgId).Find(&rules) - require.NoError(t, err) - - return rules -} - -// getDashboard retrieves a dashboard from the database for a given org, uid. -func getDashboard(t *testing.T, x *xorm.Engine, orgId int64, uid string) *dashboards.Dashboard { - dashes := make([]*dashboards.Dashboard, 0) - err := x.Table("dashboard").Where("org_id = ? AND uid = ?", orgId, uid).Find(&dashes) - require.NoError(t, err) - if len(dashes) > 1 { - t.Error("Expected only one dashboard to be returned") - } - if len(dashes) == 0 { - return nil - } - - return dashes[0] -} - -func pointer[T any](b T) *T { - return &b -} diff --git a/pkg/services/ngalert/migration/models.go b/pkg/services/ngalert/migration/models.go deleted file mode 100644 index 90553f88b70..00000000000 --- a/pkg/services/ngalert/migration/models.go +++ /dev/null @@ -1,121 +0,0 @@ -package migration - -import ( - "strings" - - pb "github.com/prometheus/alertmanager/silence/silencepb" - - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/services/folder" - migmodels "github.com/grafana/grafana/pkg/services/ngalert/migration/models" - migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" - "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/services/ngalert/store" - "github.com/grafana/grafana/pkg/services/secrets" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/util" -) - -// OrgMigration is a helper struct for migrating alerts for a single org. It contains state, services, and caches. -type OrgMigration struct { - cfg *setting.Cfg - log log.Logger - - migrationStore migrationStore.Store - encryptionService secrets.Service - - orgID int64 - seenUIDs Deduplicator - silences []*pb.MeshSilence - alertRuleTitleDedup map[string]Deduplicator // Folder -> Deduplicator (Title). - - // cache for folders created for dashboards that have custom permissions - folderCache map[string]*folder.Folder - generalAlertingFolder *folder.Folder - - state *migmodels.OrgMigrationState -} - -// newOrgMigration creates a new OrgMigration for the given orgID. -func (ms *MigrationService) newOrgMigration(orgID int64) *OrgMigration { - return &OrgMigration{ - cfg: ms.cfg, - log: ms.log.New("orgID", orgID), - - migrationStore: ms.migrationStore, - encryptionService: ms.encryptionService, - - orgID: orgID, - // We deduplicate for case-insensitive matching in MySQL-compatible backend flavours because they use case-insensitive collation. - seenUIDs: Deduplicator{set: make(map[string]struct{}), caseInsensitive: ms.migrationStore.CaseInsensitive()}, - silences: make([]*pb.MeshSilence, 0), - alertRuleTitleDedup: make(map[string]Deduplicator), - - folderCache: make(map[string]*folder.Folder), - - state: &migmodels.OrgMigrationState{ - OrgID: orgID, - CreatedFolders: make([]string, 0), - }, - } -} - -func (om *OrgMigration) AlertTitleDeduplicator(folderUID string) Deduplicator { - if _, ok := om.alertRuleTitleDedup[folderUID]; !ok { - om.alertRuleTitleDedup[folderUID] = Deduplicator{ - set: make(map[string]struct{}), - caseInsensitive: om.migrationStore.CaseInsensitive(), - maxLen: store.AlertDefinitionMaxTitleLength, - } - } - return om.alertRuleTitleDedup[folderUID] -} - -type AlertPair struct { - AlertRule *models.AlertRule - DashAlert *migrationStore.DashAlert -} - -// Deduplicator is a wrapper around map[string]struct{} and util.GenerateShortUID() which aims help maintain and generate -// unique strings (such as uids or titles). if caseInsensitive is true, all uniqueness is determined in a -// case-insensitive manner. if maxLen is greater than 0, all strings will be truncated to maxLen before being checked in -// contains and dedup will always return a string of length maxLen or less. -type Deduplicator struct { - set map[string]struct{} - caseInsensitive bool - maxLen int -} - -// contains checks whether the given string has already been seen by this Deduplicator. -func (s *Deduplicator) contains(u string) bool { - dedup := u - if s.caseInsensitive { - dedup = strings.ToLower(dedup) - } - if s.maxLen > 0 && len(dedup) > s.maxLen { - dedup = dedup[:s.maxLen] - } - _, seen := s.set[dedup] - return seen -} - -// deduplicate returns a unique string based on the given string by appending a uuid to it. Will truncate the given string if -// the resulting string would be longer than maxLen. -func (s *Deduplicator) deduplicate(dedup string) string { - uid := util.GenerateShortUID() - if s.maxLen > 0 && len(dedup)+1+len(uid) > s.maxLen { - trunc := s.maxLen - 1 - len(uid) - dedup = dedup[:trunc] - } - - return dedup + "_" + uid -} - -// add adds the given string to the Deduplicator. -func (s *Deduplicator) add(uid string) { - dedup := uid - if s.caseInsensitive { - dedup = strings.ToLower(dedup) - } - s.set[dedup] = struct{}{} -} diff --git a/pkg/services/ngalert/migration/models/state.go b/pkg/services/ngalert/migration/models/state.go deleted file mode 100644 index e7c5b78120b..00000000000 --- a/pkg/services/ngalert/migration/models/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package models - -// OrgMigrationState contains information about the state of an org migration. -type OrgMigrationState struct { - OrgID int64 `json:"orgId"` - CreatedFolders []string `json:"createdFolders"` -} diff --git a/pkg/services/ngalert/migration/permissions.go b/pkg/services/ngalert/migration/permissions.go deleted file mode 100644 index ce6f139b880..00000000000 --- a/pkg/services/ngalert/migration/permissions.go +++ /dev/null @@ -1,153 +0,0 @@ -package migration - -import ( - "context" - "errors" - "fmt" - - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/services/accesscontrol" - "github.com/grafana/grafana/pkg/services/auth/identity" - "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/datasources" - "github.com/grafana/grafana/pkg/services/folder" - "github.com/grafana/grafana/pkg/services/org" - "github.com/grafana/grafana/pkg/services/user" -) - -const DASHBOARD_FOLDER = "%s Alerts - %s" - -// MaxFolderName is the maximum length of the folder name generated using DASHBOARD_FOLDER format -const MaxFolderName = 255 - -var ( - migratorPermissions = []accesscontrol.Permission{ - {Action: dashboards.ActionFoldersRead, Scope: dashboards.ScopeFoldersAll}, - {Action: dashboards.ActionDashboardsRead, Scope: dashboards.ScopeDashboardsAll}, - {Action: dashboards.ActionFoldersPermissionsRead, Scope: dashboards.ScopeFoldersAll}, - {Action: dashboards.ActionDashboardsPermissionsRead, Scope: dashboards.ScopeDashboardsAll}, - {Action: dashboards.ActionFoldersCreate}, - {Action: dashboards.ActionDashboardsCreate, Scope: dashboards.ScopeFoldersAll}, - {Action: datasources.ActionRead, Scope: datasources.ScopeAll}, - } - generalAlertingFolderTitle = "General Alerting" -) - -// getMigrationUser returns a background user for the given orgID with permissions to execute migration-related tasks. -func getMigrationUser(orgID int64) identity.Requester { - return accesscontrol.BackgroundUser("ngalert_migration", orgID, org.RoleAdmin, migratorPermissions) -} - -// getAlertFolderNameFromDashboard generates a folder name for alerts that belong to a dashboard. Formats the string according to DASHBOARD_FOLDER format. -// If the resulting string exceeds the migrations.MaxTitleLength, the dashboard title is stripped to be at the maximum length -func getAlertFolderNameFromDashboard(dash *dashboards.Dashboard) string { - maxLen := MaxFolderName - len(fmt.Sprintf(DASHBOARD_FOLDER, "", dash.UID)) - title := dash.Title - if len(title) > maxLen { - title = title[:maxLen] - } - return fmt.Sprintf(DASHBOARD_FOLDER, title, dash.UID) // include UID to the name to avoid collision -} - -func (om *OrgMigration) getOrCreateMigratedFolder(ctx context.Context, log log.Logger, dashID int64) (*dashboards.Dashboard, *folder.Folder, error) { - dash, err := om.migrationStore.GetDashboard(ctx, om.orgID, dashID) - if err != nil { - if errors.Is(err, dashboards.ErrFolderNotFound) { - return nil, nil, fmt.Errorf("dashboard with ID %v under organisation %d not found: %w", dashID, om.orgID, err) - } - return nil, nil, fmt.Errorf("failed to get dashboard with ID %v under organisation %d: %w", dashID, om.orgID, err) - } - l := log.New( - "dashboardTitle", dash.Title, - "dashboardUID", dash.UID, - ) - - var migratedFolder *folder.Folder - switch { - case dash.HasACL: - folderName := getAlertFolderNameFromDashboard(dash) - f, ok := om.folderCache[folderName] - if !ok { - l.Info("create a new folder for alerts that belongs to dashboard because it has custom permissions", "folder", folderName) - // create folder and assign the permissions of the dashboard (included default and inherited) - f, err = om.createFolder(ctx, om.orgID, folderName) - if err != nil { - return nil, nil, fmt.Errorf("create new folder: %w", err) - } - permissions, err := om.migrationStore.GetACL(ctx, dash.OrgID, dash.ID) - if err != nil { - return nil, nil, fmt.Errorf("failed to get dashboard %d under organisation %d permissions: %w", dash.ID, dash.OrgID, err) - } - err = om.migrationStore.SetACL(ctx, f.OrgID, f.ID, permissions) - if err != nil { - return nil, nil, fmt.Errorf("failed to set folder %d under organisation %d permissions: %w", f.ID, f.OrgID, err) - } - om.folderCache[folderName] = f - } - migratedFolder = f - case dash.FolderID > 0: - // get folder if exists - f, err := om.migrationStore.GetFolder(ctx, &folder.GetFolderQuery{ID: &dash.FolderID, OrgID: dash.OrgID, SignedInUser: getMigrationUser(dash.OrgID)}) - if err != nil { - // If folder does not exist then the dashboard is an orphan and we migrate the alert to the general folder. - l.Warn("Failed to find folder for dashboard. Migrate rule to the default folder", "missing_folder_id", dash.FolderID, "error", err) - migratedFolder, err = om.getOrCreateGeneralFolder(ctx, dash.OrgID) - if err != nil { - return nil, nil, err - } - } else { - migratedFolder = f - } - default: - migratedFolder, err = om.getOrCreateGeneralFolder(ctx, dash.OrgID) - if err != nil { - return nil, nil, err - } - } - - if migratedFolder.UID == "" { - return nil, nil, fmt.Errorf("empty folder identifier") - } - - return dash, migratedFolder, nil -} - -// getOrCreateGeneralFolder returns the general folder under the specific organisation -// If the general folder does not exist it creates it. -func (om *OrgMigration) getOrCreateGeneralFolder(ctx context.Context, orgID int64) (*folder.Folder, error) { - if om.generalAlertingFolder != nil { - return om.generalAlertingFolder, nil - } - f, err := om.migrationStore.GetFolder(ctx, &folder.GetFolderQuery{OrgID: orgID, Title: &generalAlertingFolderTitle, SignedInUser: getMigrationUser(orgID)}) - if err != nil { - if errors.Is(err, dashboards.ErrFolderNotFound) { - // create folder - generalAlertingFolder, err := om.createFolder(ctx, orgID, generalAlertingFolderTitle) - if err != nil { - return nil, fmt.Errorf("create general alerting folder '%s': %w", generalAlertingFolderTitle, err) - } - om.generalAlertingFolder = generalAlertingFolder - return om.generalAlertingFolder, nil - } - return nil, fmt.Errorf("get general alerting folder '%s': %w", generalAlertingFolderTitle, err) - } - om.generalAlertingFolder = f - - return om.generalAlertingFolder, nil -} - -// createFolder creates a new folder with given permissions. -func (om *OrgMigration) createFolder(ctx context.Context, orgID int64, title string) (*folder.Folder, error) { - f, err := om.migrationStore.CreateFolder(ctx, &folder.CreateFolderCommand{ - OrgID: orgID, - Title: title, - SignedInUser: getMigrationUser(orgID).(*user.SignedInUser), - }) - if err != nil { - return nil, err - } - - om.state.CreatedFolders = append(om.state.CreatedFolders, f.UID) - - return f, nil -} diff --git a/pkg/services/ngalert/migration/securejsondata.go b/pkg/services/ngalert/migration/securejsondata.go deleted file mode 100644 index 9de357bb00a..00000000000 --- a/pkg/services/ngalert/migration/securejsondata.go +++ /dev/null @@ -1,31 +0,0 @@ -package migration - -import ( - "os" - - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/util" -) - -// SecureJsonData is used to store encrypted data (for example in data_source table). Only values are separately -// encrypted. -type SecureJsonData map[string][]byte - -var seclogger = log.New("securejsondata") - -// Decrypt returns map of the same type but where the all the values are decrypted. Opposite of what -// GetEncryptedJsonData is doing. -func (s SecureJsonData) Decrypt() map[string]string { - decrypted := make(map[string]string) - for key, data := range s { - decryptedData, err := util.Decrypt(data, setting.SecretKey) - if err != nil { - seclogger.Error(err.Error()) - os.Exit(1) - } - - decrypted[key] = string(decryptedData) - } - return decrypted -} diff --git a/pkg/services/ngalert/migration/service.go b/pkg/services/ngalert/migration/service.go deleted file mode 100644 index 876a0b2532a..00000000000 --- a/pkg/services/ngalert/migration/service.go +++ /dev/null @@ -1,137 +0,0 @@ -package migration - -import ( - "context" - "fmt" - "time" - - "github.com/grafana/grafana/pkg/infra/db" - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/infra/serverlock" - migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" - "github.com/grafana/grafana/pkg/services/secrets" - "github.com/grafana/grafana/pkg/setting" -) - -// actionName is the unique row-level lock name for serverlock.ServerLockService. -const actionName = "alerting migration" - -//nolint:stylecheck -var ForceMigrationError = fmt.Errorf("Grafana has already been migrated to Unified Alerting. Any alert rules created while using Unified Alerting will be deleted by rolling back. Set force_migration=true in your grafana.ini and restart Grafana to roll back and delete Unified Alerting configuration data.") - -type MigrationService struct { - lock *serverlock.ServerLockService - cfg *setting.Cfg - log log.Logger - store db.DB - migrationStore migrationStore.Store - - encryptionService secrets.Service -} - -func ProvideService( - lock *serverlock.ServerLockService, - cfg *setting.Cfg, - store db.DB, - migrationStore migrationStore.Store, - encryptionService secrets.Service, -) (*MigrationService, error) { - return &MigrationService{ - lock: lock, - log: log.New("ngalert.migration"), - cfg: cfg, - store: store, - migrationStore: migrationStore, - encryptionService: encryptionService, - }, nil -} - -// Run starts the migration. This will either migrate from legacy alerting to unified alerting or revert the migration. -// If the migration status in the kvstore is not set and unified alerting is enabled, the migration will be executed. -// If the migration status in the kvstore is set and both unified alerting is disabled and ForceMigration is set to true, the migration will be reverted. -func (ms *MigrationService) Run(ctx context.Context) error { - var errMigration error - errLock := ms.lock.LockExecuteAndRelease(ctx, actionName, time.Minute*10, func(context.Context) { - ms.log.Info("Starting") - errMigration = ms.store.InTransaction(ctx, func(ctx context.Context) error { - migrated, err := ms.migrationStore.IsMigrated(ctx) - if err != nil { - return fmt.Errorf("getting migration status: %w", err) - } - if migrated == ms.cfg.UnifiedAlerting.IsEnabled() { - // Nothing to do. - ms.log.Info("No migrations to run") - return nil - } - - if migrated { - // If legacy alerting is also disabled, there is nothing to do - if setting.AlertingEnabled != nil && !*setting.AlertingEnabled { - return nil - } - - // Safeguard to prevent data loss when reverting from UA to LA. - if !ms.cfg.ForceMigration { - return ForceMigrationError - } - - // Revert migration - ms.log.Info("Reverting legacy migration") - err := ms.migrationStore.RevertAllOrgs(ctx) - if err != nil { - return fmt.Errorf("reverting migration: %w", err) - } - ms.log.Info("Legacy migration reverted") - return nil - } - - ms.log.Info("Starting legacy migration") - err = ms.migrateAllOrgs(ctx) - if err != nil { - return fmt.Errorf("executing migration: %w", err) - } - - err = ms.migrationStore.SetMigrated(ctx, true) - if err != nil { - return fmt.Errorf("setting migration status: %w", err) - } - - ms.log.Info("Completed legacy migration") - return nil - }) - }) - if errLock != nil { - ms.log.Warn("Server lock for alerting migration already exists") - return nil - } - if errMigration != nil { - return fmt.Errorf("migration failed: %w", errMigration) - } - return nil -} - -// IsDisabled returns true if the cfg is nil. -func (ms *MigrationService) IsDisabled() bool { - return ms.cfg == nil -} - -// migrateAllOrgs executes the migration for all orgs. -func (ms *MigrationService) migrateAllOrgs(ctx context.Context) error { - orgs, err := ms.migrationStore.GetAllOrgs(ctx) - if err != nil { - return fmt.Errorf("get orgs: %w", err) - } - - for _, o := range orgs { - om := ms.newOrgMigration(o.ID) - if err := om.migrateOrg(ctx); err != nil { - return fmt.Errorf("migrate org %d: %w", o.ID, err) - } - - err = om.migrationStore.SetOrgMigrationState(ctx, o.ID, om.state) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/services/ngalert/migration/service_test.go b/pkg/services/ngalert/migration/service_test.go deleted file mode 100644 index 4bc386a5fe5..00000000000 --- a/pkg/services/ngalert/migration/service_test.go +++ /dev/null @@ -1,190 +0,0 @@ -package migration - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/infra/db" - legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" - "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/setting" -) - -// TestServiceRevert tests migration revert. -func TestServiceRevert(t *testing.T) { - alerts := []*legacymodels.Alert{ - createAlert(t, 1, 1, 1, "alert1", []string{"notifier1"}), - } - channels := []*legacymodels.AlertNotification{ - createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), - } - dashes := []*dashboards.Dashboard{ - createDashboard(t, 1, 1, "dash1-1", 5, nil), - createDashboard(t, 2, 1, "dash2-1", 5, nil), - createDashboard(t, 8, 1, "dash-in-general-1", 0, nil), - } - folders := []*dashboards.Dashboard{ - createFolder(t, 5, 1, "folder5-1"), - } - - t.Run("revert deletes UA resources", func(t *testing.T) { - sqlStore := db.InitTestDB(t) - x := sqlStore.GetEngine() - - setupLegacyAlertsTables(t, x, channels, alerts, folders, dashes) - - dashCount, err := x.Table("dashboard").Count(&dashboards.Dashboard{}) - require.NoError(t, err) - require.Equal(t, int64(4), dashCount) - - // Run migration. - ctx := context.Background() - cfg := &setting.Cfg{ - ForceMigration: true, - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: pointer(true), - }, - } - service := NewTestMigrationService(t, sqlStore, cfg) - - err = service.migrationStore.SetMigrated(ctx, false) - require.NoError(t, err) - - err = service.Run(ctx) - require.NoError(t, err) - - // Verify migration was run. - migrated, err := service.migrationStore.IsMigrated(ctx) - require.NoError(t, err) - require.Equal(t, true, migrated) - - // Currently, we fill in some random data for tables that aren't populated during migration. - _, err = x.Table("ngalert_configuration").Insert(models.AdminConfiguration{}) - require.NoError(t, err) - _, err = x.Table("alert_instance").Insert(models.AlertInstance{ - AlertInstanceKey: models.AlertInstanceKey{ - RuleOrgID: 1, - RuleUID: "alert1", - LabelsHash: "", - }, - CurrentState: models.InstanceStateNormal, - CurrentStateSince: time.Now(), - CurrentStateEnd: time.Now(), - LastEvalTime: time.Now(), - }) - require.NoError(t, err) - - // Verify various UA resources exist - tables := []string{ - "alert_rule", - "alert_rule_version", - "alert_configuration", - "ngalert_configuration", - "alert_instance", - } - for _, table := range tables { - count, err := x.Table(table).Count() - require.NoError(t, err) - require.True(t, count > 0, "table %s should have at least one row", table) - } - - // Revert migration. - service.cfg.UnifiedAlerting.Enabled = pointer(false) - err = service.Run(context.Background()) - require.NoError(t, err) - - // Verify revert was run. - migrated, err = service.migrationStore.IsMigrated(ctx) - require.NoError(t, err) - require.Equal(t, false, migrated) - - // Verify various UA resources are gone - for _, table := range tables { - count, err := x.Table(table).Count() - require.NoError(t, err) - require.Equal(t, int64(0), count, "table %s should have no rows", table) - } - }) - - t.Run("revert deletes folders created during migration", func(t *testing.T) { - sqlStore := db.InitTestDB(t) - x := sqlStore.GetEngine() - alerts = []*legacymodels.Alert{ - createAlert(t, 1, 8, 1, "alert1", []string{"notifier1"}), - } - setupLegacyAlertsTables(t, x, channels, alerts, folders, dashes) - - dashCount, err := x.Table("dashboard").Count(&dashboards.Dashboard{}) - require.NoError(t, err) - require.Equal(t, int64(4), dashCount) - - // Run migration. - ctx := context.Background() - cfg := &setting.Cfg{ - ForceMigration: true, - UnifiedAlerting: setting.UnifiedAlertingSettings{ - Enabled: pointer(true), - }, - } - service := NewTestMigrationService(t, sqlStore, cfg) - - err = service.migrationStore.SetMigrated(ctx, false) - require.NoError(t, err) - - err = service.Run(ctx) - require.NoError(t, err) - - // Verify migration was run. - migrated, err := service.migrationStore.IsMigrated(ctx) - require.NoError(t, err) - require.Equal(t, true, migrated) - - // Verify we created some folders. - newDashCount, err := x.Table("dashboard").Count(&dashboards.Dashboard{}) - require.NoError(t, err) - require.Truef(t, newDashCount > dashCount, "newDashCount: %d should be greater than dashCount: %d", newDashCount, dashCount) - - // Check that dashboards and folders from before migration still exist. - require.NotNil(t, getDashboard(t, x, 1, "dash1-1")) - require.NotNil(t, getDashboard(t, x, 1, "dash2-1")) - require.NotNil(t, getDashboard(t, x, 1, "dash-in-general-1")) - - state, err := service.migrationStore.GetOrgMigrationState(ctx, 1) - require.NoError(t, err) - - // Verify list of created folders. - require.NotEmpty(t, state.CreatedFolders) - for _, uid := range state.CreatedFolders { - require.NotNil(t, getDashboard(t, x, 1, uid)) - } - - // Revert migration. - service.cfg.UnifiedAlerting.Enabled = pointer(false) - err = service.Run(context.Background()) - require.NoError(t, err) - - // Verify revert was run. - migrated, err = service.migrationStore.IsMigrated(ctx) - require.NoError(t, err) - require.Equal(t, false, migrated) - - // Verify we are back to the original count. - newDashCount, err = x.Table("dashboard").Count(&dashboards.Dashboard{}) - require.NoError(t, err) - require.Equalf(t, dashCount, newDashCount, "newDashCount: %d should be equal to dashCount: %d after revert", newDashCount, dashCount) - - // Check that dashboards and folders from before migration still exist. - require.NotNil(t, getDashboard(t, x, 1, "dash1-1")) - require.NotNil(t, getDashboard(t, x, 1, "dash2-1")) - require.NotNil(t, getDashboard(t, x, 1, "dash-in-general-1")) - - // Check that folders created during migration are gone. - for _, uid := range state.CreatedFolders { - require.Nil(t, getDashboard(t, x, 1, uid)) - } - }) -} diff --git a/pkg/services/ngalert/migration/store/database.go b/pkg/services/ngalert/migration/store/database.go deleted file mode 100644 index 99589b57f49..00000000000 --- a/pkg/services/ngalert/migration/store/database.go +++ /dev/null @@ -1,511 +0,0 @@ -package store - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/grafana/grafana/pkg/infra/db" - "github.com/grafana/grafana/pkg/infra/kvstore" - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/services/accesscontrol" - legacyalerting "github.com/grafana/grafana/pkg/services/alerting" - legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" - "github.com/grafana/grafana/pkg/services/auth/identity" - "github.com/grafana/grafana/pkg/services/dashboards" - "github.com/grafana/grafana/pkg/services/datasources" - "github.com/grafana/grafana/pkg/services/folder" - apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" - migmodels "github.com/grafana/grafana/pkg/services/ngalert/migration/models" - "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/services/ngalert/notifier" - "github.com/grafana/grafana/pkg/services/ngalert/store" - "github.com/grafana/grafana/pkg/services/org" - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" - "github.com/grafana/grafana/pkg/services/user" - "github.com/grafana/grafana/pkg/setting" -) - -// Store is the database abstraction for migration persistence. -type Store interface { - InsertAlertRules(ctx context.Context, rules ...models.AlertRule) error - - SaveAlertmanagerConfiguration(ctx context.Context, orgID int64, amConfig *apimodels.PostableUserConfig) error - - GetAllOrgs(ctx context.Context) ([]*org.OrgDTO, error) - - GetDatasource(ctx context.Context, datasourceID int64, user identity.Requester) (*datasources.DataSource, error) - - GetNotificationChannels(ctx context.Context, orgID int64) ([]*legacymodels.AlertNotification, error) - - GetOrgDashboardAlerts(ctx context.Context, orgID int64) (map[int64][]*DashAlert, int, error) - - GetACL(ctx context.Context, orgID int64, dashID int64) ([]*DashboardACL, error) - SetACL(ctx context.Context, orgID int64, dashboardID int64, items []*DashboardACL) error - - GetDashboard(ctx context.Context, orgID int64, id int64) (*dashboards.Dashboard, error) - GetFolder(ctx context.Context, cmd *folder.GetFolderQuery) (*folder.Folder, error) - CreateFolder(ctx context.Context, cmd *folder.CreateFolderCommand) (*folder.Folder, error) - - IsMigrated(ctx context.Context) (bool, error) - SetMigrated(ctx context.Context, migrated bool) error - GetOrgMigrationState(ctx context.Context, orgID int64) (*migmodels.OrgMigrationState, error) - SetOrgMigrationState(ctx context.Context, orgID int64, summary *migmodels.OrgMigrationState) error - - RevertAllOrgs(ctx context.Context) error - - CaseInsensitive() bool -} - -type migrationStore struct { - store db.DB - cfg *setting.Cfg - log log.Logger - kv kvstore.KVStore - alertingStore *store.DBstore - dashboardService dashboards.DashboardService - folderService folder.Service - dataSourceCache datasources.CacheService - orgService org.Service - legacyAlertNotificationService *legacyalerting.AlertNotificationService -} - -// MigrationStore implements the Store interface. -var _ Store = (*migrationStore)(nil) - -func ProvideMigrationStore( - cfg *setting.Cfg, - sqlStore db.DB, - kv kvstore.KVStore, - alertingStore *store.DBstore, - dashboardService dashboards.DashboardService, - folderService folder.Service, - dataSourceCache datasources.CacheService, - orgService org.Service, - legacyAlertNotificationService *legacyalerting.AlertNotificationService, -) (Store, error) { - return &migrationStore{ - log: log.New("ngalert.migration-store"), - cfg: cfg, - store: sqlStore, - kv: kv, - alertingStore: alertingStore, - dashboardService: dashboardService, - folderService: folderService, - dataSourceCache: dataSourceCache, - orgService: orgService, - legacyAlertNotificationService: legacyAlertNotificationService, - }, nil -} - -// KVNamespace is the kvstore namespace used for the migration status. -const KVNamespace = "ngalert.migration" - -// migratedKey is the kvstore key used for the migration status. -const migratedKey = "migrated" - -// stateKey is the kvstore key used for the OrgMigrationState. -const stateKey = "stateKey" - -const anyOrg = 0 - -// IsMigrated returns the migration status from the kvstore. -func (ms *migrationStore) IsMigrated(ctx context.Context) (bool, error) { - kv := kvstore.WithNamespace(ms.kv, anyOrg, KVNamespace) - content, exists, err := kv.Get(ctx, migratedKey) - if err != nil { - return false, err - } - - if !exists { - return false, nil - } - - return strconv.ParseBool(content) -} - -// SetMigrated sets the migration status in the kvstore. -func (ms *migrationStore) SetMigrated(ctx context.Context, migrated bool) error { - kv := kvstore.WithNamespace(ms.kv, anyOrg, KVNamespace) - return kv.Set(ctx, migratedKey, strconv.FormatBool(migrated)) -} - -// GetOrgMigrationState returns a summary of a previous migration. -func (ms *migrationStore) GetOrgMigrationState(ctx context.Context, orgID int64) (*migmodels.OrgMigrationState, error) { - kv := kvstore.WithNamespace(ms.kv, orgID, KVNamespace) - content, exists, err := kv.Get(ctx, stateKey) - if err != nil { - return nil, err - } - - if !exists { - return &migmodels.OrgMigrationState{OrgID: orgID}, nil - } - - var summary migmodels.OrgMigrationState - err = json.Unmarshal([]byte(content), &summary) - if err != nil { - return nil, err - } - - return &summary, nil -} - -// SetOrgMigrationState sets the summary of a previous migration. -func (ms *migrationStore) SetOrgMigrationState(ctx context.Context, orgID int64, summary *migmodels.OrgMigrationState) error { - kv := kvstore.WithNamespace(ms.kv, orgID, KVNamespace) - raw, err := json.Marshal(summary) - if err != nil { - return err - } - - return kv.Set(ctx, stateKey, string(raw)) -} - -func (ms *migrationStore) InsertAlertRules(ctx context.Context, rules ...models.AlertRule) error { - if ms.store.GetDialect().DriverName() == migrator.Postgres { - // Postgresql which will automatically rollback the whole transaction on constraint violation. - // So, for postgresql, insertions will execute in a subtransaction. - err := ms.store.InTransaction(ctx, func(subCtx context.Context) error { - _, err := ms.alertingStore.InsertAlertRules(subCtx, rules) - if err != nil { - return err - } - return nil - }) - if err != nil { - return err - } - } else { - _, err := ms.alertingStore.InsertAlertRules(ctx, rules) - if err != nil { - return err - } - } - - return nil -} - -func (ms *migrationStore) SaveAlertmanagerConfiguration(ctx context.Context, orgID int64, amConfig *apimodels.PostableUserConfig) error { - rawAmConfig, err := json.Marshal(amConfig) - if err != nil { - return err - } - - cmd := models.SaveAlertmanagerConfigurationCmd{ - AlertmanagerConfiguration: string(rawAmConfig), - ConfigurationVersion: fmt.Sprintf("v%d", models.AlertConfigurationVersion), - Default: false, - OrgID: orgID, - LastApplied: 0, - } - return ms.alertingStore.SaveAlertmanagerConfiguration(ctx, &cmd) -} - -// revertPermissions are the permissions required for the background user to revert the migration. -var revertPermissions = []accesscontrol.Permission{ - {Action: dashboards.ActionFoldersDelete, Scope: dashboards.ScopeFoldersAll}, - {Action: dashboards.ActionFoldersRead, Scope: dashboards.ScopeFoldersAll}, -} - -// RevertAllOrgs reverts the migration, deleting all unified alerting resources such as alert rules, alertmanager configurations, and silence files. -// In addition, it will delete all folders and permissions originally created by this migration, these are stored in the kvstore. -func (ms *migrationStore) RevertAllOrgs(ctx context.Context) error { - return ms.store.WithTransactionalDbSession(ctx, func(sess *db.Session) error { - if _, err := sess.Exec("DELETE FROM alert_rule"); err != nil { - return err - } - - if _, err := sess.Exec("DELETE FROM alert_rule_version"); err != nil { - return err - } - - orgs, err := ms.GetAllOrgs(ctx) - if err != nil { - return fmt.Errorf("get orgs: %w", err) - } - for _, o := range orgs { - if err := ms.DeleteMigratedFolders(ctx, o.ID); err != nil { - return err - } - } - - if _, err := sess.Exec("DELETE FROM alert_configuration"); err != nil { - return err - } - - if _, err := sess.Exec("DELETE FROM ngalert_configuration"); err != nil { - return err - } - - if _, err := sess.Exec("DELETE FROM alert_instance"); err != nil { - return err - } - - if _, err := sess.Exec("DELETE FROM kv_store WHERE namespace = ?", notifier.KVNamespace); err != nil { - return err - } - - if _, err := sess.Exec("DELETE FROM kv_store WHERE namespace = ?", KVNamespace); err != nil { - return err - } - - files, err := filepath.Glob(filepath.Join(ms.cfg.DataPath, "alerting", "*", "silences")) - if err != nil { - return err - } - for _, f := range files { - if err := os.Remove(f); err != nil { - ms.log.Error("Failed to remove silence file", "file", f, "err", err) - } - } - - err = ms.SetMigrated(ctx, false) - if err != nil { - return fmt.Errorf("setting migration status: %w", err) - } - - return nil - }) -} - -// DeleteMigratedFolders deletes all folders created by the previous migration run for the given org. This includes all folder permissions. -// If the folder is not empty of all descendants the operation will fail and return an error. -func (ms *migrationStore) DeleteMigratedFolders(ctx context.Context, orgID int64) error { - summary, err := ms.GetOrgMigrationState(ctx, orgID) - if err != nil { - return err - } - return ms.DeleteFolders(ctx, orgID, summary.CreatedFolders...) -} - -// DeleteFolders deletes the folders from the given orgs with the given UIDs. This includes all folder permissions. -// If the folder is not empty of all descendants the operation will fail and return an error. -func (ms *migrationStore) DeleteFolders(ctx context.Context, orgID int64, uids ...string) error { - if len(uids) == 0 { - return nil - } - - usr := accesscontrol.BackgroundUser("ngalert_migration_revert", orgID, org.RoleAdmin, revertPermissions) - for _, folderUID := range uids { - cmd := folder.DeleteFolderCommand{ - UID: folderUID, - OrgID: orgID, - SignedInUser: usr.(*user.SignedInUser), - } - err := ms.folderService.Delete(ctx, &cmd) // Also handles permissions and other related entities. - if err != nil { - return err - } - } - return nil -} - -func (ms *migrationStore) GetDashboard(ctx context.Context, orgID int64, id int64) (*dashboards.Dashboard, error) { - return ms.dashboardService.GetDashboard(ctx, &dashboards.GetDashboardQuery{ID: id, OrgID: orgID}) -} - -func (ms *migrationStore) GetAllOrgs(ctx context.Context) ([]*org.OrgDTO, error) { - orgQuery := &org.SearchOrgsQuery{} - return ms.orgService.Search(ctx, orgQuery) -} - -func (ms *migrationStore) GetDatasource(ctx context.Context, datasourceID int64, user identity.Requester) (*datasources.DataSource, error) { - return ms.dataSourceCache.GetDatasource(ctx, datasourceID, user, false) -} - -// GetNotificationChannels returns all channels for this org. -func (ms *migrationStore) GetNotificationChannels(ctx context.Context, orgID int64) ([]*legacymodels.AlertNotification, error) { - return ms.legacyAlertNotificationService.GetAllAlertNotifications(ctx, &legacymodels.GetAllAlertNotificationsQuery{ - OrgID: orgID, - }) -} - -func (ms *migrationStore) GetFolder(ctx context.Context, cmd *folder.GetFolderQuery) (*folder.Folder, error) { - return ms.folderService.Get(ctx, cmd) -} - -func (ms *migrationStore) CreateFolder(ctx context.Context, cmd *folder.CreateFolderCommand) (*folder.Folder, error) { - return ms.folderService.Create(ctx, cmd) -} - -// based on SQLStore.GetDashboardACLInfoList() -func (ms *migrationStore) GetACL(ctx context.Context, orgID, dashboardID int64) ([]*DashboardACL, error) { - var err error - - falseStr := ms.store.GetDialect().BooleanStr(false) - - result := make([]*DashboardACL, 0) - rawSQL := ` - -- get distinct permissions for the dashboard and its parent folder - SELECT DISTINCT - da.id, - da.user_id, - da.team_id, - da.permission, - da.role - FROM dashboard as d - LEFT JOIN dashboard folder on folder.id = d.folder_id - LEFT JOIN dashboard_acl AS da ON - da.dashboard_id = d.id OR - da.dashboard_id = d.folder_id OR - ( - -- include default permissions -- - da.org_id = -1 AND ( - (folder.id IS NOT NULL AND folder.has_acl = ` + falseStr + `) OR - (folder.id IS NULL AND d.has_acl = ` + falseStr + `) - ) - ) - WHERE d.org_id = ? AND d.id = ? AND da.id IS NOT NULL - ORDER BY da.id ASC - ` - err = ms.store.WithDbSession(ctx, func(sess *db.Session) error { - return sess.SQL(rawSQL, orgID, dashboardID).Find(&result) - }) - if err != nil { - return nil, err - } - return result, err -} - -// based on SQLStore.UpdateDashboardACL() -// it should be called from inside a transaction -func (ms *migrationStore) SetACL(ctx context.Context, orgID int64, dashboardID int64, items []*DashboardACL) error { - if dashboardID <= 0 { - return fmt.Errorf("folder id must be greater than zero for a folder permission") - } - return ms.store.WithDbSession(ctx, func(sess *db.Session) error { - // userPermissionsMap is a map keeping the highest permission per user - // for handling conficting inherited (folder) and non-inherited (dashboard) user permissions - userPermissionsMap := make(map[int64]*DashboardACL, len(items)) - // teamPermissionsMap is a map keeping the highest permission per team - // for handling conficting inherited (folder) and non-inherited (dashboard) team permissions - teamPermissionsMap := make(map[int64]*DashboardACL, len(items)) - for _, item := range items { - if item.UserID != 0 { - acl, ok := userPermissionsMap[item.UserID] - if !ok { - userPermissionsMap[item.UserID] = item - } else { - if item.Permission > acl.Permission { - // the higher permission wins - userPermissionsMap[item.UserID] = item - } - } - } - - if item.TeamID != 0 { - acl, ok := teamPermissionsMap[item.TeamID] - if !ok { - teamPermissionsMap[item.TeamID] = item - } else { - if item.Permission > acl.Permission { - // the higher permission wins - teamPermissionsMap[item.TeamID] = item - } - } - } - } - - type keyType struct { - UserID int64 `xorm:"user_id"` - TeamID int64 `xorm:"team_id"` - Role RoleType - Permission permissionType - } - // seen keeps track of inserted perrmissions to avoid duplicates (due to inheritance) - seen := make(map[keyType]struct{}, len(items)) - for _, item := range items { - if item.UserID == 0 && item.TeamID == 0 && (item.Role == nil || !item.Role.IsValid()) { - return dashboards.ErrDashboardACLInfoMissing - } - - // ignore duplicate user permissions - if item.UserID != 0 { - acl, ok := userPermissionsMap[item.UserID] - if ok { - if acl.Id != item.Id { - continue - } - } - } - - // ignore duplicate team permissions - if item.TeamID != 0 { - acl, ok := teamPermissionsMap[item.TeamID] - if ok { - if acl.Id != item.Id { - continue - } - } - } - - key := keyType{UserID: item.UserID, TeamID: item.TeamID, Role: "", Permission: item.Permission} - if item.Role != nil { - key.Role = *item.Role - } - if _, ok := seen[key]; ok { - continue - } - - // unset Id so that the new record will get a different one - item.Id = 0 - item.OrgID = orgID - item.DashboardID = dashboardID - item.Created = time.Now() - item.Updated = time.Now() - - sess.Nullable("user_id", "team_id") - if _, err := sess.Insert(item); err != nil { - return err - } - seen[key] = struct{}{} - } - - // Update dashboard HasACL flag - dashboard := dashboards.Dashboard{HasACL: true} - _, err := sess.Cols("has_acl").Where("id=?", dashboardID).Update(&dashboard) - - return err - }) -} - -// GetOrgDashboardAlerts loads all legacy dashboard alerts for the given org mapped by dashboard id. -func (ms *migrationStore) GetOrgDashboardAlerts(ctx context.Context, orgID int64) (map[int64][]*DashAlert, int, error) { - var alerts []legacymodels.Alert - err := ms.store.WithDbSession(ctx, func(sess *db.Session) error { - return sess.SQL("select * from alert WHERE org_id = ? AND dashboard_id IN (SELECT id from dashboard)", orgID).Find(&alerts) - }) - if err != nil { - return nil, 0, err - } - - mappedAlerts := make(map[int64][]*DashAlert) - for i := range alerts { - alert := alerts[i] - - rawSettings, err := json.Marshal(alert.Settings) - if err != nil { - return nil, 0, fmt.Errorf("get settings for alert rule ID:%d, name:'%s', orgID:%d: %w", alert.ID, alert.Name, alert.OrgID, err) - } - var parsedSettings DashAlertSettings - err = json.Unmarshal(rawSettings, &parsedSettings) - if err != nil { - return nil, 0, fmt.Errorf("parse settings for alert rule ID:%d, name:'%s', orgID:%d: %w", alert.ID, alert.Name, alert.OrgID, err) - } - - mappedAlerts[alert.DashboardID] = append(mappedAlerts[alert.DashboardID], &DashAlert{ - Alert: &alerts[i], - ParsedSettings: &parsedSettings, - }) - } - return mappedAlerts, len(alerts), nil -} - -func (ms *migrationStore) CaseInsensitive() bool { - return ms.store.GetDialect().SupportEngine() -} diff --git a/pkg/services/ngalert/migration/store/models.go b/pkg/services/ngalert/migration/store/models.go deleted file mode 100644 index 7053e4199ed..00000000000 --- a/pkg/services/ngalert/migration/store/models.go +++ /dev/null @@ -1,91 +0,0 @@ -package store - -import ( - "encoding/json" - "time" - - legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" -) - -type RoleType string - -const ( - RoleNone RoleType = "None" - RoleViewer RoleType = "Viewer" - RoleEditor RoleType = "Editor" - RoleAdmin RoleType = "Admin" -) - -func (r RoleType) IsValid() bool { - return r == RoleViewer || r == RoleAdmin || r == RoleEditor || r == RoleNone -} - -type permissionType int - -type DashboardACL struct { - // nolint:stylecheck - Id int64 - OrgID int64 `xorm:"org_id"` - DashboardID int64 `xorm:"dashboard_id"` - - UserID int64 `xorm:"user_id"` - TeamID int64 `xorm:"team_id"` - Role *RoleType // pointer to be nullable - Permission permissionType - - Created time.Time - Updated time.Time -} - -func (p DashboardACL) TableName() string { return "dashboard_acl" } - -// uidOrID for both uid and ID, primarily used for mapping legacy channel to migrated receiver. -type UidOrID any - -type DashAlert struct { - *legacymodels.Alert - ParsedSettings *DashAlertSettings -} - -// dashAlertSettings is a type for the JSON that is in the settings field of -// the alert table. -type DashAlertSettings struct { - NoDataState string `json:"noDataState"` - ExecutionErrorState string `json:"executionErrorState"` - Conditions []DashAlertCondition `json:"conditions"` - AlertRuleTags any `json:"alertRuleTags"` - Notifications []DashAlertNot `json:"notifications"` -} - -// dashAlertNot is the object that represents the Notifications array in -// dashAlertSettings -type DashAlertNot struct { - UID string `json:"uid,omitempty"` - ID int64 `json:"id,omitempty"` -} - -// dashAlertingConditionJSON is like classic.ClassicConditionJSON except that it -// includes the model property with the query. -type DashAlertCondition struct { - Evaluator ConditionEvalJSON `json:"evaluator"` - - Operator struct { - Type string `json:"type"` - } `json:"operator"` - - Query struct { - Params []string `json:"params"` - DatasourceID int64 `json:"datasourceId"` - Model json.RawMessage - } `json:"query"` - - Reducer struct { - // Params []any `json:"params"` (Unused) - Type string `json:"type"` - } -} - -type ConditionEvalJSON struct { - Params []float64 `json:"params"` - Type string `json:"type"` // e.g. "gt" -} diff --git a/pkg/services/ngalert/migration/store/testing.go b/pkg/services/ngalert/migration/store/testing.go deleted file mode 100644 index 7f912000f92..00000000000 --- a/pkg/services/ngalert/migration/store/testing.go +++ /dev/null @@ -1,57 +0,0 @@ -package store - -import ( - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/bus" - "github.com/grafana/grafana/pkg/infra/localcache" - "github.com/grafana/grafana/pkg/infra/log/logtest" - "github.com/grafana/grafana/pkg/infra/tracing" - legacyalerting "github.com/grafana/grafana/pkg/services/alerting" - "github.com/grafana/grafana/pkg/services/datasources/guardian" - datasourceService "github.com/grafana/grafana/pkg/services/datasources/service" - encryptionservice "github.com/grafana/grafana/pkg/services/encryption/service" - "github.com/grafana/grafana/pkg/services/folder/folderimpl" - "github.com/grafana/grafana/pkg/services/ngalert/store" - "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" - "github.com/grafana/grafana/pkg/services/ngalert/testutil" - "github.com/grafana/grafana/pkg/services/org/orgimpl" - "github.com/grafana/grafana/pkg/services/quota/quotatest" - "github.com/grafana/grafana/pkg/services/sqlstore" - "github.com/grafana/grafana/pkg/setting" -) - -func NewTestMigrationStore(t *testing.T, sqlStore *sqlstore.SQLStore, cfg *setting.Cfg) *migrationStore { - if cfg.UnifiedAlerting.BaseInterval == 0 { - cfg.UnifiedAlerting.BaseInterval = time.Second * 10 - } - alertingStore := store.DBstore{ - SQLStore: sqlStore, - Cfg: cfg.UnifiedAlerting, - } - bus := bus.ProvideBus(tracing.InitializeTracerForTest()) - folderStore := folderimpl.ProvideDashboardFolderStore(sqlStore) - dashboardService, dashboardStore := testutil.SetupDashboardService(t, sqlStore, folderStore, cfg) - folderService := testutil.SetupFolderService(t, cfg, sqlStore, dashboardStore, folderStore, bus) - - quotaService := "atest.FakeQuotaService{} - orgService, err := orgimpl.ProvideService(sqlStore, cfg, quotaService) - require.NoError(t, err) - - cache := localcache.ProvideService() - return &migrationStore{ - log: &logtest.Fake{}, - cfg: cfg, - store: sqlStore, - kv: fakes.NewFakeKVStore(t), - alertingStore: &alertingStore, - dashboardService: dashboardService, - folderService: folderService, - dataSourceCache: datasourceService.ProvideCacheService(cache, sqlStore, guardian.ProvideGuardian()), - orgService: orgService, - legacyAlertNotificationService: legacyalerting.ProvideService(sqlStore, encryptionservice.SetupTestService(t), nil), - } -} diff --git a/pkg/services/ngalert/migration/testing.go b/pkg/services/ngalert/migration/testing.go deleted file mode 100644 index 4f5fcaf7769..00000000000 --- a/pkg/services/ngalert/migration/testing.go +++ /dev/null @@ -1,29 +0,0 @@ -package migration - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/infra/serverlock" - "github.com/grafana/grafana/pkg/infra/tracing" - migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" - fake_secrets "github.com/grafana/grafana/pkg/services/secrets/fakes" - "github.com/grafana/grafana/pkg/services/sqlstore" - "github.com/grafana/grafana/pkg/setting" -) - -func NewTestMigrationService(t *testing.T, sqlStore *sqlstore.SQLStore, cfg *setting.Cfg) *MigrationService { - if cfg == nil { - cfg = setting.NewCfg() - } - ms, err := ProvideService( - serverlock.ProvideService(sqlStore, tracing.InitializeTracerForTest()), - cfg, - sqlStore, - migrationStore.NewTestMigrationStore(t, sqlStore, cfg), - fake_secrets.NewFakeSecretsService(), - ) - require.NoError(t, err) - return ms -} diff --git a/pkg/services/ngalert/migration/ualert.go b/pkg/services/ngalert/migration/ualert.go deleted file mode 100644 index 085960d14de..00000000000 --- a/pkg/services/ngalert/migration/ualert.go +++ /dev/null @@ -1,115 +0,0 @@ -package migration - -import ( - "context" - "fmt" - - apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" - migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" - "github.com/grafana/grafana/pkg/services/ngalert/models" -) - -func (om *OrgMigration) migrateAlerts(ctx context.Context, alerts []*migrationStore.DashAlert, dashboardUID string, folderUID string) ([]*AlertPair, error) { - log := om.log.New( - "dashboardUID", dashboardUID, - "newFolderUID", folderUID, - ) - - pairs := make([]*AlertPair, 0, len(alerts)) - for _, da := range alerts { - al := log.New("ruleID", da.ID, "ruleName", da.Name) - alertRule, err := om.migrateAlert(ctx, al, da, dashboardUID, folderUID) - if err != nil { - return nil, fmt.Errorf("migrate alert: %w", err) - } - pairs = append(pairs, &AlertPair{AlertRule: alertRule, DashAlert: da}) - } - - return pairs, nil -} - -func (om *OrgMigration) migrateDashboard(ctx context.Context, dashID int64, alerts []*migrationStore.DashAlert) ([]*AlertPair, error) { - dash, newFolder, err := om.getOrCreateMigratedFolder(ctx, om.log, dashID) - if err != nil { - return nil, fmt.Errorf("get or create migrated folder: %w", err) - } - - pairs, err := om.migrateAlerts(ctx, alerts, dash.UID, newFolder.UID) - if err != nil { - return nil, fmt.Errorf("migrate and save alerts: %w", err) - } - - return pairs, nil -} - -func (om *OrgMigration) migrateOrgAlerts(ctx context.Context) ([]*AlertPair, error) { - mappedAlerts, cnt, err := om.migrationStore.GetOrgDashboardAlerts(ctx, om.orgID) - if err != nil { - return nil, fmt.Errorf("load alerts: %w", err) - } - om.log.Info("Alerts found to migrate", "alerts", cnt) - - pairs := make([]*AlertPair, 0, cnt) - for dashID, alerts := range mappedAlerts { - dashPairs, err := om.migrateDashboard(ctx, dashID, alerts) - if err != nil { - return nil, fmt.Errorf("migrate and save dashboard '%d': %w", dashID, err) - } - pairs = append(pairs, dashPairs...) - } - return pairs, nil -} - -func (om *OrgMigration) migrateOrgChannels(ctx context.Context, pairs []*AlertPair) (*apimodels.PostableUserConfig, error) { - channels, err := om.migrationStore.GetNotificationChannels(ctx, om.orgID) - if err != nil { - return nil, fmt.Errorf("load notification channels: %w", err) - } - - amConfig, err := om.migrateChannels(channels, pairs) - if err != nil { - return nil, err - } - - return amConfig, nil -} - -func (om *OrgMigration) migrateOrg(ctx context.Context) error { - om.log.Info("Migrating alerts for organisation") - - pairs, err := om.migrateOrgAlerts(ctx) - if err != nil { - return fmt.Errorf("migrate alerts: %w", err) - } - - // This must happen before we insert the rules into the database because it modifies the alert labels. This will - // be changed in the future when we improve how notification policies are created. - amConfig, err := om.migrateOrgChannels(ctx, pairs) - if err != nil { - return fmt.Errorf("migrate channels: %w", err) - } - - if err := om.writeSilencesFile(); err != nil { - return fmt.Errorf("write silence file for org %d: %w", om.orgID, err) - } - - if len(pairs) > 0 { - om.log.Debug("Inserting migrated alert rules", "count", len(pairs)) - rules := make([]models.AlertRule, 0, len(pairs)) - for _, p := range pairs { - rules = append(rules, *p.AlertRule) - } - err := om.migrationStore.InsertAlertRules(ctx, rules...) - if err != nil { - return fmt.Errorf("insert alert rules: %w", err) - } - } - - if amConfig != nil { - if err := om.migrationStore.SaveAlertmanagerConfiguration(ctx, om.orgID, amConfig); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index f73ddbfbc8a..521106afd3a 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -319,7 +319,6 @@ func subscribeToFolderChanges(logger log.Logger, bus bus.Bus, dbStore api.RuleSt // Run starts the scheduler and Alertmanager. func (ng *AlertNG) Run(ctx context.Context) error { ng.Log.Debug("Starting") - ng.stateManager.Warm(ctx, ng.store) children, subCtx := errgroup.WithContext(ctx) diff --git a/pkg/services/ngalert/notifier/alertmanager_test.go b/pkg/services/ngalert/notifier/alertmanager_test.go index e0a14bda177..6b9569c99c5 100644 --- a/pkg/services/ngalert/notifier/alertmanager_test.go +++ b/pkg/services/ngalert/notifier/alertmanager_test.go @@ -13,7 +13,6 @@ import ( "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/ngalert/metrics" "github.com/grafana/grafana/pkg/services/ngalert/store" - "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" "github.com/grafana/grafana/pkg/services/secrets/database" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/setting" @@ -38,7 +37,7 @@ func setupAMTest(t *testing.T) *alertmanager { DashboardService: dashboards.NewFakeDashboardService(t), } - kvStore := fakes.NewFakeKVStore(t) + kvStore := NewFakeKVStore(t) secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(sqlStore)) decryptFn := secretsService.GetDecryptedValue am, err := newAlertmanager(context.Background(), 1, cfg, s, kvStore, &NilPeer{}, decryptFn, nil, m) diff --git a/pkg/services/ngalert/notifier/channels_config/available_channels.go b/pkg/services/ngalert/notifier/channels_config/available_channels.go index 45072be5009..2af9a0cc00e 100644 --- a/pkg/services/ngalert/notifier/channels_config/available_channels.go +++ b/pkg/services/ngalert/notifier/channels_config/available_channels.go @@ -1,7 +1,6 @@ package channels_config import ( - "fmt" "os" alertingOpsgenie "github.com/grafana/alerting/receivers/opsgenie" @@ -1341,20 +1340,3 @@ func GetAvailableNotifiers() []*NotifierPlugin { }, } } - -// GetSecretKeysForContactPointType returns settings keys of contact point of the given type that are expected to be secrets. Returns error is contact point type is not known. -func GetSecretKeysForContactPointType(contactPointType string) ([]string, error) { - notifiers := GetAvailableNotifiers() - for _, n := range notifiers { - if n.Type == contactPointType { - var secureFields []string - for _, field := range n.Options { - if field.Secure { - secureFields = append(secureFields, field.PropertyName) - } - } - return secureFields, nil - } - } - return nil, fmt.Errorf("no secrets configured for type '%s'", contactPointType) -} diff --git a/pkg/services/ngalert/notifier/file_store_test.go b/pkg/services/ngalert/notifier/file_store_test.go index 92182396fa0..255a7e65716 100644 --- a/pkg/services/ngalert/notifier/file_store_test.go +++ b/pkg/services/ngalert/notifier/file_store_test.go @@ -7,12 +7,10 @@ import ( "testing" "github.com/stretchr/testify/require" - - "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" ) func TestFileStore_FilepathFor_DirectoryNotExist(t *testing.T) { - store := fakes.NewFakeKVStore(t) + store := NewFakeKVStore(t) workingDir := filepath.Join(t.TempDir(), "notexistdir") fs := NewFileStore(1, store, workingDir) filekey := "silences" @@ -33,7 +31,7 @@ func TestFileStore_FilepathFor_DirectoryNotExist(t *testing.T) { } } func TestFileStore_FilepathFor(t *testing.T) { - store := fakes.NewFakeKVStore(t) + store := NewFakeKVStore(t) workingDir := t.TempDir() fs := NewFileStore(1, store, workingDir) filekey := "silences" @@ -75,7 +73,7 @@ func TestFileStore_FilepathFor(t *testing.T) { } func TestFileStore_Persist(t *testing.T) { - store := fakes.NewFakeKVStore(t) + store := NewFakeKVStore(t) state := &fakeState{data: "something to marshal"} workingDir := t.TempDir() fs := NewFileStore(1, store, workingDir) @@ -84,9 +82,9 @@ func TestFileStore_Persist(t *testing.T) { size, err := fs.Persist(context.Background(), filekey, state) require.NoError(t, err) require.Equal(t, int64(20), size) - store.Mtx.Lock() - require.Len(t, store.Store, 1) - store.Mtx.Unlock() + store.mtx.Lock() + require.Len(t, store.store, 1) + store.mtx.Unlock() v, ok, err := store.Get(context.Background(), 1, KVNamespace, filekey) require.NoError(t, err) require.True(t, ok) diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go index b92678a833e..93b6ac1a715 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go @@ -19,7 +19,6 @@ import ( "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/provisioning" "github.com/grafana/grafana/pkg/services/ngalert/store" - ngfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/setting" @@ -32,7 +31,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) { } tmpDir := t.TempDir() - kvStore := ngfakes.NewFakeKVStore(t) + kvStore := NewFakeKVStore(t) provStore := provisioning.NewFakeProvisioningStore() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue @@ -166,7 +165,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T) } tmpDir := t.TempDir() - kvStore := ngfakes.NewFakeKVStore(t) + kvStore := NewFakeKVStore(t) provStore := provisioning.NewFakeProvisioningStore() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue @@ -260,7 +259,7 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) { DataPath: tmpDir, UnifiedAlerting: setting.UnifiedAlertingSettings{AlertmanagerConfigPollInterval: 3 * time.Minute, DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration()}, // do not poll in tests. } - kvStore := ngfakes.NewFakeKVStore(t) + kvStore := NewFakeKVStore(t) provStore := provisioning.NewFakeProvisioningStore() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue @@ -311,7 +310,7 @@ func TestMultiOrgAlertmanager_ActivateHistoricalConfiguration(t *testing.T) { DataPath: tmpDir, UnifiedAlerting: setting.UnifiedAlertingSettings{AlertmanagerConfigPollInterval: 3 * time.Minute, DefaultConfiguration: defaultConfig}, // do not poll in tests. } - kvStore := ngfakes.NewFakeKVStore(t) + kvStore := NewFakeKVStore(t) provStore := provisioning.NewFakeProvisioningStore() secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) decryptFn := secretsService.GetDecryptedValue diff --git a/pkg/services/ngalert/notifier/testing.go b/pkg/services/ngalert/notifier/testing.go index ba57d944574..cab6386561d 100644 --- a/pkg/services/ngalert/notifier/testing.go +++ b/pkg/services/ngalert/notifier/testing.go @@ -5,9 +5,12 @@ import ( "crypto/md5" "errors" "fmt" + "strings" + "sync" "testing" "time" + "github.com/grafana/grafana/pkg/infra/kvstore" "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/store" @@ -192,6 +195,98 @@ func (f *FakeOrgStore) GetOrgs(_ context.Context) ([]int64, error) { return f.orgs, nil } +type FakeKVStore struct { + mtx sync.Mutex + store map[int64]map[string]map[string]string +} + +func NewFakeKVStore(t *testing.T) *FakeKVStore { + t.Helper() + + return &FakeKVStore{ + store: map[int64]map[string]map[string]string{}, + } +} + +func (fkv *FakeKVStore) Get(_ context.Context, orgId int64, namespace string, key string) (string, bool, error) { + fkv.mtx.Lock() + defer fkv.mtx.Unlock() + org, ok := fkv.store[orgId] + if !ok { + return "", false, nil + } + k, ok := org[namespace] + if !ok { + return "", false, nil + } + + v, ok := k[key] + if !ok { + return "", false, nil + } + + return v, true, nil +} +func (fkv *FakeKVStore) Set(_ context.Context, orgId int64, namespace string, key string, value string) error { + fkv.mtx.Lock() + defer fkv.mtx.Unlock() + org, ok := fkv.store[orgId] + if !ok { + fkv.store[orgId] = map[string]map[string]string{} + } + _, ok = org[namespace] + if !ok { + fkv.store[orgId][namespace] = map[string]string{} + } + + fkv.store[orgId][namespace][key] = value + + return nil +} +func (fkv *FakeKVStore) Del(_ context.Context, orgId int64, namespace string, key string) error { + fkv.mtx.Lock() + defer fkv.mtx.Unlock() + org, ok := fkv.store[orgId] + if !ok { + return nil + } + _, ok = org[namespace] + if !ok { + return nil + } + + delete(fkv.store[orgId][namespace], key) + + return nil +} + +func (fkv *FakeKVStore) Keys(ctx context.Context, orgID int64, namespace string, keyPrefix string) ([]kvstore.Key, error) { + fkv.mtx.Lock() + defer fkv.mtx.Unlock() + var keys []kvstore.Key + for orgIDFromStore, namespaceMap := range fkv.store { + if orgID != kvstore.AllOrganizations && orgID != orgIDFromStore { + continue + } + if keyMap, exists := namespaceMap[namespace]; exists { + for k := range keyMap { + if strings.HasPrefix(k, keyPrefix) { + keys = append(keys, kvstore.Key{ + OrgId: orgIDFromStore, + Namespace: namespace, + Key: keyPrefix, + }) + } + } + } + } + return keys, nil +} + +func (fkv *FakeKVStore) GetAll(ctx context.Context, orgId int64, namespace string) (map[int64]map[string]string, error) { + return nil, nil +} + type fakeState struct { data string } diff --git a/pkg/services/ngalert/provisioning/contactpoints.go b/pkg/services/ngalert/provisioning/contactpoints.go index 7695c6df58d..d20415fc634 100644 --- a/pkg/services/ngalert/provisioning/contactpoints.go +++ b/pkg/services/ngalert/provisioning/contactpoints.go @@ -242,7 +242,7 @@ func (ecp *ContactPointService) UpdateContactPoint(ctx context.Context, orgID in if err != nil { return err } - secretKeys, err := channels_config.GetSecretKeysForContactPointType(contactPoint.Type) + secretKeys, err := GetSecretKeysForContactPointType(contactPoint.Type) if err != nil { return fmt.Errorf("%w: %s", ErrValidation, err.Error()) } @@ -531,10 +531,27 @@ func ValidateContactPoint(ctx context.Context, e apimodels.EmbeddedContactPoint, return nil } +// GetSecretKeysForContactPointType returns settings keys of contact point of the given type that are expected to be secrets. Returns error is contact point type is not known. +func GetSecretKeysForContactPointType(contactPointType string) ([]string, error) { + notifiers := channels_config.GetAvailableNotifiers() + for _, n := range notifiers { + if n.Type == contactPointType { + var secureFields []string + for _, field := range n.Options { + if field.Secure { + secureFields = append(secureFields, field.PropertyName) + } + } + return secureFields, nil + } + } + return nil, fmt.Errorf("no secrets configured for type '%s'", contactPointType) +} + // RemoveSecretsForContactPoint removes all secrets from the contact point's settings and returns them as a map. Returns error if contact point type is not known. func RemoveSecretsForContactPoint(e *apimodels.EmbeddedContactPoint) (map[string]string, error) { s := map[string]string{} - secretKeys, err := channels_config.GetSecretKeysForContactPointType(e.Type) + secretKeys, err := GetSecretKeysForContactPointType(e.Type) if err != nil { return nil, err } diff --git a/pkg/services/ngalert/sender/router_test.go b/pkg/services/ngalert/sender/router_test.go index 77bc7b168c4..1414ebf5d26 100644 --- a/pkg/services/ngalert/sender/router_test.go +++ b/pkg/services/ngalert/sender/router_test.go @@ -25,7 +25,6 @@ import ( "github.com/grafana/grafana/pkg/services/ngalert/notifier" "github.com/grafana/grafana/pkg/services/ngalert/provisioning" "github.com/grafana/grafana/pkg/services/ngalert/store" - "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" fake_secrets "github.com/grafana/grafana/pkg/services/secrets/fakes" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/setting" @@ -406,7 +405,7 @@ func createMultiOrgAlertmanager(t *testing.T, orgs []int64) *notifier.MultiOrgAl } cfgStore := notifier.NewFakeConfigStore(t, make(map[int64]*models.AlertConfiguration)) - kvStore := fakes.NewFakeKVStore(t) + kvStore := notifier.NewFakeKVStore(t) registry := prometheus.NewPedanticRegistry() m := metrics.NewNGAlert(registry) secretsService := secretsManager.SetupTestService(t, fake_secrets.NewFakeSecretsStore()) diff --git a/pkg/services/ngalert/tests/fakes/kvstore.go b/pkg/services/ngalert/tests/fakes/kvstore.go deleted file mode 100644 index 7ef60184b83..00000000000 --- a/pkg/services/ngalert/tests/fakes/kvstore.go +++ /dev/null @@ -1,102 +0,0 @@ -package fakes - -import ( - "context" - "strings" - "sync" - "testing" - - "github.com/grafana/grafana/pkg/infra/kvstore" -) - -type FakeKVStore struct { - Mtx sync.Mutex - Store map[int64]map[string]map[string]string -} - -func NewFakeKVStore(t *testing.T) *FakeKVStore { - t.Helper() - - return &FakeKVStore{ - Store: map[int64]map[string]map[string]string{}, - } -} - -func (fkv *FakeKVStore) Get(_ context.Context, orgId int64, namespace string, key string) (string, bool, error) { - fkv.Mtx.Lock() - defer fkv.Mtx.Unlock() - org, ok := fkv.Store[orgId] - if !ok { - return "", false, nil - } - k, ok := org[namespace] - if !ok { - return "", false, nil - } - - v, ok := k[key] - if !ok { - return "", false, nil - } - - return v, true, nil -} -func (fkv *FakeKVStore) Set(_ context.Context, orgId int64, namespace string, key string, value string) error { - fkv.Mtx.Lock() - defer fkv.Mtx.Unlock() - org, ok := fkv.Store[orgId] - if !ok { - fkv.Store[orgId] = map[string]map[string]string{} - } - _, ok = org[namespace] - if !ok { - fkv.Store[orgId][namespace] = map[string]string{} - } - - fkv.Store[orgId][namespace][key] = value - - return nil -} -func (fkv *FakeKVStore) Del(_ context.Context, orgId int64, namespace string, key string) error { - fkv.Mtx.Lock() - defer fkv.Mtx.Unlock() - org, ok := fkv.Store[orgId] - if !ok { - return nil - } - _, ok = org[namespace] - if !ok { - return nil - } - - delete(fkv.Store[orgId][namespace], key) - - return nil -} - -func (fkv *FakeKVStore) Keys(ctx context.Context, orgID int64, namespace string, keyPrefix string) ([]kvstore.Key, error) { - fkv.Mtx.Lock() - defer fkv.Mtx.Unlock() - var keys []kvstore.Key - for orgIDFromStore, namespaceMap := range fkv.Store { - if orgID != kvstore.AllOrganizations && orgID != orgIDFromStore { - continue - } - if keyMap, exists := namespaceMap[namespace]; exists { - for k := range keyMap { - if strings.HasPrefix(k, keyPrefix) { - keys = append(keys, kvstore.Key{ - OrgId: orgIDFromStore, - Namespace: namespace, - Key: keyPrefix, - }) - } - } - } - } - return keys, nil -} - -func (fkv *FakeKVStore) GetAll(ctx context.Context, orgId int64, namespace string) (map[int64]map[string]string, error) { - return nil, nil -} diff --git a/pkg/services/sqlstore/migrations/external_alertmanagers.go b/pkg/services/sqlstore/migrations/external_alertmanagers.go index 78d91192a9d..ceea68ead7a 100644 --- a/pkg/services/sqlstore/migrations/external_alertmanagers.go +++ b/pkg/services/sqlstore/migrations/external_alertmanagers.go @@ -3,16 +3,14 @@ package migrations import ( "fmt" "net/url" - "os" "time" "xorm.io/xorm" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/sqlstore/migrations/ualert" "github.com/grafana/grafana/pkg/services/sqlstore/migrator" - "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" ) @@ -77,9 +75,9 @@ func (e externalAlertmanagerToDatasources) Exec(sess *xorm.Session, mg *migrator ds.BasicAuth = true ds.BasicAuthUser = u.User.Username() if password, ok := u.User.Password(); ok { - ds.SecureJsonData = getEncryptedJsonData(map[string]string{ + ds.SecureJsonData = ualert.GetEncryptedJsonData(map[string]string{ "basicAuthPassword": password, - }, log.New("securejsondata")) + }) } } @@ -126,22 +124,3 @@ func generateNewDatasourceUid(sess *xorm.Session, orgId int64) (string, error) { return "", datasources.ErrDataSourceFailedGenerateUniqueUid } - -// SecureJsonData is used to store encrypted data (for example in data_source table). Only values are separately -// encrypted. -type secureJsonData map[string][]byte - -// getEncryptedJsonData returns map where all keys are encrypted. -func getEncryptedJsonData(sjd map[string]string, log log.Logger) secureJsonData { - encrypted := make(secureJsonData) - for key, data := range sjd { - encryptedData, err := util.Encrypt([]byte(data), setting.SecretKey) - if err != nil { - log.Error(err.Error()) - os.Exit(1) - } - - encrypted[key] = encryptedData - } - return encrypted -} diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index 60b2def385d..56df4323cc9 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -54,9 +54,10 @@ func (*OSSMigrations) AddMigration(mg *Migrator) { addCacheMigration(mg) addShortURLMigrations(mg) ualert.AddTablesMigrations(mg) + ualert.AddDashAlertMigration(mg) addLibraryElementsMigrations(mg) - ualert.FixEarlyMigration(mg) + ualert.RerunDashAlertMigration(mg) addSecretsMigration(mg) addKVStoreMigrations(mg) ualert.AddDashboardUIDPanelIDMigration(mg) @@ -75,6 +76,7 @@ func (*OSSMigrations) AddMigration(mg *Migrator) { addEntityEventsTableMigration(mg) addPublicDashboardMigration(mg) + ualert.CreateDefaultFoldersForAlertingMigration(mg) addDbFileStorageMigration(mg) accesscontrol.AddManagedPermissionsMigration(mg, accesscontrol.ManagedPermissionsMigrationID) @@ -99,9 +101,6 @@ func (*OSSMigrations) AddMigration(mg *Migrator) { anonservice.AddMigration(mg) signingkeys.AddMigration(mg) - - ualert.MigrationServiceMigration(mg) - ualert.CreatedFoldersMigration(mg) } func addStarMigrations(mg *Migrator) { diff --git a/pkg/services/ngalert/migration/alert_rule.go b/pkg/services/sqlstore/migrations/ualert/alert_rule.go similarity index 56% rename from pkg/services/ngalert/migration/alert_rule.go rename to pkg/services/sqlstore/migrations/ualert/alert_rule.go index 2ef2ec03fe9..2039c7e1922 100644 --- a/pkg/services/ngalert/migration/alert_rule.go +++ b/pkg/services/sqlstore/migrations/ualert/alert_rule.go @@ -1,7 +1,6 @@ -package migration +package ualert import ( - "context" "encoding/json" "fmt" "time" @@ -9,11 +8,8 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" - migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/services/ngalert/store" "github.com/grafana/grafana/pkg/tsdb/graphite" - "github.com/grafana/grafana/pkg/util" ) const ( @@ -23,7 +19,78 @@ const ( ContactLabel = "__contacts__" ) -func addMigrationInfo(da *migrationStore.DashAlert, dashboardUID string) (map[string]string, map[string]string) { +type alertRule struct { + ID int64 `xorm:"pk autoincr 'id'"` + OrgID int64 `xorm:"org_id"` + Title string + Condition string + Data []alertQuery + IntervalSeconds int64 + Version int64 + UID string `xorm:"uid"` + NamespaceUID string `xorm:"namespace_uid"` + RuleGroup string + RuleGroupIndex int `xorm:"rule_group_idx"` + NoDataState string + ExecErrState string + For duration + Updated time.Time + Annotations map[string]string + Labels map[string]string + IsPaused bool +} + +type alertRuleVersion struct { + RuleOrgID int64 `xorm:"rule_org_id"` + RuleUID string `xorm:"rule_uid"` + RuleNamespaceUID string `xorm:"rule_namespace_uid"` + RuleGroup string + RuleGroupIndex int `xorm:"rule_group_idx"` + ParentVersion int64 + RestoredFrom int64 + Version int64 + + Created time.Time + Title string + Condition string + Data []alertQuery + IntervalSeconds int64 + NoDataState string + ExecErrState string + // ideally this field should have been apimodels.ApiDuration + // but this is currently not possible because of circular dependencies + For duration + Annotations map[string]string + Labels map[string]string + IsPaused bool +} + +func (a *alertRule) makeVersion() *alertRuleVersion { + return &alertRuleVersion{ + RuleOrgID: a.OrgID, + RuleUID: a.UID, + RuleNamespaceUID: a.NamespaceUID, + RuleGroup: a.RuleGroup, + RuleGroupIndex: a.RuleGroupIndex, + ParentVersion: 0, + RestoredFrom: 0, + Version: 1, + + Created: time.Now().UTC(), + Title: a.Title, + Condition: a.Condition, + Data: a.Data, + IntervalSeconds: a.IntervalSeconds, + NoDataState: a.NoDataState, + ExecErrState: a.ExecErrState, + For: a.For, + Annotations: a.Annotations, + Labels: map[string]string{}, + IsPaused: a.IsPaused, + } +} + +func addMigrationInfo(da *dashAlert) (map[string]string, map[string]string) { tagsMap := simplejson.NewFromAny(da.ParsedSettings.AlertRuleTags).MustMap() lbls := make(map[string]string, len(tagsMap)) @@ -32,22 +99,15 @@ func addMigrationInfo(da *migrationStore.DashAlert, dashboardUID string) (map[st } annotations := make(map[string]string, 3) - annotations[ngmodels.DashboardUIDAnnotation] = dashboardUID - annotations[ngmodels.PanelIDAnnotation] = fmt.Sprintf("%v", da.PanelID) - annotations["__alertId__"] = fmt.Sprintf("%v", da.ID) + annotations[ngmodels.DashboardUIDAnnotation] = da.DashboardUID + annotations[ngmodels.PanelIDAnnotation] = fmt.Sprintf("%v", da.PanelId) + annotations["__alertId__"] = fmt.Sprintf("%v", da.Id) return lbls, annotations } -// MigrateAlert migrates a single dashboard alert from legacy alerting to unified alerting. -func (om *OrgMigration) migrateAlert(ctx context.Context, l log.Logger, da *migrationStore.DashAlert, dashboardUID string, folderUID string) (*ngmodels.AlertRule, error) { - l.Debug("Migrating alert rule to Unified Alerting") - cond, err := transConditions(ctx, da, om.migrationStore) - if err != nil { - return nil, fmt.Errorf("transform conditions: %w", err) - } - - lbls, annotations := addMigrationInfo(da, dashboardUID) +func (m *migration) makeAlertRule(l log.Logger, cond condition, da dashAlert, folderUID string) (*alertRule, error) { + lbls, annotations := addMigrationInfo(&da) message := MigrateTmpl(l.New("field", "message"), da.Message) annotations["message"] = message @@ -57,38 +117,33 @@ func (om *OrgMigration) migrateAlert(ctx context.Context, l log.Logger, da *migr return nil, fmt.Errorf("failed to migrate alert rule queries: %w", err) } + uid, err := m.seenUIDs.generateUid() + if err != nil { + return nil, fmt.Errorf("failed to migrate alert rule: %w", err) + } + + name := normalizeRuleName(da.Name, uid) + isPaused := false if da.State == "paused" { isPaused = true } - // Here we ensure that the alert rule title is unique within the folder. - dedupSet := om.AlertTitleDeduplicator(folderUID) - name := truncateRuleName(da.Name) - if dedupSet.contains(name) { - dedupedName := dedupSet.deduplicate(name) - l.Debug("Duplicate alert rule name detected, renaming", "old_name", name, "new_name", dedupedName) - name = dedupedName - } - dedupSet.add(name) - - ar := &ngmodels.AlertRule{ - OrgID: da.OrgID, - Title: name, - UID: util.GenerateShortUID(), + ar := &alertRule{ + OrgID: da.OrgId, + Title: name, // TODO: Make sure all names are unique, make new name on constraint insert error. + UID: uid, Condition: cond.Condition, Data: data, IntervalSeconds: ruleAdjustInterval(da.Frequency), Version: 1, NamespaceUID: folderUID, // Folder already created, comes from env var. - DashboardUID: &dashboardUID, - PanelID: &da.PanelID, RuleGroup: name, - For: da.For, + For: duration(da.For), Updated: time.Now().UTC(), Annotations: annotations, Labels: lbls, - RuleGroupIndex: 1, // Every rule is in its own group. + RuleGroupIndex: 1, IsPaused: isPaused, NoDataState: transNoData(l, da.ParsedSettings.NoDataState), ExecErrState: transExecErr(l, da.ParsedSettings.ExecutionErrorState), @@ -98,24 +153,20 @@ func (om *OrgMigration) migrateAlert(ctx context.Context, l log.Logger, da *migr n, v := getLabelForSilenceMatching(ar.UID) ar.Labels[n] = v - if da.ParsedSettings.ExecutionErrorState == string(legacymodels.ExecutionErrorKeepState) { - if err := om.addErrorSilence(ar); err != nil { - om.log.Error("Alert migration error: failed to create silence for Error", "rule_name", ar.Title, "err", err) - } + if err := m.addErrorSilence(da, ar); err != nil { + m.mg.Logger.Error("Alert migration error: failed to create silence for Error", "rule_name", ar.Title, "err", err) } - if da.ParsedSettings.NoDataState == string(legacymodels.NoDataKeepState) { - if err := om.addNoDataSilence(ar); err != nil { - om.log.Error("Alert migration error: failed to create silence for NoData", "rule_name", ar.Title, "err", err) - } + if err := m.addNoDataSilence(da, ar); err != nil { + m.mg.Logger.Error("Alert migration error: failed to create silence for NoData", "rule_name", ar.Title, "err", err) } return ar, nil } // migrateAlertRuleQueries attempts to fix alert rule queries so they can work in unified alerting. Queries of some data sources are not compatible with unified alerting. -func migrateAlertRuleQueries(l log.Logger, data []ngmodels.AlertQuery) ([]ngmodels.AlertQuery, error) { - result := make([]ngmodels.AlertQuery, 0, len(data)) +func migrateAlertRuleQueries(l log.Logger, data []alertQuery) ([]alertQuery, error) { + result := make([]alertQuery, 0, len(data)) for _, d := range data { // queries that are expression are not relevant, skip them. if d.DatasourceUID == expressionDatasourceUID { @@ -223,6 +274,55 @@ func isPrometheusQuery(queryData map[string]json.RawMessage) (bool, error) { return datasource.Type == "prometheus", nil } +type alertQuery struct { + // RefID is the unique identifier of the query, set by the frontend call. + RefID string `json:"refId"` + + // QueryType is an optional identifier for the type of query. + // It can be used to distinguish different types of queries. + QueryType string `json:"queryType"` + + // RelativeTimeRange is the relative Start and End of the query as sent by the frontend. + RelativeTimeRange relativeTimeRange `json:"relativeTimeRange"` + + DatasourceUID string `json:"datasourceUid"` + + // JSON is the raw JSON query and includes the above properties as well as custom properties. + Model json.RawMessage `json:"model"` +} + +// RelativeTimeRange is the per query start and end time +// for requests. +type relativeTimeRange struct { + From duration `json:"from"` + To duration `json:"to"` +} + +// duration is a type used for marshalling durations. +type duration time.Duration + +func (d duration) String() string { + return time.Duration(d).String() +} + +func (d duration) MarshalJSON() ([]byte, error) { + return json.Marshal(time.Duration(d).Seconds()) +} + +func (d *duration) UnmarshalJSON(b []byte) error { + var v any + if err := json.Unmarshal(b, &v); err != nil { + return err + } + switch value := v.(type) { + case float64: + *d = duration(time.Duration(value) * time.Second) + return nil + default: + return fmt.Errorf("invalid duration %v", v) + } +} + func ruleAdjustInterval(freq int64) int64 { // 10 corresponds to the SchedulerCfg, but TODO not worrying about fetching for now. var baseFreq int64 = 10 @@ -232,47 +332,50 @@ func ruleAdjustInterval(freq int64) int64 { return freq - (freq % baseFreq) } -func transNoData(l log.Logger, s string) ngmodels.NoDataState { +func transNoData(l log.Logger, s string) string { switch legacymodels.NoDataOption(s) { case legacymodels.NoDataSetOK: - return ngmodels.OK // values from ngalert/models/rule + return string(ngmodels.OK) // values from ngalert/models/rule case "", legacymodels.NoDataSetNoData: - return ngmodels.NoData + return string(ngmodels.NoData) case legacymodels.NoDataSetAlerting: - return ngmodels.Alerting + return string(ngmodels.Alerting) case legacymodels.NoDataKeepState: - return ngmodels.NoData // "keep last state" translates to no data because we now emit a special alert when the state is "noData". The result is that the evaluation will not return firing and instead we'll raise the special alert. + return string(ngmodels.NoData) // "keep last state" translates to no data because we now emit a special alert when the state is "noData". The result is that the evaluation will not return firing and instead we'll raise the special alert. default: l.Warn("Unable to translate execution of NoData state. Using default execution", "old", s, "new", ngmodels.NoData) - return ngmodels.NoData + return string(ngmodels.NoData) } } -func transExecErr(l log.Logger, s string) ngmodels.ExecutionErrorState { +func transExecErr(l log.Logger, s string) string { switch legacymodels.ExecutionErrorOption(s) { case "", legacymodels.ExecutionErrorSetAlerting: - return ngmodels.AlertingErrState + return string(ngmodels.AlertingErrState) case legacymodels.ExecutionErrorKeepState: // Keep last state is translated to error as we now emit a // DatasourceError alert when the state is error - return ngmodels.ErrorErrState + return string(ngmodels.ErrorErrState) case legacymodels.ExecutionErrorSetOk: - return ngmodels.OkErrState + return string(ngmodels.OkErrState) default: l.Warn("Unable to translate execution of Error state. Using default execution", "old", s, "new", ngmodels.ErrorErrState) - return ngmodels.ErrorErrState + return string(ngmodels.ErrorErrState) } } -// truncateRuleName truncates the rule name to the maximum allowed length. -func truncateRuleName(daName string) string { - if len(daName) > store.AlertDefinitionMaxTitleLength { - return daName[:store.AlertDefinitionMaxTitleLength] +func normalizeRuleName(daName string, uid string) string { + // If we have to truncate, we're losing data and so there is higher risk of uniqueness conflicts. + // Append the UID to the suffix to forcibly break any collisions. + if len(daName) > DefaultFieldMaxLength { + trunc := DefaultFieldMaxLength - 1 - len(uid) + daName = daName[:trunc] + "_" + uid } + return daName } -func extractChannelIDs(d *migrationStore.DashAlert) (channelUids []migrationStore.UidOrID) { +func extractChannelIDs(d dashAlert) (channelUids []uidOrID) { // Extracting channel UID/ID. for _, ui := range d.ParsedSettings.Notifications { if ui.UID != "" { diff --git a/pkg/services/ngalert/migration/alert_rule_test.go b/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go similarity index 54% rename from pkg/services/ngalert/migration/alert_rule_test.go rename to pkg/services/sqlstore/migrations/ualert/alert_rule_test.go index 8ffa233d79a..c8f2aff7f4a 100644 --- a/pkg/services/ngalert/migration/alert_rule_test.go +++ b/pkg/services/sqlstore/migrations/ualert/alert_rule_test.go @@ -1,7 +1,7 @@ -package migration +package ualert import ( - "context" + "encoding/json" "strings" "testing" @@ -9,12 +9,8 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log/logtest" - legacymodels "github.com/grafana/grafana/pkg/services/alerting/models" - migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/services/ngalert/store" ) func TestMigrateAlertRuleQueries(t *testing.T) { @@ -77,7 +73,7 @@ func TestMigrateAlertRuleQueries(t *testing.T) { t.Run(tt.name, func(t *testing.T) { model, err := tt.input.Encode() require.NoError(t, err) - queries, err := migrateAlertRuleQueries(&logtest.Fake{}, []models.AlertQuery{{Model: model}}) + queries, err := migrateAlertRuleQueries(&logtest.Fake{}, []alertQuery{{Model: model}}) if tt.err != nil { require.Error(t, err) require.EqualError(t, err, tt.err.Error()) @@ -95,30 +91,30 @@ func TestMigrateAlertRuleQueries(t *testing.T) { func TestAddMigrationInfo(t *testing.T) { tt := []struct { name string - alert *migrationStore.DashAlert - dashboard string + tagsJSON string expectedLabels map[string]string expectedAnnotations map[string]string }{ { name: "when alert rule tags are a JSON array, they're ignored.", - alert: &migrationStore.DashAlert{Alert: &legacymodels.Alert{ID: 43, PanelID: 42}, ParsedSettings: &migrationStore.DashAlertSettings{AlertRuleTags: []string{"one", "two", "three", "four"}}}, - dashboard: "dashboard", + tagsJSON: `{ "alertRuleTags": ["one", "two", "three", "four"] }`, expectedLabels: map[string]string{}, - expectedAnnotations: map[string]string{"__alertId__": "43", "__dashboardUid__": "dashboard", "__panelId__": "42"}, + expectedAnnotations: map[string]string{"__alertId__": "0", "__dashboardUid__": "", "__panelId__": "0"}, }, { name: "when alert rule tags are a JSON object", - alert: &migrationStore.DashAlert{Alert: &legacymodels.Alert{ID: 43, PanelID: 42}, ParsedSettings: &migrationStore.DashAlertSettings{AlertRuleTags: map[string]any{"key": "value", "key2": "value2"}}}, - dashboard: "dashboard", + tagsJSON: `{ "alertRuleTags": { "key": "value", "key2": "value2" } }`, expectedLabels: map[string]string{"key": "value", "key2": "value2"}, - expectedAnnotations: map[string]string{"__alertId__": "43", "__dashboardUid__": "dashboard", "__panelId__": "42"}, + expectedAnnotations: map[string]string{"__alertId__": "0", "__dashboardUid__": "", "__panelId__": "0"}, }, } for _, tc := range tt { t.Run(tc.name, func(t *testing.T) { - labels, annotations := addMigrationInfo(tc.alert, tc.dashboard) + var settings dashAlertSettings + require.NoError(t, json.Unmarshal([]byte(tc.tagsJSON), &settings)) + + labels, annotations := addMigrationInfo(&dashAlert{ParsedSettings: &settings}) require.Equal(t, tc.expectedLabels, labels) require.Equal(t, tc.expectedAnnotations, annotations) }) @@ -126,14 +122,13 @@ func TestAddMigrationInfo(t *testing.T) { } func TestMakeAlertRule(t *testing.T) { - sqlStore := db.InitTestDB(t) t.Run("when mapping rule names", func(t *testing.T) { t.Run("leaves basic names untouched", func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) + m := newTestMigration(t) da := createTestDashAlert() + cnd := createTestDashAlertCondition() - ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") + ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") require.NoError(t, err) require.Equal(t, da.Name, ar.Title) @@ -141,93 +136,73 @@ func TestMakeAlertRule(t *testing.T) { }) t.Run("truncates very long names to max length", func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) + m := newTestMigration(t) da := createTestDashAlert() - da.Name = strings.Repeat("a", store.AlertDefinitionMaxTitleLength+1) + da.Name = strings.Repeat("a", DefaultFieldMaxLength+1) + cnd := createTestDashAlertCondition() - ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") + ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") require.NoError(t, err) - require.Len(t, ar.Title, store.AlertDefinitionMaxTitleLength) - }) - - t.Run("deduplicate names in same org and folder", func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) - da := createTestDashAlert() - da.Name = strings.Repeat("a", store.AlertDefinitionMaxTitleLength+1) - - ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") - - require.NoError(t, err) - require.Len(t, ar.Title, store.AlertDefinitionMaxTitleLength) - - da = createTestDashAlert() - da.Name = strings.Repeat("a", store.AlertDefinitionMaxTitleLength+1) - - ar, err = m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") - - require.NoError(t, err) - require.Len(t, ar.Title, store.AlertDefinitionMaxTitleLength) + require.Len(t, ar.Title, DefaultFieldMaxLength) parts := strings.SplitN(ar.Title, "_", 2) require.Len(t, parts, 2) require.Greater(t, len(parts[1]), 8, "unique identifier should be longer than 9 characters") - require.Equal(t, store.AlertDefinitionMaxTitleLength-1, len(parts[0])+len(parts[1]), "truncated name + underscore + unique identifier should together be DefaultFieldMaxLength") + require.Equal(t, DefaultFieldMaxLength-1, len(parts[0])+len(parts[1]), "truncated name + underscore + unique identifier should together be DefaultFieldMaxLength") require.Equal(t, ar.Title, ar.RuleGroup) }) }) t.Run("alert is not paused", func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) + m := newTestMigration(t) da := createTestDashAlert() + cnd := createTestDashAlertCondition() - ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") + ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") require.NoError(t, err) require.False(t, ar.IsPaused) }) t.Run("paused dash alert is paused", func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) + m := newTestMigration(t) da := createTestDashAlert() da.State = "paused" + cnd := createTestDashAlertCondition() - ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") + ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") require.NoError(t, err) require.True(t, ar.IsPaused) }) t.Run("use default if execution of NoData is not known", func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) + m := newTestMigration(t) da := createTestDashAlert() da.ParsedSettings.NoDataState = uuid.NewString() + cnd := createTestDashAlertCondition() - ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") + ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") require.Nil(t, err) - require.Equal(t, models.NoData, ar.NoDataState) + require.Equal(t, string(models.NoData), ar.NoDataState) }) t.Run("use default if execution of Error is not known", func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) + m := newTestMigration(t) da := createTestDashAlert() da.ParsedSettings.ExecutionErrorState = uuid.NewString() + cnd := createTestDashAlertCondition() - ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") + ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") require.Nil(t, err) - require.Equal(t, models.ErrorErrState, ar.ExecErrState) + require.Equal(t, string(models.ErrorErrState), ar.ExecErrState) }) t.Run("migrate message template", func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - m := service.newOrgMigration(1) + m := newTestMigration(t) da := createTestDashAlert() da.Message = "Instance ${instance} is down" + cnd := createTestDashAlertCondition() - ar, err := m.migrateAlert(context.Background(), &logtest.Fake{}, &da, "dashboard", "folder") + ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder") require.Nil(t, err) expected := "{{- $mergedLabels := mergeLabelValues $values -}}\n" + @@ -236,12 +211,16 @@ func TestMakeAlertRule(t *testing.T) { }) } -func createTestDashAlert() migrationStore.DashAlert { - return migrationStore.DashAlert{ - Alert: &legacymodels.Alert{ - ID: 1, - Name: "test", - }, - ParsedSettings: &migrationStore.DashAlertSettings{}, +func createTestDashAlert() dashAlert { + return dashAlert{ + Id: 1, + Name: "test", + ParsedSettings: &dashAlertSettings{}, + } +} + +func createTestDashAlertCondition() condition { + return condition{ + Condition: "A", } } diff --git a/pkg/services/sqlstore/migrations/ualert/channel.go b/pkg/services/sqlstore/migrations/ualert/channel.go new file mode 100644 index 00000000000..1268ec92e59 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/channel.go @@ -0,0 +1,512 @@ +package ualert + +import ( + "crypto/md5" + "encoding/base64" + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + "time" + + "github.com/prometheus/alertmanager/pkg/labels" + "github.com/prometheus/common/model" + + "github.com/grafana/grafana/pkg/components/simplejson" + ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" +) + +const ( + // DisabledRepeatInterval is a large duration that will be used as a pseudo-disable in case a legacy channel doesn't have SendReminders enabled. + DisabledRepeatInterval = model.Duration(time.Duration(8736) * time.Hour) // 1y +) + +type notificationChannel struct { + ID int64 `xorm:"id"` + OrgID int64 `xorm:"org_id"` + Uid string `xorm:"uid"` + Name string `xorm:"name"` + Type string `xorm:"type"` + DisableResolveMessage bool `xorm:"disable_resolve_message"` + IsDefault bool `xorm:"is_default"` + Settings *simplejson.Json `xorm:"settings"` + SecureSettings SecureJsonData `xorm:"secure_settings"` + SendReminder bool `xorm:"send_reminder"` + Frequency model.Duration `xorm:"frequency"` +} + +// channelsPerOrg maps notification channels per organisation +type channelsPerOrg map[int64][]*notificationChannel + +// channelMap maps notification channels per organisation +type defaultChannelsPerOrg map[int64][]*notificationChannel + +// uidOrID for both uid and ID, primarily used for mapping legacy channel to migrated receiver. +type uidOrID any + +// channelReceiver is a convenience struct that contains a notificationChannel and its corresponding migrated PostableApiReceiver. +type channelReceiver struct { + channel *notificationChannel + receiver *PostableApiReceiver +} + +// setupAlertmanagerConfigs creates Alertmanager configs with migrated receivers and routes. +func (m *migration) setupAlertmanagerConfigs(rulesPerOrg map[int64]map[*alertRule][]uidOrID) (amConfigsPerOrg, error) { + // allChannels: channelUID -> channelConfig + allChannelsPerOrg, defaultChannelsPerOrg, err := m.getNotificationChannelMap() + if err != nil { + return nil, fmt.Errorf("failed to load notification channels: %w", err) + } + + amConfigPerOrg := make(amConfigsPerOrg, len(allChannelsPerOrg)) + for orgID, channels := range allChannelsPerOrg { + amConfig := &PostableUserConfig{ + AlertmanagerConfig: PostableApiAlertingConfig{ + Receivers: make([]*PostableApiReceiver, 0), + }, + } + amConfigPerOrg[orgID] = amConfig + + // Create all newly migrated receivers from legacy notification channels. + receiversMap, receivers, err := m.createReceivers(channels) + if err != nil { + return nil, fmt.Errorf("failed to create receiver in orgId %d: %w", orgID, err) + } + + // No need to create an Alertmanager configuration if there are no receivers left that aren't obsolete. + if len(receivers) == 0 { + m.mg.Logger.Warn("No available receivers", "orgId", orgID) + continue + } + + for _, cr := range receivers { + amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, cr.receiver) + } + + defaultReceivers := make(map[string]struct{}) + defaultChannels, ok := defaultChannelsPerOrg[orgID] + if ok { + // If the organization has default channels build a map of default receivers, used to create alert-specific routes later. + for _, c := range defaultChannels { + defaultReceivers[c.Name] = struct{}{} + } + } + defaultReceiver, defaultRoute, err := m.createDefaultRouteAndReceiver(defaultChannels) + if err != nil { + return nil, fmt.Errorf("failed to create default route & receiver in orgId %d: %w", orgID, err) + } + amConfig.AlertmanagerConfig.Route = defaultRoute + if defaultReceiver != nil { + amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, defaultReceiver) + } + + for _, cr := range receivers { + route, err := createRoute(cr) + if err != nil { + return nil, fmt.Errorf("failed to create route for receiver %s in orgId %d: %w", cr.receiver.Name, orgID, err) + } + + amConfigPerOrg[orgID].AlertmanagerConfig.Route.Routes = append(amConfigPerOrg[orgID].AlertmanagerConfig.Route.Routes, route) + } + + for ar, channelUids := range rulesPerOrg[orgID] { + filteredReceiverNames := m.filterReceiversForAlert(ar.Title, channelUids, receiversMap, defaultReceivers) + + if len(filteredReceiverNames) != 0 { + // Only create a contact label if there are specific receivers, otherwise it defaults to the root-level route. + ar.Labels[ContactLabel] = contactListToString(filteredReceiverNames) + } + } + + // Validate the alertmanager configuration produced, this gives a chance to catch bad configuration at migration time. + // Validation between legacy and unified alerting can be different (e.g. due to bug fixes) so this would fail the migration in that case. + if err := m.validateAlertmanagerConfig(amConfig); err != nil { + return nil, fmt.Errorf("failed to validate AlertmanagerConfig in orgId %d: %w", orgID, err) + } + } + + return amConfigPerOrg, nil +} + +// contactListToString creates a sorted string representation of a given map (set) of receiver names. Each name will be comma-separated and double-quoted. Names should not contain double quotes. +func contactListToString(m map[string]any) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, quote(k)) + } + sort.Strings(keys) + + return strings.Join(keys, ",") +} + +// quote will surround the given string in double quotes. +func quote(s string) string { + return `"` + s + `"` +} + +// getNotificationChannelMap returns a map of all channelUIDs to channel config as well as a separate map for just those channels that are default. +// For any given Organization, all channels in defaultChannelsPerOrg should also exist in channelsPerOrg. +func (m *migration) getNotificationChannelMap() (channelsPerOrg, defaultChannelsPerOrg, error) { + q := ` + SELECT id, + org_id, + uid, + name, + type, + disable_resolve_message, + is_default, + settings, + secure_settings, + send_reminder, + frequency + FROM + alert_notification + ` + allChannels := []notificationChannel{} + err := m.sess.SQL(q).Find(&allChannels) + if err != nil { + return nil, nil, err + } + + if len(allChannels) == 0 { + return nil, nil, nil + } + + allChannelsMap := make(channelsPerOrg) + defaultChannelsMap := make(defaultChannelsPerOrg) + for i, c := range allChannels { + if c.Type == "hipchat" || c.Type == "sensu" { + m.mg.Logger.Error("Alert migration error: discontinued notification channel found", "type", c.Type, "name", c.Name, "uid", c.Uid) + continue + } + + allChannelsMap[c.OrgID] = append(allChannelsMap[c.OrgID], &allChannels[i]) + + if c.IsDefault { + defaultChannelsMap[c.OrgID] = append(defaultChannelsMap[c.OrgID], &allChannels[i]) + } + } + + return allChannelsMap, defaultChannelsMap, nil +} + +// Create a notifier (PostableGrafanaReceiver) from a legacy notification channel +func (m *migration) createNotifier(c *notificationChannel) (*PostableGrafanaReceiver, error) { + uid, err := m.determineChannelUid(c) + if err != nil { + return nil, err + } + + settings, secureSettings, err := migrateSettingsToSecureSettings(c.Type, c.Settings, c.SecureSettings) + if err != nil { + return nil, err + } + + return &PostableGrafanaReceiver{ + UID: uid, + Name: c.Name, + Type: c.Type, + DisableResolveMessage: c.DisableResolveMessage, + Settings: settings, + SecureSettings: secureSettings, + }, nil +} + +// Create one receiver for every unique notification channel. +func (m *migration) createReceivers(allChannels []*notificationChannel) (map[uidOrID]*PostableApiReceiver, []channelReceiver, error) { + receivers := make([]channelReceiver, 0, len(allChannels)) + receiversMap := make(map[uidOrID]*PostableApiReceiver) + + set := make(map[string]struct{}) // Used to deduplicate sanitized names. + for _, c := range allChannels { + notifier, err := m.createNotifier(c) + if err != nil { + return nil, nil, err + } + + // We remove double quotes because this character will be used as the separator in the ContactLabel. To prevent partial matches in the Route Matcher we choose to sanitize them early on instead of complicating the Matcher regex. + sanitizedName := strings.ReplaceAll(c.Name, `"`, `_`) + // There can be name collisions after we sanitize. We check for this and attempt to make the name unique again using a short hash of the original name. + if _, ok := set[sanitizedName]; ok { + sanitizedName = sanitizedName + fmt.Sprintf("_%.3x", md5.Sum([]byte(c.Name))) + m.mg.Logger.Warn("Alert contains duplicate contact name after sanitization, appending unique suffix", "type", c.Type, "name", c.Name, "new_name", sanitizedName, "uid", c.Uid) + } + notifier.Name = sanitizedName + + set[sanitizedName] = struct{}{} + + cr := channelReceiver{ + channel: c, + receiver: &PostableApiReceiver{ + Name: sanitizedName, // Channel name is unique within an Org. + GrafanaManagedReceivers: []*PostableGrafanaReceiver{notifier}, + }, + } + + receivers = append(receivers, cr) + + // Store receivers for creating routes from alert rules later. + if c.Uid != "" { + receiversMap[c.Uid] = cr.receiver + } + if c.ID != 0 { + // In certain circumstances, the alert rule uses ID instead of uid. So, we add this to be able to lookup by ID in case. + receiversMap[c.ID] = cr.receiver + } + } + + return receiversMap, receivers, nil +} + +// Create the root-level route with the default receiver. If no new receiver is created specifically for the root-level route, the returned receiver will be nil. +func (m *migration) createDefaultRouteAndReceiver(defaultChannels []*notificationChannel) (*PostableApiReceiver, *Route, error) { + defaultReceiverName := "autogen-contact-point-default" + defaultRoute := &Route{ + Receiver: defaultReceiverName, + Routes: make([]*Route, 0), + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, // To keep parity with pre-migration notifications. + RepeatInterval: nil, + } + newDefaultReceiver := &PostableApiReceiver{ + Name: defaultReceiverName, + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + } + + // Return early if there are no default channels + if len(defaultChannels) == 0 { + return newDefaultReceiver, defaultRoute, nil + } + + repeatInterval := DisabledRepeatInterval // If no channels have SendReminders enabled, we will use this large value as a pseudo-disable. + if len(defaultChannels) > 1 { + // If there are more than one default channels we create a separate contact group that is used only in the root policy. This is to simplify the migrated notification policy structure. + // If we ever allow more than one receiver per route this won't be necessary. + for _, c := range defaultChannels { + // Need to create a new notifier to prevent uid conflict. + defaultNotifier, err := m.createNotifier(c) + if err != nil { + return nil, nil, err + } + + newDefaultReceiver.GrafanaManagedReceivers = append(newDefaultReceiver.GrafanaManagedReceivers, defaultNotifier) + + // Choose the lowest send reminder duration from all the notifiers to use for default route. + if c.SendReminder && c.Frequency < repeatInterval { + repeatInterval = c.Frequency + } + } + } else { + // If there is only a single default channel, we don't need a separate receiver to hold it. We can reuse the existing receiver for that single notifier. + defaultRoute.Receiver = defaultChannels[0].Name + if defaultChannels[0].SendReminder { + repeatInterval = defaultChannels[0].Frequency + } + + // No need to create a new receiver. + newDefaultReceiver = nil + } + defaultRoute.RepeatInterval = &repeatInterval + + return newDefaultReceiver, defaultRoute, nil +} + +// Create one route per contact point, matching based on ContactLabel. +func createRoute(cr channelReceiver) (*Route, error) { + // We create a regex matcher so that each alert rule need only have a single ContactLabel entry for all contact points it sends to. + // For example, if an alert needs to send to contact1 and contact2 it will have ContactLabel=`"contact1","contact2"` and will match both routes looking + // for `.*"contact1".*` and `.*"contact2".*`. + + // We quote and escape here to ensure the regex will correctly match the ContactLabel on the alerts. + name := fmt.Sprintf(`.*%s.*`, regexp.QuoteMeta(quote(cr.receiver.Name))) + mat, err := labels.NewMatcher(labels.MatchRegexp, ContactLabel, name) + if err != nil { + return nil, err + } + + repeatInterval := DisabledRepeatInterval + if cr.channel.SendReminder { + repeatInterval = cr.channel.Frequency + } + + return &Route{ + Receiver: cr.receiver.Name, + ObjectMatchers: ObjectMatchers{mat}, + Continue: true, // We continue so that each sibling contact point route can separately match. + RepeatInterval: &repeatInterval, + }, nil +} + +// Filter receivers to select those that were associated to the given rule as channels. +func (m *migration) filterReceiversForAlert(name string, channelIDs []uidOrID, receivers map[uidOrID]*PostableApiReceiver, defaultReceivers map[string]struct{}) map[string]any { + if len(channelIDs) == 0 { + // If there are no channels associated, we use the default route. + return nil + } + + // Filter receiver names. + filteredReceiverNames := make(map[string]any) + for _, uidOrId := range channelIDs { + recv, ok := receivers[uidOrId] + if ok { + filteredReceiverNames[recv.Name] = struct{}{} // Deduplicate on contact point name. + } else { + m.mg.Logger.Warn("Alert linked to obsolete notification channel, ignoring", "alert", name, "uid", uidOrId) + } + } + + coveredByDefault := func(names map[string]any) bool { + // Check if all receivers are also default ones and if so, just use the default route. + for n := range names { + if _, ok := defaultReceivers[n]; !ok { + return false + } + } + return true + } + + if len(filteredReceiverNames) == 0 || coveredByDefault(filteredReceiverNames) { + // Use the default route instead. + return nil + } + + // Add default receivers alongside rule-specific ones. + for n := range defaultReceivers { + filteredReceiverNames[n] = struct{}{} + } + + return filteredReceiverNames +} + +func (m *migration) determineChannelUid(c *notificationChannel) (string, error) { + legacyUid := c.Uid + if legacyUid == "" { + newUid, err := m.seenUIDs.generateUid() + if err != nil { + return "", err + } + m.mg.Logger.Info("Legacy notification had an empty uid, generating a new one", "id", c.ID, "uid", newUid) + return newUid, nil + } + + if m.seenUIDs.contains(legacyUid) { + newUid, err := m.seenUIDs.generateUid() + if err != nil { + return "", err + } + m.mg.Logger.Warn("Legacy notification had a UID that collides with a migrated record, generating a new one", "id", c.ID, "old", legacyUid, "new", newUid) + return newUid, nil + } + + return legacyUid, nil +} + +// Some settings were migrated from settings to secure settings in between. +// See https://grafana.com/docs/grafana/latest/installation/upgrading/#ensure-encryption-of-existing-alert-notification-channel-secrets. +// migrateSettingsToSecureSettings takes care of that. +func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json, secureSettings SecureJsonData) (*simplejson.Json, map[string]string, error) { + keys := []string{} + switch chanType { + case "slack": + keys = []string{"url", "token"} + case "pagerduty": + keys = []string{"integrationKey"} + case "webhook": + keys = []string{"password"} + case "prometheus-alertmanager": + keys = []string{"basicAuthPassword"} + case "opsgenie": + keys = []string{"apiKey"} + case "telegram": + keys = []string{"bottoken"} + case "line": + keys = []string{"token"} + case "pushover": + keys = []string{"apiToken", "userKey"} + case "threema": + keys = []string{"api_secret"} + } + + newSecureSettings := secureSettings.Decrypt() + cloneSettings := simplejson.New() + settingsMap, err := settings.Map() + if err != nil { + return nil, nil, err + } + for k, v := range settingsMap { + cloneSettings.Set(k, v) + } + for _, k := range keys { + if v, ok := newSecureSettings[k]; ok && v != "" { + continue + } + + sv := cloneSettings.Get(k).MustString() + if sv != "" { + newSecureSettings[k] = sv + cloneSettings.Del(k) + } + } + + encryptedData := GetEncryptedJsonData(newSecureSettings) + for k, v := range encryptedData { + newSecureSettings[k] = base64.StdEncoding.EncodeToString(v) + } + + return cloneSettings, newSecureSettings, nil +} + +// Below is a snapshot of all the config and supporting functions imported +// to avoid vendoring those packages. + +type PostableUserConfig struct { + TemplateFiles map[string]string `yaml:"template_files" json:"template_files"` + AlertmanagerConfig PostableApiAlertingConfig `yaml:"alertmanager_config" json:"alertmanager_config"` +} + +type amConfigsPerOrg = map[int64]*PostableUserConfig + +type PostableApiAlertingConfig struct { + Route *Route `yaml:"route,omitempty" json:"route,omitempty"` + Templates []string `yaml:"templates" json:"templates"` + Receivers []*PostableApiReceiver `yaml:"receivers,omitempty" json:"receivers,omitempty"` +} + +type Route struct { + Receiver string `yaml:"receiver,omitempty" json:"receiver,omitempty"` + ObjectMatchers ObjectMatchers `yaml:"object_matchers,omitempty" json:"object_matchers,omitempty"` + Routes []*Route `yaml:"routes,omitempty" json:"routes,omitempty"` + Continue bool `yaml:"continue,omitempty" json:"continue,omitempty"` + GroupByStr []string `yaml:"group_by,omitempty" json:"group_by,omitempty"` + RepeatInterval *model.Duration `yaml:"repeat_interval,omitempty" json:"repeat_interval,omitempty"` +} + +type ObjectMatchers labels.Matchers + +// MarshalJSON implements the json.Marshaler interface for Matchers. Vendored from definitions.ObjectMatchers. +func (m ObjectMatchers) MarshalJSON() ([]byte, error) { + if len(m) == 0 { + return nil, nil + } + result := make([][3]string, len(m)) + for i, matcher := range m { + result[i] = [3]string{matcher.Name, matcher.Type.String(), matcher.Value} + } + return json.Marshal(result) +} + +type PostableApiReceiver struct { + Name string `yaml:"name" json:"name"` + GrafanaManagedReceivers []*PostableGrafanaReceiver `yaml:"grafana_managed_receiver_configs,omitempty" json:"grafana_managed_receiver_configs,omitempty"` +} + +type PostableGrafanaReceiver CreateAlertNotificationCommand + +type CreateAlertNotificationCommand struct { + UID string `json:"uid"` + Name string `json:"name"` + Type string `json:"type"` + DisableResolveMessage bool `json:"disableResolveMessage"` + Settings *simplejson.Json `json:"settings"` + SecureSettings map[string]string `json:"secureSettings"` +} diff --git a/pkg/services/sqlstore/migrations/ualert/channel_test.go b/pkg/services/sqlstore/migrations/ualert/channel_test.go new file mode 100644 index 00000000000..a31405bd94b --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/channel_test.go @@ -0,0 +1,471 @@ +package ualert + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/prometheus/alertmanager/pkg/labels" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/components/simplejson" + ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" +) + +func TestFilterReceiversForAlert(t *testing.T) { + tc := []struct { + name string + channelIds []uidOrID + receivers map[uidOrID]*PostableApiReceiver + defaultReceivers map[string]struct{} + expected map[string]any + }{ + { + name: "when an alert has multiple channels, each should filter for the correct receiver", + channelIds: []uidOrID{"uid1", "uid2"}, + receivers: map[uidOrID]*PostableApiReceiver{ + "uid1": { + Name: "recv1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid2": { + Name: "recv2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid3": { + Name: "recv3", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + }, + defaultReceivers: map[string]struct{}{}, + expected: map[string]any{ + "recv1": struct{}{}, + "recv2": struct{}{}, + }, + }, + { + name: "when default receivers exist, they should be added to an alert's filtered receivers", + channelIds: []uidOrID{"uid1"}, + receivers: map[uidOrID]*PostableApiReceiver{ + "uid1": { + Name: "recv1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid2": { + Name: "recv2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid3": { + Name: "recv3", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + }, + defaultReceivers: map[string]struct{}{ + "recv2": {}, + }, + expected: map[string]any{ + "recv1": struct{}{}, // From alert + "recv2": struct{}{}, // From default + }, + }, + { + name: "when an alert has a channels associated by ID instead of UID, it should be included", + channelIds: []uidOrID{int64(42)}, + receivers: map[uidOrID]*PostableApiReceiver{ + int64(42): { + Name: "recv1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + }, + defaultReceivers: map[string]struct{}{}, + expected: map[string]any{ + "recv1": struct{}{}, + }, + }, + { + name: "when an alert's receivers are covered by the defaults, return nil to use default receiver downstream", + channelIds: []uidOrID{"uid1"}, + receivers: map[uidOrID]*PostableApiReceiver{ + "uid1": { + Name: "recv1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid2": { + Name: "recv2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + "uid3": { + Name: "recv3", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + }, + defaultReceivers: map[string]struct{}{ + "recv1": {}, + "recv2": {}, + }, + expected: nil, // recv1 is already a default + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + m := newTestMigration(t) + res := m.filterReceiversForAlert("", tt.channelIds, tt.receivers, tt.defaultReceivers) + + require.Equal(t, tt.expected, res) + }) + } +} + +func TestCreateRoute(t *testing.T) { + tc := []struct { + name string + channel *notificationChannel + recv *PostableApiReceiver + expected *Route + }{ + { + name: "when a receiver is passed in, the route should regex match based on quoted name with continue=true", + channel: ¬ificationChannel{}, + recv: &PostableApiReceiver{ + Name: "recv1", + }, + expected: &Route{ + Receiver: "recv1", + ObjectMatchers: ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, + Routes: nil, + Continue: true, + GroupByStr: nil, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }, + }, + { + name: "notification channel should be escaped for regex in the matcher", + channel: ¬ificationChannel{}, + recv: &PostableApiReceiver{ + Name: `. ^ $ * + - ? ( ) [ ] { } \ |`, + }, + expected: &Route{ + Receiver: `. ^ $ * + - ? ( ) [ ] { } \ |`, + ObjectMatchers: ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"\. \^ \$ \* \+ - \? \( \) \[ \] \{ \} \\ \|".*`}}, + Routes: nil, + Continue: true, + GroupByStr: nil, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }, + }, + { + name: "when a channel has sendReminder=true, the route should use the frequency in repeat interval", + channel: ¬ificationChannel{SendReminder: true, Frequency: model.Duration(time.Duration(42) * time.Hour)}, + recv: &PostableApiReceiver{ + Name: "recv1", + }, + expected: &Route{ + Receiver: "recv1", + ObjectMatchers: ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, + Routes: nil, + Continue: true, + GroupByStr: nil, + RepeatInterval: durationPointer(model.Duration(time.Duration(42) * time.Hour)), + }, + }, + { + name: "when a channel has sendReminder=false, the route should ignore the frequency in repeat interval and use DisabledRepeatInterval", + channel: ¬ificationChannel{SendReminder: false, Frequency: model.Duration(time.Duration(42) * time.Hour)}, + recv: &PostableApiReceiver{ + Name: "recv1", + }, + expected: &Route{ + Receiver: "recv1", + ObjectMatchers: ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}}, + Routes: nil, + Continue: true, + GroupByStr: nil, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + res, err := createRoute(channelReceiver{ + channel: tt.channel, + receiver: tt.recv, + }) + require.NoError(t, err) + + // Order of nested routes is not guaranteed. + cOpt := []cmp.Option{ + cmpopts.SortSlices(func(a, b *Route) bool { + if a.Receiver != b.Receiver { + return a.Receiver < b.Receiver + } + return a.ObjectMatchers[0].Value < b.ObjectMatchers[0].Value + }), + cmpopts.IgnoreUnexported(Route{}, labels.Matcher{}), + } + + if !cmp.Equal(tt.expected, res, cOpt...) { + t.Errorf("Unexpected Route: %v", cmp.Diff(tt.expected, res, cOpt...)) + } + }) + } +} + +func createNotChannel(t *testing.T, uid string, id int64, name string) *notificationChannel { + t.Helper() + return ¬ificationChannel{Uid: uid, ID: id, Name: name, Settings: simplejson.New()} +} + +func createNotChannelWithReminder(t *testing.T, uid string, id int64, name string, frequency model.Duration) *notificationChannel { + t.Helper() + return ¬ificationChannel{Uid: uid, ID: id, Name: name, SendReminder: true, Frequency: frequency, Settings: simplejson.New()} +} + +func TestCreateReceivers(t *testing.T) { + tc := []struct { + name string + allChannels []*notificationChannel + defaultChannels []*notificationChannel + expRecvMap map[uidOrID]*PostableApiReceiver + expRecv []channelReceiver + expErr error + }{ + { + name: "when given notification channels migrate them to receivers", + allChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name1"), createNotChannel(t, "uid2", int64(2), "name2")}, + expRecvMap: map[uidOrID]*PostableApiReceiver{ + "uid1": { + Name: "name1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}}, + }, + "uid2": { + Name: "name2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name2"}}, + }, + int64(1): { + Name: "name1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}}, + }, + int64(2): { + Name: "name2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name2"}}, + }, + }, + expRecv: []channelReceiver{ + { + channel: createNotChannel(t, "uid1", int64(1), "name1"), + receiver: &PostableApiReceiver{ + Name: "name1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}}, + }, + }, + { + channel: createNotChannel(t, "uid2", int64(2), "name2"), + receiver: &PostableApiReceiver{ + Name: "name2", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name2"}}, + }, + }, + }, + }, + { + name: "when given notification channel contains double quote sanitize with underscore", + allChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name\"1")}, + expRecvMap: map[uidOrID]*PostableApiReceiver{ + "uid1": { + Name: "name_1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, + }, + int64(1): { + Name: "name_1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, + }, + }, + expRecv: []channelReceiver{ + { + channel: createNotChannel(t, "uid1", int64(1), "name\"1"), + receiver: &PostableApiReceiver{ + Name: "name_1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, + }, + }, + }, + }, + { + name: "when given notification channels collide after sanitization add short hash to end", + allChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name\"1"), createNotChannel(t, "uid2", int64(2), "name_1")}, + expRecvMap: map[uidOrID]*PostableApiReceiver{ + "uid1": { + Name: "name_1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, + }, + "uid2": { + Name: "name_1_dba13d", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1_dba13d"}}, + }, + int64(1): { + Name: "name_1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, + }, + int64(2): { + Name: "name_1_dba13d", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1_dba13d"}}, + }, + }, + expRecv: []channelReceiver{ + { + channel: createNotChannel(t, "uid1", int64(1), "name\"1"), + receiver: &PostableApiReceiver{ + Name: "name_1", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}}, + }, + }, + { + channel: createNotChannel(t, "uid2", int64(2), "name_1"), + receiver: &PostableApiReceiver{ + Name: "name_1_dba13d", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1_dba13d"}}, + }, + }, + }, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + m := newTestMigration(t) + recvMap, recvs, err := m.createReceivers(tt.allChannels) + if tt.expErr != nil { + require.Error(t, err) + require.EqualError(t, err, tt.expErr.Error()) + return + } + + require.NoError(t, err) + + // We ignore certain fields for the purposes of this test + for _, recv := range recvs { + for _, not := range recv.receiver.GrafanaManagedReceivers { + not.UID = "" + not.Settings = nil + not.SecureSettings = nil + } + } + + require.Equal(t, tt.expRecvMap, recvMap) + require.ElementsMatch(t, tt.expRecv, recvs) + }) + } +} + +func TestCreateDefaultRouteAndReceiver(t *testing.T) { + tc := []struct { + name string + amConfig *PostableUserConfig + defaultChannels []*notificationChannel + expRecv *PostableApiReceiver + expRoute *Route + expErr error + }{ + { + name: "when given multiple default notification channels migrate them to a single receiver", + defaultChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name1"), createNotChannel(t, "uid2", int64(2), "name2")}, + expRecv: &PostableApiReceiver{ + Name: "autogen-contact-point-default", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}, {Name: "name2"}}, + }, + expRoute: &Route{ + Receiver: "autogen-contact-point-default", + Routes: make([]*Route, 0), + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }, + }, + { + name: "when given multiple default notification channels migrate them to a single receiver with RepeatInterval set to be the minimum of all channel frequencies", + defaultChannels: []*notificationChannel{ + createNotChannelWithReminder(t, "uid1", int64(1), "name1", model.Duration(42)), + createNotChannelWithReminder(t, "uid2", int64(2), "name2", model.Duration(100000)), + }, + expRecv: &PostableApiReceiver{ + Name: "autogen-contact-point-default", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name1"}, {Name: "name2"}}, + }, + expRoute: &Route{ + Receiver: "autogen-contact-point-default", + Routes: make([]*Route, 0), + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + RepeatInterval: durationPointer(model.Duration(42)), + }, + }, + { + name: "when given no default notification channels create a single empty receiver for default", + defaultChannels: []*notificationChannel{}, + expRecv: &PostableApiReceiver{ + Name: "autogen-contact-point-default", + GrafanaManagedReceivers: []*PostableGrafanaReceiver{}, + }, + expRoute: &Route{ + Receiver: "autogen-contact-point-default", + Routes: make([]*Route, 0), + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + RepeatInterval: nil, + }, + }, + { + name: "when given a single default notification channels don't create a new default receiver", + defaultChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name1")}, + expRecv: nil, + expRoute: &Route{ + Receiver: "name1", + Routes: make([]*Route, 0), + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + RepeatInterval: durationPointer(DisabledRepeatInterval), + }, + }, + { + name: "when given a single default notification channel with SendReminder=true, use the channels Frequency as the RepeatInterval", + defaultChannels: []*notificationChannel{createNotChannelWithReminder(t, "uid1", int64(1), "name1", model.Duration(42))}, + expRecv: nil, + expRoute: &Route{ + Receiver: "name1", + Routes: make([]*Route, 0), + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + RepeatInterval: durationPointer(model.Duration(42)), + }, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + m := newTestMigration(t) + recv, route, err := m.createDefaultRouteAndReceiver(tt.defaultChannels) + if tt.expErr != nil { + require.Error(t, err) + require.EqualError(t, err, tt.expErr.Error()) + return + } + + require.NoError(t, err) + + // We ignore certain fields for the purposes of this test + if recv != nil { + for _, not := range recv.GrafanaManagedReceivers { + not.UID = "" + not.Settings = nil + not.SecureSettings = nil + } + } + + require.Equal(t, tt.expRecv, recv) + require.Equal(t, tt.expRoute, route) + }) + } +} + +func durationPointer(d model.Duration) *model.Duration { + return &d +} diff --git a/pkg/services/ngalert/migration/cond_trans.go b/pkg/services/sqlstore/migrations/ualert/cond_trans.go similarity index 87% rename from pkg/services/ngalert/migration/cond_trans.go rename to pkg/services/sqlstore/migrations/ualert/cond_trans.go index a2a05681d3d..e67af621210 100644 --- a/pkg/services/ngalert/migration/cond_trans.go +++ b/pkg/services/sqlstore/migrations/ualert/cond_trans.go @@ -1,9 +1,7 @@ -package migration +package ualert import ( - "context" "encoding/json" - "errors" "fmt" "sort" "strings" @@ -11,22 +9,12 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/services/datasources" - migrationStore "github.com/grafana/grafana/pkg/services/ngalert/migration/store" - ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/tsdb/legacydata" "github.com/grafana/grafana/pkg/tsdb/legacydata/interval" "github.com/grafana/grafana/pkg/util" ) -// It is defined in pkg/expr/service.go as "DatasourceType" -const expressionDatasourceUID = "__expr__" - -//nolint:gocyclo -func transConditions(ctx context.Context, alert *migrationStore.DashAlert, store migrationStore.Store) (*condition, error) { - // TODO: needs a significant refactor to reduce complexity. - usr := getMigrationUser(alert.OrgID) - set := alert.ParsedSettings - +func transConditions(set dashAlertSettings, orgID int64, dsUIDMap dsUIDLookup) (*condition, error) { refIDtoCondIdx := make(map[string][]int) // a map of original refIds to their corresponding condition index for i, cond := range set.Conditions { if len(cond.Query.Params) != 3 { @@ -135,16 +123,8 @@ func transConditions(ctx context.Context, alert *migrationStore.DashAlert, store } } - // Could have an alert saved but datasource deleted, so can not require match. - dsUid := "" - if ds, err := store.GetDatasource(ctx, set.Conditions[condIdx].Query.DatasourceID, usr); err == nil { - dsUid = ds.UID - } else { - if !errors.Is(err, datasources.ErrDataSourceNotFound) { - return nil, err - } - } - + // one could have an alert saved but datasource deleted, so can not require match. + dsUID := dsUIDMap.GetUID(orgID, set.Conditions[condIdx].Query.DatasourceID) queryObj["refId"] = refID // See services/alerting/conditions/query.go's newQueryCondition @@ -173,11 +153,11 @@ func transConditions(ctx context.Context, alert *migrationStore.DashAlert, store return nil, err } - alertQuery := ngmodels.AlertQuery{ + alertQuery := alertQuery{ RefID: refID, Model: encodedObj, RelativeTimeRange: *rTR, - DatasourceUID: dsUid, + DatasourceUID: dsUID, QueryType: queryType, } newCond.Data = append(newCond.Data, alertQuery) @@ -188,7 +168,7 @@ func transConditions(ctx context.Context, alert *migrationStore.DashAlert, store conditions := make([]classicConditionJSON, len(set.Conditions)) for i, cond := range set.Conditions { newCond := classicConditionJSON{} - newCond.Evaluator = migrationStore.ConditionEvalJSON{ + newCond.Evaluator = conditionEvalJSON{ Type: cond.Evaluator.Type, Params: cond.Evaluator.Params, } @@ -204,7 +184,7 @@ func transConditions(ctx context.Context, alert *migrationStore.DashAlert, store return nil, err } newCond.Condition = ccRefID // set the alert condition to point to the classic condition - newCond.OrgID = alert.OrgID + newCond.OrgID = orgID exprModel := struct { Type string `json:"type"` @@ -221,7 +201,7 @@ func transConditions(ctx context.Context, alert *migrationStore.DashAlert, store return nil, err } - ccAlertQuery := ngmodels.AlertQuery{ + ccAlertQuery := alertQuery{ RefID: ccRefID, Model: exprModelJSON, DatasourceUID: expressionDatasourceUID, @@ -243,7 +223,7 @@ type condition struct { OrgID int64 `json:"-"` // Data is an array of data source queries and/or server side expressions. - Data []ngmodels.AlertQuery `json:"data"` + Data []alertQuery `json:"data"` } const alpha = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" @@ -270,7 +250,7 @@ func getNewRefID(refIDs map[string][]int) (string, error) { // getRelativeDuration turns the alerting durations for dashboard conditions // into a relative time range. -func getRelativeDuration(rawFrom, rawTo string) (*ngmodels.RelativeTimeRange, error) { +func getRelativeDuration(rawFrom, rawTo string) (*relativeTimeRange, error) { fromD, err := getFrom(rawFrom) if err != nil { return nil, err @@ -280,9 +260,9 @@ func getRelativeDuration(rawFrom, rawTo string) (*ngmodels.RelativeTimeRange, er if err != nil { return nil, err } - return &ngmodels.RelativeTimeRange{ - From: ngmodels.Duration(fromD), - To: ngmodels.Duration(toD), + return &relativeTimeRange{ + From: duration(fromD), + To: duration(toD), }, nil } @@ -317,7 +297,7 @@ func getTo(to string) (time.Duration, error) { } type classicConditionJSON struct { - Evaluator migrationStore.ConditionEvalJSON `json:"evaluator"` + Evaluator conditionEvalJSON `json:"evaluator"` Operator struct { Type string `json:"type"` diff --git a/pkg/services/sqlstore/migrations/ualert/created_folders_mig.go b/pkg/services/sqlstore/migrations/ualert/created_folders_mig.go deleted file mode 100644 index 1d4f1da80c0..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/created_folders_mig.go +++ /dev/null @@ -1,83 +0,0 @@ -package ualert - -import ( - "encoding/json" - "fmt" - "time" - - "xorm.io/xorm" - - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" -) - -// stateKey is a vendored migrationStore.stateKey. -var stateKey = "stateKey" - -// CreatedFoldersMigration moves the record of created folders during legacy migration from Dashboard created_by=-8 -// to the kvstore. If there are no dashboards with created_by=-.8, then nothing needs to be done. -func CreatedFoldersMigration(mg *migrator.Migrator) { - mg.AddMigration("migrate record of created folders during legacy migration to kvstore", &createdFoldersToKVStore{}) -} - -type createdFoldersToKVStore struct { - migrator.MigrationBase -} - -func (c createdFoldersToKVStore) SQL(migrator.Dialect) string { - return codeMigration -} - -func (c createdFoldersToKVStore) Exec(sess *xorm.Session, mg *migrator.Migrator) error { - var results []struct { - UID string `xorm:"uid"` - OrgID int64 `xorm:"org_id"` - } - folderCreatedBy := -8 - if err := sess.SQL("select * from dashboard where created_by = ?", folderCreatedBy).Find(&results); err != nil { - return err - } - - if len(results) == 0 { - mg.Logger.Debug("no dashboards with created_by=-8, nothing to set in kvstore") - return nil - } - - type orgMigrationState struct { - OrgID int64 `json:"orgId"` - CreatedFolders []string `json:"createdFolders"` - } - states := make(map[int64]*orgMigrationState) - for _, r := range results { - if _, ok := states[r.OrgID]; !ok { - states[r.OrgID] = &orgMigrationState{ - OrgID: r.OrgID, - CreatedFolders: []string{}, - } - } - states[r.OrgID].CreatedFolders = append(states[r.OrgID].CreatedFolders, r.UID) - } - - now := time.Now() - for _, state := range states { - raw, err := json.Marshal(state) - if err != nil { - return err - } - - orgId := state.OrgID - entry := kvStoreV1Entry{ - OrgID: &orgId, - Namespace: &KVNamespace, - Key: &stateKey, - Value: string(raw), - Created: now, - Updated: now, - } - if _, errCreate := sess.Table("kv_store").Insert(&entry); errCreate != nil { - mg.Logger.Error("failed to insert record of created folders to kvstore", "err", errCreate) - return fmt.Errorf("failed to insert record of created folders to kvstore: %w", errCreate) - } - } - - return nil -} diff --git a/pkg/services/sqlstore/migrations/ualert/dash_alert.go b/pkg/services/sqlstore/migrations/ualert/dash_alert.go new file mode 100644 index 00000000000..cf385464999 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/dash_alert.go @@ -0,0 +1,130 @@ +package ualert + +import ( + "encoding/json" + "fmt" + "time" +) + +type dashAlert struct { + Id int64 + OrgId int64 + DashboardId int64 + PanelId int64 + Name string + Message string + Frequency int64 + For time.Duration + State string + + Settings json.RawMessage + ParsedSettings *dashAlertSettings + DashboardUID string // Set from separate call +} + +var slurpDashSQL = ` +SELECT id, + org_id, + dashboard_id, + panel_id, + org_id, + name, + message, + frequency, + %s, + state, + settings +FROM + alert +WHERE org_id IN (SELECT id from org) + AND dashboard_id IN (SELECT id from dashboard) +` + +// slurpDashAlerts loads all alerts from the alert database table into +// the dashAlert type. If there are alerts that belong to either organization or dashboard that does not exist, those alerts will not be returned/ +// Additionally it unmarshals the json settings for the alert into the +// ParsedSettings property of the dash alert. +func (m *migration) slurpDashAlerts() ([]dashAlert, error) { + dashAlerts := []dashAlert{} + err := m.sess.SQL(fmt.Sprintf(slurpDashSQL, m.mg.Dialect.Quote("for"))).Find(&dashAlerts) + + if err != nil { + return nil, err + } + + for i := range dashAlerts { + err = json.Unmarshal(dashAlerts[i].Settings, &dashAlerts[i].ParsedSettings) + if err != nil { + da := dashAlerts[i] + return nil, fmt.Errorf("failed to parse alert rule ID:%d, name:'%s', orgID:%d: %w", da.Id, da.Name, da.OrgId, err) + } + } + + return dashAlerts, nil +} + +// dashAlertSettings is a type for the JSON that is in the settings field of +// the alert table. +type dashAlertSettings struct { + NoDataState string `json:"noDataState"` + ExecutionErrorState string `json:"executionErrorState"` + Conditions []dashAlertCondition `json:"conditions"` + AlertRuleTags any `json:"alertRuleTags"` + Notifications []dashAlertNot `json:"notifications"` +} + +// dashAlertNot is the object that represents the Notifications array in +// dashAlertSettings +type dashAlertNot struct { + UID string `json:"uid,omitempty"` + ID int64 `json:"id,omitempty"` +} + +// dashAlertingConditionJSON is like classic.ClassicConditionJSON except that it +// includes the model property with the query. +type dashAlertCondition struct { + Evaluator conditionEvalJSON `json:"evaluator"` + + Operator struct { + Type string `json:"type"` + } `json:"operator"` + + Query struct { + Params []string `json:"params"` + DatasourceID int64 `json:"datasourceId"` + Model json.RawMessage + } `json:"query"` + + Reducer struct { + // Params []any `json:"params"` (Unused) + Type string `json:"type"` + } +} + +type conditionEvalJSON struct { + Params []float64 `json:"params"` + Type string `json:"type"` // e.g. "gt" +} + +// slurpDashUIDs returns a map of [orgID, dashboardId] -> dashUID. +func (m *migration) slurpDashUIDs() (map[[2]int64]string, error) { + dashIDs := []struct { + OrgID int64 `xorm:"org_id"` + ID int64 `xorm:"id"` + UID string `xorm:"uid"` + }{} + + err := m.sess.SQL(`SELECT org_id, id, uid FROM dashboard`).Find(&dashIDs) + + if err != nil { + return nil, err + } + + idToUID := make(map[[2]int64]string, len(dashIDs)) + + for _, ds := range dashIDs { + idToUID[[2]int64{ds.OrgID, ds.ID}] = ds.UID + } + + return idToUID, nil +} diff --git a/pkg/services/sqlstore/migrations/ualert/dashboard.go b/pkg/services/sqlstore/migrations/ualert/dashboard.go new file mode 100644 index 00000000000..d7440f60b02 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/dashboard.go @@ -0,0 +1,108 @@ +package ualert + +import ( + "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/slugify" +) + +type dashboard struct { + Id int64 + Uid string + Slug string + OrgId int64 + GnetId int64 + Version int + PluginId string + + Created time.Time + Updated time.Time + + UpdatedBy int64 + CreatedBy int64 + FolderId int64 + IsFolder bool + HasACL bool `xorm:"has_acl"` + + Title string + Data *simplejson.Json +} + +func (d *dashboard) setUid(uid string) { + d.Uid = uid + d.Data.Set("uid", uid) +} + +func (d *dashboard) setVersion(version int) { + d.Version = version + d.Data.Set("version", version) +} + +// UpdateSlug updates the slug +func (d *dashboard) updateSlug() { + title := d.Data.Get("title").MustString() + d.Slug = slugify.Slugify(title) +} + +func newDashboardFromJson(data *simplejson.Json) *dashboard { + dash := &dashboard{} + dash.Data = data + dash.Title = dash.Data.Get("title").MustString() + dash.updateSlug() + update := false + + if id, err := dash.Data.Get("id").Float64(); err == nil { + dash.Id = int64(id) + update = true + } + + if uid, err := dash.Data.Get("uid").String(); err == nil { + dash.Uid = uid + update = true + } + + if version, err := dash.Data.Get("version").Float64(); err == nil && update { + dash.Version = int(version) + dash.Updated = time.Now() + } else { + dash.Data.Set("version", 0) + dash.Created = time.Now() + dash.Updated = time.Now() + } + + if gnetId, err := dash.Data.Get("gnetId").Float64(); err == nil { + dash.GnetId = int64(gnetId) + } + + return dash +} + +type saveFolderCommand struct { + Dashboard *simplejson.Json `json:"dashboard" binding:"Required"` + UserId int64 `json:"userId"` + Message string `json:"message"` + OrgId int64 `json:"-"` + RestoredFrom int `json:"-"` + PluginId string `json:"-"` + FolderId int64 `json:"folderId"` + IsFolder bool `json:"isFolder"` +} + +// GetDashboardModel turns the command into the saveable model +func (cmd *saveFolderCommand) getDashboardModel() *dashboard { + dash := newDashboardFromJson(cmd.Dashboard) + userId := cmd.UserId + + if userId == 0 { + userId = -1 + } + + dash.UpdatedBy = userId + dash.OrgId = cmd.OrgId + dash.PluginId = cmd.PluginId + dash.IsFolder = cmd.IsFolder + dash.FolderId = cmd.FolderId + dash.updateSlug() + return dash +} diff --git a/pkg/services/sqlstore/migrations/ualert/datasource.go b/pkg/services/sqlstore/migrations/ualert/datasource.go new file mode 100644 index 00000000000..03027a4d747 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/datasource.go @@ -0,0 +1,31 @@ +package ualert + +type dsUIDLookup map[[2]int64]string + +// GetUID fetch thes datasource UID based on orgID+datasourceID +func (d dsUIDLookup) GetUID(orgID, datasourceID int64) string { + return d[[2]int64{orgID, datasourceID}] +} + +// slurpDSIDs returns a map of [orgID, dataSourceId] -> UID. +func (m *migration) slurpDSIDs() (dsUIDLookup, error) { + dsIDs := []struct { + OrgID int64 `xorm:"org_id"` + ID int64 `xorm:"id"` + UID string `xorm:"uid"` + }{} + + err := m.sess.SQL(`SELECT org_id, id, uid FROM data_source`).Find(&dsIDs) + + if err != nil { + return nil, err + } + + idToUID := make(dsUIDLookup, len(dsIDs)) + + for _, ds := range dsIDs { + idToUID[[2]int64{ds.OrgID, ds.ID}] = ds.UID + } + + return idToUID, nil +} diff --git a/pkg/services/sqlstore/migrations/ualert/migration_service_migration.go b/pkg/services/sqlstore/migrations/ualert/migration_service_migration.go deleted file mode 100644 index ad491b794fc..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/migration_service_migration.go +++ /dev/null @@ -1,69 +0,0 @@ -package ualert - -import ( - "fmt" - "os" - "strconv" - "time" - - "xorm.io/xorm" - - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" -) - -// KVNamespace is a vendored migration.KVNamespace. -var KVNamespace = "ngalert.migration" - -// migratedKey is a vendored migration.migratedKey. -var migratedKey = "migrated" - -// MigrationServiceMigration moves the legacy alert migration status from the migration log to kvstore. -func MigrationServiceMigration(mg *migrator.Migrator) { - mg.AddMigration("set legacy alert migration status in kvstore", &migrationLogToKVStore{}) -} - -type migrationLogToKVStore struct { - migrator.MigrationBase -} - -func (c migrationLogToKVStore) SQL(migrator.Dialect) string { - return codeMigration -} - -func (c migrationLogToKVStore) Exec(sess *xorm.Session, mg *migrator.Migrator) error { - logs, err := mg.GetMigrationLog() - if err != nil { - mg.Logger.Error("alert migration failure: could not get migration log", "error", err) - os.Exit(1) - } - - _, migrationRun := logs[migTitle] - - var anyOrg int64 = 0 - now := time.Now() - entry := kvStoreV1Entry{ - OrgID: &anyOrg, - Namespace: &KVNamespace, - Key: &migratedKey, - Value: strconv.FormatBool(migrationRun), - Created: now, - Updated: now, - } - if _, errCreate := sess.Table("kv_store").Insert(&entry); errCreate != nil { - mg.Logger.Error("failed to insert migration status to kvstore", "err", errCreate) - return fmt.Errorf("failed to insert migration status to kvstore: %w", errCreate) - } - return nil -} - -// kvStoreV1Entry is a vendored kvstore.Item. -type kvStoreV1Entry struct { - ID int64 `xorm:"pk autoincr 'id'"` - OrgID *int64 `xorm:"org_id"` - Namespace *string `xorm:"namespace"` - Key *string `xorm:"key"` - Value string `xorm:"value"` - - Created time.Time `xorm:"created"` - Updated time.Time `xorm:"updated"` -} diff --git a/pkg/services/sqlstore/migrations/ualert/migration_test.go b/pkg/services/sqlstore/migrations/ualert/migration_test.go new file mode 100644 index 00000000000..0c76f6321bf --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/migration_test.go @@ -0,0 +1,847 @@ +package ualert_test + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/prometheus/alertmanager/pkg/labels" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + "gopkg.in/ini.v1" + "xorm.io/xorm" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/services/alerting/models" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/datasources" + ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/services/sqlstore/migrations" + "github.com/grafana/grafana/pkg/services/sqlstore/migrations/ualert" + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/services/sqlstore/sqlutil" + "github.com/grafana/grafana/pkg/setting" +) + +// TestAddDashAlertMigration tests the AddDashAlertMigration wrapper method that decides when to run the migration based on migration status and settings. +func TestAddDashAlertMigration(t *testing.T) { + x := setupTestDB(t) + + tc := []struct { + name string + config *setting.Cfg + isMigrationRun bool + shouldPanic bool + expected []string // set of migration titles + }{ + { + name: "when unified alerting enabled and migration not already run, then add main migration and clear rmMigration log entry", + config: &setting.Cfg{ + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: boolPointer(true), + }, + }, + isMigrationRun: false, + expected: []string{fmt.Sprintf(ualert.ClearMigrationEntryTitle, ualert.RmMigTitle), ualert.MigTitle}, + }, + { + name: "when unified alerting disabled and migration is already run, then add rmMigration and clear main migration log entry", + config: &setting.Cfg{ + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: boolPointer(false), + }, + ForceMigration: true, + }, + isMigrationRun: true, + expected: []string{fmt.Sprintf(ualert.ClearMigrationEntryTitle, ualert.MigTitle), ualert.RmMigTitle}, + }, + { + name: "when unified alerting disabled, migration is already run and force migration is disabled, then the migration should panic", + config: &setting.Cfg{ + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: boolPointer(false), + }, + ForceMigration: false, + }, + isMigrationRun: true, + expected: []string{fmt.Sprintf(ualert.ClearMigrationEntryTitle, ualert.MigTitle), ualert.RmMigTitle}, + }, + { + name: "when unified alerting enabled and migration is already run, then do nothing", + config: &setting.Cfg{ + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: boolPointer(true), + }, + }, + isMigrationRun: true, + expected: []string{}, + }, + { + name: "when unified alerting disabled and migration is not already run, then do nothing", + config: &setting.Cfg{ + UnifiedAlerting: setting.UnifiedAlertingSettings{ + Enabled: boolPointer(false), + }, + }, + isMigrationRun: false, + expected: []string{}, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + defer func() { + // if the code should panic, make sure it has + if r := recover(); r == nil && tt.shouldPanic { + t.Errorf("The code did not panic") + } + }() + if tt.isMigrationRun { + log := migrator.MigrationLog{ + MigrationID: ualert.MigTitle, + SQL: "", + Timestamp: time.Now(), + Success: true, + } + _, err := x.Insert(log) + require.NoError(t, err) + } else { + _, err := x.Exec("DELETE FROM migration_log WHERE migration_id = ?", ualert.MigTitle) + require.NoError(t, err) + } + + mg := migrator.NewMigrator(x, tt.config) + + ualert.AddDashAlertMigration(mg) + require.Equal(t, tt.expected, mg.GetMigrationIDs(false)) + }) + } +} + +// TestAMConfigMigration tests the execution of the main DashAlertMigration specifically for migrations of channels and routes. +func TestAMConfigMigration(t *testing.T) { + // Run initial migration to have a working DB. + x := setupTestDB(t) + + tc := []struct { + name string + legacyChannels []*models.AlertNotification + alerts []*models.Alert + + expected map[int64]*ualert.PostableUserConfig + expErr error + }{ + { + name: "general multi-org, multi-alert, multi-channel migration", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), + createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, false), + createAlertNotification(t, int64(2), "notifier4", "email", emailSettings, false), + createAlertNotification(t, int64(2), "notifier5", "slack", slackSettings, false), + createAlertNotification(t, int64(2), "notifier6", "opsgenie", opsgenieSettings, true), // default + }, + alerts: []*models.Alert{ + createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1"}), + createAlert(t, int64(1), int64(1), int64(2), "alert2", []string{"notifier2", "notifier3"}), + createAlert(t, int64(1), int64(2), int64(3), "alert3", []string{"notifier3"}), + createAlert(t, int64(2), int64(3), int64(1), "alert4", []string{"notifier4"}), + createAlert(t, int64(2), int64(3), int64(2), "alert5", []string{"notifier4", "notifier5", "notifier6"}), + createAlert(t, int64(2), int64(4), int64(3), "alert6", []string{}), + }, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + {Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + {Receiver: "notifier3", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier3".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + }, + RepeatInterval: nil, + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, + {Name: "notifier3", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}, + {Name: "autogen-contact-point-default"}, // empty default + }, + }, + }, + int64(2): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "notifier6", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier4", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier4".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + {Receiver: "notifier5", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier5".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + {Receiver: "notifier6", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier6".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + }, + RepeatInterval: durationPointer(ualert.DisabledRepeatInterval), + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier4", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier4", Type: "email"}}}, + {Name: "notifier5", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier5", Type: "slack"}}}, + {Name: "notifier6", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier6", Type: "opsgenie"}}}, + }, + }, + }, + }, + }, + { + name: "when no default channel, create empty autogen-contact-point-default", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + }, + alerts: []*models.Alert{}, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + }, + RepeatInterval: nil, + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "autogen-contact-point-default"}, + }, + }, + }, + }, + }, + { + name: "when single default channel, don't create autogen-contact-point-default", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), + }, + alerts: []*models.Alert{}, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "notifier1", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + }, + RepeatInterval: durationPointer(ualert.DisabledRepeatInterval), + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + }, + }, + }, + }, + }, + { + name: "when single default channel with SendReminder, use channel Frequency as RepeatInterval", + legacyChannels: []*models.AlertNotification{ + createAlertNotificationWithReminder(t, int64(1), "notifier1", "email", emailSettings, true, true, time.Duration(1)*time.Hour), + }, + alerts: []*models.Alert{}, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "notifier1", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour))}, + }, + RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour)), + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + }, + }, + }, + }, + }, + { + name: "when multiple default channels, add them to autogen-contact-point-default as well", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), + createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, true), + }, + alerts: []*models.Alert{}, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + {Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + }, + RepeatInterval: durationPointer(ualert.DisabledRepeatInterval), + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, + {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}, + }, + }, + }, + }, + }, + { + name: "when multiple default channels with SendReminder, use minimum channel frequency as RepeatInterval", + legacyChannels: []*models.AlertNotification{ + createAlertNotificationWithReminder(t, int64(1), "notifier1", "email", emailSettings, true, true, time.Duration(1)*time.Hour), + createAlertNotificationWithReminder(t, int64(1), "notifier2", "slack", slackSettings, true, true, time.Duration(30)*time.Minute), + }, + alerts: []*models.Alert{}, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(1) * time.Hour))}, + {Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(model.Duration(time.Duration(30) * time.Minute))}, + }, + RepeatInterval: durationPointer(model.Duration(time.Duration(30) * time.Minute)), + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, + {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier2", Type: "slack"}}}, + }, + }, + }, + }, + }, + { + name: "when default channels exist alongside non-default, add only defaults to autogen-contact-point-default", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, true), // default + createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), + createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, true), // default + }, + alerts: []*models.Alert{}, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + {Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + {Receiver: "notifier3", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier3".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + }, + RepeatInterval: durationPointer(ualert.DisabledRepeatInterval), + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, + {Name: "notifier3", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier3", Type: "opsgenie"}}}, + {Name: "autogen-contact-point-default", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}, {Name: "notifier3", Type: "opsgenie"}}}}, + }, + }, + }, + }, + { + name: "when alerts share channels, only create one receiver per legacy channel", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), + }, + alerts: []*models.Alert{ + createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1"}), + createAlert(t, int64(1), int64(1), int64(1), "alert2", []string{"notifier1", "notifier2"}), + }, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + {Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + }, + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "notifier2", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier2", Type: "slack"}}}, + {Name: "autogen-contact-point-default"}, + }, + }, + }, + }, + }, + { + name: "when channel not linked to any alerts, still create a receiver for it", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + }, + alerts: []*models.Alert{}, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + }, + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "autogen-contact-point-default"}, + }, + }, + }, + }, + }, + { + name: "when unsupported channels, do not migrate them", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + createAlertNotification(t, int64(1), "notifier2", "hipchat", "", false), + createAlertNotification(t, int64(1), "notifier3", "sensu", "", false), + }, + alerts: []*models.Alert{}, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + }, + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "autogen-contact-point-default"}, + }, + }, + }, + }, + }, + { + name: "when unsupported channel linked to alert, do not migrate only that channel", + legacyChannels: []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + createAlertNotification(t, int64(1), "notifier2", "sensu", "", false), + }, + alerts: []*models.Alert{ + createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1", "notifier2"}), + }, + expected: map[int64]*ualert.PostableUserConfig{ + int64(1): { + AlertmanagerConfig: ualert.PostableApiAlertingConfig{ + Route: &ualert.Route{ + Receiver: "autogen-contact-point-default", + GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel}, + Routes: []*ualert.Route{ + {Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true, RepeatInterval: durationPointer(ualert.DisabledRepeatInterval)}, + }, + }, + Receivers: []*ualert.PostableApiReceiver{ + {Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}}, + {Name: "autogen-contact-point-default"}, + }, + }, + }, + }, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + defer teardown(t, x) + setupLegacyAlertsTables(t, x, tt.legacyChannels, tt.alerts) + runDashAlertMigrationTestRun(t, x) + + for orgId := range tt.expected { + amConfig := getAlertmanagerConfig(t, x, orgId) + + // Order of nested GrafanaManagedReceivers is not guaranteed. + cOpt := []cmp.Option{ + cmpopts.IgnoreUnexported(ualert.PostableApiReceiver{}), + cmpopts.IgnoreFields(ualert.PostableGrafanaReceiver{}, "UID", "Settings", "SecureSettings"), + cmpopts.SortSlices(func(a, b *ualert.PostableGrafanaReceiver) bool { return a.Name < b.Name }), + cmpopts.SortSlices(func(a, b *ualert.PostableApiReceiver) bool { return a.Name < b.Name }), + } + if !cmp.Equal(tt.expected[orgId].AlertmanagerConfig.Receivers, amConfig.AlertmanagerConfig.Receivers, cOpt...) { + t.Errorf("Unexpected Receivers: %v", cmp.Diff(tt.expected[orgId].AlertmanagerConfig.Receivers, amConfig.AlertmanagerConfig.Receivers, cOpt...)) + } + + // Order of routes is not guaranteed. + cOpt = []cmp.Option{ + cmpopts.SortSlices(func(a, b *ualert.Route) bool { + if a.Receiver != b.Receiver { + return a.Receiver < b.Receiver + } + return a.ObjectMatchers[0].Value < b.ObjectMatchers[0].Value + }), + cmpopts.IgnoreUnexported(ualert.Route{}, labels.Matcher{}), + } + if !cmp.Equal(tt.expected[orgId].AlertmanagerConfig.Route, amConfig.AlertmanagerConfig.Route, cOpt...) { + t.Errorf("Unexpected Route: %v", cmp.Diff(tt.expected[orgId].AlertmanagerConfig.Route, amConfig.AlertmanagerConfig.Route, cOpt...)) + } + } + }) + } +} + +// TestDashAlertMigration tests the execution of the main DashAlertMigration specifically for migrations of models. +func TestDashAlertMigration(t *testing.T) { + // Run initial migration to have a working DB. + x := setupTestDB(t) + + t.Run("when DashAlertMigration create ContactLabel on migrated AlertRules", func(t *testing.T) { + defer teardown(t, x) + legacyChannels := []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false), + createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false), + createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, false), + createAlertNotification(t, int64(2), "notifier4", "email", emailSettings, false), + createAlertNotification(t, int64(2), "notifier5", "slack", slackSettings, false), + createAlertNotification(t, int64(2), "notifier6", "opsgenie", opsgenieSettings, true), // default + } + alerts := []*models.Alert{ + createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1"}), + createAlert(t, int64(1), int64(1), int64(2), "alert2", []string{"notifier2", "notifier3"}), + createAlert(t, int64(1), int64(2), int64(3), "alert3", []string{"notifier3"}), + createAlert(t, int64(2), int64(3), int64(1), "alert4", []string{"notifier4"}), + createAlert(t, int64(2), int64(3), int64(2), "alert5", []string{"notifier4", "notifier5", "notifier6"}), + createAlert(t, int64(2), int64(4), int64(3), "alert6", []string{}), + } + expected := map[int64]map[string]*ngModels.AlertRule{ + int64(1): { + "alert1": {Labels: map[string]string{ualert.ContactLabel: `"notifier1"`}}, + "alert2": {Labels: map[string]string{ualert.ContactLabel: `"notifier2","notifier3"`}}, + "alert3": {Labels: map[string]string{ualert.ContactLabel: `"notifier3"`}}, + }, + int64(2): { + "alert4": {Labels: map[string]string{ualert.ContactLabel: `"notifier4","notifier6"`}}, + "alert5": {Labels: map[string]string{ualert.ContactLabel: `"notifier4","notifier5","notifier6"`}}, + "alert6": {Labels: map[string]string{}}, + }, + } + setupLegacyAlertsTables(t, x, legacyChannels, alerts) + runDashAlertMigrationTestRun(t, x) + + for orgId := range expected { + rules := getAlertRules(t, x, orgId) + expectedRulesMap := expected[orgId] + require.Len(t, rules, len(expectedRulesMap)) + for _, r := range rules { + require.Equal(t, expectedRulesMap[r.Title].Labels[ualert.ContactLabel], r.Labels[ualert.ContactLabel]) + } + } + }) + + t.Run("when DashAlertMigration create ContactLabel with sanitized name if name contains double quote", func(t *testing.T) { + defer teardown(t, x) + legacyChannels := []*models.AlertNotification{ + createAlertNotification(t, int64(1), "notif\"ier1", "email", emailSettings, false), + } + alerts := []*models.Alert{ + createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notif\"ier1"}), + } + expected := map[int64]map[string]*ngModels.AlertRule{ + int64(1): { + "alert1": {Labels: map[string]string{ualert.ContactLabel: `"notif_ier1"`}}, + }, + } + setupLegacyAlertsTables(t, x, legacyChannels, alerts) + runDashAlertMigrationTestRun(t, x) + + for orgId := range expected { + rules := getAlertRules(t, x, orgId) + expectedRulesMap := expected[orgId] + require.Len(t, rules, len(expectedRulesMap)) + for _, r := range rules { + require.Equal(t, expectedRulesMap[r.Title].Labels[ualert.ContactLabel], r.Labels[ualert.ContactLabel]) + } + } + }) + + t.Run("when folder is missing put alert in General folder", func(t *testing.T) { + o := createOrg(t, 1) + folder1 := createDashboard(t, 1, o.ID, "folder-1") + folder1.IsFolder = true + dash1 := createDashboard(t, 3, o.ID, "dash1") + dash1.FolderID = folder1.ID + dash2 := createDashboard(t, 4, o.ID, "dash2") + dash2.FolderID = 22 // missing folder + + a1 := createAlert(t, o.ID, dash1.ID, int64(1), "alert-1", []string{}) + a2 := createAlert(t, o.ID, dash2.ID, int64(1), "alert-2", []string{}) + + _, err := x.Insert(o, folder1, dash1, dash2, a1, a2) + require.NoError(t, err) + + runDashAlertMigrationTestRun(t, x) + + rules := getAlertRules(t, x, o.ID) + require.Len(t, rules, 2) + + var generalFolder dashboards.Dashboard + _, err = x.Table(&dashboards.Dashboard{}).Where("title = ? AND org_id = ?", ualert.GENERAL_FOLDER, o.ID).Get(&generalFolder) + require.NoError(t, err) + + require.NotNil(t, generalFolder) + + for _, rule := range rules { + var expectedFolder dashboards.Dashboard + if rule.Title == a1.Name { + expectedFolder = *folder1 + } else { + expectedFolder = generalFolder + } + require.Equal(t, expectedFolder.UID, rule.NamespaceUID) + } + }) +} + +const ( + emailSettings = `{"addresses": "test"}` + slackSettings = `{"recipient": "test", "token": "test"}` + opsgenieSettings = `{"apiKey": "test"}` +) + +// setupTestDB prepares the sqlite database and runs OSS migrations to initialize the schemas. +func setupTestDB(t *testing.T) *xorm.Engine { + t.Helper() + testDB := sqlutil.SQLite3TestDB() + + x, err := xorm.NewEngine(testDB.DriverName, testDB.ConnStr) + require.NoError(t, err) + + err = migrator.NewDialect(x.DriverName()).CleanDB(x) + require.NoError(t, err) + + mg := migrator.NewMigrator(x, &setting.Cfg{Raw: ini.Empty()}) + migrations := &migrations.OSSMigrations{} + migrations.AddMigration(mg) + + err = mg.Start(false, 0) + require.NoError(t, err) + + return x +} + +var ( + now = time.Now() +) + +// createAlertNotificationWithReminder creates a legacy alert notification channel for inserting into the test database. +func createAlertNotificationWithReminder(t *testing.T, orgId int64, uid string, channelType string, settings string, defaultChannel bool, sendReminder bool, frequency time.Duration) *models.AlertNotification { + t.Helper() + settingsJson := simplejson.New() + if settings != "" { + s, err := simplejson.NewJson([]byte(settings)) + if err != nil { + t.Fatalf("Failed to unmarshal alert notification json: %v", err) + } + settingsJson = s + } + + return &models.AlertNotification{ + OrgID: orgId, + UID: uid, + Name: uid, // Same as uid to make testing easier. + Type: channelType, + DisableResolveMessage: false, + IsDefault: defaultChannel, + Settings: settingsJson, + SecureSettings: make(map[string][]byte), + Created: now, + Updated: now, + SendReminder: sendReminder, + Frequency: frequency, + } +} + +// createAlertNotification creates a legacy alert notification channel for inserting into the test database. +func createAlertNotification(t *testing.T, orgId int64, uid string, channelType string, settings string, defaultChannel bool) *models.AlertNotification { + return createAlertNotificationWithReminder(t, orgId, uid, channelType, settings, defaultChannel, false, time.Duration(0)) +} + +// createAlert creates a legacy alert rule for inserting into the test database. +func createAlert(t *testing.T, orgId int64, dashboardId int64, panelsId int64, name string, notifierUids []string) *models.Alert { + t.Helper() + + var settings = simplejson.New() + if len(notifierUids) != 0 { + notifiers := make([]interface{}, 0) + for _, n := range notifierUids { + notifiers = append(notifiers, struct { + Uid string + }{Uid: n}) + } + + settings.Set("notifications", notifiers) + } + + return &models.Alert{ + OrgID: orgId, + DashboardID: dashboardId, + PanelID: panelsId, + Name: name, + Message: "message", + Frequency: int64(60), + For: time.Duration(time.Duration(60).Seconds()), + State: models.AlertStateOK, + Settings: settings, + NewStateDate: now, + Created: now, + Updated: now, + } +} + +// createDashboard creates a dashboard for inserting into the test database. +func createDashboard(t *testing.T, id int64, orgId int64, uid string) *dashboards.Dashboard { + t.Helper() + return &dashboards.Dashboard{ + ID: id, + OrgID: orgId, + UID: uid, + Created: now, + Updated: now, + Title: uid, // Not tested, needed to satisfy contraint. + } +} + +// createDatasource creates a ddatasource for inserting into the test database. +func createDatasource(t *testing.T, id int64, orgId int64, uid string) *datasources.DataSource { + t.Helper() + return &datasources.DataSource{ + ID: id, + OrgID: orgId, + UID: uid, + Created: now, + Updated: now, + Name: uid, // Not tested, needed to satisfy contraint. + } +} + +func createOrg(t *testing.T, id int64) *org.Org { + t.Helper() + return &org.Org{ + ID: id, + Version: 1, + Name: fmt.Sprintf("org_%d", id), + Created: time.Now(), + Updated: time.Now(), + } +} + +// teardown cleans the input tables between test cases. +func teardown(t *testing.T, x *xorm.Engine) { + _, err := x.Exec("DELETE from org") + require.NoError(t, err) + _, err = x.Exec("DELETE from alert") + require.NoError(t, err) + _, err = x.Exec("DELETE from alert_notification") + require.NoError(t, err) + _, err = x.Exec("DELETE from dashboard") + require.NoError(t, err) + _, err = x.Exec("DELETE from data_source") + require.NoError(t, err) +} + +// setupDashAlertMigrationTestRun runs DashAlertMigration for a new test run. +func runDashAlertMigrationTestRun(t *testing.T, x *xorm.Engine) { + _, errDeleteMig := x.Exec("DELETE FROM migration_log WHERE migration_id = ?", ualert.MigTitle) + require.NoError(t, errDeleteMig) + + alertMigrator := migrator.NewMigrator(x, &setting.Cfg{}) + alertMigrator.AddMigration(ualert.RmMigTitle, &ualert.RmMigration{}) + ualert.AddDashAlertMigration(alertMigrator) + + errRunningMig := alertMigrator.Start(false, 0) + require.NoError(t, errRunningMig) +} + +// setupLegacyAlertsTables inserts data into the legacy alerting tables that is needed for testing the migration. +func setupLegacyAlertsTables(t *testing.T, x *xorm.Engine, legacyChannels []*models.AlertNotification, alerts []*models.Alert) { + t.Helper() + + orgs := []org.Org{ + *createOrg(t, 1), + *createOrg(t, 2), + } + + // Setup dashboards. + dashboards := []dashboards.Dashboard{ + *createDashboard(t, 1, 1, "dash1-1"), + *createDashboard(t, 2, 1, "dash2-1"), + *createDashboard(t, 3, 2, "dash3-2"), + *createDashboard(t, 4, 2, "dash4-2"), + } + _, errDashboards := x.Insert(dashboards) + require.NoError(t, errDashboards) + + // Setup data_sources. + dataSources := []datasources.DataSource{ + *createDatasource(t, 1, 1, "ds1-1"), + *createDatasource(t, 2, 1, "ds2-1"), + *createDatasource(t, 3, 2, "ds3-2"), + *createDatasource(t, 4, 2, "ds4-2"), + } + + _, errOrgs := x.Insert(orgs) + require.NoError(t, errOrgs) + + _, errDataSourcess := x.Insert(dataSources) + require.NoError(t, errDataSourcess) + + if len(legacyChannels) > 0 { + _, channelErr := x.Insert(legacyChannels) + require.NoError(t, channelErr) + } + + if len(alerts) > 0 { + _, alertErr := x.Insert(alerts) + require.NoError(t, alertErr) + } +} + +// getAlertmanagerConfig retreives the Alertmanager Config from the database for a given orgId. +func getAlertmanagerConfig(t *testing.T, x *xorm.Engine, orgId int64) *ualert.PostableUserConfig { + amConfig := "" + _, err := x.Table("alert_configuration").Where("org_id = ?", orgId).Cols("alertmanager_configuration").Get(&amConfig) + require.NoError(t, err) + + config := ualert.PostableUserConfig{} + err = json.Unmarshal([]byte(amConfig), &config) + require.NoError(t, err) + return &config +} + +// getAlertmanagerConfig retreives the Alertmanager Config from the database for a given orgId. +func getAlertRules(t *testing.T, x *xorm.Engine, orgId int64) []*ngModels.AlertRule { + rules := make([]*ngModels.AlertRule, 0) + err := x.Table("alert_rule").Where("org_id = ?", orgId).Find(&rules) + require.NoError(t, err) + + return rules +} + +func boolPointer(b bool) *bool { + return &b +} + +func durationPointer(d model.Duration) *model.Duration { + return &d +} diff --git a/pkg/services/sqlstore/migrations/ualert/permissions.go b/pkg/services/sqlstore/migrations/ualert/permissions.go new file mode 100644 index 00000000000..49cad344c7d --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/permissions.go @@ -0,0 +1,284 @@ +package ualert + +import ( + "fmt" + "time" + + "xorm.io/xorm" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/metrics" + "github.com/grafana/grafana/pkg/services/dashboards" + dashver "github.com/grafana/grafana/pkg/services/dashboardversion" + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/util" +) + +type roleType string + +const ( + RoleNone roleType = "None" + RoleViewer roleType = "Viewer" + RoleEditor roleType = "Editor" + RoleAdmin roleType = "Admin" +) + +func (r roleType) IsValid() bool { + return r == RoleViewer || r == RoleAdmin || r == RoleEditor || r == RoleNone +} + +type permissionType int + +type dashboardACL struct { + // nolint:stylecheck + Id int64 + OrgID int64 `xorm:"org_id"` + DashboardID int64 `xorm:"dashboard_id"` + + UserID int64 `xorm:"user_id"` + TeamID int64 `xorm:"team_id"` + Role *roleType // pointer to be nullable + Permission permissionType + + Created time.Time + Updated time.Time +} + +func (p dashboardACL) TableName() string { return "dashboard_acl" } + +type folderHelper struct { + sess *xorm.Session + mg *migrator.Migrator +} + +// getOrCreateGeneralFolder returns the general folder under the specific organisation +// If the general folder does not exist it creates it. +func (m *folderHelper) getOrCreateGeneralFolder(orgID int64) (*dashboard, error) { + // there is a unique constraint on org_id, folder_id, title + // there are no nested folders so the parent folder id is always 0 + dashboard := dashboard{OrgId: orgID, FolderId: 0, Title: GENERAL_FOLDER} + has, err := m.sess.Get(&dashboard) + if err != nil { + return nil, err + } else if !has { + // create folder + return m.createGeneralFolder(orgID) + } + return &dashboard, nil +} + +func (m *folderHelper) createGeneralFolder(orgID int64) (*dashboard, error) { + return m.createFolder(orgID, GENERAL_FOLDER) +} + +// returns the folder of the given dashboard (if exists) +func (m *folderHelper) getFolder(dash dashboard, da dashAlert) (dashboard, error) { + // get folder if exists + folder := dashboard{} + if dash.FolderId > 0 { + exists, err := m.sess.Where("id=?", dash.FolderId).Get(&folder) + if err != nil { + return folder, fmt.Errorf("failed to get folder %d: %w", dash.FolderId, err) + } + if !exists { + return folder, fmt.Errorf("folder with id %v not found", dash.FolderId) + } + if !folder.IsFolder { + return folder, fmt.Errorf("id %v is a dashboard not a folder", dash.FolderId) + } + } + return folder, nil +} + +// based on sqlstore.saveDashboard() +// it should be called from inside a transaction +func (m *folderHelper) createFolder(orgID int64, title string) (*dashboard, error) { + cmd := saveFolderCommand{ + OrgId: orgID, + FolderId: 0, + IsFolder: true, + Dashboard: simplejson.NewFromAny(map[string]any{ + "title": title, + }), + } + dash := cmd.getDashboardModel() + dash.setUid(util.GenerateShortUID()) + + parentVersion := dash.Version + dash.setVersion(1) + dash.Created = time.Now() + dash.CreatedBy = FOLDER_CREATED_BY + dash.Updated = time.Now() + dash.UpdatedBy = FOLDER_CREATED_BY + metrics.MApiDashboardInsert.Inc() + + if _, err := m.sess.Insert(dash); err != nil { + return nil, err + } + + dashVersion := &dashver.DashboardVersion{ + DashboardID: dash.Id, + ParentVersion: parentVersion, + RestoredFrom: cmd.RestoredFrom, + Version: dash.Version, + Created: time.Now(), + CreatedBy: dash.UpdatedBy, + Message: cmd.Message, + Data: dash.Data, + } + + // insert version entry + if _, err := m.sess.Insert(dashVersion); err != nil { + return nil, err + } + return dash, nil +} + +// based on SQLStore.UpdateDashboardACL() +// it should be called from inside a transaction +func (m *folderHelper) setACL(orgID int64, dashboardID int64, items []*dashboardACL) error { + if dashboardID <= 0 { + return fmt.Errorf("folder id must be greater than zero for a folder permission") + } + + // userPermissionsMap is a map keeping the highest permission per user + // for handling conficting inherited (folder) and non-inherited (dashboard) user permissions + userPermissionsMap := make(map[int64]*dashboardACL, len(items)) + // teamPermissionsMap is a map keeping the highest permission per team + // for handling conficting inherited (folder) and non-inherited (dashboard) team permissions + teamPermissionsMap := make(map[int64]*dashboardACL, len(items)) + for _, item := range items { + if item.UserID != 0 { + acl, ok := userPermissionsMap[item.UserID] + if !ok { + userPermissionsMap[item.UserID] = item + } else { + if item.Permission > acl.Permission { + // the higher permission wins + userPermissionsMap[item.UserID] = item + } + } + } + + if item.TeamID != 0 { + acl, ok := teamPermissionsMap[item.TeamID] + if !ok { + teamPermissionsMap[item.TeamID] = item + } else { + if item.Permission > acl.Permission { + // the higher permission wins + teamPermissionsMap[item.TeamID] = item + } + } + } + } + + type keyType struct { + UserID int64 `xorm:"user_id"` + TeamID int64 `xorm:"team_id"` + Role roleType + Permission permissionType + } + // seen keeps track of inserted perrmissions to avoid duplicates (due to inheritance) + seen := make(map[keyType]struct{}, len(items)) + for _, item := range items { + if item.UserID == 0 && item.TeamID == 0 && (item.Role == nil || !item.Role.IsValid()) { + return dashboards.ErrDashboardACLInfoMissing + } + + // ignore duplicate user permissions + if item.UserID != 0 { + acl, ok := userPermissionsMap[item.UserID] + if ok { + if acl.Id != item.Id { + continue + } + } + } + + // ignore duplicate team permissions + if item.TeamID != 0 { + acl, ok := teamPermissionsMap[item.TeamID] + if ok { + if acl.Id != item.Id { + continue + } + } + } + + key := keyType{UserID: item.UserID, TeamID: item.TeamID, Role: "", Permission: item.Permission} + if item.Role != nil { + key.Role = *item.Role + } + if _, ok := seen[key]; ok { + continue + } + + // unset Id so that the new record will get a different one + item.Id = 0 + item.OrgID = orgID + item.DashboardID = dashboardID + item.Created = time.Now() + item.Updated = time.Now() + + m.sess.Nullable("user_id", "team_id") + if _, err := m.sess.Insert(item); err != nil { + return err + } + seen[key] = struct{}{} + } + + // Update dashboard HasACL flag + dashboard := dashboards.Dashboard{HasACL: true} + _, err := m.sess.Cols("has_acl").Where("id=?", dashboardID).Update(&dashboard) + return err +} + +// based on SQLStore.GetDashboardACLInfoList() +func (m *folderHelper) getACL(orgID, dashboardID int64) ([]*dashboardACL, error) { + var err error + + falseStr := m.mg.Dialect.BooleanStr(false) + + result := make([]*dashboardACL, 0) + rawSQL := ` + -- get distinct permissions for the dashboard and its parent folder + SELECT DISTINCT + da.id, + da.user_id, + da.team_id, + da.permission, + da.role + FROM dashboard as d + LEFT JOIN dashboard folder on folder.id = d.folder_id + LEFT JOIN dashboard_acl AS da ON + da.dashboard_id = d.id OR + da.dashboard_id = d.folder_id OR + ( + -- include default permissions -- + da.org_id = -1 AND ( + (folder.id IS NOT NULL AND folder.has_acl = ` + falseStr + `) OR + (folder.id IS NULL AND d.has_acl = ` + falseStr + `) + ) + ) + WHERE d.org_id = ? AND d.id = ? AND da.id IS NOT NULL + ORDER BY da.id ASC + ` + err = m.sess.SQL(rawSQL, orgID, dashboardID).Find(&result) + return result, err +} + +// getOrgsThatHaveFolders returns a unique list of organization ID that have at least one folder +func (m *folderHelper) getOrgsIDThatHaveFolders() (map[int64]struct{}, error) { + // get folder if exists + var rows []int64 + err := m.sess.Table(&dashboard{}).Where("is_folder=?", true).Distinct("org_id").Find(&rows) + if err != nil { + return nil, err + } + result := make(map[int64]struct{}, len(rows)) + for _, s := range rows { + result[s] = struct{}{} + } + return result, nil +} diff --git a/pkg/services/sqlstore/migrations/ualert/rule_group_index_migration.go b/pkg/services/sqlstore/migrations/ualert/rule_group_index_migration.go deleted file mode 100644 index 611bf66e290..00000000000 --- a/pkg/services/sqlstore/migrations/ualert/rule_group_index_migration.go +++ /dev/null @@ -1,209 +0,0 @@ -package ualert - -import ( - "encoding/json" - "fmt" - "time" - - "xorm.io/xorm" - - ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/services/sqlstore/migrator" -) - -// UpdateRuleGroupIndexMigration updates a new field rule_group_index for alert rules that belong to a group with more than 1 alert. -func UpdateRuleGroupIndexMigration(mg *migrator.Migrator) { - mg.AddMigration("update group index for alert rules", &updateRulesOrderInGroup{}) -} - -type updateRulesOrderInGroup struct { - migrator.MigrationBase -} - -func (c updateRulesOrderInGroup) SQL(migrator.Dialect) string { - return codeMigration -} - -func (c updateRulesOrderInGroup) Exec(sess *xorm.Session, migrator *migrator.Migrator) error { - var rows []*alertRule - if err := sess.Table(alertRule{}).Asc("id").Find(&rows); err != nil { - return fmt.Errorf("failed to read the list of alert rules: %w", err) - } - - if len(rows) == 0 { - migrator.Logger.Debug("No rules to migrate.") - return nil - } - - groups := map[ngmodels.AlertRuleGroupKey][]*alertRule{} - - for _, row := range rows { - groupKey := ngmodels.AlertRuleGroupKey{ - OrgID: row.OrgID, - NamespaceUID: row.NamespaceUID, - RuleGroup: row.RuleGroup, - } - groups[groupKey] = append(groups[groupKey], row) - } - - toUpdate := make([]*alertRule, 0, len(rows)) - - for _, rules := range groups { - for i, rule := range rules { - if rule.RuleGroupIndex == i+1 { - continue - } - rule.RuleGroupIndex = i + 1 - toUpdate = append(toUpdate, rule) - } - } - - if len(toUpdate) == 0 { - migrator.Logger.Debug("No rules to upgrade group index") - return nil - } - - updated := time.Now() - versions := make([]interface{}, 0, len(toUpdate)) - - for _, rule := range toUpdate { - rule.Updated = updated - version := rule.makeVersion() - version.Version = rule.Version + 1 - version.ParentVersion = rule.Version - rule.Version++ - _, err := sess.ID(rule.ID).Cols("version", "updated", "rule_group_idx").Update(rule) - if err != nil { - migrator.Logger.Error("failed to update alert rule", "uid", rule.UID, "err", err) - return fmt.Errorf("unable to update alert rules with group index: %w", err) - } - migrator.Logger.Debug("updated group index for alert rule", "rule_uid", rule.UID) - versions = append(versions, version) - } - _, err := sess.Insert(versions...) - if err != nil { - migrator.Logger.Error("failed to insert changes to alert_rule_version", "err", err) - return fmt.Errorf("unable to update alert rules with group index: %w", err) - } - return nil -} - -type alertRule struct { - ID int64 `xorm:"pk autoincr 'id'"` - OrgID int64 `xorm:"org_id"` - Title string - Condition string - Data []alertQuery - IntervalSeconds int64 - Version int64 - UID string `xorm:"uid"` - NamespaceUID string `xorm:"namespace_uid"` - RuleGroup string - RuleGroupIndex int `xorm:"rule_group_idx"` - NoDataState string - ExecErrState string - For duration - Updated time.Time - Annotations map[string]string - Labels map[string]string - IsPaused bool -} - -type alertRuleVersion struct { - RuleOrgID int64 `xorm:"rule_org_id"` - RuleUID string `xorm:"rule_uid"` - RuleNamespaceUID string `xorm:"rule_namespace_uid"` - RuleGroup string - RuleGroupIndex int `xorm:"rule_group_idx"` - ParentVersion int64 - RestoredFrom int64 - Version int64 - - Created time.Time - Title string - Condition string - Data []alertQuery - IntervalSeconds int64 - NoDataState string - ExecErrState string - // ideally this field should have been apimodels.ApiDuration - // but this is currently not possible because of circular dependencies - For duration - Annotations map[string]string - Labels map[string]string - IsPaused bool -} - -func (a *alertRule) makeVersion() *alertRuleVersion { - return &alertRuleVersion{ - RuleOrgID: a.OrgID, - RuleUID: a.UID, - RuleNamespaceUID: a.NamespaceUID, - RuleGroup: a.RuleGroup, - RuleGroupIndex: a.RuleGroupIndex, - ParentVersion: 0, - RestoredFrom: 0, - Version: 1, - - Created: time.Now().UTC(), - Title: a.Title, - Condition: a.Condition, - Data: a.Data, - IntervalSeconds: a.IntervalSeconds, - NoDataState: a.NoDataState, - ExecErrState: a.ExecErrState, - For: a.For, - Annotations: a.Annotations, - Labels: map[string]string{}, - IsPaused: a.IsPaused, - } -} - -type alertQuery struct { - // RefID is the unique identifier of the query, set by the frontend call. - RefID string `json:"refId"` - - // QueryType is an optional identifier for the type of query. - // It can be used to distinguish different types of queries. - QueryType string `json:"queryType"` - - // RelativeTimeRange is the relative Start and End of the query as sent by the frontend. - RelativeTimeRange relativeTimeRange `json:"relativeTimeRange"` - - DatasourceUID string `json:"datasourceUid"` - - // JSON is the raw JSON query and includes the above properties as well as custom properties. - Model json.RawMessage `json:"model"` -} - -// RelativeTimeRange is the per query start and end time -// for requests. -type relativeTimeRange struct { - From duration `json:"from"` - To duration `json:"to"` -} - -// duration is a type used for marshalling durations. -type duration time.Duration - -func (d duration) String() string { - return time.Duration(d).String() -} - -func (d duration) MarshalJSON() ([]byte, error) { - return json.Marshal(time.Duration(d).Seconds()) -} - -func (d *duration) UnmarshalJSON(b []byte) error { - var v interface{} - if err := json.Unmarshal(b, &v); err != nil { - return err - } - switch value := v.(type) { - case float64: - *d = duration(time.Duration(value) * time.Second) - return nil - default: - return fmt.Errorf("invalid duration %v", v) - } -} diff --git a/pkg/services/sqlstore/migrations/ualert/securejsondata.go b/pkg/services/sqlstore/migrations/ualert/securejsondata.go new file mode 100644 index 00000000000..20279f10e33 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/securejsondata.go @@ -0,0 +1,60 @@ +package ualert + +import ( + "os" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" +) + +// SecureJsonData is used to store encrypted data (for example in data_source table). Only values are separately +// encrypted. +type SecureJsonData map[string][]byte + +var seclogger = log.New("securejsondata") + +// DecryptedValue returns single decrypted value from SecureJsonData. Similar to normal map access second return value +// is true if the key exists and false if not. +func (s SecureJsonData) DecryptedValue(key string) (string, bool) { + if value, ok := s[key]; ok { + decryptedData, err := util.Decrypt(value, setting.SecretKey) + if err != nil { + seclogger.Error(err.Error()) + os.Exit(1) + } + return string(decryptedData), true + } + return "", false +} + +// Decrypt returns map of the same type but where the all the values are decrypted. Opposite of what +// GetEncryptedJsonData is doing. +func (s SecureJsonData) Decrypt() map[string]string { + decrypted := make(map[string]string) + for key, data := range s { + decryptedData, err := util.Decrypt(data, setting.SecretKey) + if err != nil { + seclogger.Error(err.Error()) + os.Exit(1) + } + + decrypted[key] = string(decryptedData) + } + return decrypted +} + +// GetEncryptedJsonData returns map where all keys are encrypted. +func GetEncryptedJsonData(sjd map[string]string) SecureJsonData { + encrypted := make(SecureJsonData) + for key, data := range sjd { + encryptedData, err := util.Encrypt([]byte(data), setting.SecretKey) + if err != nil { + seclogger.Error(err.Error()) + os.Exit(1) + } + + encrypted[key] = encryptedData + } + return encrypted +} diff --git a/pkg/services/ngalert/migration/silences.go b/pkg/services/sqlstore/migrations/ualert/silences.go similarity index 70% rename from pkg/services/ngalert/migration/silences.go rename to pkg/services/sqlstore/migrations/ualert/silences.go index 0263605c0a8..69d1a2b320c 100644 --- a/pkg/services/ngalert/migration/silences.go +++ b/pkg/services/sqlstore/migrations/ualert/silences.go @@ -1,4 +1,4 @@ -package migration +package ualert import ( "bytes" @@ -16,7 +16,7 @@ import ( pb "github.com/prometheus/alertmanager/silence/silencepb" "github.com/prometheus/common/model" - "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" ) const ( @@ -26,10 +26,14 @@ const ( ErrorAlertName = "DatasourceError" ) -func (om *OrgMigration) addErrorSilence(rule *models.AlertRule) error { +func (m *migration) addErrorSilence(da dashAlert, rule *alertRule) error { + if da.ParsedSettings.ExecutionErrorState != "keep_state" { + return nil + } + uid, err := uuid.NewRandom() if err != nil { - return errors.New("create uuid for silence") + return errors.New("failed to create uuid for silence") } s := &pb.MeshSilence{ @@ -54,14 +58,21 @@ func (om *OrgMigration) addErrorSilence(rule *models.AlertRule) error { }, ExpiresAt: time.Now().AddDate(1, 0, 0), // 1 year } - om.silences = append(om.silences, s) + if _, ok := m.silences[da.OrgId]; !ok { + m.silences[da.OrgId] = make([]*pb.MeshSilence, 0) + } + m.silences[da.OrgId] = append(m.silences[da.OrgId], s) return nil } -func (om *OrgMigration) addNoDataSilence(rule *models.AlertRule) error { +func (m *migration) addNoDataSilence(da dashAlert, rule *alertRule) error { + if da.ParsedSettings.NoDataState != "keep_state" { + return nil + } + uid, err := uuid.NewRandom() if err != nil { - return errors.New("create uuid for silence") + return errors.New("failed to create uuid for silence") } s := &pb.MeshSilence{ @@ -86,20 +97,28 @@ func (om *OrgMigration) addNoDataSilence(rule *models.AlertRule) error { }, ExpiresAt: time.Now().AddDate(1, 0, 0), // 1 year. } - om.silences = append(om.silences, s) + _, ok := m.silences[da.OrgId] + if !ok { + m.silences[da.OrgId] = make([]*pb.MeshSilence, 0) + } + m.silences[da.OrgId] = append(m.silences[da.OrgId], s) return nil } -func (om *OrgMigration) writeSilencesFile() error { +func (m *migration) writeSilencesFile(orgID int64) error { var buf bytes.Buffer - om.log.Debug("Writing silences file", "silences", len(om.silences)) - for _, e := range om.silences { + orgSilences, ok := m.silences[orgID] + if !ok { + return nil + } + + for _, e := range orgSilences { if _, err := pbutil.WriteDelimited(&buf, e); err != nil { return err } } - f, err := openReplace(silencesFileNameForOrg(om.cfg.DataPath, om.orgID)) + f, err := openReplace(silencesFileNameForOrg(m.mg, orgID)) if err != nil { return err } @@ -111,8 +130,12 @@ func (om *OrgMigration) writeSilencesFile() error { return f.Close() } -func silencesFileNameForOrg(dataPath string, orgID int64) string { - return filepath.Join(dataPath, "alerting", strconv.Itoa(int(orgID)), "silences") +func getSilenceFileNamesForAllOrgs(mg *migrator.Migrator) ([]string, error) { + return filepath.Glob(filepath.Join(mg.Cfg.DataPath, "alerting", "*", "silences")) +} + +func silencesFileNameForOrg(mg *migrator.Migrator, orgID int64) string { + return filepath.Join(mg.Cfg.DataPath, "alerting", strconv.Itoa(int(orgID)), "silences") } // replaceFile wraps a file that is moved to another filename on closing. diff --git a/pkg/services/sqlstore/migrations/ualert/tables.go b/pkg/services/sqlstore/migrations/ualert/tables.go index b6babe3409d..78add4d1ce5 100644 --- a/pkg/services/sqlstore/migrations/ualert/tables.go +++ b/pkg/services/sqlstore/migrations/ualert/tables.go @@ -492,6 +492,9 @@ func addAlertImageMigrations(mg *migrator.Migrator) { } func extractAlertmanagerConfigurationHistoryMigration(mg *migrator.Migrator) { + if !mg.Cfg.UnifiedAlerting.IsEnabled() { + return + } // Since it's not always consistent as to what state the org ID indexes are in, just drop them all and rebuild from scratch. // This is not expensive since this table is guaranteed to have a small number of rows. mg.AddMigration("drop non-unique orgID index on alert_configuration", migrator.NewDropIndexMigration(migrator.Table{Name: "alert_configuration"}, &migrator.Index{Cols: []string{"org_id"}})) diff --git a/pkg/services/ngalert/migration/template.go b/pkg/services/sqlstore/migrations/ualert/template.go similarity index 99% rename from pkg/services/ngalert/migration/template.go rename to pkg/services/sqlstore/migrations/ualert/template.go index fb218d582c9..08eb3dfbc43 100644 --- a/pkg/services/ngalert/migration/template.go +++ b/pkg/services/sqlstore/migrations/ualert/template.go @@ -1,7 +1,7 @@ // This file contains code that parses templates from old alerting into a sequence // of tokens. Each token can be either a string literal or a variable. -package migration +package ualert import ( "bytes" diff --git a/pkg/services/ngalert/migration/template_test.go b/pkg/services/sqlstore/migrations/ualert/template_test.go similarity index 99% rename from pkg/services/ngalert/migration/template_test.go rename to pkg/services/sqlstore/migrations/ualert/template_test.go index 88786af5e46..f95fbc757f3 100644 --- a/pkg/services/ngalert/migration/template_test.go +++ b/pkg/services/sqlstore/migrations/ualert/template_test.go @@ -1,12 +1,11 @@ -package migration +package ualert import ( "fmt" "testing" - "github.com/stretchr/testify/assert" - "github.com/grafana/grafana/pkg/infra/log" + "github.com/stretchr/testify/assert" ) func TestTokenString(t *testing.T) { diff --git a/pkg/services/sqlstore/migrations/ualert/testing.go b/pkg/services/sqlstore/migrations/ualert/testing.go new file mode 100644 index 00000000000..9a74c5d04d9 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/testing.go @@ -0,0 +1,25 @@ +package ualert + +import ( + "testing" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/prometheus/alertmanager/silence/silencepb" +) + +// newTestMigration generates an empty migration to use in tests. +func newTestMigration(t *testing.T) *migration { + t.Helper() + + return &migration{ + mg: &migrator.Migrator{ + + Logger: log.New("test"), + }, + seenUIDs: uidSet{ + set: make(map[string]struct{}), + }, + silences: make(map[int64][]*silencepb.MeshSilence), + } +} diff --git a/pkg/services/sqlstore/migrations/ualert/ualert.go b/pkg/services/sqlstore/migrations/ualert/ualert.go index d59b187ff7c..65080ebe3c3 100644 --- a/pkg/services/sqlstore/migrations/ualert/ualert.go +++ b/pkg/services/sqlstore/migrations/ualert/ualert.go @@ -1,21 +1,48 @@ package ualert import ( + "context" + "encoding/json" + "errors" "fmt" "os" "path/filepath" "strconv" + "strings" + "time" + alertingNotify "github.com/grafana/alerting/notify" + pb "github.com/prometheus/alertmanager/silence/silencepb" "xorm.io/xorm" ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" ) +const GENERAL_FOLDER = "General Alerting" +const DASHBOARD_FOLDER = "%s Alerts - %s" + +// MaxFolderName is the maximum length of the folder name generated using DASHBOARD_FOLDER format +const MaxFolderName = 255 + +// FOLDER_CREATED_BY us used to track folders created by this migration +// during alert migration cleanup. +const FOLDER_CREATED_BY = -8 + +const KV_NAMESPACE = "alertmanager" + var migTitle = "move dashboard alerts to unified alerting" +var rmMigTitle = "remove unified alerting data" + +const clearMigrationEntryTitle = "clear migration entry %q" const codeMigration = "code migration" +// It is defined in pkg/expr/service.go as "DatasourceType" +const expressionDatasourceUID = "__expr__" + type MigrationError struct { AlertId int64 Err error @@ -27,16 +54,99 @@ func (e MigrationError) Error() string { func (e *MigrationError) Unwrap() error { return e.Err } -// FixEarlyMigration fixes UA configs created before 8.2 with org_id=0 and moves some files like __default__.tmpl. -// The only use of this migration is when a user enabled ng-alerting before 8.2. -func FixEarlyMigration(mg *migrator.Migrator) { +func AddDashAlertMigration(mg *migrator.Migrator) { + logs, err := mg.GetMigrationLog() + if err != nil { + mg.Logger.Error("Alert migration failure: could not get migration log", "error", err) + os.Exit(1) + } + + _, migrationRun := logs[migTitle] + + switch { + // If unified alerting is enabled and the upgrade migration has not been run + case mg.Cfg.UnifiedAlerting.IsEnabled() && !migrationRun: + // Remove the migration entry that removes all unified alerting data. This is so when the feature + // flag is removed in future the "remove unified alerting data" migration will be run again. + mg.AddMigration(fmt.Sprintf(clearMigrationEntryTitle, rmMigTitle), &clearMigrationEntry{ + migrationID: rmMigTitle, + }) + if err != nil { + mg.Logger.Error("Alert migration error: could not clear alert migration for removing data", "error", err) + } + mg.AddMigration(migTitle, &migration{ + // We deduplicate for case-insensitive matching in MySQL-compatible backend flavours because they use case-insensitive collation. + seenUIDs: uidSet{set: make(map[string]struct{}), caseInsensitive: mg.Dialect.SupportEngine()}, + silences: make(map[int64][]*pb.MeshSilence), + }) + // If unified alerting is disabled and upgrade migration has been run + case !mg.Cfg.UnifiedAlerting.IsEnabled() && migrationRun: + // If legacy alerting is also disabled, there is nothing to do + if setting.AlertingEnabled != nil && !*setting.AlertingEnabled { + return + } + + // Safeguard to prevent data loss when migrating from UA to LA + if !mg.Cfg.ForceMigration { + panic("Grafana has already been migrated to Unified Alerting.\nAny alert rules created while using Unified Alerting will be deleted by rolling back.\n\nSet force_migration=true in your grafana.ini and restart Grafana to roll back and delete Unified Alerting configuration data.") + } + // Remove the migration entry that creates unified alerting data. This is so when the feature + // flag is enabled in the future the migration "move dashboard alerts to unified alerting" will be run again. + mg.AddMigration(fmt.Sprintf(clearMigrationEntryTitle, migTitle), &clearMigrationEntry{ + migrationID: migTitle, + }) + if err != nil { + mg.Logger.Error("Alert migration error: could not clear dashboard alert migration", "error", err) + } + mg.AddMigration(rmMigTitle, &rmMigration{}) + } +} + +// RerunDashAlertMigration force the dashboard alert migration to run +// to make sure that the Alertmanager configurations will be created for each organisation +func RerunDashAlertMigration(mg *migrator.Migrator) { + logs, err := mg.GetMigrationLog() + if err != nil { + mg.Logger.Error("Alert migration failure: could not get migration log", "error", err) + os.Exit(1) + } + cloneMigTitle := fmt.Sprintf("clone %s", migTitle) - mg.AddMigration(cloneMigTitle, &upgradeNgAlerting{}) + + _, migrationRun := logs[cloneMigTitle] + ngEnabled := mg.Cfg.UnifiedAlerting.IsEnabled() + + switch { + case ngEnabled && !migrationRun: + // The only use of this migration is when a user enabled ng-alerting before 8.2. + mg.AddMigration(cloneMigTitle, &upgradeNgAlerting{}) + // if user disables the feature flag and enables it back. + // This migration does not need to be run because the original migration AddDashAlertMigration does what's needed + } } func AddDashboardUIDPanelIDMigration(mg *migrator.Migrator) { + logs, err := mg.GetMigrationLog() + if err != nil { + mg.Logger.Error("Alert migration failure: could not get migration log", "error", err) + os.Exit(1) + } + migrationID := "update dashboard_uid and panel_id from existing annotations" - mg.AddMigration(migrationID, &updateDashboardUIDPanelIDMigration{}) + _, migrationRun := logs[migrationID] + ngEnabled := mg.Cfg.UnifiedAlerting.IsEnabled() + undoMigrationID := "undo " + migrationID + + if ngEnabled && !migrationRun { + // If ngalert is enabled and the migration has not been run then run it. + mg.AddMigration(migrationID, &updateDashboardUIDPanelIDMigration{}) + } else if !ngEnabled && migrationRun { + // If ngalert is disabled and the migration has been run then remove it + // from the migration log so it will run if ngalert is re-enabled. + mg.AddMigration(undoMigrationID, &clearMigrationEntry{ + migrationID: migrationID, + }) + } } // updateDashboardUIDPanelIDMigration sets the dashboard_uid and panel_id columns @@ -88,6 +198,327 @@ func (m *updateDashboardUIDPanelIDMigration) Exec(sess *xorm.Session, mg *migrat return nil } +// clearMigrationEntry removes an entry fromt the migration_log table. +// This migration is not recorded in the migration_log so that it can re-run several times. +type clearMigrationEntry struct { + migrator.MigrationBase + + migrationID string +} + +func (m *clearMigrationEntry) SQL(dialect migrator.Dialect) string { + return "clear migration entry code migration" +} + +func (m *clearMigrationEntry) Exec(sess *xorm.Session, mg *migrator.Migrator) error { + _, err := sess.SQL(`DELETE from migration_log where migration_id = ?`, m.migrationID).Query() + if err != nil { + return fmt.Errorf("failed to clear migration entry %v: %w", m.migrationID, err) + } + return nil +} + +func (m *clearMigrationEntry) SkipMigrationLog() bool { + return true +} + +type migration struct { + migrator.MigrationBase + // session and mg are attached for convenience. + sess *xorm.Session + mg *migrator.Migrator + + seenUIDs uidSet + silences map[int64][]*pb.MeshSilence +} + +func (m *migration) SQL(dialect migrator.Dialect) string { + return codeMigration +} + +//nolint:gocyclo +func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { + m.sess = sess + m.mg = mg + + dashAlerts, err := m.slurpDashAlerts() + if err != nil { + return err + } + mg.Logger.Info("Alerts found to migrate", "alerts", len(dashAlerts)) + + // [orgID, dataSourceId] -> UID + dsIDMap, err := m.slurpDSIDs() + if err != nil { + return err + } + + // [orgID, dashboardId] -> dashUID + dashIDMap, err := m.slurpDashUIDs() + if err != nil { + return err + } + + // cache for folders created for dashboards that have custom permissions + folderCache := make(map[string]*dashboard) + // cache for the general folders + generalFolderCache := make(map[int64]*dashboard) + + folderHelper := folderHelper{ + sess: sess, + mg: mg, + } + + gf := func(dash dashboard, da dashAlert) (*dashboard, error) { + f, ok := generalFolderCache[dash.OrgId] + if !ok { + // get or create general folder + f, err = folderHelper.getOrCreateGeneralFolder(dash.OrgId) + if err != nil { + return nil, MigrationError{ + Err: fmt.Errorf("failed to get or create general folder under organisation %d: %w", dash.OrgId, err), + AlertId: da.Id, + } + } + generalFolderCache[dash.OrgId] = f + } + // No need to assign default permissions to general folder + // because they are included to the query result if it's a folder with no permissions + // https://github.com/grafana/grafana/blob/076e2ce06a6ecf15804423fcc8dca1b620a321e5/pkg/services/sqlstore/dashboard_acl.go#L109 + return f, nil + } + + // Per org map of newly created rules to which notification channels it should send to. + rulesPerOrg := make(map[int64]map[*alertRule][]uidOrID) + + for _, da := range dashAlerts { + l := mg.Logger.New("ruleID", da.Id, "ruleName", da.Name, "dashboardUID", da.DashboardUID, "orgID", da.OrgId) + l.Debug("Migrating alert rule to Unified Alerting") + newCond, err := transConditions(*da.ParsedSettings, da.OrgId, dsIDMap) + if err != nil { + return err + } + + da.DashboardUID = dashIDMap[[2]int64{da.OrgId, da.DashboardId}] + + // get dashboard + dash := dashboard{} + exists, err := m.sess.Where("org_id=? AND uid=?", da.OrgId, da.DashboardUID).Get(&dash) + if err != nil { + return MigrationError{ + Err: fmt.Errorf("failed to get dashboard %s under organisation %d: %w", da.DashboardUID, da.OrgId, err), + AlertId: da.Id, + } + } + if !exists { + return MigrationError{ + Err: fmt.Errorf("dashboard with UID %v under organisation %d not found: %w", da.DashboardUID, da.OrgId, err), + AlertId: da.Id, + } + } + + var folder *dashboard + switch { + case dash.HasACL: + folderName := getAlertFolderNameFromDashboard(&dash) + f, ok := folderCache[folderName] + if !ok { + l.Info("Create a new folder for alerts that belongs to dashboard because it has custom permissions", "folder", folderName) + // create folder and assign the permissions of the dashboard (included default and inherited) + f, err = folderHelper.createFolder(dash.OrgId, folderName) + if err != nil { + return MigrationError{ + Err: fmt.Errorf("failed to create folder: %w", err), + AlertId: da.Id, + } + } + permissions, err := folderHelper.getACL(dash.OrgId, dash.Id) + if err != nil { + return MigrationError{ + Err: fmt.Errorf("failed to get dashboard %d under organisation %d permissions: %w", dash.Id, dash.OrgId, err), + AlertId: da.Id, + } + } + err = folderHelper.setACL(f.OrgId, f.Id, permissions) + if err != nil { + return MigrationError{ + Err: fmt.Errorf("failed to set folder %d under organisation %d permissions: %w", f.Id, f.OrgId, err), + AlertId: da.Id, + } + } + folderCache[folderName] = f + } + folder = f + case dash.FolderId > 0: + // get folder if exists + f, err := folderHelper.getFolder(dash, da) + if err != nil { + // If folder does not exist then the dashboard is an orphan and we migrate the alert to the general folder. + l.Warn("Failed to find folder for dashboard. Migrate rule to the default folder", "rule_name", da.Name, "dashboard_uid", da.DashboardUID, "missing_folder_id", dash.FolderId) + folder, err = gf(dash, da) + if err != nil { + return err + } + } else { + folder = &f + } + default: + folder, err = gf(dash, da) + if err != nil { + return err + } + } + + if folder.Uid == "" { + return MigrationError{ + Err: fmt.Errorf("empty folder identifier"), + AlertId: da.Id, + } + } + rule, err := m.makeAlertRule(l, *newCond, da, folder.Uid) + if err != nil { + return fmt.Errorf("failed to migrate alert rule '%s' [ID:%d, DashboardUID:%s, orgID:%d]: %w", da.Name, da.Id, da.DashboardUID, da.OrgId, err) + } + + if _, ok := rulesPerOrg[rule.OrgID]; !ok { + rulesPerOrg[rule.OrgID] = make(map[*alertRule][]uidOrID) + } + if _, ok := rulesPerOrg[rule.OrgID][rule]; !ok { + rulesPerOrg[rule.OrgID][rule] = extractChannelIDs(da) + } else { + return MigrationError{ + Err: fmt.Errorf("duplicate generated rule UID"), + AlertId: da.Id, + } + } + } + + for orgID := range rulesPerOrg { + if err := m.writeSilencesFile(orgID); err != nil { + m.mg.Logger.Error("Alert migration error: failed to write silence file", "err", err) + } + } + + amConfigPerOrg, err := m.setupAlertmanagerConfigs(rulesPerOrg) + if err != nil { + return err + } + + err = m.insertRules(mg, rulesPerOrg) + if err != nil { + return err + } + + for orgID, amConfig := range amConfigPerOrg { + if err := m.writeAlertmanagerConfig(orgID, amConfig); err != nil { + return err + } + } + + return nil +} + +func (m *migration) insertRules(mg *migrator.Migrator, rulesPerOrg map[int64]map[*alertRule][]uidOrID) error { + for _, rules := range rulesPerOrg { + for rule := range rules { + var err error + if strings.HasPrefix(mg.Dialect.DriverName(), migrator.Postgres) { + err = mg.InTransaction(func(sess *xorm.Session) error { + _, err := sess.Insert(rule) + return err + }) + } else { + _, err = m.sess.Insert(rule) + } + if err != nil { + // TODO better error handling, if constraint + rule.Title += fmt.Sprintf(" %v", rule.UID) + rule.RuleGroup += fmt.Sprintf(" %v", rule.UID) + + _, err = m.sess.Insert(rule) + if err != nil { + return err + } + } + + // create entry in alert_rule_version + _, err = m.sess.Insert(rule.makeVersion()) + if err != nil { + return err + } + } + } + return nil +} + +func (m *migration) writeAlertmanagerConfig(orgID int64, amConfig *PostableUserConfig) error { + rawAmConfig, err := json.Marshal(amConfig) + if err != nil { + return err + } + + // remove an existing configuration, which could have been left during switching back to legacy alerting + _, _ = m.sess.Delete(AlertConfiguration{OrgID: orgID}) + + // We don't need to apply the configuration, given the multi org alertmanager will do an initial sync before the server is ready. + _, err = m.sess.Insert(AlertConfiguration{ + AlertmanagerConfiguration: string(rawAmConfig), + // Since we are migration for a snapshot of the code, it is always going to migrate to + // the v1 config. + ConfigurationVersion: "v1", + OrgID: orgID, + }) + if err != nil { + return err + } + + return nil +} + +// validateAlertmanagerConfig validates the alertmanager configuration produced by the migration against the receivers. +func (m *migration) validateAlertmanagerConfig(config *PostableUserConfig) error { + for _, r := range config.AlertmanagerConfig.Receivers { + for _, gr := range r.GrafanaManagedReceivers { + data, err := gr.Settings.MarshalJSON() + if err != nil { + return err + } + var ( + cfg = &alertingNotify.GrafanaIntegrationConfig{ + UID: gr.UID, + Name: gr.Name, + Type: gr.Type, + DisableResolveMessage: gr.DisableResolveMessage, + Settings: data, + SecureSettings: gr.SecureSettings, + } + ) + + // decryptFunc represents the legacy way of decrypting data. Before the migration, we don't need any new way, + // given that the previous alerting will never support it. + decryptFunc := func(_ context.Context, sjd map[string][]byte, key string, fallback string) string { + if value, ok := sjd[key]; ok { + decryptedData, err := util.Decrypt(value, setting.SecretKey) + if err != nil { + m.mg.Logger.Warn("Unable to decrypt key '%s' for %s receiver with uid %s, returning fallback.", key, gr.Type, gr.UID) + return fallback + } + return string(decryptedData) + } + return fallback + } + _, err = alertingNotify.BuildReceiverConfiguration(context.Background(), &alertingNotify.APIReceiver{ + GrafanaIntegrations: alertingNotify.GrafanaIntegrations{Integrations: []*alertingNotify.GrafanaIntegrationConfig{cfg}}, + }, decryptFunc) + if err != nil { + return err + } + } + } + + return nil +} + type AlertConfiguration struct { ID int64 `xorm:"pk autoincr 'id'"` OrgID int64 `xorm:"org_id"` @@ -97,6 +528,84 @@ type AlertConfiguration struct { CreatedAt int64 `xorm:"created"` } +// rmMigration removes Grafana 8 alert data +type rmMigration struct { + migrator.MigrationBase +} + +func (m *rmMigration) SQL(dialect migrator.Dialect) string { + return codeMigration +} + +func (m *rmMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error { + _, err := sess.Exec("delete from alert_rule") + if err != nil { + return err + } + + _, err = sess.Exec("delete from alert_rule_version") + if err != nil { + return err + } + + _, err = sess.Exec("delete from dashboard_acl where dashboard_id IN (select id from dashboard where created_by = ?)", FOLDER_CREATED_BY) + if err != nil { + return err + } + + _, err = sess.Exec("delete from dashboard where created_by = ?", FOLDER_CREATED_BY) + if err != nil { + return err + } + + _, err = sess.Exec("delete from alert_configuration") + if err != nil { + return err + } + + _, err = sess.Exec("delete from ngalert_configuration") + if err != nil { + return err + } + + _, err = sess.Exec("delete from alert_instance") + if err != nil { + return err + } + + exists, err := sess.IsTableExist("kv_store") + if err != nil { + return err + } + + if exists { + _, err = sess.Exec("delete from kv_store where namespace = ?", KV_NAMESPACE) + if err != nil { + return err + } + } + + files, err := getSilenceFileNamesForAllOrgs(mg) + if err != nil { + return err + } + for _, f := range files { + if err := os.Remove(f); err != nil { + mg.Logger.Error("Alert migration error: failed to remove silence file", "file", f, "err", err) + } + } + + return nil +} + +// rmMigrationWithoutLogging is similar migration to rmMigration +// but is not recorded in the migration_log table so that it can rerun in the future +type rmMigrationWithoutLogging = rmMigration + +func (m *rmMigrationWithoutLogging) SkipMigrationLog() bool { + return true +} + type upgradeNgAlerting struct { migrator.MigrationBase } @@ -226,3 +735,191 @@ func (u *upgradeNgAlerting) updateAlertmanagerFiles(orgId int64, migrator *migra func (u *upgradeNgAlerting) SQL(migrator.Dialect) string { return codeMigration } + +// getAlertFolderNameFromDashboard generates a folder name for alerts that belong to a dashboard. Formats the string according to DASHBOARD_FOLDER format. +// If the resulting string exceeds the migrations.MaxTitleLength, the dashboard title is stripped to be at the maximum length +func getAlertFolderNameFromDashboard(dash *dashboard) string { + maxLen := MaxFolderName - len(fmt.Sprintf(DASHBOARD_FOLDER, "", dash.Uid)) + title := dash.Title + if len(title) > maxLen { + title = title[:maxLen] + } + return fmt.Sprintf(DASHBOARD_FOLDER, title, dash.Uid) // include UID to the name to avoid collision +} + +// CreateDefaultFoldersForAlertingMigration creates a folder dedicated for alerting if no folders exist +func CreateDefaultFoldersForAlertingMigration(mg *migrator.Migrator) { + if !mg.Cfg.UnifiedAlerting.IsEnabled() { + return + } + mg.AddMigration("create default alerting folders", &createDefaultFoldersForAlertingMigration{}) +} + +type createDefaultFoldersForAlertingMigration struct { + migrator.MigrationBase +} + +func (c createDefaultFoldersForAlertingMigration) Exec(sess *xorm.Session, migrator *migrator.Migrator) error { + helper := folderHelper{ + sess: sess, + mg: migrator, + } + + var rows []struct { + Id int64 + Name string + } + + if err := sess.Table("org").Cols("id", "name").Find(&rows); err != nil { + return fmt.Errorf("failed to read the list of organizations: %w", err) + } + + orgsWithFolders, err := helper.getOrgsIDThatHaveFolders() + if err != nil { + return fmt.Errorf("failed to list organizations that have at least one folder: %w", err) + } + + for _, row := range rows { + // if there's at least one folder in the org or if alerting is disabled for that org, skip adding the default folder + if _, ok := orgsWithFolders[row.Id]; ok { + migrator.Logger.Debug("Skip adding default alerting folder because organization already has at least one folder", "org_id", row.Id) + continue + } + if _, ok := migrator.Cfg.UnifiedAlerting.DisabledOrgs[row.Id]; ok { + migrator.Logger.Debug("Skip adding default alerting folder because alerting is disabled for the organization ", "org_id", row.Id) + continue + } + folder, err := helper.createGeneralFolder(row.Id) + if err != nil { + return fmt.Errorf("failed to create the default alerting folder for organization %s (ID: %d): %w", row.Name, row.Id, err) + } + migrator.Logger.Info("Created the default folder for alerting", "org_id", row.Id, "folder_name", folder.Title, "folder_uid", folder.Uid) + } + return nil +} + +func (c createDefaultFoldersForAlertingMigration) SQL(migrator.Dialect) string { + return codeMigration +} + +// UpdateRuleGroupIndexMigration updates a new field rule_group_index for alert rules that belong to a group with more than 1 alert. +func UpdateRuleGroupIndexMigration(mg *migrator.Migrator) { + if !mg.Cfg.UnifiedAlerting.IsEnabled() { + return + } + mg.AddMigration("update group index for alert rules", &updateRulesOrderInGroup{}) +} + +type updateRulesOrderInGroup struct { + migrator.MigrationBase +} + +func (c updateRulesOrderInGroup) SQL(migrator.Dialect) string { + return codeMigration +} + +func (c updateRulesOrderInGroup) Exec(sess *xorm.Session, migrator *migrator.Migrator) error { + var rows []*alertRule + if err := sess.Table(alertRule{}).Asc("id").Find(&rows); err != nil { + return fmt.Errorf("failed to read the list of alert rules: %w", err) + } + + if len(rows) == 0 { + migrator.Logger.Debug("No rules to migrate.") + return nil + } + + groups := map[ngmodels.AlertRuleGroupKey][]*alertRule{} + + for _, row := range rows { + groupKey := ngmodels.AlertRuleGroupKey{ + OrgID: row.OrgID, + NamespaceUID: row.NamespaceUID, + RuleGroup: row.RuleGroup, + } + groups[groupKey] = append(groups[groupKey], row) + } + + toUpdate := make([]*alertRule, 0, len(rows)) + + for _, rules := range groups { + for i, rule := range rules { + if rule.RuleGroupIndex == i+1 { + continue + } + rule.RuleGroupIndex = i + 1 + toUpdate = append(toUpdate, rule) + } + } + + if len(toUpdate) == 0 { + migrator.Logger.Debug("No rules to upgrade group index") + return nil + } + + updated := time.Now() + versions := make([]any, 0, len(toUpdate)) + + for _, rule := range toUpdate { + rule.Updated = updated + version := rule.makeVersion() + version.Version = rule.Version + 1 + version.ParentVersion = rule.Version + rule.Version++ + _, err := sess.ID(rule.ID).Cols("version", "updated", "rule_group_idx").Update(rule) + if err != nil { + migrator.Logger.Error("Failed to update alert rule", "uid", rule.UID, "err", err) + return fmt.Errorf("unable to update alert rules with group index: %w", err) + } + migrator.Logger.Debug("Updated group index for alert rule", "rule_uid", rule.UID) + versions = append(versions, version) + } + _, err := sess.Insert(versions...) + if err != nil { + migrator.Logger.Error("Failed to insert changes to alert_rule_version", "err", err) + return fmt.Errorf("unable to update alert rules with group index: %w", err) + } + return nil +} + +// uidSet is a wrapper around map[string]struct{} and util.GenerateShortUID() which aims help generate uids in quick +// succession while taking into consideration case sensitivity requirements. if caseInsensitive is true, all generated +// uids must also be unique when compared in a case-insensitive manner. +type uidSet struct { + set map[string]struct{} + caseInsensitive bool +} + +// contains checks whether the given uid has already been generated in this uidSet. +func (s *uidSet) contains(uid string) bool { + dedup := uid + if s.caseInsensitive { + dedup = strings.ToLower(dedup) + } + _, seen := s.set[dedup] + return seen +} + +// add adds the given uid to the uidSet. +func (s *uidSet) add(uid string) { + dedup := uid + if s.caseInsensitive { + dedup = strings.ToLower(dedup) + } + s.set[dedup] = struct{}{} +} + +// generateUid will generate a new unique uid that is not already contained in the uidSet. +// If it fails to create one that has not already been generated it will make multiple, but not unlimited, attempts. +// If all attempts are exhausted an error will be returned. +func (s *uidSet) generateUid() (string, error) { + for i := 0; i < 5; i++ { + gen := util.GenerateShortUID() + if !s.contains(gen) { + s.add(gen) + return gen, nil + } + } + + return "", errors.New("failed to generate UID") +} diff --git a/pkg/services/ngalert/migration/ualert_test.go b/pkg/services/sqlstore/migrations/ualert/ualert_test.go similarity index 53% rename from pkg/services/ngalert/migration/ualert_test.go rename to pkg/services/sqlstore/migrations/ualert/ualert_test.go index 2f4cd49f90b..98a56c07c6f 100644 --- a/pkg/services/ngalert/migration/ualert_test.go +++ b/pkg/services/sqlstore/migrations/ualert/ualert_test.go @@ -1,33 +1,74 @@ -package migration +package ualert import ( + "encoding/base64" "encoding/json" "fmt" + "sort" "strings" "testing" + "github.com/prometheus/alertmanager/pkg/labels" "github.com/stretchr/testify/require" - "github.com/grafana/grafana/pkg/infra/db" - "github.com/grafana/grafana/pkg/services/dashboards" - apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/util" ) +var MigTitle = migTitle +var RmMigTitle = rmMigTitle +var ClearMigrationEntryTitle = clearMigrationEntryTitle + +type RmMigration = rmMigration + +// UnmarshalJSON implements the json.Unmarshaler interface for Matchers. Vendored from definitions.ObjectMatchers. +func (m *ObjectMatchers) UnmarshalJSON(data []byte) error { + var rawMatchers [][3]string + if err := json.Unmarshal(data, &rawMatchers); err != nil { + return err + } + for _, rawMatcher := range rawMatchers { + var matchType labels.MatchType + switch rawMatcher[1] { + case "=": + matchType = labels.MatchEqual + case "!=": + matchType = labels.MatchNotEqual + case "=~": + matchType = labels.MatchRegexp + case "!~": + matchType = labels.MatchNotRegexp + default: + return fmt.Errorf("unsupported match type %q in matcher", rawMatcher[1]) + } + + rawMatcher[2] = strings.TrimPrefix(rawMatcher[2], "\"") + rawMatcher[2] = strings.TrimSuffix(rawMatcher[2], "\"") + + matcher, err := labels.NewMatcher(matchType, rawMatcher[0], rawMatcher[2]) + if err != nil { + return err + } + *m = append(*m, matcher) + } + sort.Sort(labels.Matchers(*m)) + return nil +} + func Test_validateAlertmanagerConfig(t *testing.T) { tc := []struct { name string - receivers []*apimodels.PostableGrafanaReceiver + receivers []*PostableGrafanaReceiver err error }{ { name: "when a slack receiver does not have a valid URL - it should error", - receivers: []*apimodels.PostableGrafanaReceiver{ + receivers: []*PostableGrafanaReceiver{ { UID: "test-uid", Name: "SlackWithBadURL", Type: "slack", - Settings: mustRawMessage(map[string]any{}), + Settings: simplejson.NewFromAny(map[string]interface{}{}), SecureSettings: map[string]string{"url": invalidUri}, }, }, @@ -35,38 +76,36 @@ func Test_validateAlertmanagerConfig(t *testing.T) { }, { name: "when a slack receiver has an invalid recipient - it should not error", - receivers: []*apimodels.PostableGrafanaReceiver{ + receivers: []*PostableGrafanaReceiver{ { UID: util.GenerateShortUID(), Name: "SlackWithBadRecipient", Type: "slack", - Settings: mustRawMessage(map[string]any{"recipient": "this passes"}), + Settings: simplejson.NewFromAny(map[string]interface{}{"recipient": "this passes"}), SecureSettings: map[string]string{"url": "http://webhook.slack.com/myuser"}, }, }, }, { name: "when the configuration is valid - it should not error", - receivers: []*apimodels.PostableGrafanaReceiver{ + receivers: []*PostableGrafanaReceiver{ { UID: util.GenerateShortUID(), Name: "SlackWithBadURL", Type: "slack", - Settings: mustRawMessage(map[string]interface{}{"recipient": "#a-good-channel"}), + Settings: simplejson.NewFromAny(map[string]interface{}{"recipient": "#a-good-channel"}), SecureSettings: map[string]string{"url": "http://webhook.slack.com/myuser"}, }, }, }, } - sqlStore := db.InitTestDB(t) for _, tt := range tc { t.Run(tt.name, func(t *testing.T) { - service := NewTestMigrationService(t, sqlStore, nil) - mg := service.newOrgMigration(1) + mg := newTestMigration(t) config := configFromReceivers(t, tt.receivers) - require.NoError(t, encryptSecureSettings(config, mg)) // make sure we encrypt the settings + require.NoError(t, config.EncryptSecureSettings()) // make sure we encrypt the settings err := mg.validateAlertmanagerConfig(config) if tt.err != nil { require.Error(t, err) @@ -78,24 +117,24 @@ func Test_validateAlertmanagerConfig(t *testing.T) { } } -func configFromReceivers(t *testing.T, receivers []*apimodels.PostableGrafanaReceiver) *apimodels.PostableUserConfig { +func configFromReceivers(t *testing.T, receivers []*PostableGrafanaReceiver) *PostableUserConfig { t.Helper() - return &apimodels.PostableUserConfig{ - AlertmanagerConfig: apimodels.PostableApiAlertingConfig{ - Receivers: []*apimodels.PostableApiReceiver{ - {PostableGrafanaReceivers: apimodels.PostableGrafanaReceivers{GrafanaManagedReceivers: receivers}}, + return &PostableUserConfig{ + AlertmanagerConfig: PostableApiAlertingConfig{ + Receivers: []*PostableApiReceiver{ + {GrafanaManagedReceivers: receivers}, }, }, } } -func encryptSecureSettings(c *apimodels.PostableUserConfig, m *OrgMigration) error { +func (c *PostableUserConfig) EncryptSecureSettings() error { for _, r := range c.AlertmanagerConfig.Receivers { for _, gr := range r.GrafanaManagedReceivers { - err := m.encryptSecureSettings(gr.SecureSettings) - if err != nil { - return err + encryptedData := GetEncryptedJsonData(gr.SecureSettings) + for k, v := range encryptedData { + gr.SecureSettings[k] = base64.StdEncoding.EncodeToString(v) } } } @@ -106,13 +145,13 @@ const invalidUri = "�6�M��)uk譹1(�h`$�o�N>mĕ����cS2�dh func Test_getAlertFolderNameFromDashboard(t *testing.T) { t.Run("should include full title", func(t *testing.T) { - dash := &dashboards.Dashboard{ - UID: util.GenerateShortUID(), + dash := &dashboard{ + Uid: util.GenerateShortUID(), Title: "TEST", } folder := getAlertFolderNameFromDashboard(dash) require.Contains(t, folder, dash.Title) - require.Contains(t, folder, dash.UID) + require.Contains(t, folder, dash.Uid) }) t.Run("should cut title to the length", func(t *testing.T) { title := "" @@ -124,25 +163,25 @@ func Test_getAlertFolderNameFromDashboard(t *testing.T) { } } - dash := &dashboards.Dashboard{ - UID: util.GenerateShortUID(), + dash := &dashboard{ + Uid: util.GenerateShortUID(), Title: title, } folder := getAlertFolderNameFromDashboard(dash) require.Len(t, folder, MaxFolderName) - require.Contains(t, folder, dash.UID) + require.Contains(t, folder, dash.Uid) }) } func Test_shortUIDCaseInsensitiveConflicts(t *testing.T) { - s := Deduplicator{ + s := uidSet{ set: make(map[string]struct{}), caseInsensitive: true, } // 10000 uids seems to be enough to cause a collision in almost every run if using util.GenerateShortUID directly. for i := 0; i < 10000; i++ { - s.add(util.GenerateShortUID()) + _, _ = s.generateUid() } // check if any are case-insensitive duplicates. @@ -153,8 +192,3 @@ func Test_shortUIDCaseInsensitiveConflicts(t *testing.T) { require.Equal(t, len(s.set), len(deduped)) } - -func mustRawMessage[T any](s T) apimodels.RawMessage { - js, _ := json.Marshal(s) - return js -} From ea50695cfe780f24748519f82b82284fb2c43867 Mon Sep 17 00:00:00 2001 From: Virginia Cepeda Date: Wed, 11 Oct 2023 23:10:23 +0100 Subject: [PATCH 070/102] Alerting: add help texts to insights (#76366) * Change texts and add spacing between sections * Minor text changes * Display timeseries with 0 values if result is empty --- .../alerting/unified/home/Insights.tsx | 98 ++++++++++++------- .../unified/insights/SectionFooter.tsx | 17 ++++ .../unified/insights/SectionSubheader.tsx | 6 +- .../insights/grafana/AlertsByStateScene.tsx | 23 ++++- .../grafana/EvalSuccessVsFailuresScene.tsx | 2 +- .../unified/insights/grafana/Firing.tsx | 2 +- .../insights/grafana/InstanceStatusScene.tsx | 3 +- .../grafana/MissedIterationsScene.tsx | 2 +- .../grafana/MostFiredInstancesTable.tsx | 4 +- .../unified/insights/grafana/Paused.tsx | 2 +- .../insights/grafana/RulesByEvaluation.tsx | 2 +- .../grafana/RulesByEvaluationPercentage.tsx | 2 +- .../alertmanager/NotificationsScene.tsx | 4 +- .../alertmanager/SilencesByStateScene.tsx | 2 +- .../unified/insights/mimir/AlertsByState.tsx | 2 +- .../unified/insights/mimir/InvalidConfig.tsx | 2 +- .../unified/insights/mimir/Notifications.tsx | 2 +- .../unified/insights/mimir/Silences.tsx | 2 +- ...upEvaluationDurationIntervalRatioScene.tsx | 2 +- .../RuleGroupEvaluationDurationScene.tsx | 2 +- .../perGroup/RuleGroupEvaluationsScene.tsx | 2 +- .../mimir/perGroup/RuleGroupIntervalScene.tsx | 2 +- .../mimir/perGroup/RulesPerGroupScene.tsx | 2 +- .../rules/EvalSuccessVsFailuresScene.tsx | 2 +- .../unified/insights/mimir/rules/Firing.tsx | 2 +- .../insights/mimir/rules/InstancesByState.tsx | 2 +- .../rules/InstancesPercentageByState.tsx | 2 +- .../mimir/rules/MissedIterationsScene.tsx | 2 +- .../insights/mimir/rules/MostFiredRules.tsx | 2 +- .../unified/insights/mimir/rules/Pending.tsx | 2 +- 30 files changed, 128 insertions(+), 73 deletions(-) create mode 100644 public/app/features/alerting/unified/insights/SectionFooter.tsx diff --git a/public/app/features/alerting/unified/home/Insights.tsx b/public/app/features/alerting/unified/home/Insights.tsx index 289b77c7def..33ec4e4f9b7 100644 --- a/public/app/features/alerting/unified/home/Insights.tsx +++ b/public/app/features/alerting/unified/home/Insights.tsx @@ -17,6 +17,7 @@ import { VariableValueSelectors, } from '@grafana/scenes'; +import { SectionFooter } from '../insights/SectionFooter'; import { SectionSubheader } from '../insights/SectionSubheader'; import { getGrafanaInstancesByStateScene } from '../insights/grafana/AlertsByStateScene'; import { getGrafanaEvalSuccessVsFailuresScene } from '../insights/grafana/EvalSuccessVsFailuresScene'; @@ -127,15 +128,6 @@ export function getInsightsScenes() { ); } - if (showMimirAlertmanager) { - categories.push( - new SceneFlexItem({ - ySizing: 'content', - body: getCloudScenes(), - }) - ); - } - if (showMimirManaged) { categories.push( new SceneFlexItem({ @@ -154,12 +146,21 @@ export function getInsightsScenes() { ); } + if (showMimirAlertmanager) { + categories.push( + new SceneFlexItem({ + ySizing: 'content', + body: getCloudScenes(), + }) + ); + } + return new EmbeddedScene({ $timeRange: THIS_WEEK_TIME_RANGE, controls: [ new SceneReactObject({ component: SectionSubheader, - props: { children:
    Monitor the status of your system.
    , datasources: [] }, + props: { children:
    Monitor the status of your system.
    }, }), new SceneControlsSpacer(), new SceneTimePicker({}), @@ -180,18 +181,12 @@ function getGrafanaManagedScenes() { body: new SceneFlexLayout({ direction: 'column', children: [ - new SceneFlexItem({ - body: new SceneReactObject({ - component: SectionSubheader, - props: { children:
    Grafana-managed rules
    , datasources: [] }, - }), - }), new SceneFlexLayout({ direction: 'column', children: [ new SceneFlexLayout({ children: [ - getMostFiredInstancesScene(ashDs, 'Top 10 firing instances this week'), + getMostFiredInstancesScene(ashDs, 'Top 10 firing instances'), getFiringGrafanaAlertsScene(cloudUsageDs, 'Firing rules'), getPausedGrafanaAlertsScene(cloudUsageDs, 'Paused rules'), ], @@ -206,14 +201,34 @@ function getGrafanaManagedScenes() { new SceneFlexLayout({ height: '400px', children: [ - getInstanceStatByStatusScene(cloudUsageDs, 'Alerting instances', 'alerting'), - getInstanceStatByStatusScene(cloudUsageDs, 'Pending instances', 'pending'), + getInstanceStatByStatusScene( + cloudUsageDs, + 'Firing instances', + 'The number of currently firing alert rule instances', + 'alerting' + ), + getInstanceStatByStatusScene( + cloudUsageDs, + 'Pending instances', + 'The number of currently pending alert rule instances', + 'pending' + ), ], }), new SceneFlexLayout({ children: [ - getInstanceStatByStatusScene(cloudUsageDs, 'No data instances', 'nodata'), - getInstanceStatByStatusScene(cloudUsageDs, 'Error instances', 'error'), + getInstanceStatByStatusScene( + cloudUsageDs, + 'No data instances', + 'The current number of alert rule instances in No data state', + 'nodata' + ), + getInstanceStatByStatusScene( + cloudUsageDs, + 'Error instances', + 'The current number of alert rule instances in Error state', + 'error' + ), ], }), ], @@ -234,6 +249,9 @@ function getGrafanaManagedScenes() { }), ], }), + new SceneReactObject({ + component: SectionFooter, + }), ], }), }); @@ -247,18 +265,15 @@ function getGrafanaAlertmanagerScenes() { body: new SceneFlexLayout({ direction: 'column', children: [ - new SceneFlexItem({ - body: new SceneReactObject({ - component: SectionSubheader, - props: { children:
    Grafana Alertmanager
    , datasources: [] }, - }), - }), new SceneFlexLayout({ children: [ - getGrafanaAlertmanagerNotificationsScene(cloudUsageDs, 'Notifications'), + getGrafanaAlertmanagerNotificationsScene(cloudUsageDs, 'Notification delivery'), getGrafanaAlertmanagerSilencesScene(cloudUsageDs, 'Silences'), ], }), + new SceneReactObject({ + component: SectionFooter, + }), ], }), }); @@ -275,13 +290,13 @@ function getCloudScenes() { new SceneFlexItem({ body: new SceneReactObject({ component: SectionSubheader, - props: { children:
    Mimir Alertmanager
    , datasources: [cloudUsageDs] }, + props: { datasources: [cloudUsageDs] }, }), }), new SceneFlexLayout({ children: [ - getAlertsByStateScene(cloudUsageDs, 'Alerts by state'), - getNotificationsScene(cloudUsageDs, 'Notifications'), + getAlertsByStateScene(cloudUsageDs, 'Firing alerts by state'), + getNotificationsScene(cloudUsageDs, 'Notification delivery'), ], }), new SceneFlexLayout({ @@ -290,6 +305,9 @@ function getCloudScenes() { getInvalidConfigScene(cloudUsageDs, 'Invalid configuration'), ], }), + new SceneReactObject({ + component: SectionFooter, + }), ], }), }); @@ -306,28 +324,31 @@ function getMimirManagedRulesScenes() { new SceneFlexItem({ body: new SceneReactObject({ component: SectionSubheader, - props: { children:
    Mimir-managed rules
    , datasources: [grafanaCloudPromDs, cloudUsageDs] }, + props: { datasources: [grafanaCloudPromDs, cloudUsageDs] }, }), }), new SceneFlexLayout({ children: [ - getMostFiredRulesScene(grafanaCloudPromDs, 'Top 10 firing rules this week'), + getMostFiredRulesScene(grafanaCloudPromDs, 'Top 10 firing rules'), getFiringCloudAlertsScene(grafanaCloudPromDs, 'Firing instances'), getPendingCloudAlertsScene(grafanaCloudPromDs, 'Pending instances'), ], }), new SceneFlexLayout({ children: [ - getInstancesByStateScene(grafanaCloudPromDs, 'Count of alert instances by state'), - getInstancesPercentageByStateScene(grafanaCloudPromDs, '% of alert instances by State'), + getInstancesByStateScene(grafanaCloudPromDs, 'Firing and pending alert instances'), + getInstancesPercentageByStateScene(grafanaCloudPromDs, '% of alert instances by state'), ], }), new SceneFlexLayout({ children: [ getEvalSuccessVsFailuresScene(cloudUsageDs, 'Evaluation success vs failures'), - getMissedIterationsScene(cloudUsageDs, 'Iterations missed'), + getMissedIterationsScene(cloudUsageDs, 'Missed evaluations'), ], }), + new SceneReactObject({ + component: SectionFooter, + }), ], }), }); @@ -351,7 +372,7 @@ function getMimirManagedRulesPerGroupScenes() { new SceneFlexItem({ body: new SceneReactObject({ component: SectionSubheader, - props: { children:
    Mimir-managed Rules - Per Rule Group
    , datasources: [cloudUsageDs] }, + props: { datasources: [cloudUsageDs] }, }), }), new SceneFlexLayout({ @@ -367,6 +388,9 @@ function getMimirManagedRulesPerGroupScenes() { getRuleGroupEvaluationDurationIntervalRatioScene(cloudUsageDs, 'Evaluation duration / interval ratio'), ], }), + new SceneReactObject({ + component: SectionFooter, + }), ], }), $variables: new SceneVariableSet({ diff --git a/public/app/features/alerting/unified/insights/SectionFooter.tsx b/public/app/features/alerting/unified/insights/SectionFooter.tsx new file mode 100644 index 00000000000..337352ea18c --- /dev/null +++ b/public/app/features/alerting/unified/insights/SectionFooter.tsx @@ -0,0 +1,17 @@ +import { css } from '@emotion/css'; +import React from 'react'; + +import { GrafanaTheme2 } from '@grafana/data/src/themes'; +import { useStyles2 } from '@grafana/ui'; + +export function SectionFooter({ children }: React.PropsWithChildren<{}>) { + const styles = useStyles2(getStyles); + + return
    {children &&
    {children}
    }
    ; +} + +const getStyles = (theme: GrafanaTheme2) => ({ + sectionFooter: css({ + marginBottom: theme.spacing(2), + }), +}); diff --git a/public/app/features/alerting/unified/insights/SectionSubheader.tsx b/public/app/features/alerting/unified/insights/SectionSubheader.tsx index 7c0a9dda0a3..3490ed2a01c 100644 --- a/public/app/features/alerting/unified/insights/SectionSubheader.tsx +++ b/public/app/features/alerting/unified/insights/SectionSubheader.tsx @@ -11,13 +11,13 @@ import { DataSourcesInfo } from './DataSourcesInfo'; export function SectionSubheader({ children, datasources, -}: React.PropsWithChildren<{ datasources: DataSourceInformation[] }>) { +}: React.PropsWithChildren<{ datasources?: DataSourceInformation[] }>) { const styles = useStyles2(getStyles); return (
    -
    {children}
    - + {children} + {datasources && }
    ); } diff --git a/public/app/features/alerting/unified/insights/grafana/AlertsByStateScene.tsx b/public/app/features/alerting/unified/insights/grafana/AlertsByStateScene.tsx index 39c205ab789..2d3703dfed4 100644 --- a/public/app/features/alerting/unified/insights/grafana/AlertsByStateScene.tsx +++ b/public/app/features/alerting/unified/insights/grafana/AlertsByStateScene.tsx @@ -1,6 +1,6 @@ import React from 'react'; -import { PanelBuilders, SceneFlexItem, SceneQueryRunner } from '@grafana/scenes'; +import { PanelBuilders, SceneDataTransformer, SceneFlexItem, SceneQueryRunner } from '@grafana/scenes'; import { DataSourceRef, GraphDrawStyle, TooltipDisplayMode } from '@grafana/schema'; import { overrideToFixedColor, PANEL_STYLES } from '../../home/Insights'; @@ -19,19 +19,32 @@ export function getGrafanaInstancesByStateScene(datasource: DataSourceRef, panel ], }); + const transformation = new SceneDataTransformer({ + $data: query, + transformations: [ + { + id: 'renameByRegex', + options: { + regex: 'alerting', + renamePattern: 'firing', + }, + }, + ], + }); + return new SceneFlexItem({ ...PANEL_STYLES, height: '400px', body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) - .setData(query) + .setDescription('A breakdown of all of your alert rule instances based on state') + .setData(transformation) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) .setOverrides((b) => b - .matchFieldsWithName('alerting') - .overrideColor(overrideToFixedColor('alerting')) + .matchFieldsWithName('firing') + .overrideColor(overrideToFixedColor('firing')) .matchFieldsWithName('normal') .overrideColor(overrideToFixedColor('normal')) .matchFieldsWithName('pending') diff --git a/public/app/features/alerting/unified/insights/grafana/EvalSuccessVsFailuresScene.tsx b/public/app/features/alerting/unified/insights/grafana/EvalSuccessVsFailuresScene.tsx index 315a4e20209..51f8acabf87 100644 --- a/public/app/features/alerting/unified/insights/grafana/EvalSuccessVsFailuresScene.tsx +++ b/public/app/features/alerting/unified/insights/grafana/EvalSuccessVsFailuresScene.tsx @@ -29,7 +29,7 @@ export function getGrafanaEvalSuccessVsFailuresScene(datasource: DataSourceRef, ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of successful and failed alert rule evaluations') .setData(query) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) diff --git a/public/app/features/alerting/unified/insights/grafana/Firing.tsx b/public/app/features/alerting/unified/insights/grafana/Firing.tsx index 25b9a9702c0..bbe6070be4a 100644 --- a/public/app/features/alerting/unified/insights/grafana/Firing.tsx +++ b/public/app/features/alerting/unified/insights/grafana/Firing.tsx @@ -23,7 +23,7 @@ export function getFiringGrafanaAlertsScene(datasource: DataSourceRef, panelTitl ...PANEL_STYLES, body: PanelBuilders.stat() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of currently firing alert rules') .setData(query) .setThresholds({ mode: ThresholdsMode.Absolute, diff --git a/public/app/features/alerting/unified/insights/grafana/InstanceStatusScene.tsx b/public/app/features/alerting/unified/insights/grafana/InstanceStatusScene.tsx index 451d7f105d9..3600fd336f8 100644 --- a/public/app/features/alerting/unified/insights/grafana/InstanceStatusScene.tsx +++ b/public/app/features/alerting/unified/insights/grafana/InstanceStatusScene.tsx @@ -8,6 +8,7 @@ import { InsightsRatingModal } from '../RatingModal'; export function getInstanceStatByStatusScene( datasource: DataSourceRef, panelTitle: string, + panelDescription: string, status: 'alerting' | 'pending' | 'nodata' | 'normal' | 'error' ) { const query = new SceneQueryRunner({ @@ -26,7 +27,7 @@ export function getInstanceStatByStatusScene( height: '100%', body: PanelBuilders.stat() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription(panelDescription) .setData(query) .setOverrides((b) => b.matchFieldsWithName(status).overrideColor(overrideToFixedColor(status))) .setNoValue('0') diff --git a/public/app/features/alerting/unified/insights/grafana/MissedIterationsScene.tsx b/public/app/features/alerting/unified/insights/grafana/MissedIterationsScene.tsx index 27e4d909615..2055c5002dd 100644 --- a/public/app/features/alerting/unified/insights/grafana/MissedIterationsScene.tsx +++ b/public/app/features/alerting/unified/insights/grafana/MissedIterationsScene.tsx @@ -58,7 +58,7 @@ export function getGrafanaMissedIterationsScene(datasource: DataSourceRef, panel ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of missed iterations per evaluation group') .setData(transformation) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) diff --git a/public/app/features/alerting/unified/insights/grafana/MostFiredInstancesTable.tsx b/public/app/features/alerting/unified/insights/grafana/MostFiredInstancesTable.tsx index 78229731ec8..d3b63ab5d8c 100644 --- a/public/app/features/alerting/unified/insights/grafana/MostFiredInstancesTable.tsx +++ b/public/app/features/alerting/unified/insights/grafana/MostFiredInstancesTable.tsx @@ -88,7 +88,7 @@ export function getMostFiredInstancesScene(datasource: DataSourceRef, panelTitle }, renameByName: { labels_alertname: 'Alert rule name', - 'Value #A': 'Fires this week', + 'Value #A': 'Number of fires', }, }, }, @@ -99,7 +99,7 @@ export function getMostFiredInstancesScene(datasource: DataSourceRef, panelTitle ...PANEL_STYLES, body: PanelBuilders.table() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The alert rule instances that have fired the most') .setData(transformation) .setNoValue('No new alerts fired last week') .setHeaderActions() diff --git a/public/app/features/alerting/unified/insights/grafana/Paused.tsx b/public/app/features/alerting/unified/insights/grafana/Paused.tsx index fb73894dd2a..5dcab79b5be 100644 --- a/public/app/features/alerting/unified/insights/grafana/Paused.tsx +++ b/public/app/features/alerting/unified/insights/grafana/Paused.tsx @@ -23,7 +23,7 @@ export function getPausedGrafanaAlertsScene(datasource: DataSourceRef, panelTitl ...PANEL_STYLES, body: PanelBuilders.stat() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of current paused alert rules') .setData(query) .setThresholds({ mode: ThresholdsMode.Absolute, diff --git a/public/app/features/alerting/unified/insights/grafana/RulesByEvaluation.tsx b/public/app/features/alerting/unified/insights/grafana/RulesByEvaluation.tsx index 57a16d7eafc..c331bc01927 100644 --- a/public/app/features/alerting/unified/insights/grafana/RulesByEvaluation.tsx +++ b/public/app/features/alerting/unified/insights/grafana/RulesByEvaluation.tsx @@ -23,7 +23,7 @@ export function getGrafanaRulesByEvaluationScene(datasource: DataSourceRef, pane ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('See how many of your alert rules are paused or active') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) diff --git a/public/app/features/alerting/unified/insights/grafana/RulesByEvaluationPercentage.tsx b/public/app/features/alerting/unified/insights/grafana/RulesByEvaluationPercentage.tsx index 3491c67f598..826b3461206 100644 --- a/public/app/features/alerting/unified/insights/grafana/RulesByEvaluationPercentage.tsx +++ b/public/app/features/alerting/unified/insights/grafana/RulesByEvaluationPercentage.tsx @@ -23,7 +23,7 @@ export function getGrafanaRulesByEvaluationPercentageScene(datasource: DataSourc ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('See what percentage of your alert rules are paused or active') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setCustomFieldConfig('fillOpacity', 45) diff --git a/public/app/features/alerting/unified/insights/grafana/alertmanager/NotificationsScene.tsx b/public/app/features/alerting/unified/insights/grafana/alertmanager/NotificationsScene.tsx index 6f018f1492e..da4b51d0e13 100644 --- a/public/app/features/alerting/unified/insights/grafana/alertmanager/NotificationsScene.tsx +++ b/public/app/features/alerting/unified/insights/grafana/alertmanager/NotificationsScene.tsx @@ -12,7 +12,7 @@ export function getGrafanaAlertmanagerNotificationsScene(datasource: DataSourceR queries: [ { refId: 'A', - expr: 'grafanacloud_grafana_instance_alerting_notifications_total:rate5m - grafanacloud_grafana_instance_alerting_notifications_failed_total:rate5m', + expr: '(grafanacloud_grafana_instance_alerting_notifications_total:rate5m - grafanacloud_grafana_instance_alerting_notifications_failed_total:rate5m) or vector(0)', range: true, legendFormat: 'success', }, @@ -29,7 +29,7 @@ export function getGrafanaAlertmanagerNotificationsScene(datasource: DataSourceR ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of successful and failed notifications') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) diff --git a/public/app/features/alerting/unified/insights/grafana/alertmanager/SilencesByStateScene.tsx b/public/app/features/alerting/unified/insights/grafana/alertmanager/SilencesByStateScene.tsx index a4c908df7ee..b29b5894550 100644 --- a/public/app/features/alerting/unified/insights/grafana/alertmanager/SilencesByStateScene.tsx +++ b/public/app/features/alerting/unified/insights/grafana/alertmanager/SilencesByStateScene.tsx @@ -23,7 +23,7 @@ export function getGrafanaAlertmanagerSilencesScene(datasource: DataSourceRef, p ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of silences by state') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) diff --git a/public/app/features/alerting/unified/insights/mimir/AlertsByState.tsx b/public/app/features/alerting/unified/insights/mimir/AlertsByState.tsx index 0132bb3b4bc..1f3ebb7deea 100644 --- a/public/app/features/alerting/unified/insights/mimir/AlertsByState.tsx +++ b/public/app/features/alerting/unified/insights/mimir/AlertsByState.tsx @@ -23,7 +23,7 @@ export function getAlertsByStateScene(datasource: DataSourceRef, panelTitle: str ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('A breakdown of all of your firing alert rule instances based on state') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) diff --git a/public/app/features/alerting/unified/insights/mimir/InvalidConfig.tsx b/public/app/features/alerting/unified/insights/mimir/InvalidConfig.tsx index 6e9c5bce46b..401c5303965 100644 --- a/public/app/features/alerting/unified/insights/mimir/InvalidConfig.tsx +++ b/public/app/features/alerting/unified/insights/mimir/InvalidConfig.tsx @@ -23,7 +23,7 @@ export function getInvalidConfigScene(datasource: DataSourceRef, panelTitle: str ...PANEL_STYLES, body: PanelBuilders.stat() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The current state of your alertmanager configuration') .setData(query) .setUnit('bool_yes_no') .setOption('graphMode', BigValueGraphMode.None) diff --git a/public/app/features/alerting/unified/insights/mimir/Notifications.tsx b/public/app/features/alerting/unified/insights/mimir/Notifications.tsx index 24aaeb7a13e..933107dd599 100644 --- a/public/app/features/alerting/unified/insights/mimir/Notifications.tsx +++ b/public/app/features/alerting/unified/insights/mimir/Notifications.tsx @@ -29,7 +29,7 @@ export function getNotificationsScene(datasource: DataSourceRef, panelTitle: str ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of successful and failed notifications') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) diff --git a/public/app/features/alerting/unified/insights/mimir/Silences.tsx b/public/app/features/alerting/unified/insights/mimir/Silences.tsx index 6aee610cb1e..416c51061d4 100644 --- a/public/app/features/alerting/unified/insights/mimir/Silences.tsx +++ b/public/app/features/alerting/unified/insights/mimir/Silences.tsx @@ -23,7 +23,7 @@ export function getSilencesScene(datasource: DataSourceRef, panelTitle: string) ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of silences by state') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) diff --git a/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationDurationIntervalRatioScene.tsx b/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationDurationIntervalRatioScene.tsx index 85e1254f2da..9292de7952a 100644 --- a/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationDurationIntervalRatioScene.tsx +++ b/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationDurationIntervalRatioScene.tsx @@ -23,7 +23,7 @@ export function getRuleGroupEvaluationDurationIntervalRatioScene(datasource: Dat ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The percentage of interval time spent evaluating') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) diff --git a/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationDurationScene.tsx b/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationDurationScene.tsx index c1dcde74f37..b183efc9eda 100644 --- a/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationDurationScene.tsx +++ b/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationDurationScene.tsx @@ -23,7 +23,7 @@ export function getRuleGroupEvaluationDurationScene(datasource: DataSourceRef, p ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('How long it took to evaluate the rule group') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setUnit('s') diff --git a/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationsScene.tsx b/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationsScene.tsx index dc8e8007b9b..b35ac1d113b 100644 --- a/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationsScene.tsx +++ b/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupEvaluationsScene.tsx @@ -29,7 +29,7 @@ export function getRuleGroupEvaluationsScene(datasource: DataSourceRef, panelTit ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of successful and failed evaluations for the rule group') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) diff --git a/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupIntervalScene.tsx b/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupIntervalScene.tsx index ed47c208dbc..524e597e6a2 100644 --- a/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupIntervalScene.tsx +++ b/public/app/features/alerting/unified/insights/mimir/perGroup/RuleGroupIntervalScene.tsx @@ -23,7 +23,7 @@ export function getRuleGroupIntervalScene(datasource: DataSourceRef, panelTitle: ...PANEL_STYLES, body: PanelBuilders.stat() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The current and historical rule group evaluation interval') .setData(query) .setUnit('s') .setOption('graphMode', BigValueGraphMode.Area) diff --git a/public/app/features/alerting/unified/insights/mimir/perGroup/RulesPerGroupScene.tsx b/public/app/features/alerting/unified/insights/mimir/perGroup/RulesPerGroupScene.tsx index 947b3839ebc..3be05e4debe 100644 --- a/public/app/features/alerting/unified/insights/mimir/perGroup/RulesPerGroupScene.tsx +++ b/public/app/features/alerting/unified/insights/mimir/perGroup/RulesPerGroupScene.tsx @@ -23,7 +23,7 @@ export function getRulesPerGroupScene(datasource: DataSourceRef, panelTitle: str ...PANEL_STYLES, body: PanelBuilders.stat() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The current and historical number of alert rules in the rule group') .setData(query) .setUnit('none') .setOption('graphMode', BigValueGraphMode.Area) diff --git a/public/app/features/alerting/unified/insights/mimir/rules/EvalSuccessVsFailuresScene.tsx b/public/app/features/alerting/unified/insights/mimir/rules/EvalSuccessVsFailuresScene.tsx index 8d948a65af2..792445ec767 100644 --- a/public/app/features/alerting/unified/insights/mimir/rules/EvalSuccessVsFailuresScene.tsx +++ b/public/app/features/alerting/unified/insights/mimir/rules/EvalSuccessVsFailuresScene.tsx @@ -29,7 +29,7 @@ export function getEvalSuccessVsFailuresScene(datasource: DataSourceRef, panelTi ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of successful and failed alert rule evaluations') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) diff --git a/public/app/features/alerting/unified/insights/mimir/rules/Firing.tsx b/public/app/features/alerting/unified/insights/mimir/rules/Firing.tsx index 8c60ba3e63f..46547869bf0 100644 --- a/public/app/features/alerting/unified/insights/mimir/rules/Firing.tsx +++ b/public/app/features/alerting/unified/insights/mimir/rules/Firing.tsx @@ -23,7 +23,7 @@ export function getFiringCloudAlertsScene(datasource: DataSourceRef, panelTitle: ...PANEL_STYLES, body: PanelBuilders.stat() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of currently firing alert rule instances') .setData(query) .setThresholds({ mode: ThresholdsMode.Absolute, diff --git a/public/app/features/alerting/unified/insights/mimir/rules/InstancesByState.tsx b/public/app/features/alerting/unified/insights/mimir/rules/InstancesByState.tsx index 48984b3d9b4..42ab455b650 100644 --- a/public/app/features/alerting/unified/insights/mimir/rules/InstancesByState.tsx +++ b/public/app/features/alerting/unified/insights/mimir/rules/InstancesByState.tsx @@ -23,7 +23,7 @@ export function getInstancesByStateScene(datasource: DataSourceRef, panelTitle: ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of firing and pending alert rule instances') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) diff --git a/public/app/features/alerting/unified/insights/mimir/rules/InstancesPercentageByState.tsx b/public/app/features/alerting/unified/insights/mimir/rules/InstancesPercentageByState.tsx index 129bfb316f8..d23b001b6fd 100644 --- a/public/app/features/alerting/unified/insights/mimir/rules/InstancesPercentageByState.tsx +++ b/public/app/features/alerting/unified/insights/mimir/rules/InstancesPercentageByState.tsx @@ -23,7 +23,7 @@ export function getInstancesPercentageByStateScene(datasource: DataSourceRef, pa ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('See what percentage of your alert rules are firing versus pending') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setCustomFieldConfig('fillOpacity', 45) diff --git a/public/app/features/alerting/unified/insights/mimir/rules/MissedIterationsScene.tsx b/public/app/features/alerting/unified/insights/mimir/rules/MissedIterationsScene.tsx index 9eacc01bb77..16d32fa5941 100644 --- a/public/app/features/alerting/unified/insights/mimir/rules/MissedIterationsScene.tsx +++ b/public/app/features/alerting/unified/insights/mimir/rules/MissedIterationsScene.tsx @@ -23,7 +23,7 @@ export function getMissedIterationsScene(datasource: DataSourceRef, panelTitle: ...PANEL_STYLES, body: PanelBuilders.timeseries() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of evaluations missed') .setData(query) .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) diff --git a/public/app/features/alerting/unified/insights/mimir/rules/MostFiredRules.tsx b/public/app/features/alerting/unified/insights/mimir/rules/MostFiredRules.tsx index ca07b14f0ef..df2f00f7c38 100644 --- a/public/app/features/alerting/unified/insights/mimir/rules/MostFiredRules.tsx +++ b/public/app/features/alerting/unified/insights/mimir/rules/MostFiredRules.tsx @@ -43,7 +43,7 @@ export function getMostFiredRulesScene(datasource: DataSourceRef, panelTitle: st ...PANEL_STYLES, body: PanelBuilders.table() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The alert rules that have fired the most') .setData(transformation) .setHeaderActions() .build(), diff --git a/public/app/features/alerting/unified/insights/mimir/rules/Pending.tsx b/public/app/features/alerting/unified/insights/mimir/rules/Pending.tsx index 00a357d0828..a792f31ce43 100644 --- a/public/app/features/alerting/unified/insights/mimir/rules/Pending.tsx +++ b/public/app/features/alerting/unified/insights/mimir/rules/Pending.tsx @@ -23,7 +23,7 @@ export function getPendingCloudAlertsScene(datasource: DataSourceRef, panelTitle ...PANEL_STYLES, body: PanelBuilders.stat() .setTitle(panelTitle) - .setDescription(panelTitle) + .setDescription('The number of currently pending alert rule instances') .setData(query) .setThresholds({ mode: ThresholdsMode.Absolute, From d003ffe439850f36a984866d0a16750db3699908 Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Wed, 11 Oct 2023 18:28:26 -0400 Subject: [PATCH 071/102] Update origin annotation names (#76301) * Update origin annotation names k8s does not support annotation names with multiple slashes in them, so this PR updates the origin annotations to match the format for updated and created annotations. * fix tests --- pkg/kinds/general.go | 8 ++++---- pkg/services/dashboards/models_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/kinds/general.go b/pkg/kinds/general.go index 94022921448..74c7a7f6fb0 100644 --- a/pkg/kinds/general.go +++ b/pkg/kinds/general.go @@ -53,10 +53,10 @@ const annoKeyFolder = "grafana.com/folder" const annoKeySlug = "grafana.com/slug" // Identify where values came from -const annoKeyOriginName = "grafana.com/origin/name" -const annoKeyOriginPath = "grafana.com/origin/path" -const annoKeyOriginKey = "grafana.com/origin/key" -const annoKeyOriginTime = "grafana.com/origin/time" +const annoKeyOriginName = "grafana.com/originName" +const annoKeyOriginPath = "grafana.com/originPath" +const annoKeyOriginKey = "grafana.com/originKey" +const annoKeyOriginTime = "grafana.com/originTime" func (m *GrafanaResourceMetadata) GetUpdatedTimestamp() *time.Time { v, ok := m.Annotations[annoKeyUpdatedTimestamp] diff --git a/pkg/services/dashboards/models_test.go b/pkg/services/dashboards/models_test.go index 1e34a927ddd..dcbc4102956 100644 --- a/pkg/services/dashboards/models_test.go +++ b/pkg/services/dashboards/models_test.go @@ -123,8 +123,8 @@ func TestResourceConversion(t *testing.T) { "annotations": { "grafana.com/createdBy": "user:10", "grafana.com/folder": "folder:1234", - "grafana.com/origin/key": "plugin-xyz", - "grafana.com/origin/name": "plugin", + "grafana.com/originKey": "plugin-xyz", + "grafana.com/originName": "plugin", "grafana.com/slug": "test-dash", "grafana.com/updatedBy": "user:11", "grafana.com/updatedTimestamp": "2010-01-01T08:00:00Z" From a12cb8cbf3a9b33841b2f2cb1522be11de78c86a Mon Sep 17 00:00:00 2001 From: kay delaney <45561153+kaydelaney@users.noreply.github.com> Date: Thu, 12 Oct 2023 00:30:50 +0100 Subject: [PATCH 072/102] LibraryPanels: Add RBAC support (#73475) --- .../feature-toggles/index.md | 1 + .../src/types/featureToggles.gen.ts | 1 + pkg/api/accesscontrol.go | 83 +++++++++++++- pkg/services/accesscontrol/models.go | 6 + .../ossaccesscontrol/permissions_services.go | 6 +- pkg/services/featuremgmt/registry.go | 8 ++ pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + pkg/services/folder/folderimpl/folder_test.go | 6 +- pkg/services/libraryelements/accesscontrol.go | 69 ++++++++++++ pkg/services/libraryelements/api.go | 70 ++++++++++-- pkg/services/libraryelements/database.go | 27 +++-- .../libraryelements/libraryelements.go | 8 +- .../librarypanels/librarypanels_test.go | 2 +- .../accesscontrol/dashboard_permissions.go | 105 ++++++++++++++++++ .../sqlstore/migrations/migrations.go | 1 + 16 files changed, 370 insertions(+), 28 deletions(-) create mode 100644 pkg/services/libraryelements/accesscontrol.go diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 1e1d429e4bf..6b3c5206fea 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -136,6 +136,7 @@ Experimental features might be changed or removed without prior notice. | `dashgpt` | Enable AI powered features in dashboards | | `sseGroupByDatasource` | Send query to the same datasource in a single request when using server side expressions | | `requestInstrumentationStatusSource` | Include a status source label for request metrics and logs | +| `libraryPanelRBAC` | Enables RBAC support for library panels | | `wargamesTesting` | Placeholder feature flag for internal testing | | `alertingInsights` | Show the new alerting insights landing page | | `externalCorePlugins` | Allow core plugins to be loaded as external | diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 35a753964bc..4ef472d4b50 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -125,6 +125,7 @@ export interface FeatureToggles { newBrowseDashboards?: boolean; sseGroupByDatasource?: boolean; requestInstrumentationStatusSource?: boolean; + libraryPanelRBAC?: boolean; lokiRunQueriesInParallel?: boolean; wargamesTesting?: boolean; alertingInsights?: boolean; diff --git a/pkg/api/accesscontrol.go b/pkg/api/accesscontrol.go index c2cbec08a44..e3d5265229a 100644 --- a/pkg/api/accesscontrol.go +++ b/pkg/api/accesscontrol.go @@ -7,6 +7,8 @@ import ( contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/libraryelements" "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginaccesscontrol" "github.com/grafana/grafana/pkg/tsdb/grafanads" @@ -408,6 +410,76 @@ func (hs *HTTPServer) declareFixedRoles() error { Grants: []string{"Admin"}, } + libraryPanelsCreatorRole := ac.RoleRegistration{ + Role: ac.RoleDTO{ + Name: "fixed:library.panels:creator", + DisplayName: "Library panel creator", + Description: "Create library panel in general folder.", + Group: "Library panels", + Permissions: []ac.Permission{ + {Action: dashboards.ActionFoldersRead, Scope: dashboards.ScopeFoldersProvider.GetResourceScopeUID(ac.GeneralFolderUID)}, + {Action: libraryelements.ActionLibraryPanelsCreate, Scope: dashboards.ScopeFoldersProvider.GetResourceScopeUID(ac.GeneralFolderUID)}, + }, + }, + Grants: []string{"Editor"}, + } + + libraryPanelsReaderRole := ac.RoleRegistration{ + Role: ac.RoleDTO{ + Name: "fixed:library.panels:reader", + DisplayName: "Library panel reader", + Description: "Read all library panels.", + Group: "Library panels", + Permissions: []ac.Permission{ + {Action: libraryelements.ActionLibraryPanelsRead, Scope: libraryelements.ScopeLibraryPanelsAll}, + }, + }, + Grants: []string{"Admin"}, + } + + libraryPanelsGeneralReaderRole := ac.RoleRegistration{ + Role: ac.RoleDTO{ + Name: "fixed:library.panels:general.reader", + DisplayName: "Library panel general reader", + Description: "Read all library panels in general folder.", + Group: "Library panels", + Permissions: []ac.Permission{ + {Action: libraryelements.ActionLibraryPanelsRead, Scope: dashboards.ScopeFoldersProvider.GetResourceScopeUID(ac.GeneralFolderUID)}, + }, + }, + Grants: []string{"Viewer"}, + } + + libraryPanelsWriterRole := ac.RoleRegistration{ + Role: ac.RoleDTO{ + Name: "fixed:library.panels:writer", + DisplayName: "Library panel writer", + Group: "Library panels", + Description: "Create, read, write or delete all library panels and their permissions.", + Permissions: ac.ConcatPermissions(libraryPanelsReaderRole.Role.Permissions, []ac.Permission{ + {Action: libraryelements.ActionLibraryPanelsWrite, Scope: libraryelements.ScopeLibraryPanelsAll}, + {Action: libraryelements.ActionLibraryPanelsDelete, Scope: libraryelements.ScopeLibraryPanelsAll}, + {Action: libraryelements.ActionLibraryPanelsCreate, Scope: libraryelements.ScopeLibraryPanelsAll}, + }), + }, + Grants: []string{"Admin"}, + } + + libraryPanelsGeneralWriterRole := ac.RoleRegistration{ + Role: ac.RoleDTO{ + Name: "fixed:library.panels:general.writer", + DisplayName: "Library panel general writer", + Group: "Library panels", + Description: "Create, read, write or delete all library panels and their permissions in the general folder.", + Permissions: ac.ConcatPermissions(libraryPanelsGeneralReaderRole.Role.Permissions, []ac.Permission{ + {Action: libraryelements.ActionLibraryPanelsWrite, Scope: dashboards.ScopeFoldersProvider.GetResourceScopeUID(ac.GeneralFolderUID)}, + {Action: libraryelements.ActionLibraryPanelsDelete, Scope: dashboards.ScopeFoldersProvider.GetResourceScopeUID(ac.GeneralFolderUID)}, + {Action: libraryelements.ActionLibraryPanelsCreate, Scope: dashboards.ScopeFoldersProvider.GetResourceScopeUID(ac.GeneralFolderUID)}, + }), + }, + Grants: []string{"Editor"}, + } + publicDashboardsWriterRole := ac.RoleRegistration{ Role: ac.RoleDTO{ Name: "fixed:dashboards.public:writer", @@ -447,15 +519,18 @@ func (hs *HTTPServer) declareFixedRoles() error { Grants: []string{"Admin"}, } - return hs.accesscontrolService.DeclareFixedRoles( - provisioningWriterRole, datasourcesReaderRole, builtInDatasourceReader, datasourcesWriterRole, + roles := []ac.RoleRegistration{provisioningWriterRole, datasourcesReaderRole, builtInDatasourceReader, datasourcesWriterRole, datasourcesIdReaderRole, orgReaderRole, orgWriterRole, orgMaintainerRole, teamsCreatorRole, teamsWriterRole, datasourcesExplorerRole, annotationsReaderRole, dashboardAnnotationsWriterRole, annotationsWriterRole, dashboardsCreatorRole, dashboardsReaderRole, dashboardsWriterRole, foldersCreatorRole, foldersReaderRole, foldersWriterRole, apikeyReaderRole, apikeyWriterRole, - publicDashboardsWriterRole, featuremgmtReaderRole, featuremgmtWriterRole, - ) + publicDashboardsWriterRole, featuremgmtReaderRole, featuremgmtWriterRole} + if hs.Features.IsEnabled(featuremgmt.FlagLibraryPanelRBAC) { + roles = append(roles, libraryPanelsCreatorRole, libraryPanelsReaderRole, libraryPanelsWriterRole, libraryPanelsGeneralReaderRole, libraryPanelsGeneralWriterRole) + } + + return hs.accesscontrolService.DeclareFixedRoles(roles...) } // Metadata helpers diff --git a/pkg/services/accesscontrol/models.go b/pkg/services/accesscontrol/models.go index f09ef0b4fec..a5b0a6ed08b 100644 --- a/pkg/services/accesscontrol/models.go +++ b/pkg/services/accesscontrol/models.go @@ -468,6 +468,12 @@ const ( // Feature Management actions ActionFeatureManagementRead = "featuremgmt.read" ActionFeatureManagementWrite = "featuremgmt.write" + + // Library Panel actions + ActionLibraryPanelsCreate = "library.panels:create" + ActionLibraryPanelsRead = "library.panels:read" + ActionLibraryPanelsWrite = "library.panels:write" + ActionLibraryPanelsDelete = "library.panels:delete" ) var ( diff --git a/pkg/services/accesscontrol/ossaccesscontrol/permissions_services.go b/pkg/services/accesscontrol/ossaccesscontrol/permissions_services.go index 3b469539b71..35c4b9cd657 100644 --- a/pkg/services/accesscontrol/ossaccesscontrol/permissions_services.go +++ b/pkg/services/accesscontrol/ossaccesscontrol/permissions_services.go @@ -14,6 +14,7 @@ import ( "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/folder" + "github.com/grafana/grafana/pkg/services/libraryelements" "github.com/grafana/grafana/pkg/services/licensing" "github.com/grafana/grafana/pkg/services/serviceaccounts" "github.com/grafana/grafana/pkg/services/serviceaccounts/retriever" @@ -191,7 +192,7 @@ type FolderPermissionsService struct { *resourcepermissions.Service } -var FolderViewActions = []string{dashboards.ActionFoldersRead, accesscontrol.ActionAlertingRuleRead} +var FolderViewActions = []string{dashboards.ActionFoldersRead, accesscontrol.ActionAlertingRuleRead, libraryelements.ActionLibraryPanelsRead} var FolderEditActions = append(FolderViewActions, []string{ dashboards.ActionFoldersWrite, dashboards.ActionFoldersDelete, @@ -199,6 +200,9 @@ var FolderEditActions = append(FolderViewActions, []string{ accesscontrol.ActionAlertingRuleCreate, accesscontrol.ActionAlertingRuleUpdate, accesscontrol.ActionAlertingRuleDelete, + libraryelements.ActionLibraryPanelsCreate, + libraryelements.ActionLibraryPanelsWrite, + libraryelements.ActionLibraryPanelsDelete, }...) var FolderAdminActions = append(FolderEditActions, []string{dashboards.ActionFoldersPermissionsRead, dashboards.ActionFoldersPermissionsWrite}...) diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 28354e32caf..fd84f4c3a0a 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -746,6 +746,14 @@ var ( FrontendOnly: false, Owner: grafanaPluginsPlatformSquad, }, + { + Name: "libraryPanelRBAC", + Description: "Enables RBAC support for library panels", + Stage: FeatureStageExperimental, + FrontendOnly: false, + Owner: grafanaDashboardsSquad, + RequiresRestart: true, + }, { Name: "lokiRunQueriesInParallel", Description: "Enables running Loki queries in parallel", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 651b7a48fa9..8aa3465a746 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -106,6 +106,7 @@ reportingRetries,preview,@grafana/sharing-squad,false,false,true,false newBrowseDashboards,GA,@grafana/grafana-frontend-platform,false,false,false,true sseGroupByDatasource,experimental,@grafana/observability-metrics,false,false,false,false requestInstrumentationStatusSource,experimental,@grafana/plugins-platform-backend,false,false,false,false +libraryPanelRBAC,experimental,@grafana/dashboards-squad,false,false,true,false lokiRunQueriesInParallel,privatePreview,@grafana/observability-logs,false,false,false,false wargamesTesting,experimental,@grafana/hosted-grafana-team,false,false,false,false alertingInsights,experimental,@grafana/alerting-squad,false,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 820b0db4bdf..c50cdb739b8 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -435,6 +435,10 @@ const ( // Include a status source label for request metrics and logs FlagRequestInstrumentationStatusSource = "requestInstrumentationStatusSource" + // FlagLibraryPanelRBAC + // Enables RBAC support for library panels + FlagLibraryPanelRBAC = "libraryPanelRBAC" + // FlagLokiRunQueriesInParallel // Enables running Loki queries in parallel FlagLokiRunQueriesInParallel = "lokiRunQueriesInParallel" diff --git a/pkg/services/folder/folderimpl/folder_test.go b/pkg/services/folder/folderimpl/folder_test.go index f9d85b577e9..0d2e5eaa6db 100644 --- a/pkg/services/folder/folderimpl/folder_test.go +++ b/pkg/services/folder/folderimpl/folder_test.go @@ -406,7 +406,7 @@ func TestIntegrationNestedFolderService(t *testing.T) { alertStore, err := ngstore.ProvideDBStore(cfg, featuresFlagOn, db, serviceWithFlagOn, ac, dashSrv) require.NoError(t, err) - elementService := libraryelements.ProvideService(cfg, db, routeRegister, serviceWithFlagOn, featuresFlagOn) + elementService := libraryelements.ProvideService(cfg, db, routeRegister, serviceWithFlagOn, featuresFlagOn, ac) lps, err := librarypanels.ProvideService(cfg, db, routeRegister, elementService, serviceWithFlagOn) require.NoError(t, err) @@ -481,7 +481,7 @@ func TestIntegrationNestedFolderService(t *testing.T) { alertStore, err := ngstore.ProvideDBStore(cfg, featuresFlagOff, db, serviceWithFlagOff, ac, dashSrv) require.NoError(t, err) - elementService := libraryelements.ProvideService(cfg, db, routeRegister, serviceWithFlagOff, featuresFlagOff) + elementService := libraryelements.ProvideService(cfg, db, routeRegister, serviceWithFlagOff, featuresFlagOff, ac) lps, err := librarypanels.ProvideService(cfg, db, routeRegister, elementService, serviceWithFlagOff) require.NoError(t, err) @@ -602,7 +602,7 @@ func TestIntegrationNestedFolderService(t *testing.T) { CanEditValue: true, }) - elementService := libraryelements.ProvideService(cfg, db, routeRegister, tc.service, tc.featuresFlag) + elementService := libraryelements.ProvideService(cfg, db, routeRegister, tc.service, tc.featuresFlag, ac) lps, err := librarypanels.ProvideService(cfg, db, routeRegister, elementService, tc.service) require.NoError(t, err) diff --git a/pkg/services/libraryelements/accesscontrol.go b/pkg/services/libraryelements/accesscontrol.go new file mode 100644 index 00000000000..4760f9cf2ae --- /dev/null +++ b/pkg/services/libraryelements/accesscontrol.go @@ -0,0 +1,69 @@ +package libraryelements + +import ( + "context" + "errors" + "strings" + + "github.com/grafana/grafana/pkg/infra/appcontext" + ac "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/folder" + "github.com/grafana/grafana/pkg/services/libraryelements/model" +) + +const ( + ScopeLibraryPanelsRoot = "library.panels" + ScopeLibraryPanelsPrefix = "library.panels:uid:" + + ActionLibraryPanelsCreate = "library.panels:create" + ActionLibraryPanelsRead = "library.panels:read" + ActionLibraryPanelsWrite = "library.panels:write" + ActionLibraryPanelsDelete = "library.panels:delete" +) + +var ( + ScopeLibraryPanelsProvider = ac.NewScopeProvider(ScopeLibraryPanelsRoot) + + ScopeLibraryPanelsAll = ScopeLibraryPanelsProvider.GetResourceAllScope() +) + +var ( + ErrNoElementsFound = errors.New("library element not found") + ErrElementNameNotUnique = errors.New("several library elements with the same name were found") +) + +// LibraryPanelUIDScopeResolver provides a ScopeAttributeResolver that is able to convert a scope prefixed with "library.panels:uid:" +// into uid based scopes for a library panel and its associated folder hierarchy +func LibraryPanelUIDScopeResolver(l *LibraryElementService, folderSvc folder.Service) (string, ac.ScopeAttributeResolver) { + prefix := ScopeLibraryPanelsProvider.GetResourceScopeUID("") + return prefix, ac.ScopeAttributeResolverFunc(func(ctx context.Context, orgID int64, scope string) ([]string, error) { + if !strings.HasPrefix(scope, prefix) { + return nil, ac.ErrInvalidScope + } + + uid, err := ac.ParseScopeUID(scope) + if err != nil { + return nil, err + } + + user, err := appcontext.User(ctx) + if err != nil { + return nil, err + } + + libElDTO, err := l.getLibraryElementByUid(ctx, user, model.GetLibraryElementCommand{ + UID: uid, + FolderName: dashboards.RootFolderName, + }) + if err != nil { + return nil, err + } + + inheritedScopes, err := dashboards.GetInheritedScopes(ctx, orgID, libElDTO.FolderUID, folderSvc) + if err != nil { + return nil, err + } + return append(inheritedScopes, dashboards.ScopeFoldersProvider.GetResourceScopeUID(libElDTO.FolderUID), ScopeLibraryPanelsProvider.GetResourceScopeUID(uid)), nil + }) +} diff --git a/pkg/services/libraryelements/api.go b/pkg/services/libraryelements/api.go index 4b09a49a773..5bb3a03612a 100644 --- a/pkg/services/libraryelements/api.go +++ b/pkg/services/libraryelements/api.go @@ -6,23 +6,38 @@ import ( "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/api/routing" - "github.com/grafana/grafana/pkg/middleware" + ac "github.com/grafana/grafana/pkg/services/accesscontrol" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/folder" "github.com/grafana/grafana/pkg/services/libraryelements/model" "github.com/grafana/grafana/pkg/web" ) func (l *LibraryElementService) registerAPIEndpoints() { + authorize := ac.Middleware(l.AccessControl) + l.RouteRegister.Group("/api/library-elements", func(entities routing.RouteRegister) { - entities.Post("/", middleware.ReqSignedIn, routing.Wrap(l.createHandler)) - entities.Delete("/:uid", middleware.ReqSignedIn, routing.Wrap(l.deleteHandler)) - entities.Get("/", middleware.ReqSignedIn, routing.Wrap(l.getAllHandler)) - entities.Get("/:uid", middleware.ReqSignedIn, routing.Wrap(l.getHandler)) - entities.Get("/:uid/connections/", middleware.ReqSignedIn, routing.Wrap(l.getConnectionsHandler)) - entities.Get("/name/:name", middleware.ReqSignedIn, routing.Wrap(l.getByNameHandler)) - entities.Patch("/:uid", middleware.ReqSignedIn, routing.Wrap(l.patchHandler)) + uidScope := ScopeLibraryPanelsProvider.GetResourceScopeUID(ac.Parameter(":uid")) + + if l.features.IsEnabled(featuremgmt.FlagLibraryPanelRBAC) { + entities.Post("/", authorize(ac.EvalPermission(ActionLibraryPanelsCreate)), routing.Wrap(l.createHandler)) + entities.Delete("/:uid", authorize(ac.EvalPermission(ActionLibraryPanelsDelete, uidScope)), routing.Wrap(l.deleteHandler)) + entities.Get("/", authorize(ac.EvalPermission(ActionLibraryPanelsRead)), routing.Wrap(l.getAllHandler)) + entities.Get("/:uid", authorize(ac.EvalPermission(ActionLibraryPanelsRead, uidScope)), routing.Wrap(l.getHandler)) + entities.Get("/:uid/connections/", authorize(ac.EvalPermission(ActionLibraryPanelsRead, uidScope)), routing.Wrap(l.getConnectionsHandler)) + entities.Get("/name/:name", routing.Wrap(l.getByNameHandler)) + entities.Patch("/:uid", authorize(ac.EvalPermission(ActionLibraryPanelsWrite, uidScope)), routing.Wrap(l.patchHandler)) + } else { + entities.Post("/", routing.Wrap(l.createHandler)) + entities.Delete("/:uid", routing.Wrap(l.deleteHandler)) + entities.Get("/", routing.Wrap(l.getAllHandler)) + entities.Get("/:uid", routing.Wrap(l.getHandler)) + entities.Get("/:uid/connections/", routing.Wrap(l.getConnectionsHandler)) + entities.Get("/name/:name", routing.Wrap(l.getByNameHandler)) + entities.Patch("/:uid", routing.Wrap(l.patchHandler)) + } }) } @@ -56,7 +71,8 @@ func (l *LibraryElementService) createHandler(c *contextmodel.ReqContext) respon cmd.FolderID = folder.ID } } - element, err := l.CreateElement(c.Req.Context(), c.SignedInUser, cmd) + + element, err := l.createLibraryElement(c.Req.Context(), c.SignedInUser, cmd) if err != nil { return toLibraryElementError(err, "Failed to create library element") } @@ -109,6 +125,7 @@ func (l *LibraryElementService) deleteHandler(c *contextmodel.ReqContext) respon // Responses: // 200: getLibraryElementResponse // 401: unauthorisedError +// 403: forbiddenError // 404: notFoundError // 500: internalServerError func (l *LibraryElementService) getHandler(c *contextmodel.ReqContext) response.Response { @@ -154,6 +171,14 @@ func (l *LibraryElementService) getAllHandler(c *contextmodel.ReqContext) respon return toLibraryElementError(err, "Failed to get library elements") } + if l.features.IsEnabled(featuremgmt.FlagLibraryPanelRBAC) { + filteredPanels, err := l.filterLibraryPanelsByPermission(c, elementsResult.Elements) + if err != nil { + return toLibraryElementError(err, "Failed to evaluate permissions") + } + elementsResult.Elements = filteredPanels + } + return response.JSON(http.StatusOK, model.LibraryElementSearchResponse{Result: elementsResult}) } @@ -216,6 +241,7 @@ func (l *LibraryElementService) patchHandler(c *contextmodel.ReqContext) respons // Responses: // 200: getLibraryElementConnectionsResponse // 401: unauthorisedError +// 403: forbiddenError // 404: notFoundError // 500: internalServerError func (l *LibraryElementService) getConnectionsHandler(c *contextmodel.ReqContext) response.Response { @@ -244,7 +270,31 @@ func (l *LibraryElementService) getByNameHandler(c *contextmodel.ReqContext) res return toLibraryElementError(err, "Failed to get library element") } - return response.JSON(http.StatusOK, model.LibraryElementArrayResponse{Result: elements}) + if l.features.IsEnabled(featuremgmt.FlagLibraryPanelRBAC) { + filteredElements, err := l.filterLibraryPanelsByPermission(c, elements) + if err != nil { + return toLibraryElementError(err, err.Error()) + } + + return response.JSON(http.StatusOK, model.LibraryElementArrayResponse{Result: filteredElements}) + } else { + return response.JSON(http.StatusOK, model.LibraryElementArrayResponse{Result: elements}) + } +} + +func (l *LibraryElementService) filterLibraryPanelsByPermission(c *contextmodel.ReqContext, elements []model.LibraryElementDTO) ([]model.LibraryElementDTO, error) { + filteredPanels := make([]model.LibraryElementDTO, 0) + for _, p := range elements { + allowed, err := l.AccessControl.Evaluate(c.Req.Context(), c.SignedInUser, ac.EvalPermission(ActionLibraryPanelsRead, ScopeLibraryPanelsProvider.GetResourceScopeUID(p.UID))) + if err != nil { + return nil, err + } + if allowed { + filteredPanels = append(filteredPanels, p) + } + } + + return filteredPanels, nil } func toLibraryElementError(err error, message string) response.Response { diff --git a/pkg/services/libraryelements/database.go b/pkg/services/libraryelements/database.go index 2932bfa3e17..73df537ec8b 100644 --- a/pkg/services/libraryelements/database.go +++ b/pkg/services/libraryelements/database.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/kinds/librarypanel" + ac "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/auth/identity" "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/featuremgmt" @@ -82,7 +83,7 @@ func syncFieldsWithModel(libraryElement *model.LibraryElement) error { return nil } -func getLibraryElement(dialect migrator.Dialect, session *db.Session, uid string, orgID int64) (model.LibraryElementWithMeta, error) { +func GetLibraryElement(dialect migrator.Dialect, session *db.Session, uid string, orgID int64) (model.LibraryElementWithMeta, error) { elements := make([]model.LibraryElementWithMeta, 0) sql := selectLibraryElementDTOWithMeta + ", coalesce(dashboard.title, 'General') AS folder_name" + @@ -161,8 +162,18 @@ func (l *LibraryElementService) createLibraryElement(c context.Context, signedIn } err = l.SQLStore.WithTransactionalDbSession(c, func(session *db.Session) error { - if err := l.requireEditPermissionsOnFolder(c, signedInUser, cmd.FolderID); err != nil { - return err + if l.features.IsEnabled(featuremgmt.FlagLibraryPanelRBAC) { + allowed, err := l.AccessControl.Evaluate(c, signedInUser, ac.EvalPermission(ActionLibraryPanelsCreate, dashboards.ScopeFoldersProvider.GetResourceScopeUID(*cmd.FolderUID))) + if !allowed { + return fmt.Errorf("insufficient permissions for creating library panel in folder with UID %s", *cmd.FolderUID) + } + if err != nil { + return err + } + } else { + if err := l.requireEditPermissionsOnFolder(c, signedInUser, cmd.FolderID); err != nil { + return err + } } if _, err := session.Insert(&element); err != nil { if l.SQLStore.GetDialect().IsUniqueConstraintViolation(err) { @@ -208,7 +219,7 @@ func (l *LibraryElementService) createLibraryElement(c context.Context, signedIn func (l *LibraryElementService) deleteLibraryElement(c context.Context, signedInUser identity.Requester, uid string) (int64, error) { var elementID int64 err := l.SQLStore.WithTransactionalDbSession(c, func(session *db.Session) error { - element, err := getLibraryElement(l.SQLStore.GetDialect(), session, uid, signedInUser.GetOrgID()) + element, err := GetLibraryElement(l.SQLStore.GetDialect(), session, uid, signedInUser.GetOrgID()) if err != nil { return err } @@ -520,7 +531,7 @@ func (l *LibraryElementService) patchLibraryElement(c context.Context, signedInU return model.LibraryElementDTO{}, err } err := l.SQLStore.WithTransactionalDbSession(c, func(session *db.Session) error { - elementInDB, err := getLibraryElement(l.SQLStore.GetDialect(), session, uid, signedInUser.GetOrgID()) + elementInDB, err := GetLibraryElement(l.SQLStore.GetDialect(), session, uid, signedInUser.GetOrgID()) if err != nil { return err } @@ -537,7 +548,7 @@ func (l *LibraryElementService) patchLibraryElement(c context.Context, signedInU return model.ErrLibraryElementUIDTooLong } - _, err := getLibraryElement(l.SQLStore.GetDialect(), session, updateUID, signedInUser.GetOrgID()) + _, err := GetLibraryElement(l.SQLStore.GetDialect(), session, updateUID, signedInUser.GetOrgID()) if !errors.Is(err, model.ErrLibraryElementNotFound) { return model.ErrLibraryElementAlreadyExists } @@ -634,7 +645,7 @@ func (l *LibraryElementService) getConnections(c context.Context, signedInUser i } err = l.SQLStore.WithDbSession(c, func(session *db.Session) error { - element, err := getLibraryElement(l.SQLStore.GetDialect(), session, uid, signedInUser.GetOrgID()) + element, err := GetLibraryElement(l.SQLStore.GetDialect(), session, uid, signedInUser.GetOrgID()) if err != nil { return err } @@ -737,7 +748,7 @@ func (l *LibraryElementService) connectElementsToDashboardID(c context.Context, return err } for _, elementUID := range elementUIDs { - element, err := getLibraryElement(l.SQLStore.GetDialect(), session, elementUID, signedInUser.GetOrgID()) + element, err := GetLibraryElement(l.SQLStore.GetDialect(), session, elementUID, signedInUser.GetOrgID()) if err != nil { return err } diff --git a/pkg/services/libraryelements/libraryelements.go b/pkg/services/libraryelements/libraryelements.go index 65f60cbc7f2..864f951ac75 100644 --- a/pkg/services/libraryelements/libraryelements.go +++ b/pkg/services/libraryelements/libraryelements.go @@ -7,6 +7,7 @@ import ( "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/auth/identity" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/folder" @@ -14,7 +15,7 @@ import ( "github.com/grafana/grafana/pkg/setting" ) -func ProvideService(cfg *setting.Cfg, sqlStore db.DB, routeRegister routing.RouteRegister, folderService folder.Service, features featuremgmt.FeatureToggles) *LibraryElementService { +func ProvideService(cfg *setting.Cfg, sqlStore db.DB, routeRegister routing.RouteRegister, folderService folder.Service, features featuremgmt.FeatureToggles, ac accesscontrol.AccessControl) *LibraryElementService { l := &LibraryElementService{ Cfg: cfg, SQLStore: sqlStore, @@ -22,8 +23,12 @@ func ProvideService(cfg *setting.Cfg, sqlStore db.DB, routeRegister routing.Rout folderService: folderService, log: log.New("library-elements"), features: features, + AccessControl: ac, } + l.registerAPIEndpoints() + ac.RegisterScopeAttributeResolver(LibraryPanelUIDScopeResolver(l, l.folderService)) + return l } @@ -45,6 +50,7 @@ type LibraryElementService struct { folderService folder.Service log log.Logger features featuremgmt.FeatureToggles + AccessControl accesscontrol.AccessControl } var _ Service = (*LibraryElementService)(nil) diff --git a/pkg/services/librarypanels/librarypanels_test.go b/pkg/services/librarypanels/librarypanels_test.go index d64c2cd4d82..2b3bae99b2a 100644 --- a/pkg/services/librarypanels/librarypanels_test.go +++ b/pkg/services/librarypanels/librarypanels_test.go @@ -830,7 +830,7 @@ func testScenario(t *testing.T, desc string, fn func(t *testing.T, sc scenarioCo features := featuremgmt.WithFeatures() folderService := folderimpl.ProvideService(ac, bus.ProvideBus(tracing.InitializeTracerForTest()), cfg, dashboardStore, folderStore, sqlStore, features) - elementService := libraryelements.ProvideService(cfg, sqlStore, routing.NewRouteRegister(), folderService, featuremgmt.WithFeatures()) + elementService := libraryelements.ProvideService(cfg, sqlStore, routing.NewRouteRegister(), folderService, featuremgmt.WithFeatures(), ac) service := LibraryPanelService{ Cfg: cfg, SQLStore: sqlStore, diff --git a/pkg/services/sqlstore/migrations/accesscontrol/dashboard_permissions.go b/pkg/services/sqlstore/migrations/accesscontrol/dashboard_permissions.go index 51099bb4e7d..083a6fe2196 100644 --- a/pkg/services/sqlstore/migrations/accesscontrol/dashboard_permissions.go +++ b/pkg/services/sqlstore/migrations/accesscontrol/dashboard_permissions.go @@ -555,6 +555,111 @@ func (m *managedFolderAlertActionsRepeatMigrator) Exec(sess *xorm.Session, mg *m return nil } +const managedFolderLibraryPanelActionsMigratorID = "managed folder permissions library panel actions migration" + +func AddManagedFolderLibraryPanelActionsMigration(mg *migrator.Migrator) { + mg.AddMigration(managedFolderLibraryPanelActionsMigratorID, &managedFolderLibraryPanelActionsMigrator{}) +} + +type managedFolderLibraryPanelActionsMigrator struct { + migrator.MigrationBase +} + +func (m *managedFolderLibraryPanelActionsMigrator) SQL(dialect migrator.Dialect) string { + return CodeMigrationSQL +} + +// TODO: Refactor with alerts migration +func (m *managedFolderLibraryPanelActionsMigrator) Exec(sess *xorm.Session, mg *migrator.Migrator) error { + var ids []any + if err := sess.SQL("SELECT id FROM role WHERE name LIKE 'managed:%'").Find(&ids); err != nil { + return err + } + + if len(ids) == 0 { + return nil + } + + var permissions []ac.Permission + if err := sess.SQL("SELECT role_id, action, scope FROM permission WHERE role_id IN(?"+strings.Repeat(" ,?", len(ids)-1)+") AND scope LIKE 'folders:%'", ids...).Find(&permissions); err != nil { + return err + } + + mapped := make(map[int64]map[string][]ac.Permission, len(ids)-1) + for _, p := range permissions { + if mapped[p.RoleID] == nil { + mapped[p.RoleID] = make(map[string][]ac.Permission) + } + mapped[p.RoleID][p.Scope] = append(mapped[p.RoleID][p.Scope], p) + } + + var toAdd []ac.Permission + now := time.Now() + + for id, a := range mapped { + for scope, p := range a { + if hasFolderView(p) { + if !hasAction(ac.ActionLibraryPanelsRead, p) { + toAdd = append(toAdd, ac.Permission{ + RoleID: id, + Updated: now, + Created: now, + Scope: scope, + Action: ac.ActionLibraryPanelsRead, + }) + } + } + + if hasFolderAdmin(p) || hasFolderEdit(p) { + if !hasAction(ac.ActionLibraryPanelsCreate, p) { + toAdd = append(toAdd, ac.Permission{ + RoleID: id, + Updated: now, + Created: now, + Scope: scope, + Action: ac.ActionLibraryPanelsCreate, + }) + } + if !hasAction(ac.ActionLibraryPanelsDelete, p) { + toAdd = append(toAdd, ac.Permission{ + RoleID: id, + Updated: now, + Created: now, + Scope: scope, + Action: ac.ActionLibraryPanelsDelete, + }) + } + if !hasAction(ac.ActionLibraryPanelsWrite, p) { + toAdd = append(toAdd, ac.Permission{ + RoleID: id, + Updated: now, + Created: now, + Scope: scope, + Action: ac.ActionLibraryPanelsWrite, + }) + } + } + } + } + + if len(toAdd) == 0 { + return nil + } + + err := batch(len(toAdd), batchSize, func(start, end int) error { + if _, err := sess.InsertMulti(toAdd[start:end]); err != nil { + return err + } + return nil + }) + + if err != nil { + return err + } + + return nil +} + func hasFolderAdmin(permissions []ac.Permission) bool { return hasActions(folderPermissionTranslation[dashboards.PERMISSION_ADMIN], permissions) } diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index 56df4323cc9..4272a5cafff 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -89,6 +89,7 @@ func (*OSSMigrations) AddMigration(mg *Migrator) { accesscontrol.AddAdminOnlyMigration(mg) accesscontrol.AddSeedAssignmentMigrations(mg) accesscontrol.AddManagedFolderAlertActionsRepeatFixedMigration(mg) + accesscontrol.AddManagedFolderLibraryPanelActionsMigration(mg) AddExternalAlertmanagerToDatasourceMigration(mg) From 65a94d9e752963257de817fb54fbd65d5dee93e0 Mon Sep 17 00:00:00 2001 From: Leon Sorokin Date: Thu, 12 Oct 2023 00:05:21 -0500 Subject: [PATCH 073/102] ZoomPlugin: No-op on modifier keys (#76398) --- .../src/components/uPlot/plugins/ZoomPlugin.tsx | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/grafana-ui/src/components/uPlot/plugins/ZoomPlugin.tsx b/packages/grafana-ui/src/components/uPlot/plugins/ZoomPlugin.tsx index 415d000d303..02f539dd3ee 100644 --- a/packages/grafana-ui/src/components/uPlot/plugins/ZoomPlugin.tsx +++ b/packages/grafana-ui/src/components/uPlot/plugins/ZoomPlugin.tsx @@ -11,6 +11,8 @@ interface ZoomPluginProps { // min px width that triggers zoom const MIN_ZOOM_DIST = 5; +const maybeZoomAction = (e?: MouseEvent | null) => e != null && !e.ctrlKey && !e.metaKey; + /** * @alpha */ @@ -21,9 +23,13 @@ export const ZoomPlugin = ({ onZoom, config, withZoomY = false }: ZoomPluginProp if (withZoomY) { config.addHook('init', (u) => { - u.root!.addEventListener( + u.over!.addEventListener( 'mousedown', (e) => { + if (!maybeZoomAction(e)) { + return; + } + if (e.button === 0 && e.shiftKey) { yDrag = true; @@ -45,6 +51,10 @@ export const ZoomPlugin = ({ onZoom, config, withZoomY = false }: ZoomPluginProp } config.addHook('setSelect', (u) => { + if (!maybeZoomAction(u.cursor!.event)) { + return; + } + if (withZoomY && yDrag) { if (u.select.height >= MIN_ZOOM_DIST) { for (let key in u.scales!) { @@ -76,6 +86,10 @@ export const ZoomPlugin = ({ onZoom, config, withZoomY = false }: ZoomPluginProp config.setCursor({ bind: { dblclick: (u) => () => { + if (!maybeZoomAction(u.cursor!.event)) { + return null; + } + if (withZoomY && yZoomed) { for (let key in u.scales!) { if (key !== 'x') { From 4dfffa3c32089f0d888f1b62690ae031c38e6079 Mon Sep 17 00:00:00 2001 From: Victor Marin <36818606+mdvictor@users.noreply.github.com> Date: Thu, 12 Oct 2023 08:43:32 +0300 Subject: [PATCH 074/102] Add context to transformation editor (#76317) * Add context to editor so field calculation does not return undefined variable and shows on next transform * fix bug where default alias would get interpolated --- .../transformations/transformers/calculateField.ts | 6 +++++- .../TransformationsEditor/TransformationEditor.tsx | 13 ++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/grafana-data/src/transformations/transformers/calculateField.ts b/packages/grafana-data/src/transformations/transformers/calculateField.ts index d73404a2196..e749dfecf72 100644 --- a/packages/grafana-data/src/transformations/transformers/calculateField.ts +++ b/packages/grafana-data/src/transformations/transformers/calculateField.ts @@ -315,7 +315,11 @@ export function getNameFromOptions(options: CalculateFieldTransformerOptions) { } case CalculateFieldMode.BinaryOperation: { const { binary } = options; - return `${binary?.left ?? ''} ${binary?.operator ?? ''} ${binary?.right ?? ''}`; + const alias = `${binary?.left ?? ''} ${binary?.operator ?? ''} ${binary?.right ?? ''}`; + + //Remove $ signs as they will be interpolated and cause issues. Variables can still be used + //in alias but shouldn't in the autogenerated name + return alias.replace(/\$/g, ''); } case CalculateFieldMode.ReduceRow: { diff --git a/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx b/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx index 71d60048059..83a0fa747a2 100644 --- a/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx +++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx @@ -9,8 +9,10 @@ import { transformDataFrame, TransformerRegistryItem, getFrameMatchers, + DataTransformContext, } from '@grafana/data'; import { selectors } from '@grafana/e2e-selectors'; +import { getTemplateSrv } from '@grafana/runtime'; import { Icon, JSONFormatter, useStyles2 } from '@grafana/ui'; import { TransformationsEditorTransformation } from './types'; @@ -42,14 +44,19 @@ export const TransformationEditor = ({ const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined; const inputTransforms = configs.slice(0, index).map((t) => t.transformation); const outputTransforms = configs.slice(index, index + 1).map((t) => t.transformation); - const inputSubscription = transformDataFrame(inputTransforms, data).subscribe((v) => { + + const ctx: DataTransformContext = { + interpolate: (v: string) => getTemplateSrv().replace(v), + }; + + const inputSubscription = transformDataFrame(inputTransforms, data, ctx).subscribe((v) => { if (matcher) { v = data.filter((v) => matcher(v)); } setInput(v); }); - const outputSubscription = transformDataFrame(inputTransforms, data) - .pipe(mergeMap((before) => transformDataFrame(outputTransforms, before))) + const outputSubscription = transformDataFrame(inputTransforms, data, ctx) + .pipe(mergeMap((before) => transformDataFrame(outputTransforms, before, ctx))) .subscribe(setOutput); return function unsubscribe() { From 94c15e492631dd5465d6e81bd83f23d7c9ccfd70 Mon Sep 17 00:00:00 2001 From: Leon Sorokin Date: Thu, 12 Oct 2023 01:19:22 -0500 Subject: [PATCH 075/102] TooltipPlugin2: Raise parent grid item z-index when pinned (#76379) --- .betterer.results | 3 + .../uPlot/plugins/TooltipPlugin2.tsx | 72 +++++++++++-------- public/sass/components/_dashboard_grid.scss | 6 +- 3 files changed, 51 insertions(+), 30 deletions(-) diff --git a/.betterer.results b/.betterer.results index abfc6463bfe..3b87f1e2d77 100644 --- a/.betterer.results +++ b/.betterer.results @@ -1092,6 +1092,9 @@ exports[`better eslint`] = { [0, 0, 0, "Do not use any type assertions.", "0"], [0, 0, 0, "Do not use any type assertions.", "1"] ], + "packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin2.tsx:5381": [ + [0, 0, 0, "Do not use any type assertions.", "0"] + ], "packages/grafana-ui/src/components/uPlot/types.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"] ], diff --git a/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin2.tsx b/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin2.tsx index 6562d8aa334..9699817e6c1 100644 --- a/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin2.tsx +++ b/packages/grafana-ui/src/components/uPlot/plugins/TooltipPlugin2.tsx @@ -5,14 +5,13 @@ import uPlot from 'uplot'; import { GrafanaTheme2 } from '@grafana/data'; -import { useStyles2 } from '../../../themes/ThemeContext'; +import { useStyles2 } from '../../../themes'; import { UPlotConfigBuilder } from '../config/UPlotConfigBuilder'; import { CloseButton } from './CloseButton'; interface TooltipPlugin2Props { config: UPlotConfigBuilder; - // or via .children() render prop callback? render: ( u: uPlot, dataIdxs: Array, @@ -67,7 +66,7 @@ export const TooltipPlugin2 = ({ config, render }: TooltipPlugin2Props) => { const sizeRef = useRef(); - const className = useStyles2(getStyles).tooltipWrapper; + const styles = useStyles2(getStyles); useLayoutEffect(() => { sizeRef.current = { @@ -108,8 +107,9 @@ export const TooltipPlugin2 = ({ config, render }: TooltipPlugin2Props) => { let closestSeriesIdx: number | null = null; let pendingRender = false; + let pendingPinned = false; - const scheduleRender = () => { + const scheduleRender = (setPinned = false) => { if (!pendingRender) { // defer unrender for 100ms to reduce flickering in small gaps if (!_isHovering) { @@ -120,11 +120,43 @@ export const TooltipPlugin2 = ({ config, render }: TooltipPlugin2Props) => { pendingRender = true; } + + if (setPinned) { + pendingPinned = true; + } + }; + + // in some ways this is similar to ClickOutsideWrapper.tsx + const downEventOutside = (e: Event) => { + let isOutside = (e.target as HTMLDivElement).closest(`.${styles.tooltipWrapper}`) !== domRef.current; + + if (isOutside) { + dismiss(); + } }; const _render = () => { pendingRender = false; + if (pendingPinned) { + _style = { pointerEvents: _isPinned ? 'all' : 'none' }; + + domRef.current!.closest('.react-grid-item')?.classList.toggle('context-menu-open', _isPinned); + + // @ts-ignore + _plot!.cursor._lock = _isPinned; + + if (_isPinned) { + document.addEventListener('mousedown', downEventOutside, true); + document.addEventListener('keydown', downEventOutside, true); + } else { + document.removeEventListener('mousedown', downEventOutside, true); + document.removeEventListener('keydown', downEventOutside, true); + } + + pendingPinned = false; + } + let state: TooltipContainerState = { style: _style, isPinned: _isPinned, @@ -139,35 +171,18 @@ export const TooltipPlugin2 = ({ config, render }: TooltipPlugin2Props) => { const dismiss = () => { _isPinned = false; _isHovering = false; - _style = { pointerEvents: 'none' }; - - // @ts-ignore - _plot!.cursor._lock = _isPinned; - - scheduleRender(); + _plot!.setCursor({ left: -10, top: -10 }); + scheduleRender(true); }; config.addHook('init', (u) => { setState({ plot: (_plot = u) }); - // TODO: use cursor.lock & and mousedown/mouseup here (to prevent unlocking) + // this handles pinning u.over.addEventListener('click', (e) => { - if (_isHovering) { - if (e.target === u.over) { - _isPinned = !_isPinned; - _style = { pointerEvents: _isPinned ? 'all' : 'none' }; - scheduleRender(); - } - - // @ts-ignore - u.cursor._lock = _isPinned; - - // hack to trigger cursor to new position after unlock - // (should not be necessary after using the cursor.lock API) - if (!_isPinned) { - u.setCursor({ left: e.clientX - u.rect.left, top: e.clientY - u.rect.top }); - _isHovering = false; - } + if (_isHovering && !_isPinned && e.target === u.over) { + _isPinned = true; + scheduleRender(true); } }); }); @@ -268,7 +283,7 @@ export const TooltipPlugin2 = ({ config, render }: TooltipPlugin2Props) => { if (plot && isHovering) { return createPortal( -
    +
    {isPinned && } {contents}
    , @@ -290,5 +305,6 @@ const getStyles = (theme: GrafanaTheme2) => ({ position: 'absolute', background: theme.colors.background.secondary, boxShadow: `0 4px 8px ${theme.colors.background.primary}`, + userSelect: 'text', }), }); diff --git a/public/sass/components/_dashboard_grid.scss b/public/sass/components/_dashboard_grid.scss index a0f40c3b105..7bba52c7524 100644 --- a/public/sass/components/_dashboard_grid.scss +++ b/public/sass/components/_dashboard_grid.scss @@ -87,7 +87,7 @@ } // Hack to prevent panel overlap during drag/hover (due to descending z-index assignment) -.react-grid-item { +.react-grid-item:not(.context-menu-open, .resizing) { &:hover, &:active, &:focus { @@ -96,10 +96,12 @@ } // Hack for preventing panel menu overlapping. +.react-grid-item.context-menu-open, +.react-grid-item.resizing, .react-grid-item.resizing.panel, .react-grid-item.panel.dropdown-menu-open, .react-grid-item.react-draggable-dragging.panel { - z-index: $zindex-dropdown; + z-index: $zindex-dropdown !important; } // Disable animation on initial rendering and enable it when component has been mounted. From fbbf9b1a8fa38281a0ac95e70c10b3289ca365a8 Mon Sep 17 00:00:00 2001 From: Sonia Aguilar <33540275+soniaAguilarPeiron@users.noreply.github.com> Date: Thu, 12 Oct 2023 08:04:59 +0100 Subject: [PATCH 076/102] Alerting: Use new endpoints in the Modify Export (#75796) Co-authored-by: Konrad Lalik --- .../alerting/unified/api/alertRuleApi.ts | 22 +++ .../components/export/GrafanaModifyExport.tsx | 33 ++-- .../components/rule-editor/FolderAndGroup.tsx | 34 ++-- .../rule-editor/GrafanaEvaluationBehavior.tsx | 19 +- .../alert-rule-form/AlertRuleForm.tsx | 20 +- .../alert-rule-form/ModifyExportRuleForm.tsx | 187 ++++++++++++++---- .../getPayloadToExport.test.ts | 125 ++++++++++++ .../unified/components/rule-editor/util.ts | 6 +- .../components/rules/RuleActionsButtons.tsx | 5 +- .../alerting/unified/utils/rule-form.ts | 17 ++ 10 files changed, 377 insertions(+), 91 deletions(-) create mode 100644 public/app/features/alerting/unified/components/rule-editor/alert-rule-form/getPayloadToExport.test.ts diff --git a/public/app/features/alerting/unified/api/alertRuleApi.ts b/public/app/features/alerting/unified/api/alertRuleApi.ts index d8b54019fe4..bee885c2e0d 100644 --- a/public/app/features/alerting/unified/api/alertRuleApi.ts +++ b/public/app/features/alerting/unified/api/alertRuleApi.ts @@ -6,7 +6,10 @@ import { Annotations, GrafanaAlertStateDecision, Labels, + PostableRuleGrafanaRuleDTO, PromRulesResponse, + RulerAlertingRuleDTO, + RulerRecordingRuleDTO, RulerRuleGroupDTO, RulerRulesConfigDTO, } from 'app/types/unified-alerting-dto'; @@ -68,6 +71,13 @@ export interface Rule { export type AlertInstances = Record; +export interface ModifyExportPayload { + rules: Array; + name: string; + interval?: string | undefined; + source_tenants?: string[] | undefined; +} + export const alertRuleApi = alertingApi.injectEndpoints({ endpoints: (build) => ({ preview: build.mutation< @@ -220,5 +230,17 @@ export const alertRuleApi = alertingApi.injectEndpoints({ responseType: 'text', }), }), + exportModifiedRuleGroup: build.mutation< + string, + { payload: ModifyExportPayload; format: ExportFormats; nameSpace: string } + >({ + query: ({ payload, format, nameSpace }) => ({ + url: `/api/ruler/grafana/api/v1/rules/${nameSpace}/export/`, + params: { format: format }, + responseType: 'text', + data: payload, + method: 'POST', + }), + }), }), }); diff --git a/public/app/features/alerting/unified/components/export/GrafanaModifyExport.tsx b/public/app/features/alerting/unified/components/export/GrafanaModifyExport.tsx index 3dffb337a74..02a671b35ea 100644 --- a/public/app/features/alerting/unified/components/export/GrafanaModifyExport.tsx +++ b/public/app/features/alerting/unified/components/export/GrafanaModifyExport.tsx @@ -1,4 +1,3 @@ -import { omit } from 'lodash'; import * as React from 'react'; import { useEffect, useState } from 'react'; import { useAsync } from 'react-use'; @@ -8,11 +7,9 @@ import { Alert, LoadingPlaceholder } from '@grafana/ui'; import { GrafanaRouteComponentProps } from '../../../../../core/navigation/types'; import { useDispatch } from '../../../../../types'; -import { RuleIdentifier, RuleWithLocation } from '../../../../../types/unified-alerting'; -import { RulerRuleDTO } from '../../../../../types/unified-alerting-dto'; +import { RuleIdentifier } from '../../../../../types/unified-alerting'; import { fetchEditableRuleAction, fetchRulesSourceBuildInfoAction } from '../../state/actions'; -import { RuleFormValues } from '../../types/rule-form'; -import { rulerRuleToFormValues } from '../../utils/rule-form'; +import { formValuesFromExistingRule } from '../../utils/rule-form'; import * as ruleId from '../../utils/rule-id'; import { isGrafanaRulerRule } from '../../utils/rules'; import { createUrl } from '../../utils/url'; @@ -21,18 +18,6 @@ import { ModifyExportRuleForm } from '../rule-editor/alert-rule-form/ModifyExpor interface GrafanaModifyExportProps extends GrafanaRouteComponentProps<{ id?: string }> {} -// TODO Duplicated in AlertRuleForm -const ignoreHiddenQueries = (ruleDefinition: RuleFormValues): RuleFormValues => { - return { - ...ruleDefinition, - queries: ruleDefinition.queries?.map((query) => omit(query, 'model.hide')), - }; -}; - -function formValuesFromExistingRule(rule: RuleWithLocation) { - return ignoreHiddenQueries(rulerRuleToFormValues(rule)); -} - export default function GrafanaModifyExport({ match }: GrafanaModifyExportProps) { const dispatch = useDispatch(); @@ -105,8 +90,18 @@ export default function GrafanaModifyExport({ match }: GrafanaModifyExportProps) } return ( - - {alertRule && } + + {alertRule && ( + + )} ); } diff --git a/public/app/features/alerting/unified/components/rule-editor/FolderAndGroup.tsx b/public/app/features/alerting/unified/components/rule-editor/FolderAndGroup.tsx index c7502b41a74..6349303e795 100644 --- a/public/app/features/alerting/unified/components/rule-editor/FolderAndGroup.tsx +++ b/public/app/features/alerting/unified/components/rule-editor/FolderAndGroup.tsx @@ -28,7 +28,7 @@ import { checkForPathSeparator } from './util'; export const MAX_GROUP_RESULTS = 1000; -export const useGetGroupOptionsFromFolder = (folderTitle: string) => { +export const useFolderGroupOptions = (folderTitle: string, enableProvisionedGroups: boolean) => { const dispatch = useDispatch(); // fetch the ruler rules from the database so we can figure out what other "groups" are already defined @@ -44,13 +44,18 @@ export const useGetGroupOptionsFromFolder = (folderTitle: string) => { const folderGroups = grafanaFolders.find((f) => f.name === folderTitle)?.groups ?? []; const groupOptions = folderGroups - .map>((group) => ({ - label: group.name, - value: group.name, - description: group.interval ?? MINUTE, - // we include provisioned folders, but disable the option to select them - isDisabled: isProvisionedGroup(group), - })) + .map>((group) => { + const isProvisioned = isProvisionedGroup(group); + return { + label: group.name, + value: group.name, + description: group.interval ?? MINUTE, + // we include provisioned folders, but disable the option to select them + isDisabled: !enableProvisionedGroups ? isProvisioned : false, + isProvisioned: isProvisioned, + }; + }) + .sort(sortByLabel); return { groupOptions, loading: groupfoldersForGrafana?.loading }; @@ -70,7 +75,13 @@ const findGroupMatchingLabel = (group: SelectableValue, query: string) = return group.label?.toLowerCase().includes(query.toLowerCase()); }; -export function FolderAndGroup({ groupfoldersForGrafana }: { groupfoldersForGrafana?: RulerRulesConfigDTO | null }) { +export function FolderAndGroup({ + groupfoldersForGrafana, + enableProvisionedGroups, +}: { + groupfoldersForGrafana?: RulerRulesConfigDTO | null; + enableProvisionedGroups: boolean; +}) { const { formState: { errors }, watch, @@ -83,7 +94,7 @@ export function FolderAndGroup({ groupfoldersForGrafana }: { groupfoldersForGraf const folder = watch('folder'); const group = watch('group'); - const { groupOptions, loading } = useGetGroupOptionsFromFolder(folder?.title ?? ''); + const { groupOptions, loading } = useFolderGroupOptions(folder?.title ?? '', enableProvisionedGroups); const [isCreatingFolder, setIsCreatingFolder] = useState(false); const [isCreatingEvaluationGroup, setIsCreatingEvaluationGroup] = useState(false); @@ -213,8 +224,7 @@ export function FolderAndGroup({ groupfoldersForGrafana }: { groupfoldersForGraf getOptionLabel={(option: SelectableValue) => (
    {option.label} - {/* making the assumption here that it's provisioned when it's disabled, should probably change this */} - {option.isDisabled && ( + {option['isProvisioned'] && ( <> {' '} diff --git a/public/app/features/alerting/unified/components/rule-editor/GrafanaEvaluationBehavior.tsx b/public/app/features/alerting/unified/components/rule-editor/GrafanaEvaluationBehavior.tsx index 8039dcb468a..0d640760a14 100644 --- a/public/app/features/alerting/unified/components/rule-editor/GrafanaEvaluationBehavior.tsx +++ b/public/app/features/alerting/unified/components/rule-editor/GrafanaEvaluationBehavior.tsx @@ -16,7 +16,7 @@ import { parsePrometheusDuration } from '../../utils/time'; import { CollapseToggle } from '../CollapseToggle'; import { EditCloudGroupModal } from '../rules/EditRuleGroupModal'; -import { FolderAndGroup, useGetGroupOptionsFromFolder } from './FolderAndGroup'; +import { FolderAndGroup, useFolderGroupOptions } from './FolderAndGroup'; import { GrafanaAlertStatePicker } from './GrafanaAlertStatePicker'; import { NeedHelpInfo } from './NeedHelpInfo'; import { RuleEditorSection } from './RuleEditorSection'; @@ -59,7 +59,7 @@ const forValidationOptions = (evaluateEvery: string): RegisterOptions => ({ }); const useIsNewGroup = (folder: string, group: string) => { - const { groupOptions } = useGetGroupOptionsFromFolder(folder); + const { groupOptions } = useFolderGroupOptions(folder, false); const groupIsInGroupOptions = useCallback( (group_: string) => groupOptions.some((groupInList: SelectableValue) => groupInList.label === group_), @@ -71,9 +71,11 @@ const useIsNewGroup = (folder: string, group: string) => { function FolderGroupAndEvaluationInterval({ evaluateEvery, setEvaluateEvery, + enableProvisionedGroups, }: { evaluateEvery: string; setEvaluateEvery: (value: string) => void; + enableProvisionedGroups: boolean; }) { const styles = useStyles2(getStyles); const { watch, setValue, getValues } = useFormContext(); @@ -116,7 +118,10 @@ function FolderGroupAndEvaluationInterval({ return (
    - + {folderName && isEditingGroup && ( void; existing: boolean; + enableProvisionedGroups: boolean; }) { const styles = useStyles2(getStyles); const [showErrorHandling, setShowErrorHandling] = useState(false); @@ -222,7 +229,11 @@ export function GrafanaEvaluationBehavior({ // TODO remove "and alert condition" for recording rules - + {existing && ( diff --git a/public/app/features/alerting/unified/components/rule-editor/alert-rule-form/AlertRuleForm.tsx b/public/app/features/alerting/unified/components/rule-editor/alert-rule-form/AlertRuleForm.tsx index 31d4d91df2f..057b1cd66d8 100644 --- a/public/app/features/alerting/unified/components/rule-editor/alert-rule-form/AlertRuleForm.tsx +++ b/public/app/features/alerting/unified/components/rule-editor/alert-rule-form/AlertRuleForm.tsx @@ -1,5 +1,4 @@ import { css } from '@emotion/css'; -import { omit } from 'lodash'; import React, { useEffect, useMemo, useState } from 'react'; import { DeepMap, FieldError, FormProvider, useForm, UseFormWatch } from 'react-hook-form'; import { Link, useParams } from 'react-router-dom'; @@ -15,7 +14,6 @@ import { useCleanup } from 'app/core/hooks/useCleanup'; import { useQueryParams } from 'app/core/hooks/useQueryParams'; import { useDispatch } from 'app/types'; import { RuleWithLocation } from 'app/types/unified-alerting'; -import { RulerRuleDTO } from 'app/types/unified-alerting-dto'; import { LogMessages, trackNewAlerRuleFormError } from '../../../Analytics'; import { useUnifiedAlertingSelector } from '../../../hooks/useUnifiedAlertingSelector'; @@ -23,11 +21,12 @@ import { deleteRuleAction, saveRuleFormAction } from '../../../state/actions'; import { RuleFormType, RuleFormValues } from '../../../types/rule-form'; import { initialAsyncRequestState } from '../../../utils/redux'; import { + formValuesFromExistingRule, getDefaultFormValues, getDefaultQueries, + ignoreHiddenQueries, MINUTE, normalizeDefaultAnnotations, - rulerRuleToFormValues, } from '../../../utils/rule-form'; import * as ruleId from '../../../utils/rule-id'; import { GrafanaRuleExporter } from '../../export/GrafanaRuleExporter'; @@ -233,6 +232,7 @@ export const AlertRuleForm = ({ existing, prefill }: Props) => { evaluateEvery={evaluateEvery} setEvaluateEvery={setEvaluateEvery} existing={Boolean(existing)} + enableProvisionedGroups={false} /> )} @@ -279,17 +279,6 @@ const isCortexLokiOrRecordingRule = (watch: UseFormWatch) => { return (ruleType === RuleFormType.cloudAlerting || ruleType === RuleFormType.cloudRecording) && dataSourceName !== ''; }; -// the backend will always execute "hidden" queries, so we have no choice but to remove the property in the front-end -// to avoid confusion. The query editor shows them as "disabled" and that's a different semantic meaning. -// furthermore the "AlertingQueryRunner" calls `filterQuery` on each data source and those will skip running queries that are "hidden"." -// It seems like we have no choice but to act like "hidden" queries don't exist in alerting. -const ignoreHiddenQueries = (ruleDefinition: RuleFormValues): RuleFormValues => { - return { - ...ruleDefinition, - queries: ruleDefinition.queries?.map((query) => omit(query, 'model.hide')), - }; -}; - function formValuesFromQueryParams(ruleDefinition: string, type: RuleFormType): RuleFormValues { let ruleFromQueryParams: Partial; @@ -319,9 +308,6 @@ function formValuesFromPrefill(rule: Partial): RuleFormValues { }); } -function formValuesFromExistingRule(rule: RuleWithLocation) { - return ignoreHiddenQueries(rulerRuleToFormValues(rule)); -} const getStyles = (theme: GrafanaTheme2) => ({ buttonSpinner: css({ marginRight: theme.spacing(1), diff --git a/public/app/features/alerting/unified/components/rule-editor/alert-rule-form/ModifyExportRuleForm.tsx b/public/app/features/alerting/unified/components/rule-editor/alert-rule-form/ModifyExportRuleForm.tsx index c2764afeb15..c5033010171 100644 --- a/public/app/features/alerting/unified/components/rule-editor/alert-rule-form/ModifyExportRuleForm.tsx +++ b/public/app/features/alerting/unified/components/rule-editor/alert-rule-form/ModifyExportRuleForm.tsx @@ -1,12 +1,22 @@ -import React, { useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { FormProvider, useForm } from 'react-hook-form'; +import { useAsync } from 'react-use'; import { Stack } from '@grafana/experimental'; -import { Button, CustomScrollbar, LinkButton } from '@grafana/ui'; +import { Button, CustomScrollbar, LinkButton, LoadingPlaceholder } from '@grafana/ui'; +import { useAppNotification } from 'app/core/copy/appNotification'; +import { useQueryParams } from 'app/core/hooks/useQueryParams'; import { AppChromeUpdate } from '../../../../../../core/components/AppChrome/AppChromeUpdate'; +import { RulerRuleDTO, RulerRuleGroupDTO } from '../../../../../../types/unified-alerting-dto'; +import { alertRuleApi, ModifyExportPayload } from '../../../api/alertRuleApi'; +import { fetchRulerRulesGroup } from '../../../api/ruler'; +import { useDataSourceFeatures } from '../../../hooks/useCombinedRule'; import { RuleFormValues } from '../../../types/rule-form'; -import { MINUTE } from '../../../utils/rule-form'; +import { GRAFANA_RULES_SOURCE_NAME } from '../../../utils/datasource'; +import { formValuesToRulerGrafanaRuleDTO, MINUTE } from '../../../utils/rule-form'; +import { isGrafanaRulerRule } from '../../../utils/rules'; +import { FileExportPreview } from '../../export/FileExportPreview'; import { GrafanaExportDrawer } from '../../export/GrafanaExportDrawer'; import { allGrafanaExportProviders, ExportFormats } from '../../export/providers'; import { AlertRuleNameInput } from '../AlertRuleNameInput'; @@ -16,41 +26,49 @@ import { NotificationsStep } from '../NotificationsStep'; import { QueryAndExpressionsStep } from '../query-and-alert-condition/QueryAndExpressionsStep'; interface ModifyExportRuleFormProps { - alertUid?: string; + alertUid: string; ruleForm?: RuleFormValues; } -type ModifyExportMode = 'rule' | 'group'; - export function ModifyExportRuleForm({ ruleForm, alertUid }: ModifyExportRuleFormProps) { const formAPI = useForm({ mode: 'onSubmit', defaultValues: ruleForm, shouldFocusError: true, }); + const [queryParams] = useQueryParams(); - const existing = Boolean(ruleForm); - const returnTo = `/alerting/list`; + const existing = Boolean(ruleForm); // always should be true + const notifyApp = useAppNotification(); + const returnTo = !queryParams['returnTo'] ? '/alerting/list' : String(queryParams['returnTo']); - const [showExporter, setShowExporter] = useState(undefined); + const [exportData, setExportData] = useState(undefined); const [conditionErrorMsg, setConditionErrorMsg] = useState(''); - console.log('conditionErrorMsg', conditionErrorMsg); const [evaluateEvery, setEvaluateEvery] = useState(ruleForm?.evaluateEvery ?? MINUTE); const checkAlertCondition = (msg = '') => { setConditionErrorMsg(msg); }; + const submit = (exportData: RuleFormValues | undefined) => { + if (conditionErrorMsg !== '') { + notifyApp.error(conditionErrorMsg); + return; + } + setExportData(exportData); + }; + + const onClose = useCallback(() => { + setExportData(undefined); + }, [setExportData]); + const actionButtons = [ - + submit(undefined)}> Cancel , - , - , ]; @@ -72,6 +90,7 @@ export function ModifyExportRuleForm({ ruleForm, alertUid }: ModifyExportRuleFor evaluateEvery={evaluateEvery} setEvaluateEvery={setEvaluateEvery} existing={Boolean(existing)} + enableProvisionedGroups={true} /> {/* Step 4 & 5 */} @@ -83,32 +102,132 @@ export function ModifyExportRuleForm({ ruleForm, alertUid }: ModifyExportRuleFor
    + {exportData && } - {showExporter && ( - setShowExporter(undefined)} /> - )} ); } -interface GrafanaRuleDesignExporterProps { - onClose: () => void; - exportMode: ModifyExportMode; -} +const useGetGroup = (nameSpace: string, group: string) => { + const { dsFeatures } = useDataSourceFeatures(GRAFANA_RULES_SOURCE_NAME); -export const GrafanaRuleDesignExporter = ({ onClose, exportMode }: GrafanaRuleDesignExporterProps) => { - const [activeTab, setActiveTab] = useState('yaml'); - const title = exportMode === 'rule' ? 'Export Rule' : 'Export Group'; + const rulerConfig = dsFeatures?.rulerConfig; + + const targetGroup = useAsync(async () => { + return rulerConfig ? await fetchRulerRulesGroup(rulerConfig, nameSpace, group) : undefined; + }, [rulerConfig, nameSpace, group]); + + return targetGroup; +}; + +interface GrafanaRuleDesignExportPreviewProps { + exportFormat: ExportFormats; + onClose: () => void; + exportValues: RuleFormValues; + uid: string; +} +export const getPayloadToExport = ( + uid: string, + formValues: RuleFormValues, + existingGroup: RulerRuleGroupDTO | null | undefined +): ModifyExportPayload => { + const grafanaRuleDto = formValuesToRulerGrafanaRuleDTO(formValues); + + const updatedRule = { ...grafanaRuleDto, grafana_alert: { ...grafanaRuleDto.grafana_alert, uid: uid } }; + if (existingGroup?.rules) { + // we have to update the rule in the group in the same position if it exists, otherwise we have to add it at the end + let alreadyExistsInGroup = false; + const updatedRules = existingGroup.rules.map((rule: RulerRuleDTO) => { + if (isGrafanaRulerRule(rule) && rule.grafana_alert.uid === uid) { + alreadyExistsInGroup = true; + return updatedRule; + } else { + return rule; + } + }); + if (!alreadyExistsInGroup) { + // we have to add the updated rule at the end of the group + updatedRules.push(updatedRule); + } + return { + ...existingGroup, + rules: updatedRules, + }; + } else { + // we have to create a new group with the updated rule + return { + name: existingGroup?.name ?? '', + rules: [updatedRule], + }; + } +}; + +const useGetPayloadToExport = (values: RuleFormValues, uid: string) => { + const rulerGroupDto = useGetGroup(values.folder?.title ?? '', values.group); + const payload: ModifyExportPayload = useMemo(() => { + return getPayloadToExport(uid, values, rulerGroupDto?.value); + }, [uid, rulerGroupDto, values]); + return { payload, loadingGroup: rulerGroupDto.loading }; +}; + +const GrafanaRuleDesignExportPreview = ({ + exportFormat, + exportValues, + onClose, + uid, +}: GrafanaRuleDesignExportPreviewProps) => { + const [getExport, exportData] = alertRuleApi.endpoints.exportModifiedRuleGroup.useMutation(); + const { loadingGroup, payload } = useGetPayloadToExport(exportValues, uid); + + const nameSpace = exportValues.folder?.title ?? ''; + + useEffect(() => { + !loadingGroup && getExport({ payload, format: exportFormat, nameSpace: nameSpace }); + }, [nameSpace, exportFormat, payload, getExport, loadingGroup]); + + if (exportData.isLoading) { + return ; + } + + const downloadFileName = `modify-export-${payload.name}-${uid}-${new Date().getTime()}`; return ( - - TODO - + /> ); }; + +interface GrafanaRuleDesignExporterProps { + onClose: () => void; + exportValues: RuleFormValues; + uid: string; +} + +export const GrafanaRuleDesignExporter = React.memo( + ({ onClose, exportValues, uid }: GrafanaRuleDesignExporterProps) => { + const [activeTab, setActiveTab] = useState('yaml'); + + return ( + + + + ); + } +); + +GrafanaRuleDesignExporter.displayName = 'GrafanaRuleDesignExporter'; diff --git a/public/app/features/alerting/unified/components/rule-editor/alert-rule-form/getPayloadToExport.test.ts b/public/app/features/alerting/unified/components/rule-editor/alert-rule-form/getPayloadToExport.test.ts new file mode 100644 index 00000000000..1dd5e991d94 --- /dev/null +++ b/public/app/features/alerting/unified/components/rule-editor/alert-rule-form/getPayloadToExport.test.ts @@ -0,0 +1,125 @@ +import { RulerRuleDTO, RulerRuleGroupDTO } from 'app/types/unified-alerting-dto'; + +import { mockRulerGrafanaRule } from '../../../mocks'; +import { RuleFormValues } from '../../../types/rule-form'; +import { Annotation } from '../../../utils/constants'; +import { getDefaultFormValues } from '../../../utils/rule-form'; + +import { getPayloadToExport } from './ModifyExportRuleForm'; + +const rule1 = mockRulerGrafanaRule( + { + for: '1m', + labels: { severity: 'critical', region: 'region1' }, + annotations: { [Annotation.summary]: 'This grafana rule1' }, + }, + { uid: 'uid-rule-1', title: 'Rule1', data: [] } +); + +const rule2 = mockRulerGrafanaRule( + { + for: '1m', + labels: { severity: 'notcritical', region: 'region2' }, + annotations: { [Annotation.summary]: 'This grafana rule2' }, + }, + { uid: 'uid-rule-2', title: 'Rule2', data: [] } +); + +const rule3 = mockRulerGrafanaRule( + { + for: '1m', + labels: { severity: 'notcritical3', region: 'region3' }, + annotations: { [Annotation.summary]: 'This grafana rule2' }, + }, + { uid: 'uid-rule-3', title: 'Rule3', data: [] } +); + +// Prepare the form values for rule2 updated +const defaultValues = getDefaultFormValues(); +const formValuesForRule2Updated: RuleFormValues = { + ...defaultValues, + queries: [ + { + refId: 'A', + relativeTimeRange: { from: 900, to: 1000 }, + datasourceUid: 'dsuid', + model: { + refId: 'A', + hide: true, + }, + queryType: 'query', + }, + ], + condition: 'A', + forTime: 2455, + name: 'Rule2 updated', + labels: [{ key: 'newLabel', value: 'newLabel' }], + annotations: [{ key: 'summary', value: 'This grafana rule2 updated' }], +}; + +const expectedModifiedRule2 = (uid: string) => ({ + annotations: { + summary: 'This grafana rule2 updated', + }, + for: '5m', + grafana_alert: { + condition: 'A', + data: [ + { + datasourceUid: 'dsuid', + model: { + refId: 'A', + hide: true, + }, + queryType: 'query', + refId: 'A', + relativeTimeRange: { + from: 900, + to: 1000, + }, + }, + ], + exec_err_state: 'Error', + is_paused: false, + no_data_state: 'NoData', + title: 'Rule2 updated', + uid: uid, + }, + labels: { + newLabel: 'newLabel', + }, +}); + +describe('getPayloadFromDto', () => { + const groupDto: RulerRuleGroupDTO = { + name: 'Test Group', + rules: [rule1, rule2, rule3], + }; + + it('should return a ModifyExportPayload with the updated rule added to a group with this rule belongs, in the same position', () => { + const result = getPayloadToExport('uid-rule-2', formValuesForRule2Updated, groupDto); + expect(result).toEqual({ + name: 'Test Group', + rules: [rule1, expectedModifiedRule2('uid-rule-2'), rule3], + }); + }); + it('should return a ModifyExportPayload with the updated rule added to a non empty rule where this rule does not belong, in the last position', () => { + const result = getPayloadToExport('uid-rule-5', formValuesForRule2Updated, groupDto); + expect(result).toEqual({ + name: 'Test Group', + rules: [rule1, rule2, rule3, expectedModifiedRule2('uid-rule-5')], + }); + }); + + it('should return a ModifyExportPayload with the updated rule added to an empty group', () => { + const emptyGroupDto: RulerRuleGroupDTO = { + name: 'Empty Group', + rules: [], + }; + const result = getPayloadToExport('uid-rule-2', formValuesForRule2Updated, emptyGroupDto); + expect(result).toEqual({ + name: 'Empty Group', + rules: [expectedModifiedRule2('uid-rule-2')], + }); + }); +}); diff --git a/public/app/features/alerting/unified/components/rule-editor/util.ts b/public/app/features/alerting/unified/components/rule-editor/util.ts index f0eefd50da9..3bc456259a6 100644 --- a/public/app/features/alerting/unified/components/rule-editor/util.ts +++ b/public/app/features/alerting/unified/components/rule-editor/util.ts @@ -3,11 +3,11 @@ import { ValidateResult } from 'react-hook-form'; import { DataFrame, + isTimeSeriesFrames, + LoadingState, + PanelData, ThresholdsConfig, ThresholdsMode, - isTimeSeriesFrames, - PanelData, - LoadingState, } from '@grafana/data'; import { GraphTresholdsStyleMode } from '@grafana/schema'; import { config } from 'app/core/config'; diff --git a/public/app/features/alerting/unified/components/rules/RuleActionsButtons.tsx b/public/app/features/alerting/unified/components/rules/RuleActionsButtons.tsx index d717509921b..286c58c8d35 100644 --- a/public/app/features/alerting/unified/components/rules/RuleActionsButtons.tsx +++ b/public/app/features/alerting/unified/components/rules/RuleActionsButtons.tsx @@ -145,7 +145,6 @@ export const RuleActionsButtons = ({ rule, rulesSource }: Props) => { if (isGrafanaRulerRule(rulerRule) && canReadProvisioning) { moreActions.push(); - if (config.featureToggles.alertingModifiedExport) { moreActions.push( { icon="edit" onClick={() => locationService.push( - `/alerting/${encodeURIComponent(ruleId.stringifyIdentifier(identifier))}/modify-export` + createUrl(`/alerting/${encodeURIComponent(ruleId.stringifyIdentifier(identifier))}/modify-export`, { + returnTo: location.pathname + location.search, + }) ) } /> diff --git a/public/app/features/alerting/unified/utils/rule-form.ts b/public/app/features/alerting/unified/utils/rule-form.ts index 46b058238b9..7b3d1d26612 100644 --- a/public/app/features/alerting/unified/utils/rule-form.ts +++ b/public/app/features/alerting/unified/utils/rule-form.ts @@ -1,3 +1,5 @@ +import { omit } from 'lodash'; + import { DataQuery, DataSourceInstanceSettings, @@ -544,3 +546,18 @@ function isPromQuery(model: AlertDataQuery): model is PromQuery { export function isPromOrLokiQuery(model: AlertDataQuery): model is PromOrLokiQuery { return 'expr' in model; } + +// the backend will always execute "hidden" queries, so we have no choice but to remove the property in the front-end +// to avoid confusion. The query editor shows them as "disabled" and that's a different semantic meaning. +// furthermore the "AlertingQueryRunner" calls `filterQuery` on each data source and those will skip running queries that are "hidden"." +// It seems like we have no choice but to act like "hidden" queries don't exist in alerting. +export const ignoreHiddenQueries = (ruleDefinition: RuleFormValues): RuleFormValues => { + return { + ...ruleDefinition, + queries: ruleDefinition.queries?.map((query) => omit(query, 'model.hide')), + }; +}; + +export function formValuesFromExistingRule(rule: RuleWithLocation) { + return ignoreHiddenQueries(rulerRuleToFormValues(rule)); +} From 665dc1fa449c6d541e4f61dfa768607cf180622f Mon Sep 17 00:00:00 2001 From: Esteban Beltran Date: Thu, 12 Oct 2023 09:43:59 +0200 Subject: [PATCH 077/102] Sandbox: e2e tests for apps running inside the frontend sandbox (#76357) * Sandbox: Add tests for apps inside the sandbox * Force type to prevent flawky test --- .../frontend-sandbox-app-test/img/logo.svg | 1 + .../frontend-sandbox-app-test/module.js | 41 +++++++++ .../frontend-sandbox-app-test/plugin.json | 35 ++++++++ .../frontend-sandbox-panel-test/img/logo.svg | 1 + .../frontend-sandbox-panel.spec.ts | 4 +- .../frontend-sandbox-app.spec.ts | 85 +++++++++++++++++++ 6 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 e2e/custom-plugins/frontend-sandbox-app-test/img/logo.svg create mode 100644 e2e/custom-plugins/frontend-sandbox-app-test/module.js create mode 100644 e2e/custom-plugins/frontend-sandbox-app-test/plugin.json create mode 100644 e2e/custom-plugins/frontend-sandbox-panel-test/img/logo.svg create mode 100644 e2e/various-suite/frontend-sandbox-app.spec.ts diff --git a/e2e/custom-plugins/frontend-sandbox-app-test/img/logo.svg b/e2e/custom-plugins/frontend-sandbox-app-test/img/logo.svg new file mode 100644 index 00000000000..08ec9068d45 --- /dev/null +++ b/e2e/custom-plugins/frontend-sandbox-app-test/img/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/e2e/custom-plugins/frontend-sandbox-app-test/module.js b/e2e/custom-plugins/frontend-sandbox-app-test/module.js new file mode 100644 index 00000000000..478549ee25a --- /dev/null +++ b/e2e/custom-plugins/frontend-sandbox-app-test/module.js @@ -0,0 +1,41 @@ +/* + * This is a dummy plugin to test the frontend sandbox + * It is not meant to be used in any other way + * This file doesn't require any compilation + */ +define(['react', '@grafana/data', 'react-router-dom'], function (React, grafanaData, ReactRouterDom) { + const { AppPlugin } = grafanaData; + const { Switch, Route } = ReactRouterDom; + + function PageOne() { + return React.createElement( + 'div', + { + 'data-testid': 'sandbox-app-test-page-one', + }, + 'This is a page one' + ); + } + + function App() { + return React.createElement(Switch, null, React.createElement(Route, { component: PageOne })); + } + function AppConfig() { + return React.createElement( + 'div', + { + 'data-testid': 'sandbox-app-test-config-page', + }, + + 'This is a config page' + ); + } + + const plugin = new AppPlugin().setRootPage(App).addConfigPage({ + title: 'Configuration', + icon: 'cog', + body: AppConfig, + id: 'configuration', + }); + return { plugin }; +}); diff --git a/e2e/custom-plugins/frontend-sandbox-app-test/plugin.json b/e2e/custom-plugins/frontend-sandbox-app-test/plugin.json new file mode 100644 index 00000000000..4997ee82633 --- /dev/null +++ b/e2e/custom-plugins/frontend-sandbox-app-test/plugin.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://raw.githubusercontent.com/grafana/grafana/master/docs/sources/developers/plugins/plugin.schema.json", + "type": "app", + "name": "Sandbox app test plugin", + "id": "sandbox-app-test", + "info": { + "keywords": ["app", "sandbox"], + "description": "", + "author": { + "name": "Grafana" + }, + "logos": { + "small": "img/logo.svg", + "large": "img/logo.svg" + }, + "links": [], + "screenshots": [], + "version": "1.0.0", + "updated": "2023-06-27" + }, + "includes": [ + { + "type": "page", + "icon": "cog", + "name": "Sandbox App Page", + "path": "/plugins/sandbox-app-test", + "role": "Admin", + "addToNav": true + } + ], + "dependencies": { + "grafanaDependency": ">=10.0", + "plugins": [] + } +} diff --git a/e2e/custom-plugins/frontend-sandbox-panel-test/img/logo.svg b/e2e/custom-plugins/frontend-sandbox-panel-test/img/logo.svg new file mode 100644 index 00000000000..08ec9068d45 --- /dev/null +++ b/e2e/custom-plugins/frontend-sandbox-panel-test/img/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/e2e/panels-suite/frontend-sandbox-panel.spec.ts b/e2e/panels-suite/frontend-sandbox-panel.spec.ts index 69205f06e09..fb237f6f54e 100644 --- a/e2e/panels-suite/frontend-sandbox-panel.spec.ts +++ b/e2e/panels-suite/frontend-sandbox-panel.spec.ts @@ -57,7 +57,7 @@ describe('Panel sandbox', () => { }); cy.get('[data-testid="panel-editor-custom-editor-input"]').should('not.be.disabled'); - cy.get('[data-testid="panel-editor-custom-editor-input"]').type('x'); + cy.get('[data-testid="panel-editor-custom-editor-input"]').type('x', { force: true }); cy.get('[data-sandbox-test="panel-editor"]').should('exist'); }); }); @@ -114,7 +114,7 @@ describe('Panel sandbox', () => { }); cy.get('[data-testid="panel-editor-custom-editor-input"]').should('not.be.disabled'); - cy.get('[data-testid="panel-editor-custom-editor-input"]').type('x'); + cy.get('[data-testid="panel-editor-custom-editor-input"]').type('x', { force: true }); cy.wait(100); // small delay to prevent false positives from too fast tests cy.get('[data-sandbox-test="panel-editor"]').should('not.exist'); }); diff --git a/e2e/various-suite/frontend-sandbox-app.spec.ts b/e2e/various-suite/frontend-sandbox-app.spec.ts new file mode 100644 index 00000000000..9bcf4d49832 --- /dev/null +++ b/e2e/various-suite/frontend-sandbox-app.spec.ts @@ -0,0 +1,85 @@ +import { e2e } from '../utils'; + +const APP_ID = 'sandbox-app-test'; + +describe('Datasource sandbox', () => { + before(() => { + e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'), true); + cy.request({ + url: `${Cypress.env('BASE_URL')}/api/plugins/${APP_ID}/settings`, + method: 'POST', + body: { + enabled: true, + }, + }); + }); + beforeEach(() => { + e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'), true); + }); + + describe('App Page', () => { + describe('Sandbox disabled', () => { + beforeEach(() => { + cy.window().then((win) => { + win.localStorage.setItem('grafana.featureToggles', 'pluginsFrontendSandbox=0'); + }); + }); + + it('Loads the app page without the sandbox div wrapper', () => { + e2e.pages.Home.visit(); + e2e.components.NavBar.Toggle.button().click(); + e2e.components.NavToolbar.container().get('[aria-label="Expand section Apps"]').click(); + e2e.components.NavMenu.item().contains('Sandbox app test plugin').click(); + cy.wait(200); // wait to prevent false positives because cypress checks too fast + cy.get('div[data-plugin-sandbox="sandbox-app-test"]').should('not.exist'); + cy.get('div[data-testid="sandbox-app-test-page-one"]').should('exist'); + }); + + it('Loads the app configuration without the sandbox div wrapper', () => { + e2e.pages.Home.visit(); + e2e.components.NavBar.Toggle.button().click(); + e2e.components.NavToolbar.container().get('[aria-label="Expand section Apps"]').click(); + e2e.components.NavMenu.item().contains('Apps').click(); + cy.get('a[aria-label="Tab Sandbox App Page"]').click(); + cy.wait(200); // wait to prevent false positives because cypress checks too fast + cy.get('div[data-plugin-sandbox="sandbox-app-test"]').should('not.exist'); + cy.get('div[data-testid="sandbox-app-test-config-page"]').should('exist'); + }); + }); + + describe('Sandbox enabled', () => { + beforeEach(() => { + cy.window().then((win) => { + win.localStorage.setItem('grafana.featureToggles', 'pluginsFrontendSandbox=1'); + }); + }); + + it('Loads the app page with the sandbox div wrapper', () => { + e2e.pages.Home.visit(); + e2e.components.NavBar.Toggle.button().click(); + e2e.components.NavToolbar.container().get('[aria-label="Expand section Apps"]').click(); + e2e.components.NavMenu.item().contains('Sandbox app test plugin').click(); + cy.get('div[data-plugin-sandbox="sandbox-app-test"]').should('exist'); + cy.get('div[data-testid="sandbox-app-test-page-one"]').should('exist'); + }); + + it('Loads the app configuration with the sandbox div wrapper', () => { + e2e.pages.Home.visit(); + e2e.components.NavBar.Toggle.button().click(); + e2e.components.NavToolbar.container().get('[aria-label="Expand section Apps"]').click(); + e2e.components.NavMenu.item().contains('Apps').click(); + cy.get('a[aria-label="Tab Sandbox App Page"]').click(); + cy.get('div[data-plugin-sandbox="sandbox-app-test"]').should('exist'); + cy.get('div[data-testid="sandbox-app-test-config-page"]').should('exist'); + }); + }); + }); + + afterEach(() => { + e2e.flows.revertAllChanges(); + }); + + after(() => { + cy.clearCookies(); + }); +}); From 466f8a1f5afe34b16acd15aab3ae5cc71cc60324 Mon Sep 17 00:00:00 2001 From: Jo Date: Thu, 12 Oct 2023 10:10:54 +0200 Subject: [PATCH 078/102] Teams: Move team API to own service (#76347) * move team API to its own service * remove uneeded import * reshare pref api logic --- pkg/api/api.go | 19 ---- pkg/api/common_test.go | 1 - pkg/api/http_server.go | 4 +- pkg/api/preferences.go | 100 +--------------- .../backgroundsvcs/background_services.go | 3 +- pkg/server/wire.go | 2 + pkg/services/preference/prefapi/api.go | 107 ++++++++++++++++++ pkg/services/team/teamapi/api.go | 82 ++++++++++++++ pkg/{api => services/team/teamapi}/team.go | 67 +++++++---- .../team/teamapi}/team_members.go | 46 ++++---- .../team/teamapi}/team_members_test.go | 75 ++++++++---- .../team/teamapi}/team_test.go | 60 ++++------ 12 files changed, 343 insertions(+), 223 deletions(-) create mode 100644 pkg/services/preference/prefapi/api.go create mode 100644 pkg/services/team/teamapi/api.go rename pkg/{api => services/team/teamapi}/team.go (75%) rename pkg/{api => services/team/teamapi}/team_members.go (74%) rename pkg/{api => services/team/teamapi}/team_members_test.go (59%) rename pkg/{api => services/team/teamapi}/team_test.go (79%) diff --git a/pkg/api/api.go b/pkg/api/api.go index cad49e0c4c6..215ff274848 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -270,25 +270,6 @@ func (hs *HTTPServer) registerRoutes() { usersRoute.Post("/:id/using/:orgId", authorize(ac.EvalPermission(ac.ActionUsersWrite, userIDScope)), routing.Wrap(hs.UpdateUserActiveOrg)) }, requestmeta.SetOwner(requestmeta.TeamAuth)) - // team (admin permission required) - apiRoute.Group("/teams", func(teamsRoute routing.RouteRegister) { - teamsRoute.Post("/", authorize(ac.EvalPermission(ac.ActionTeamsCreate)), routing.Wrap(hs.CreateTeam)) - teamsRoute.Put("/:teamId", authorize(ac.EvalPermission(ac.ActionTeamsWrite, ac.ScopeTeamsID)), routing.Wrap(hs.UpdateTeam)) - teamsRoute.Delete("/:teamId", authorize(ac.EvalPermission(ac.ActionTeamsDelete, ac.ScopeTeamsID)), routing.Wrap(hs.DeleteTeamByID)) - teamsRoute.Get("/:teamId/members", authorize(ac.EvalPermission(ac.ActionTeamsPermissionsRead, ac.ScopeTeamsID)), routing.Wrap(hs.GetTeamMembers)) - teamsRoute.Post("/:teamId/members", authorize(ac.EvalPermission(ac.ActionTeamsPermissionsWrite, ac.ScopeTeamsID)), routing.Wrap(hs.AddTeamMember)) - teamsRoute.Put("/:teamId/members/:userId", authorize(ac.EvalPermission(ac.ActionTeamsPermissionsWrite, ac.ScopeTeamsID)), routing.Wrap(hs.UpdateTeamMember)) - teamsRoute.Delete("/:teamId/members/:userId", authorize(ac.EvalPermission(ac.ActionTeamsPermissionsWrite, ac.ScopeTeamsID)), routing.Wrap(hs.RemoveTeamMember)) - teamsRoute.Get("/:teamId/preferences", authorize(ac.EvalPermission(ac.ActionTeamsRead, ac.ScopeTeamsID)), routing.Wrap(hs.GetTeamPreferences)) - teamsRoute.Put("/:teamId/preferences", authorize(ac.EvalPermission(ac.ActionTeamsWrite, ac.ScopeTeamsID)), routing.Wrap(hs.UpdateTeamPreferences)) - }, requestmeta.SetOwner(requestmeta.TeamAuth)) - - // team without requirement of user to be org admin - apiRoute.Group("/teams", func(teamsRoute routing.RouteRegister) { - teamsRoute.Get("/:teamId", authorize(ac.EvalPermission(ac.ActionTeamsRead, ac.ScopeTeamsID)), routing.Wrap(hs.GetTeamByID)) - teamsRoute.Get("/search", authorize(ac.EvalPermission(ac.ActionTeamsRead)), routing.Wrap(hs.SearchTeams)) - }, requestmeta.SetOwner(requestmeta.TeamAuth)) - // org information available to all users. apiRoute.Group("/org", func(orgRoute routing.RouteRegister) { orgRoute.Get("/", authorize(ac.EvalPermission(ac.ActionOrgsRead)), routing.Wrap(hs.GetCurrentOrg)) diff --git a/pkg/api/common_test.go b/pkg/api/common_test.go index d7227f4127d..9f40753df17 100644 --- a/pkg/api/common_test.go +++ b/pkg/api/common_test.go @@ -214,7 +214,6 @@ func setupScenarioContext(t *testing.T, url string) *scenarioContext { return sc } -// FIXME: This user should not be anonymous func authedUserWithPermissions(userID, orgID int64, permissions []accesscontrol.Permission) *user.SignedInUser { return &user.SignedInUser{UserID: userID, OrgID: orgID, OrgRole: org.RoleViewer, Permissions: map[int64]map[string][]string{orgID: accesscontrol.GroupScopesByAction(permissions)}} } diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index 63e95b1afd7..4f3a9c04ef5 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -170,7 +170,6 @@ type HTTPServer struct { queryDataService query.Service serviceAccountsService serviceaccounts.Service authInfoService login.AuthInfoService - teamPermissionsService accesscontrol.TeamPermissionsService NotificationService *notifications.NotificationService DashboardService dashboards.DashboardService dashboardProvisioningService dashboards.DashboardProvisioningService @@ -236,7 +235,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi dsGuardian guardian.DatasourceGuardianProvider, alertNotificationService *alerting.AlertNotificationService, dashboardsnapshotsService dashboardsnapshots.Service, pluginSettings pluginSettings.Service, avatarCacheServer *avatar.AvatarCacheServer, preferenceService pref.Service, - teamsPermissionsService accesscontrol.TeamPermissionsService, folderPermissionsService accesscontrol.FolderPermissionsService, + folderPermissionsService accesscontrol.FolderPermissionsService, dashboardPermissionsService accesscontrol.DashboardPermissionsService, dashboardVersionService dashver.Service, starService star.Service, csrfService csrf.Service, basekinds *corekind.Base, playlistService playlist.Service, apiKeyService apikey.Service, kvStore kvstore.KVStore, @@ -319,7 +318,6 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi dashboardProvisioningService: dashboardProvisioningService, folderService: folderService, dsGuardian: dsGuardian, - teamPermissionsService: teamsPermissionsService, AlertNotificationService: alertNotificationService, dashboardsnapshotsService: dashboardsnapshotsService, PluginSettings: pluginSettings, diff --git a/pkg/api/preferences.go b/pkg/api/preferences.go index e6c71d8b775..7cbbcfd1912 100644 --- a/pkg/api/preferences.go +++ b/pkg/api/preferences.go @@ -11,6 +11,7 @@ import ( contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/dashboards" pref "github.com/grafana/grafana/pkg/services/preference" + "github.com/grafana/grafana/pkg/services/preference/prefapi" "github.com/grafana/grafana/pkg/web" ) @@ -68,56 +69,7 @@ func (hs *HTTPServer) GetUserPreferences(c *contextmodel.ReqContext) response.Re return response.Error(http.StatusInternalServerError, "Failed to get user preferences", errID) } - return hs.getPreferencesFor(c.Req.Context(), c.SignedInUser.GetOrgID(), userID, 0) -} - -func (hs *HTTPServer) getPreferencesFor(ctx context.Context, orgID, userID, teamID int64) response.Response { - prefsQuery := pref.GetPreferenceQuery{UserID: userID, OrgID: orgID, TeamID: teamID} - - preference, err := hs.preferenceService.Get(ctx, &prefsQuery) - if err != nil { - return response.Error(http.StatusInternalServerError, "Failed to get preferences", err) - } - - var dashboardUID string - - // when homedashboardID is 0, that means it is the default home dashboard, no UID would be returned in the response - if preference.HomeDashboardID != 0 { - query := dashboards.GetDashboardQuery{ID: preference.HomeDashboardID, OrgID: orgID} - queryResult, err := hs.DashboardService.GetDashboard(ctx, &query) - if err == nil { - dashboardUID = queryResult.UID - } - } - - dto := preferences.Spec{} - - if preference.WeekStart != nil && *preference.WeekStart != "" { - dto.WeekStart = preference.WeekStart - } - if preference.Theme != "" { - dto.Theme = &preference.Theme - } - if dashboardUID != "" { - dto.HomeDashboardUID = &dashboardUID - } - if preference.Timezone != "" { - dto.Timezone = &preference.Timezone - } - - if preference.JSONData != nil { - if preference.JSONData.Language != "" { - dto.Language = &preference.JSONData.Language - } - - if preference.JSONData.QueryHistory.HomeTab != "" { - dto.QueryHistory = &preferences.QueryHistoryPreference{ - HomeTab: &preference.JSONData.QueryHistory.HomeTab, - } - } - } - - return response.JSON(http.StatusOK, &dto) + return prefapi.GetPreferencesFor(c.Req.Context(), hs.DashboardService, hs.preferenceService, c.SignedInUser.GetOrgID(), userID, 0) } // swagger:route PUT /user/preferences user_preferences updateUserPreferences @@ -142,48 +94,8 @@ func (hs *HTTPServer) UpdateUserPreferences(c *contextmodel.ReqContext) response return response.Error(http.StatusInternalServerError, "Failed to update user preferences", errID) } - return hs.updatePreferencesFor(c.Req.Context(), c.SignedInUser.GetOrgID(), userID, 0, &dtoCmd) -} - -func (hs *HTTPServer) updatePreferencesFor(ctx context.Context, orgID, userID, teamId int64, dtoCmd *dtos.UpdatePrefsCmd) response.Response { - if dtoCmd.Theme != "" && !pref.IsValidThemeID(dtoCmd.Theme) { - return response.Error(http.StatusBadRequest, "Invalid theme", nil) - } - - dashboardID := dtoCmd.HomeDashboardID - if dtoCmd.HomeDashboardUID != nil { - query := dashboards.GetDashboardQuery{UID: *dtoCmd.HomeDashboardUID, OrgID: orgID} - if query.UID == "" { - // clear the value - dashboardID = 0 - } else { - queryResult, err := hs.DashboardService.GetDashboard(ctx, &query) - if err != nil { - return response.Error(http.StatusNotFound, "Dashboard not found", err) - } - dashboardID = queryResult.ID - } - } - dtoCmd.HomeDashboardID = dashboardID - - saveCmd := pref.SavePreferenceCommand{ - UserID: userID, - OrgID: orgID, - TeamID: teamId, - Theme: dtoCmd.Theme, - Language: dtoCmd.Language, - Timezone: dtoCmd.Timezone, - WeekStart: dtoCmd.WeekStart, - HomeDashboardID: dtoCmd.HomeDashboardID, - QueryHistory: dtoCmd.QueryHistory, - CookiePreferences: dtoCmd.Cookies, - } - - if err := hs.preferenceService.Save(ctx, &saveCmd); err != nil { - return response.ErrOrFallback(http.StatusInternalServerError, "Failed to save preferences", err) - } - - return response.Success("Preferences updated") + return prefapi.UpdatePreferencesFor(c.Req.Context(), hs.DashboardService, + hs.preferenceService, c.SignedInUser.GetOrgID(), userID, 0, &dtoCmd) } // swagger:route PATCH /user/preferences user_preferences patchUserPreferences @@ -262,7 +174,7 @@ func (hs *HTTPServer) patchPreferencesFor(ctx context.Context, orgID, userID, te // 403: forbiddenError // 500: internalServerError func (hs *HTTPServer) GetOrgPreferences(c *contextmodel.ReqContext) response.Response { - return hs.getPreferencesFor(c.Req.Context(), c.SignedInUser.GetOrgID(), 0, 0) + return prefapi.GetPreferencesFor(c.Req.Context(), hs.DashboardService, hs.preferenceService, c.SignedInUser.GetOrgID(), 0, 0) } // swagger:route PUT /org/preferences org_preferences updateOrgPreferences @@ -281,7 +193,7 @@ func (hs *HTTPServer) UpdateOrgPreferences(c *contextmodel.ReqContext) response. return response.Error(http.StatusBadRequest, "bad request data", err) } - return hs.updatePreferencesFor(c.Req.Context(), c.SignedInUser.GetOrgID(), 0, 0, &dtoCmd) + return prefapi.UpdatePreferencesFor(c.Req.Context(), hs.DashboardService, hs.preferenceService, c.SignedInUser.GetOrgID(), 0, 0, &dtoCmd) } // swagger:route PATCH /org/preferences org_preferences patchOrgPreferences diff --git a/pkg/registry/backgroundsvcs/background_services.go b/pkg/registry/backgroundsvcs/background_services.go index 19f8cfc0d26..8e316697aa4 100644 --- a/pkg/registry/backgroundsvcs/background_services.go +++ b/pkg/registry/backgroundsvcs/background_services.go @@ -40,6 +40,7 @@ import ( "github.com/grafana/grafana/pkg/services/store/entity" "github.com/grafana/grafana/pkg/services/store/sanitizer" "github.com/grafana/grafana/pkg/services/supportbundles/supportbundlesimpl" + "github.com/grafana/grafana/pkg/services/team/teamapi" "github.com/grafana/grafana/pkg/services/updatechecker" ) @@ -62,7 +63,7 @@ func ProvideBackgroundServiceRegistry( _ serviceaccounts.Service, _ *guardian.Provider, _ *plugindashboardsservice.DashboardUpdater, _ *sanitizer.Provider, _ *grpcserver.HealthService, _ entity.EntityStoreServer, _ *grpcserver.ReflectionService, _ *ldapapi.Service, - _ *apiregistry.Service, _ auth.IDService, + _ *apiregistry.Service, _ auth.IDService, _ *teamapi.TeamAPI, ) *BackgroundServiceRegistry { return NewBackgroundServiceRegistry( httpServer, diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 44755d416b4..a6f4567bc65 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -141,6 +141,7 @@ import ( "github.com/grafana/grafana/pkg/services/supportbundles/supportbundlesimpl" "github.com/grafana/grafana/pkg/services/tag" "github.com/grafana/grafana/pkg/services/tag/tagimpl" + "github.com/grafana/grafana/pkg/services/team/teamapi" "github.com/grafana/grafana/pkg/services/team/teamimpl" tempuser "github.com/grafana/grafana/pkg/services/temp_user" "github.com/grafana/grafana/pkg/services/temp_user/tempuserimpl" @@ -342,6 +343,7 @@ var wireBasicSet = wire.NewSet( resolver.ProvideEntityReferenceResolver, httpentitystore.ProvideHTTPEntityStore, teamimpl.ProvideService, + teamapi.ProvideTeamAPI, tempuserimpl.ProvideService, loginattemptimpl.ProvideService, wire.Bind(new(loginattempt.Service), new(*loginattemptimpl.Service)), diff --git a/pkg/services/preference/prefapi/api.go b/pkg/services/preference/prefapi/api.go new file mode 100644 index 00000000000..a77639571eb --- /dev/null +++ b/pkg/services/preference/prefapi/api.go @@ -0,0 +1,107 @@ +// shared logic between httpserver and teamapi +package prefapi + +import ( + "context" + "net/http" + + "github.com/grafana/grafana/pkg/api/dtos" + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/kinds/preferences" + "github.com/grafana/grafana/pkg/services/dashboards" + pref "github.com/grafana/grafana/pkg/services/preference" +) + +func UpdatePreferencesFor(ctx context.Context, + dashboardService dashboards.DashboardService, preferenceService pref.Service, + orgID, userID, teamId int64, dtoCmd *dtos.UpdatePrefsCmd) response.Response { + if dtoCmd.Theme != "" && !pref.IsValidThemeID(dtoCmd.Theme) { + return response.Error(http.StatusBadRequest, "Invalid theme", nil) + } + + dashboardID := dtoCmd.HomeDashboardID + if dtoCmd.HomeDashboardUID != nil { + query := dashboards.GetDashboardQuery{UID: *dtoCmd.HomeDashboardUID, OrgID: orgID} + if query.UID == "" { + // clear the value + dashboardID = 0 + } else { + queryResult, err := dashboardService.GetDashboard(ctx, &query) + if err != nil { + return response.Error(http.StatusNotFound, "Dashboard not found", err) + } + dashboardID = queryResult.ID + } + } + dtoCmd.HomeDashboardID = dashboardID + + saveCmd := pref.SavePreferenceCommand{ + UserID: userID, + OrgID: orgID, + TeamID: teamId, + Theme: dtoCmd.Theme, + Language: dtoCmd.Language, + Timezone: dtoCmd.Timezone, + WeekStart: dtoCmd.WeekStart, + HomeDashboardID: dtoCmd.HomeDashboardID, + QueryHistory: dtoCmd.QueryHistory, + CookiePreferences: dtoCmd.Cookies, + } + + if err := preferenceService.Save(ctx, &saveCmd); err != nil { + return response.ErrOrFallback(http.StatusInternalServerError, "Failed to save preferences", err) + } + + return response.Success("Preferences updated") +} + +func GetPreferencesFor(ctx context.Context, + dashboardService dashboards.DashboardService, preferenceService pref.Service, + orgID, userID, teamID int64) response.Response { + prefsQuery := pref.GetPreferenceQuery{UserID: userID, OrgID: orgID, TeamID: teamID} + + preference, err := preferenceService.Get(ctx, &prefsQuery) + if err != nil { + return response.Error(http.StatusInternalServerError, "Failed to get preferences", err) + } + + var dashboardUID string + + // when homedashboardID is 0, that means it is the default home dashboard, no UID would be returned in the response + if preference.HomeDashboardID != 0 { + query := dashboards.GetDashboardQuery{ID: preference.HomeDashboardID, OrgID: orgID} + queryResult, err := dashboardService.GetDashboard(ctx, &query) + if err == nil { + dashboardUID = queryResult.UID + } + } + + dto := preferences.Spec{} + + if preference.WeekStart != nil && *preference.WeekStart != "" { + dto.WeekStart = preference.WeekStart + } + if preference.Theme != "" { + dto.Theme = &preference.Theme + } + if dashboardUID != "" { + dto.HomeDashboardUID = &dashboardUID + } + if preference.Timezone != "" { + dto.Timezone = &preference.Timezone + } + + if preference.JSONData != nil { + if preference.JSONData.Language != "" { + dto.Language = &preference.JSONData.Language + } + + if preference.JSONData.QueryHistory.HomeTab != "" { + dto.QueryHistory = &preferences.QueryHistoryPreference{ + HomeTab: &preference.JSONData.QueryHistory.HomeTab, + } + } + } + + return response.JSON(http.StatusOK, &dto) +} diff --git a/pkg/services/team/teamapi/api.go b/pkg/services/team/teamapi/api.go new file mode 100644 index 00000000000..b744dfdaf5d --- /dev/null +++ b/pkg/services/team/teamapi/api.go @@ -0,0 +1,82 @@ +package teamapi + +import ( + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/middleware/requestmeta" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/licensing" + pref "github.com/grafana/grafana/pkg/services/preference" + "github.com/grafana/grafana/pkg/services/team" + "github.com/grafana/grafana/pkg/setting" +) + +type TeamAPI struct { + teamService team.Service + ac accesscontrol.Service + teamPermissionsService accesscontrol.TeamPermissionsService + license licensing.Licensing + cfg *setting.Cfg + preferenceService pref.Service + ds dashboards.DashboardService +} + +func ProvideTeamAPI( + routeRegister routing.RouteRegister, + teamService team.Service, + ac accesscontrol.Service, + acEvaluator accesscontrol.AccessControl, + teamPermissionsService accesscontrol.TeamPermissionsService, + license licensing.Licensing, + cfg *setting.Cfg, + preferenceService pref.Service, + ds dashboards.DashboardService, +) *TeamAPI { + tapi := &TeamAPI{ + teamService: teamService, + ac: ac, + teamPermissionsService: teamPermissionsService, + license: license, + cfg: cfg, + preferenceService: preferenceService, + ds: ds, + } + + tapi.registerRoutes(routeRegister, acEvaluator) + return tapi +} + +func (tapi *TeamAPI) registerRoutes(router routing.RouteRegister, ac accesscontrol.AccessControl) { + authorize := accesscontrol.Middleware(ac) + router.Group("/api", func(apiRoute routing.RouteRegister) { + // team (admin permission required) + apiRoute.Group("/teams", func(teamsRoute routing.RouteRegister) { + teamsRoute.Post("/", authorize(accesscontrol.EvalPermission(accesscontrol.ActionTeamsCreate)), + routing.Wrap(tapi.createTeam)) + teamsRoute.Put("/:teamId", authorize(accesscontrol.EvalPermission(accesscontrol.ActionTeamsWrite, + accesscontrol.ScopeTeamsID)), routing.Wrap(tapi.updateTeam)) + teamsRoute.Delete("/:teamId", authorize(accesscontrol.EvalPermission(accesscontrol.ActionTeamsDelete, + accesscontrol.ScopeTeamsID)), routing.Wrap(tapi.deleteTeamByID)) + teamsRoute.Get("/:teamId/members", authorize(accesscontrol.EvalPermission(accesscontrol.ActionTeamsPermissionsRead, + accesscontrol.ScopeTeamsID)), routing.Wrap(tapi.getTeamMembers)) + teamsRoute.Post("/:teamId/members", authorize(accesscontrol.EvalPermission(accesscontrol.ActionTeamsPermissionsWrite, + accesscontrol.ScopeTeamsID)), routing.Wrap(tapi.addTeamMember)) + teamsRoute.Put("/:teamId/members/:userId", authorize(accesscontrol.EvalPermission(accesscontrol.ActionTeamsPermissionsWrite, + accesscontrol.ScopeTeamsID)), routing.Wrap(tapi.updateTeamMember)) + teamsRoute.Delete("/:teamId/members/:userId", authorize(accesscontrol.EvalPermission(accesscontrol.ActionTeamsPermissionsWrite, + accesscontrol.ScopeTeamsID)), routing.Wrap(tapi.removeTeamMember)) + teamsRoute.Get("/:teamId/preferences", authorize(accesscontrol.EvalPermission(accesscontrol.ActionTeamsRead, + accesscontrol.ScopeTeamsID)), routing.Wrap(tapi.getTeamPreferences)) + teamsRoute.Put("/:teamId/preferences", authorize(accesscontrol.EvalPermission(accesscontrol.ActionTeamsWrite, + accesscontrol.ScopeTeamsID)), routing.Wrap(tapi.updateTeamPreferences)) + }, requestmeta.SetOwner(requestmeta.TeamAuth)) + + // team without requirement of user to be org admin + apiRoute.Group("/teams", func(teamsRoute routing.RouteRegister) { + teamsRoute.Get("/:teamId", authorize(accesscontrol.EvalPermission(accesscontrol.ActionTeamsRead, + accesscontrol.ScopeTeamsID)), routing.Wrap(tapi.getTeamByID)) + teamsRoute.Get("/search", authorize(accesscontrol.EvalPermission(accesscontrol.ActionTeamsRead)), + routing.Wrap(tapi.searchTeams)) + }, requestmeta.SetOwner(requestmeta.TeamAuth)) + }) +} diff --git a/pkg/api/team.go b/pkg/services/team/teamapi/team.go similarity index 75% rename from pkg/api/team.go rename to pkg/services/team/teamapi/team.go index 42f61252815..ebcde9222d9 100644 --- a/pkg/api/team.go +++ b/pkg/services/team/teamapi/team.go @@ -1,4 +1,4 @@ -package api +package teamapi import ( "errors" @@ -7,9 +7,11 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/auth/identity" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/preference/prefapi" "github.com/grafana/grafana/pkg/services/team" "github.com/grafana/grafana/pkg/services/team/sortopts" "github.com/grafana/grafana/pkg/util" @@ -26,13 +28,13 @@ import ( // 403: forbiddenError // 409: conflictError // 500: internalServerError -func (hs *HTTPServer) CreateTeam(c *contextmodel.ReqContext) response.Response { +func (tapi *TeamAPI) createTeam(c *contextmodel.ReqContext) response.Response { cmd := team.CreateTeamCommand{} if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) } - t, err := hs.teamService.CreateTeam(cmd.Name, cmd.Email, c.SignedInUser.GetOrgID()) + t, err := tapi.teamService.CreateTeam(cmd.Name, cmd.Email, c.SignedInUser.GetOrgID()) if err != nil { if errors.Is(err, team.ErrTeamNameTaken) { return response.Error(http.StatusConflict, "Team name taken", err) @@ -42,7 +44,7 @@ func (hs *HTTPServer) CreateTeam(c *contextmodel.ReqContext) response.Response { // Clear permission cache for the user who's created the team, so that new permissions are fetched for their next call // Required for cases when caller wants to immediately interact with the newly created object - hs.accesscontrolService.ClearUserPermissionCache(c.SignedInUser) + tapi.ac.ClearUserPermissionCache(c.SignedInUser) // if the request is authenticated using API tokens // the SignedInUser is an empty struct therefore @@ -55,7 +57,7 @@ func (hs *HTTPServer) CreateTeam(c *contextmodel.ReqContext) response.Response { c.Logger.Error("Could not add creator to team because user id is not a number", "error", err) break } - if err := addOrUpdateTeamMember(c.Req.Context(), hs.teamPermissionsService, userID, c.SignedInUser.GetOrgID(), + if err := addOrUpdateTeamMember(c.Req.Context(), tapi.teamPermissionsService, userID, c.SignedInUser.GetOrgID(), t.ID, dashboards.PERMISSION_ADMIN.String()); err != nil { c.Logger.Error("Could not add creator to team", "error", err) } @@ -80,7 +82,7 @@ func (hs *HTTPServer) CreateTeam(c *contextmodel.ReqContext) response.Response { // 404: notFoundError // 409: conflictError // 500: internalServerError -func (hs *HTTPServer) UpdateTeam(c *contextmodel.ReqContext) response.Response { +func (tapi *TeamAPI) updateTeam(c *contextmodel.ReqContext) response.Response { cmd := team.UpdateTeamCommand{} var err error if err := web.Bind(c.Req, &cmd); err != nil { @@ -92,7 +94,7 @@ func (hs *HTTPServer) UpdateTeam(c *contextmodel.ReqContext) response.Response { return response.Error(http.StatusBadRequest, "teamId is invalid", err) } - if err := hs.teamService.UpdateTeam(c.Req.Context(), &cmd); err != nil { + if err := tapi.teamService.UpdateTeam(c.Req.Context(), &cmd); err != nil { if errors.Is(err, team.ErrTeamNameTaken) { return response.Error(http.StatusBadRequest, "Team name taken", err) } @@ -112,14 +114,14 @@ func (hs *HTTPServer) UpdateTeam(c *contextmodel.ReqContext) response.Response { // 403: forbiddenError // 404: notFoundError // 500: internalServerError -func (hs *HTTPServer) DeleteTeamByID(c *contextmodel.ReqContext) response.Response { +func (tapi *TeamAPI) deleteTeamByID(c *contextmodel.ReqContext) response.Response { orgID := c.SignedInUser.GetOrgID() teamID, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) if err != nil { return response.Error(http.StatusBadRequest, "teamId is invalid", err) } - if err := hs.teamService.DeleteTeam(c.Req.Context(), &team.DeleteTeamCommand{OrgID: orgID, ID: teamID}); err != nil { + if err := tapi.teamService.DeleteTeam(c.Req.Context(), &team.DeleteTeamCommand{OrgID: orgID, ID: teamID}); err != nil { if errors.Is(err, team.ErrTeamNotFound) { return response.Error(http.StatusNotFound, "Failed to delete Team. ID not found", nil) } @@ -137,7 +139,7 @@ func (hs *HTTPServer) DeleteTeamByID(c *contextmodel.ReqContext) response.Respon // 401: unauthorisedError // 403: forbiddenError // 500: internalServerError -func (hs *HTTPServer) SearchTeams(c *contextmodel.ReqContext) response.Response { +func (tapi *TeamAPI) searchTeams(c *contextmodel.ReqContext) response.Response { perPage := c.QueryInt("perpage") if perPage <= 0 { perPage = 1000 @@ -159,11 +161,11 @@ func (hs *HTTPServer) SearchTeams(c *contextmodel.ReqContext) response.Response Page: page, Limit: perPage, SignedInUser: c.SignedInUser, - HiddenUsers: hs.Cfg.HiddenUsers, + HiddenUsers: tapi.cfg.HiddenUsers, SortOpts: sortOpts, } - queryResult, err := hs.teamService.SearchTeams(c.Req.Context(), &query) + queryResult, err := tapi.teamService.SearchTeams(c.Req.Context(), &query) if err != nil { return response.Error(http.StatusInternalServerError, "Failed to search Teams", err) } @@ -174,7 +176,7 @@ func (hs *HTTPServer) SearchTeams(c *contextmodel.ReqContext) response.Response teamIDs[strconv.FormatInt(team.ID, 10)] = true } - metadata := hs.getMultiAccessControlMetadata(c, "teams:id:", teamIDs) + metadata := tapi.getMultiAccessControlMetadata(c, "teams:id:", teamIDs) if len(metadata) > 0 { for _, team := range queryResult.Teams { team.AccessControl = metadata[strconv.FormatInt(team.ID, 10)] @@ -197,7 +199,7 @@ func (hs *HTTPServer) SearchTeams(c *contextmodel.ReqContext) response.Response // 403: forbiddenError // 404: notFoundError // 500: internalServerError -func (hs *HTTPServer) GetTeamByID(c *contextmodel.ReqContext) response.Response { +func (tapi *TeamAPI) getTeamByID(c *contextmodel.ReqContext) response.Response { teamId, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) if err != nil { return response.Error(http.StatusBadRequest, "teamId is invalid", err) @@ -207,10 +209,10 @@ func (hs *HTTPServer) GetTeamByID(c *contextmodel.ReqContext) response.Response OrgID: c.SignedInUser.GetOrgID(), ID: teamId, SignedInUser: c.SignedInUser, - HiddenUsers: hs.Cfg.HiddenUsers, + HiddenUsers: tapi.cfg.HiddenUsers, } - queryResult, err := hs.teamService.GetTeamByID(c.Req.Context(), &query) + queryResult, err := tapi.teamService.GetTeamByID(c.Req.Context(), &query) if err != nil { if errors.Is(err, team.ErrTeamNotFound) { return response.Error(http.StatusNotFound, "Team not found", err) @@ -220,7 +222,7 @@ func (hs *HTTPServer) GetTeamByID(c *contextmodel.ReqContext) response.Response } // Add accesscontrol metadata - queryResult.AccessControl = hs.getAccessControlMetadata(c, c.SignedInUser.GetOrgID(), "teams:id:", strconv.FormatInt(queryResult.ID, 10)) + queryResult.AccessControl = tapi.getAccessControlMetadata(c, c.SignedInUser.GetOrgID(), "teams:id:", strconv.FormatInt(queryResult.ID, 10)) queryResult.AvatarURL = dtos.GetGravatarUrlWithDefault(queryResult.Email, queryResult.Name) return response.JSON(http.StatusOK, &queryResult) @@ -234,13 +236,13 @@ func (hs *HTTPServer) GetTeamByID(c *contextmodel.ReqContext) response.Response // 200: getPreferencesResponse // 401: unauthorisedError // 500: internalServerError -func (hs *HTTPServer) GetTeamPreferences(c *contextmodel.ReqContext) response.Response { +func (tapi *TeamAPI) getTeamPreferences(c *contextmodel.ReqContext) response.Response { teamId, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) if err != nil { return response.Error(http.StatusBadRequest, "teamId is invalid", err) } - return hs.getPreferencesFor(c.Req.Context(), c.SignedInUser.GetOrgID(), 0, teamId) + return prefapi.GetPreferencesFor(c.Req.Context(), tapi.ds, tapi.preferenceService, c.SignedInUser.GetOrgID(), 0, teamId) } // swagger:route PUT /teams/{team_id}/preferences teams updateTeamPreferences @@ -252,7 +254,7 @@ func (hs *HTTPServer) GetTeamPreferences(c *contextmodel.ReqContext) response.Re // 400: badRequestError // 401: unauthorisedError // 500: internalServerError -func (hs *HTTPServer) UpdateTeamPreferences(c *contextmodel.ReqContext) response.Response { +func (tapi *TeamAPI) updateTeamPreferences(c *contextmodel.ReqContext) response.Response { dtoCmd := dtos.UpdatePrefsCmd{} if err := web.Bind(c.Req, &dtoCmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) @@ -263,7 +265,7 @@ func (hs *HTTPServer) UpdateTeamPreferences(c *contextmodel.ReqContext) response return response.Error(http.StatusBadRequest, "teamId is invalid", err) } - return hs.updatePreferencesFor(c.Req.Context(), c.SignedInUser.GetOrgID(), 0, teamId, &dtoCmd) + return prefapi.UpdatePreferencesFor(c.Req.Context(), tapi.ds, tapi.preferenceService, c.SignedInUser.GetOrgID(), 0, teamId, &dtoCmd) } // swagger:parameters updateTeamPreferences @@ -355,3 +357,26 @@ type CreateTeamResponse struct { Message string `json:"message"` } `json:"body"` } + +// getMultiAccessControlMetadata returns the accesscontrol metadata associated with a given set of resources +// Context must contain permissions in the given org (see LoadPermissionsMiddleware or AuthorizeInOrgMiddleware) +func (tapi *TeamAPI) getMultiAccessControlMetadata(c *contextmodel.ReqContext, + prefix string, resourceIDs map[string]bool) map[string]accesscontrol.Metadata { + if !c.QueryBool("accesscontrol") { + return map[string]accesscontrol.Metadata{} + } + + if len(c.SignedInUser.GetPermissions()) == 0 { + return map[string]accesscontrol.Metadata{} + } + + return accesscontrol.GetResourcesMetadata(c.Req.Context(), c.SignedInUser.GetPermissions(), prefix, resourceIDs) +} + +// Metadata helpers +// getAccessControlMetadata returns the accesscontrol metadata associated with a given resource +func (tapi *TeamAPI) getAccessControlMetadata(c *contextmodel.ReqContext, + orgID int64, prefix string, resourceID string) accesscontrol.Metadata { + ids := map[string]bool{resourceID: true} + return tapi.getMultiAccessControlMetadata(c, prefix, ids)[resourceID] +} diff --git a/pkg/api/team_members.go b/pkg/services/team/teamapi/team_members.go similarity index 74% rename from pkg/api/team_members.go rename to pkg/services/team/teamapi/team_members.go index 000206758c7..a9f996a8c11 100644 --- a/pkg/api/team_members.go +++ b/pkg/services/team/teamapi/team_members.go @@ -1,4 +1,4 @@ -package api +package teamapi import ( "context" @@ -28,7 +28,7 @@ import ( // 403: forbiddenError // 404: notFoundError // 500: internalServerError -func (hs *HTTPServer) GetTeamMembers(c *contextmodel.ReqContext) response.Response { +func (tapi *TeamAPI) getTeamMembers(c *contextmodel.ReqContext) response.Response { teamId, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) if err != nil { return response.Error(http.StatusBadRequest, "teamId is invalid", err) @@ -36,21 +36,21 @@ func (hs *HTTPServer) GetTeamMembers(c *contextmodel.ReqContext) response.Respon query := team.GetTeamMembersQuery{OrgID: c.SignedInUser.GetOrgID(), TeamID: teamId, SignedInUser: c.SignedInUser} - queryResult, err := hs.teamService.GetTeamMembers(c.Req.Context(), &query) + queryResult, err := tapi.teamService.GetTeamMembers(c.Req.Context(), &query) if err != nil { - return response.Error(500, "Failed to get Team Members", err) + return response.Error(http.StatusInternalServerError, "Failed to get Team Members", err) } filteredMembers := make([]*team.TeamMemberDTO, 0, len(queryResult)) for _, member := range queryResult { - if dtos.IsHiddenUser(member.Login, c.SignedInUser, hs.Cfg) { + if dtos.IsHiddenUser(member.Login, c.SignedInUser, tapi.cfg) { continue } member.AvatarURL = dtos.GetGravatarUrl(member.Email) member.Labels = []string{} - if hs.License.FeatureEnabled("teamgroupsync") && member.External { + if tapi.license.FeatureEnabled("teamgroupsync") && member.External { authProvider := login.GetAuthProviderLabel(member.AuthModule) member.Labels = append(member.Labels, authProvider) } @@ -71,7 +71,7 @@ func (hs *HTTPServer) GetTeamMembers(c *contextmodel.ReqContext) response.Respon // 403: forbiddenError // 404: notFoundError // 500: internalServerError -func (hs *HTTPServer) AddTeamMember(c *contextmodel.ReqContext) response.Response { +func (tapi *TeamAPI) addTeamMember(c *contextmodel.ReqContext) response.Response { cmd := team.AddTeamMemberCommand{} var err error if err := web.Bind(c.Req, &cmd); err != nil { @@ -83,17 +83,17 @@ func (hs *HTTPServer) AddTeamMember(c *contextmodel.ReqContext) response.Respons return response.Error(http.StatusBadRequest, "teamId is invalid", err) } - isTeamMember, err := hs.teamService.IsTeamMember(c.SignedInUser.GetOrgID(), cmd.TeamID, cmd.UserID) + isTeamMember, err := tapi.teamService.IsTeamMember(c.SignedInUser.GetOrgID(), cmd.TeamID, cmd.UserID) if err != nil { - return response.Error(500, "Failed to add team member.", err) + return response.Error(http.StatusInternalServerError, "Failed to add team member.", err) } if isTeamMember { - return response.Error(400, "User is already added to this team", nil) + return response.Error(http.StatusBadRequest, "User is already added to this team", nil) } - err = addOrUpdateTeamMember(c.Req.Context(), hs.teamPermissionsService, cmd.UserID, cmd.OrgID, cmd.TeamID, getPermissionName(cmd.Permission)) + err = addOrUpdateTeamMember(c.Req.Context(), tapi.teamPermissionsService, cmd.UserID, cmd.OrgID, cmd.TeamID, getPermissionName(cmd.Permission)) if err != nil { - return response.Error(500, "Failed to add Member to Team", err) + return response.Error(http.StatusInternalServerError, "Failed to add Member to Team", err) } return response.JSON(http.StatusOK, &util.DynMap{ @@ -111,7 +111,7 @@ func (hs *HTTPServer) AddTeamMember(c *contextmodel.ReqContext) response.Respons // 403: forbiddenError // 404: notFoundError // 500: internalServerError -func (hs *HTTPServer) UpdateTeamMember(c *contextmodel.ReqContext) response.Response { +func (tapi *TeamAPI) updateTeamMember(c *contextmodel.ReqContext) response.Response { cmd := team.UpdateTeamMemberCommand{} if err := web.Bind(c.Req, &cmd); err != nil { return response.Error(http.StatusBadRequest, "bad request data", err) @@ -126,17 +126,17 @@ func (hs *HTTPServer) UpdateTeamMember(c *contextmodel.ReqContext) response.Resp } orgId := c.SignedInUser.GetOrgID() - isTeamMember, err := hs.teamService.IsTeamMember(orgId, teamId, userId) + isTeamMember, err := tapi.teamService.IsTeamMember(orgId, teamId, userId) if err != nil { - return response.Error(500, "Failed to update team member.", err) + return response.Error(http.StatusInternalServerError, "Failed to update team member.", err) } if !isTeamMember { - return response.Error(404, "Team member not found.", nil) + return response.Error(http.StatusNotFound, "Team member not found.", nil) } - err = addOrUpdateTeamMember(c.Req.Context(), hs.teamPermissionsService, userId, orgId, teamId, getPermissionName(cmd.Permission)) + err = addOrUpdateTeamMember(c.Req.Context(), tapi.teamPermissionsService, userId, orgId, teamId, getPermissionName(cmd.Permission)) if err != nil { - return response.Error(500, "Failed to update team member.", err) + return response.Error(http.StatusInternalServerError, "Failed to update team member.", err) } return response.Success("Team member updated") } @@ -161,7 +161,7 @@ func getPermissionName(permission dashboards.PermissionType) string { // 403: forbiddenError // 404: notFoundError // 500: internalServerError -func (hs *HTTPServer) RemoveTeamMember(c *contextmodel.ReqContext) response.Response { +func (tapi *TeamAPI) removeTeamMember(c *contextmodel.ReqContext) response.Response { orgId := c.SignedInUser.GetOrgID() teamId, err := strconv.ParseInt(web.Params(c.Req)[":teamId"], 10, 64) if err != nil { @@ -173,16 +173,16 @@ func (hs *HTTPServer) RemoveTeamMember(c *contextmodel.ReqContext) response.Resp } teamIDString := strconv.FormatInt(teamId, 10) - if _, err := hs.teamPermissionsService.SetUserPermission(c.Req.Context(), orgId, accesscontrol.User{ID: userId}, teamIDString, ""); err != nil { + if _, err := tapi.teamPermissionsService.SetUserPermission(c.Req.Context(), orgId, accesscontrol.User{ID: userId}, teamIDString, ""); err != nil { if errors.Is(err, team.ErrTeamNotFound) { - return response.Error(404, "Team not found", nil) + return response.Error(http.StatusNotFound, "Team not found", nil) } if errors.Is(err, team.ErrTeamMemberNotFound) { - return response.Error(404, "Team member not found", nil) + return response.Error(http.StatusNotFound, "Team member not found", nil) } - return response.Error(500, "Failed to remove Member from Team", err) + return response.Error(http.StatusInternalServerError, "Failed to remove Member from Team", err) } return response.Success("Team Member removed") } diff --git a/pkg/api/team_members_test.go b/pkg/services/team/teamapi/team_members_test.go similarity index 59% rename from pkg/api/team_members_test.go rename to pkg/services/team/teamapi/team_members_test.go index 85858bc2902..da9465c538c 100644 --- a/pkg/api/team_members_test.go +++ b/pkg/services/team/teamapi/team_members_test.go @@ -1,4 +1,4 @@ -package api +package teamapi import ( "net/http" @@ -8,24 +8,52 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - ac "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/accesscontrol/acimpl" "github.com/grafana/grafana/pkg/services/accesscontrol/actest" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/licensing" + "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/services/preference/preftest" "github.com/grafana/grafana/pkg/services/team/teamtest" + "github.com/grafana/grafana/pkg/services/user" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/web/webtest" ) +func SetupAPITestServer(t *testing.T, opts ...func(a *TeamAPI)) *webtest.Server { + t.Helper() + router := routing.NewRouteRegister() + cfg := setting.NewCfg() + cfg.LDAPAuthEnabled = true + + a := ProvideTeamAPI(router, + teamtest.NewFakeService(), + actest.FakeService{}, + acimpl.ProvideAccessControl(cfg), + &actest.FakePermissionsService{}, + &licensing.OSSLicensingService{}, + cfg, + preftest.NewPreferenceServiceFake(), + dashboards.NewFakeDashboardService(t), + ) + for _, o := range opts { + o(a) + } + + server := webtest.NewServer(t, router) + + return server +} + func TestAddTeamMembersAPIEndpoint(t *testing.T) { - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = setting.NewCfg() - hs.teamService = teamtest.NewFakeService() - hs.teamPermissionsService = &actest.FakePermissionsService{} - }) + server := SetupAPITestServer(t) t.Run("should be able to add team member with correct permission", func(t *testing.T) { req := webtest.RequestWithSignedInUser( server.NewRequest(http.MethodPost, "/api/teams/1/members", strings.NewReader("{\"userId\": 1}")), - userWithPermissions(1, []ac.Permission{{Action: ac.ActionTeamsPermissionsWrite, Scope: "teams:id:1"}}), + authedUserWithPermissions(1, 1, []accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:1"}}), ) res, err := server.SendJSON(req) require.NoError(t, err) @@ -36,7 +64,7 @@ func TestAddTeamMembersAPIEndpoint(t *testing.T) { t.Run("should not be able to add team member without correct permission", func(t *testing.T) { req := webtest.RequestWithSignedInUser( server.NewRequest(http.MethodPost, "/api/teams/1/members", strings.NewReader("{\"userId\": 1}")), - userWithPermissions(1, []ac.Permission{{Action: ac.ActionTeamsPermissionsWrite, Scope: "teams:id:2"}}), + authedUserWithPermissions(1, 1, []accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:2"}}), ) res, err := server.SendJSON(req) require.NoError(t, err) @@ -46,16 +74,12 @@ func TestAddTeamMembersAPIEndpoint(t *testing.T) { } func TestGetTeamMembersAPIEndpoint(t *testing.T) { - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = setting.NewCfg() - hs.teamService = teamtest.NewFakeService() - hs.teamPermissionsService = &actest.FakePermissionsService{} - }) + server := SetupAPITestServer(t) t.Run("should be able to get team members with correct permission", func(t *testing.T) { req := webtest.RequestWithSignedInUser( server.NewGetRequest("/api/teams/1/members"), - userWithPermissions(1, []ac.Permission{{Action: ac.ActionTeamsPermissionsRead, Scope: "teams:id:1"}}), + authedUserWithPermissions(1, 1, []accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsRead, Scope: "teams:id:1"}}), ) res, err := server.SendJSON(req) require.NoError(t, err) @@ -65,7 +89,7 @@ func TestGetTeamMembersAPIEndpoint(t *testing.T) { t.Run("should not be able to get team members without correct permission", func(t *testing.T) { req := webtest.RequestWithSignedInUser( server.NewGetRequest("/api/teams/1/members"), - userWithPermissions(1, []ac.Permission{{Action: ac.ActionTeamsPermissionsRead, Scope: "teams:id:2"}}), + authedUserWithPermissions(1, 1, []accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsRead, Scope: "teams:id:2"}}), ) res, err := server.SendJSON(req) require.NoError(t, err) @@ -75,16 +99,14 @@ func TestGetTeamMembersAPIEndpoint(t *testing.T) { } func TestUpdateTeamMembersAPIEndpoint(t *testing.T) { - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = setting.NewCfg() + server := SetupAPITestServer(t, func(hs *TeamAPI) { hs.teamService = &teamtest.FakeService{ExpectedIsMember: true} - hs.teamPermissionsService = &actest.FakePermissionsService{} }) t.Run("should be able to update team member with correct permission", func(t *testing.T) { req := webtest.RequestWithSignedInUser( server.NewRequest(http.MethodPut, "/api/teams/1/members/1", strings.NewReader("{\"permission\": 1}")), - userWithPermissions(1, []ac.Permission{{Action: ac.ActionTeamsPermissionsWrite, Scope: "teams:id:1"}}), + authedUserWithPermissions(1, 1, []accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:1"}}), ) res, err := server.SendJSON(req) require.NoError(t, err) @@ -94,7 +116,7 @@ func TestUpdateTeamMembersAPIEndpoint(t *testing.T) { t.Run("should not be able to update team member without correct permission", func(t *testing.T) { req := webtest.RequestWithSignedInUser( server.NewRequest(http.MethodPut, "/api/teams/1/members/1", strings.NewReader("{\"permission\": 1}")), - userWithPermissions(1, []ac.Permission{{Action: ac.ActionTeamsPermissionsWrite, Scope: "teams:id:2"}}), + authedUserWithPermissions(1, 1, []accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:2"}}), ) res, err := server.SendJSON(req) require.NoError(t, err) @@ -104,8 +126,7 @@ func TestUpdateTeamMembersAPIEndpoint(t *testing.T) { } func TestDeleteTeamMembersAPIEndpoint(t *testing.T) { - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = setting.NewCfg() + server := SetupAPITestServer(t, func(hs *TeamAPI) { hs.teamService = &teamtest.FakeService{ExpectedIsMember: true} hs.teamPermissionsService = &actest.FakePermissionsService{} }) @@ -113,7 +134,7 @@ func TestDeleteTeamMembersAPIEndpoint(t *testing.T) { t.Run("should be able to delete team member with correct permission", func(t *testing.T) { req := webtest.RequestWithSignedInUser( server.NewRequest(http.MethodDelete, "/api/teams/1/members/1", nil), - userWithPermissions(1, []ac.Permission{{Action: ac.ActionTeamsPermissionsWrite, Scope: "teams:id:1"}}), + authedUserWithPermissions(1, 1, []accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:1"}}), ) res, err := server.SendJSON(req) require.NoError(t, err) @@ -123,7 +144,7 @@ func TestDeleteTeamMembersAPIEndpoint(t *testing.T) { t.Run("should not be able to delete member without correct permission", func(t *testing.T) { req := webtest.RequestWithSignedInUser( server.NewRequest(http.MethodDelete, "/api/teams/1/members/1", nil), - userWithPermissions(1, []ac.Permission{{Action: ac.ActionTeamsPermissionsWrite, Scope: "teams:id:2"}}), + authedUserWithPermissions(1, 1, []accesscontrol.Permission{{Action: accesscontrol.ActionTeamsPermissionsWrite, Scope: "teams:id:2"}}), ) res, err := server.SendJSON(req) require.NoError(t, err) @@ -131,3 +152,7 @@ func TestDeleteTeamMembersAPIEndpoint(t *testing.T) { require.NoError(t, res.Body.Close()) }) } + +func authedUserWithPermissions(userID, orgID int64, permissions []accesscontrol.Permission) *user.SignedInUser { + return &user.SignedInUser{UserID: userID, OrgID: orgID, OrgRole: org.RoleViewer, Permissions: map[int64]map[string][]string{orgID: accesscontrol.GroupScopesByAction(permissions)}} +} diff --git a/pkg/api/team_test.go b/pkg/services/team/teamapi/team_test.go similarity index 79% rename from pkg/api/team_test.go rename to pkg/services/team/teamapi/team_test.go index 5c6646124a7..4cd3f541046 100644 --- a/pkg/api/team_test.go +++ b/pkg/services/team/teamapi/team_test.go @@ -1,4 +1,4 @@ -package api +package teamapi import ( "fmt" @@ -10,14 +10,11 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/services/accesscontrol" - "github.com/grafana/grafana/pkg/services/accesscontrol/acimpl" - "github.com/grafana/grafana/pkg/services/accesscontrol/actest" pref "github.com/grafana/grafana/pkg/services/preference" "github.com/grafana/grafana/pkg/services/preference/preftest" "github.com/grafana/grafana/pkg/services/team" "github.com/grafana/grafana/pkg/services/team/teamtest" "github.com/grafana/grafana/pkg/services/user" - "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/web/webtest" ) @@ -31,17 +28,14 @@ const ( ) func TestTeamAPIEndpoint_CreateTeam(t *testing.T) { - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = setting.NewCfg() + server := SetupAPITestServer(t, func(hs *TeamAPI) { hs.teamService = teamtest.NewFakeService() - hs.AccessControl = acimpl.ProvideAccessControl(setting.NewCfg()) - hs.accesscontrolService = actest.FakeService{} }) input := strings.NewReader(fmt.Sprintf(teamCmd, 1)) t.Run("Access control allows creating teams with the correct permissions", func(t *testing.T) { req := server.NewPostRequest(createTeamURL, input) - req = webtest.RequestWithSignedInUser(req, userWithPermissions(1, []accesscontrol.Permission{{Action: accesscontrol.ActionTeamsCreate}})) + req = webtest.RequestWithSignedInUser(req, authedUserWithPermissions(1, 1, []accesscontrol.Permission{{Action: accesscontrol.ActionTeamsCreate}})) res, err := server.SendJSON(req) require.NoError(t, err) assert.Equal(t, http.StatusOK, res.StatusCode) @@ -51,7 +45,7 @@ func TestTeamAPIEndpoint_CreateTeam(t *testing.T) { input = strings.NewReader(fmt.Sprintf(teamCmd, 2)) t.Run("Access control prevents creating teams with the incorrect permissions", func(t *testing.T) { req := server.NewPostRequest(createTeamURL, input) - req = webtest.RequestWithSignedInUser(req, userWithPermissions(1, []accesscontrol.Permission{})) + req = webtest.RequestWithSignedInUser(req, authedUserWithPermissions(1, 1, []accesscontrol.Permission{})) res, err := server.SendJSON(req) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, res.StatusCode) @@ -60,14 +54,13 @@ func TestTeamAPIEndpoint_CreateTeam(t *testing.T) { } func TestTeamAPIEndpoint_SearchTeams(t *testing.T) { - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = setting.NewCfg() + server := SetupAPITestServer(t, func(hs *TeamAPI) { hs.teamService = teamtest.NewFakeService() }) t.Run("Access control prevents searching for teams with the incorrect permissions", func(t *testing.T) { req := server.NewGetRequest(searchTeamsURL) - req = webtest.RequestWithSignedInUser(req, userWithPermissions(1, []accesscontrol.Permission{})) + req = webtest.RequestWithSignedInUser(req, authedUserWithPermissions(1, 1, []accesscontrol.Permission{})) res, err := server.Send(req) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, res.StatusCode) @@ -76,7 +69,7 @@ func TestTeamAPIEndpoint_SearchTeams(t *testing.T) { t.Run("Access control allows searching for teams with the correct permissions", func(t *testing.T) { req := server.NewGetRequest(searchTeamsURL) - req = webtest.RequestWithSignedInUser(req, userWithPermissions(1, []accesscontrol.Permission{ + req = webtest.RequestWithSignedInUser(req, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsRead, Scope: accesscontrol.ScopeTeamsAll}, })) res, err := server.Send(req) @@ -87,8 +80,7 @@ func TestTeamAPIEndpoint_SearchTeams(t *testing.T) { } func TestTeamAPIEndpoint_GetTeamByID(t *testing.T) { - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = setting.NewCfg() + server := SetupAPITestServer(t, func(hs *TeamAPI) { hs.teamService = &teamtest.FakeService{ExpectedTeamDTO: &team.TeamDTO{}} }) @@ -96,7 +88,7 @@ func TestTeamAPIEndpoint_GetTeamByID(t *testing.T) { t.Run("Access control prevents getting a team when missing permissions", func(t *testing.T) { req := server.NewGetRequest(url) - req = webtest.RequestWithSignedInUser(req, userWithPermissions(1, []accesscontrol.Permission{})) + req = webtest.RequestWithSignedInUser(req, authedUserWithPermissions(1, 1, []accesscontrol.Permission{})) res, err := server.Send(req) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, res.StatusCode) @@ -105,7 +97,7 @@ func TestTeamAPIEndpoint_GetTeamByID(t *testing.T) { t.Run("Access control allows getting a team with the correct permissions", func(t *testing.T) { req := server.NewGetRequest(url) - req = webtest.RequestWithSignedInUser(req, userWithPermissions(1, []accesscontrol.Permission{ + req = webtest.RequestWithSignedInUser(req, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsRead, Scope: "teams:id:1"}, })) res, err := server.Send(req) @@ -116,7 +108,7 @@ func TestTeamAPIEndpoint_GetTeamByID(t *testing.T) { t.Run("Access control allows getting a team with wildcard scope", func(t *testing.T) { req := server.NewGetRequest(url) - req = webtest.RequestWithSignedInUser(req, userWithPermissions(1, []accesscontrol.Permission{ + req = webtest.RequestWithSignedInUser(req, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsRead, Scope: "teams:id:*"}, })) res, err := server.Send(req) @@ -130,8 +122,7 @@ func TestTeamAPIEndpoint_GetTeamByID(t *testing.T) { // Then the endpoint should return 200 if the user has accesscontrol.ActionTeamsWrite with teams:id:1 scope // else return 403 func TestTeamAPIEndpoint_UpdateTeam(t *testing.T) { - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = setting.NewCfg() + server := SetupAPITestServer(t, func(hs *TeamAPI) { hs.teamService = &teamtest.FakeService{ExpectedTeamDTO: &team.TeamDTO{}} }) @@ -142,7 +133,7 @@ func TestTeamAPIEndpoint_UpdateTeam(t *testing.T) { } t.Run("Access control allows updating team with the correct permissions", func(t *testing.T) { - res, err := request(1, userWithPermissions(1, []accesscontrol.Permission{ + res, err := request(1, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsWrite, Scope: "teams:id:1"}, })) require.NoError(t, err) @@ -151,7 +142,7 @@ func TestTeamAPIEndpoint_UpdateTeam(t *testing.T) { }) t.Run("Access control allows updating teams with the wildcard scope", func(t *testing.T) { - res, err := request(1, userWithPermissions(1, []accesscontrol.Permission{ + res, err := request(1, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsWrite, Scope: "teams:*"}, })) require.NoError(t, err) @@ -160,7 +151,7 @@ func TestTeamAPIEndpoint_UpdateTeam(t *testing.T) { }) t.Run("Access control prevent updating a team with wrong scope", func(t *testing.T) { - res, err := request(1, userWithPermissions(1, []accesscontrol.Permission{ + res, err := request(1, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsWrite, Scope: "teams:id:2"}, })) require.NoError(t, err) @@ -173,8 +164,7 @@ func TestTeamAPIEndpoint_UpdateTeam(t *testing.T) { // Then the endpoint should return 200 if the user has accesscontrol.ActionTeamsDelete with teams:id:1 scope // else return 403 func TestTeamAPIEndpoint_DeleteTeam(t *testing.T) { - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = setting.NewCfg() + server := SetupAPITestServer(t, func(hs *TeamAPI) { hs.teamService = &teamtest.FakeService{ExpectedTeamDTO: &team.TeamDTO{}} }) @@ -185,7 +175,7 @@ func TestTeamAPIEndpoint_DeleteTeam(t *testing.T) { } t.Run("Access control prevents deleting teams with the incorrect permissions", func(t *testing.T) { - res, err := request(1, userWithPermissions(1, []accesscontrol.Permission{ + res, err := request(1, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsDelete, Scope: "teams:id:2"}, })) require.NoError(t, err) @@ -194,7 +184,7 @@ func TestTeamAPIEndpoint_DeleteTeam(t *testing.T) { }) t.Run("Access control allows deleting teams with the correct permissions", func(t *testing.T) { - res, err := request(1, userWithPermissions(1, []accesscontrol.Permission{ + res, err := request(1, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsDelete, Scope: "teams:id:1"}, })) require.NoError(t, err) @@ -207,8 +197,7 @@ func TestTeamAPIEndpoint_DeleteTeam(t *testing.T) { // Then the endpoint should return 200 if the user has accesscontrol.ActionTeamsRead with teams:id:1 scope // else return 403 func TestTeamAPIEndpoint_GetTeamPreferences(t *testing.T) { - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = setting.NewCfg() + server := SetupAPITestServer(t, func(hs *TeamAPI) { hs.preferenceService = &preftest.FakePreferenceService{ExpectedPreference: &pref.Preference{}} }) @@ -219,7 +208,7 @@ func TestTeamAPIEndpoint_GetTeamPreferences(t *testing.T) { } t.Run("Access control allows getting team preferences with the correct permissions", func(t *testing.T) { - res, err := request(1, userWithPermissions(1, []accesscontrol.Permission{ + res, err := request(1, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsRead, Scope: "teams:id:1"}, })) require.NoError(t, err) @@ -228,7 +217,7 @@ func TestTeamAPIEndpoint_GetTeamPreferences(t *testing.T) { }) t.Run("Access control prevents getting team preferences with the incorrect permissions", func(t *testing.T) { - res, err := request(1, userWithPermissions(1, []accesscontrol.Permission{ + res, err := request(1, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsRead, Scope: "teams:id:2"}, })) require.NoError(t, err) @@ -241,8 +230,7 @@ func TestTeamAPIEndpoint_GetTeamPreferences(t *testing.T) { // Then the endpoint should return 200 if the user has accesscontrol.ActionTeamsWrite with teams:id:1 scope // else return 403 func TestTeamAPIEndpoint_UpdateTeamPreferences(t *testing.T) { - server := SetupAPITestServer(t, func(hs *HTTPServer) { - hs.Cfg = setting.NewCfg() + server := SetupAPITestServer(t, func(hs *TeamAPI) { hs.preferenceService = &preftest.FakePreferenceService{ExpectedPreference: &pref.Preference{}} }) @@ -253,7 +241,7 @@ func TestTeamAPIEndpoint_UpdateTeamPreferences(t *testing.T) { } t.Run("Access control allows updating team preferences with the correct permissions", func(t *testing.T) { - res, err := request(1, userWithPermissions(1, []accesscontrol.Permission{ + res, err := request(1, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsWrite, Scope: "teams:id:1"}, })) require.NoError(t, err) @@ -262,7 +250,7 @@ func TestTeamAPIEndpoint_UpdateTeamPreferences(t *testing.T) { }) t.Run("Access control prevents updating team preferences with the incorrect permissions", func(t *testing.T) { - res, err := request(1, userWithPermissions(1, []accesscontrol.Permission{ + res, err := request(1, authedUserWithPermissions(1, 1, []accesscontrol.Permission{ {Action: accesscontrol.ActionTeamsWrite, Scope: "teams:id:2"}, })) require.NoError(t, err) From 9bb3d64b1e07687a98f937599d336379219e3e23 Mon Sep 17 00:00:00 2001 From: Fabrizio <135109076+fabrizio-grafana@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:19:01 +0200 Subject: [PATCH 079/102] Minor fixes for TraceQL query editor (#76344) --- package.json | 2 +- .../tempo/traceql/TraceQLEditor.test.tsx | 14 +++++++++++++- .../datasource/tempo/traceql/TraceQLEditor.tsx | 10 +++++++++- .../datasource/tempo/traceql/errorHighlighting.ts | 11 +++++++++++ yarn.lock | 10 +++++----- 5 files changed, 39 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index fdbf162f159..05481e3e04b 100644 --- a/package.json +++ b/package.json @@ -248,7 +248,7 @@ "@grafana/flamegraph": "workspace:*", "@grafana/google-sdk": "0.1.1", "@grafana/lezer-logql": "0.2.0", - "@grafana/lezer-traceql": "0.0.6", + "@grafana/lezer-traceql": "0.0.7", "@grafana/monaco-logql": "^0.0.7", "@grafana/runtime": "workspace:*", "@grafana/scenes": "^1.15.0", diff --git a/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.test.tsx b/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.test.tsx index 1a99d4ed309..79c1e3a5de0 100644 --- a/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.test.tsx +++ b/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.test.tsx @@ -1,6 +1,6 @@ import { computeErrorMessage, getErrorNodes } from './errorHighlighting'; -describe('computeErrorMarkers', () => { +describe('Check for syntax errors in query', () => { it.each([ ['{span.http.status_code = }', 'Invalid value after comparison or aritmetic operator.'], ['{span.http.status_code 200}', 'Invalid comparison operator after field expression.'], @@ -59,8 +59,20 @@ describe('computeErrorMarkers', () => { ['{.foo=300} && {.foo=300} | avg(.value) =', 'Invalid value after comparison operator.'], ['{.foo=300} | max(duration) > 1hs', 'Invalid value after comparison operator.'], ['{ span.http.status_code', 'Invalid comparison operator after field expression.'], + ['abcxyz', 'Invalid query.'], ])('error message for invalid query - %s, %s', (query: string, expectedErrorMessage: string) => { const errorNode = getErrorNodes(query)[0]; expect(computeErrorMessage(errorNode)).toBe(expectedErrorMessage); }); + + it.each([ + ['123'], + ['abc'], + ['1a2b3c'], + ['{span.status = $code}'], + ['{span.${attribute} = "GET"}'], + ['{span.${attribute:format} = ${value:format} }'], + ])('valid query - %s', (query: string) => { + expect(getErrorNodes(query)).toStrictEqual([]); + }); }); diff --git a/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.tsx b/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.tsx index 42d83347505..05cb706ca9a 100644 --- a/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.tsx +++ b/public/app/plugins/datasource/tempo/traceql/TraceQLEditor.tsx @@ -67,10 +67,18 @@ export function TraceQLEditor(props: Props) { } setupAutoSize(editor); + // Parse query that might already exist (e.g., after a page refresh) + const model = editor.getModel(); + if (model) { + const errorNodes = getErrorNodes(model.getValue()); + setErrorMarkers(monaco, model, errorNodes); + } + // Register callback for query changes editor.onDidChangeModelContent((changeEvent) => { const model = editor.getModel(); - if (!model || model.getValue() === '') { + + if (!model) { return; } diff --git a/public/app/plugins/datasource/tempo/traceql/errorHighlighting.ts b/public/app/plugins/datasource/tempo/traceql/errorHighlighting.ts index dff1c7f9715..bd49a30b80c 100644 --- a/public/app/plugins/datasource/tempo/traceql/errorHighlighting.ts +++ b/public/app/plugins/datasource/tempo/traceql/errorHighlighting.ts @@ -78,6 +78,17 @@ export const computeErrorMessage = (errorNode: SyntaxNode) => { * @returns the error nodes */ export const getErrorNodes = (query: string): SyntaxNode[] => { + // Return immediately if the query is empty, to avoid raising exceptions in processing it + if (query.trim() === '') { + return []; + } + + // Check whether this is a trace ID or traceQL query by checking if it only contains hex characters + const hexOnlyRegex = /^[0-9A-Fa-f]*$/; + if (query.trim().match(hexOnlyRegex)) { + return []; + } + const tree = parser.parse(query); // Find all error nodes and compute the associated erro boundaries diff --git a/yarn.lock b/yarn.lock index 0d9cdf2b00a..0b368148317 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3281,12 +3281,12 @@ __metadata: languageName: node linkType: hard -"@grafana/lezer-traceql@npm:0.0.6": - version: 0.0.6 - resolution: "@grafana/lezer-traceql@npm:0.0.6" +"@grafana/lezer-traceql@npm:0.0.7": + version: 0.0.7 + resolution: "@grafana/lezer-traceql@npm:0.0.7" peerDependencies: "@lezer/lr": ^1.3.0 - checksum: 166a30c38f4f78e1768e80f724176b71becf3fa51414bc8bea4c2c5288a92d1f8b7872280a06c0d3f7f7608b174e5fc930e1af39339b2c2fdf627fb7ac14c6b4 + checksum: 920f8116d61907e12ca1dd51949242abf435e675105699c7b2fcd8c024999ed0ddfd3500b1802111f3ed9ad37e16f40826790658bdf94224cd5b615e20360ab9 languageName: node linkType: hard @@ -17635,7 +17635,7 @@ __metadata: "@grafana/flamegraph": "workspace:*" "@grafana/google-sdk": 0.1.1 "@grafana/lezer-logql": 0.2.0 - "@grafana/lezer-traceql": 0.0.6 + "@grafana/lezer-traceql": 0.0.7 "@grafana/monaco-logql": ^0.0.7 "@grafana/runtime": "workspace:*" "@grafana/scenes": ^1.15.0 From b6dff851276fee7659fa73ec8ec9da0273c09f24 Mon Sep 17 00:00:00 2001 From: Misi Date: Thu, 12 Oct 2023 10:20:00 +0200 Subject: [PATCH 080/102] Chore: Update grafana/saml to the latest (#76363) * Update go.mod * add missing squad to dependency --- go.mod | 6 +++--- go.sum | 18 ++---------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 7f055116c51..11835080ef7 100644 --- a/go.mod +++ b/go.mod @@ -242,7 +242,6 @@ require ( github.com/google/go-github/v45 v45.2.0 // @grafana/grafana-delivery github.com/grafana/codejen v0.0.3 // @grafana/dataviz-squad github.com/grafana/dskit v0.0.0-20230706162620-5081d8ed53e6 // @grafana/backend-platform - github.com/grafana/phlare/api v0.1.4-0.20230426005640-f90edba05413 // @grafana/observability-traces-and-profiling github.com/huandu/xstrings v1.3.1 // @grafana/partner-datasources github.com/jmoiron/sqlx v1.3.5 // @grafana/backend-platform github.com/matryer/is v1.4.0 // @grafana/grafana-as-code @@ -289,6 +288,8 @@ require ( require github.com/grafana/gofpdf v0.0.0-20231002120153-857cc45be447 // @grafana/sharing-squad +require github.com/grafana/pyroscope/api v0.2.0 // @grafana/observability-traces-and-profiling + require ( cloud.google.com/go v0.110.6 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect @@ -342,7 +343,6 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/grafana/pyroscope/api v0.2.0 // indirect github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db // indirect github.com/grafana/sqlds/v2 v2.3.10 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -490,7 +490,7 @@ require ( ) // Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream -replace github.com/crewjam/saml => github.com/grafana/saml v0.4.13-0.20230823151800-99004118fff5 +replace github.com/crewjam/saml => github.com/grafana/saml v0.4.13-0.20231011134114-94154cb1bda6 // Thema's thema CLI requires cobra, which eventually works its way down to go-hclog@v1.0.0. // Upgrading affects backend plugins: https://github.com/grafana/grafana/pull/47653#discussion_r850508593 diff --git a/go.sum b/go.sum index 5daad8e9c43..85c749bd5d8 100644 --- a/go.sum +++ b/go.sum @@ -841,9 +841,6 @@ github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pO github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/bsm/ginkgo/v2 v2.5.0 h1:aOAnND1T40wEdAtkGSkvSICWeQ8L3UASX7YVCqQx+eQ= github.com/bsm/gomega v1.20.0 h1:JhAwLmtRzXFTx2AkALSLa8ijZafntmhSoU63Ok18Uq8= -github.com/bufbuild/connect-go v1.4.1 h1:6usL3JGjKhxQpvDlizP7u8VfjAr1JkckcAUbrdcbgNY= -github.com/bufbuild/connect-go v1.4.1/go.mod h1:9iNvh/NOsfhNBUH5CtvXeVUskQO1xsrEviH7ZArwZ3I= -github.com/bufbuild/connect-go v1.9.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8= github.com/bufbuild/connect-go v1.10.0 h1:QAJ3G9A1OYQW2Jbk3DeoJbkCxuKArrvZgDt47mjdTbg= github.com/bufbuild/connect-go v1.10.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -1301,7 +1298,6 @@ github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= -github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:9wScpmSP5A3Bk8V3XHWUcJmYTh+ZnlHVyc+A4oZYS3Y= github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:56xuuqnHyryaerycW3BfssRdxQstACi0Epw/yC5E2xM= github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= @@ -1653,7 +1649,6 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4= -github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -1729,7 +1724,6 @@ github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10/go.mod h1:79YE0hCXdHa github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1788,8 +1782,6 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gotestyourself/gotestyourself v1.3.0/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= -github.com/grafana/alerting v0.0.0-20230918125844-e60d564786c9 h1:XHgUyX+q7zYJnBmtbfPyBeOg0W/ds/uw218XOP2k5v0= -github.com/grafana/alerting v0.0.0-20230918125844-e60d564786c9/go.mod h1:6BES5CyEqz7fDAG3MYvJLe0hqGwvIoGDN8A1aNrLGus= github.com/grafana/alerting v0.0.0-20231005194306-939f55779e5f h1:zz5YocNyxQ9X8Bvm2ezPOCGKxngrXX1XGLfywqk5+vU= github.com/grafana/alerting v0.0.0-20231005194306-939f55779e5f/go.mod h1:6BES5CyEqz7fDAG3MYvJLe0hqGwvIoGDN8A1aNrLGus= github.com/grafana/codejen v0.0.3 h1:tAWxoTUuhgmEqxJPOLtJoxlPBbMULFwKFOcRsPRPXDw= @@ -1816,14 +1808,10 @@ github.com/grafana/grafana-google-sdk-go v0.1.0 h1:LKGY8z2DSxKjYfr2flZsWgTRTZ6HG github.com/grafana/grafana-google-sdk-go v0.1.0/go.mod h1:Vo2TKWfDVmNTELBUM+3lkrZvFtBws0qSZdXhQxRdJrE= github.com/grafana/grafana-plugin-sdk-go v0.94.0/go.mod h1:3VXz4nCv6wH5SfgB3mlW39s+c+LetqSCjFj7xxPC5+M= github.com/grafana/grafana-plugin-sdk-go v0.114.0/go.mod h1:D7x3ah+1d4phNXpbnOaxa/osSaZlwh9/ZUnGGzegRbk= -github.com/grafana/grafana-plugin-sdk-go v0.179.0 h1:gaow50UDWLhvVlrPye4rESajpRItEecKi2+i+TE1r7Q= -github.com/grafana/grafana-plugin-sdk-go v0.179.0/go.mod h1:fPX9spPWEzyUg0BLTQbdXCCq7PVSJZGNVKfNpiTQUts= github.com/grafana/grafana-plugin-sdk-go v0.180.0 h1:+5aFolBj2H7ze7oejIUjrkmZ9lSydnsbo3e4Kn1CN1Y= github.com/grafana/grafana-plugin-sdk-go v0.180.0/go.mod h1:fPX9spPWEzyUg0BLTQbdXCCq7PVSJZGNVKfNpiTQUts= github.com/grafana/kindsys v0.0.0-20230508162304-452481b63482 h1:1YNoeIhii4UIIQpCPU+EXidnqf449d0C3ZntAEt4KSo= github.com/grafana/kindsys v0.0.0-20230508162304-452481b63482/go.mod h1:GNcfpy5+SY6RVbNGQW264gC0r336Dm+0zgQ5vt6+M8Y= -github.com/grafana/phlare/api v0.1.4-0.20230426005640-f90edba05413 h1:bBzCezZNRyYlJpXTkyZdY4fpPxHZUdyeyRWzhtw/P6I= -github.com/grafana/phlare/api v0.1.4-0.20230426005640-f90edba05413/go.mod h1:IvwuGG9xa/h96UH/exgvsfy3zE+ZpctkNT9o5aaGdrU= github.com/grafana/prometheus-alertmanager v0.25.1-0.20230918083811-3513be6760b7 h1:7gsywzIb39SYZEp9qOnNaxD4d9OOkAfJGvnRUQUtlTM= github.com/grafana/prometheus-alertmanager v0.25.1-0.20230918083811-3513be6760b7/go.mod h1:0SOsbwZb277OdNdUR+376IxcAYV3Pp+igiAOJdr/98M= github.com/grafana/pyroscope/api v0.2.0 h1:TzOxL0s6SiaLEy944ZAKgHcx/JDRJXu4O8ObwkqR6p4= @@ -1831,8 +1819,8 @@ github.com/grafana/pyroscope/api v0.2.0/go.mod h1:nhH+xai9cYFgs6lMy/+L0pKj0d5yCM github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db h1:7aN5cccjIqCLTzedH7MZzRZt5/lsAHch6Z3L2ZGn5FA= github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= -github.com/grafana/saml v0.4.13-0.20230823151800-99004118fff5 h1:F0fY72CD+fFUFQVbqVXqszdm/T3b/LUqv2EssTAwm14= -github.com/grafana/saml v0.4.13-0.20230823151800-99004118fff5/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA= +github.com/grafana/saml v0.4.13-0.20231011134114-94154cb1bda6 h1:CuCfH9nIpkrOkv1UasyLexbskquvgxsmrUy6MaJEwDM= +github.com/grafana/saml v0.4.13-0.20231011134114-94154cb1bda6/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA= github.com/grafana/sqlds/v2 v2.3.10 h1:HWKhE0vR6LoEiE+Is8CSZOgaB//D1yqb2ntkass9Fd4= github.com/grafana/sqlds/v2 v2.3.10/go.mod h1:c6ibxnxRVGxV/0YkEgvy7QpQH/lyifFyV7K/14xvdIs= github.com/grafana/tempo v1.5.1-0.20230524121406-1dc1bfe7085b h1:mDlkqgTEJuK7vjPG44f3ZMtId5AAYLWHvBVbiGqIOOQ= @@ -2604,7 +2592,6 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= @@ -2847,7 +2834,6 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= -github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/unknwon/bra v0.0.0-20200517080246-1e3013ecaff8 h1:aVGB3YnaS/JNfOW3tiHIlmNmTDg618va+eT0mVomgyI= github.com/unknwon/bra v0.0.0-20200517080246-1e3013ecaff8/go.mod h1:fVle4kNr08ydeohzYafr20oZzbAkhQT39gKK/pFQ5M4= github.com/unknwon/com v1.0.1 h1:3d1LTxD+Lnf3soQiD4Cp/0BRB+Rsa/+RTvz8GMMzIXs= From d00410560a3fb6a39c8d3fa9b4121b92d7c04b7b Mon Sep 17 00:00:00 2001 From: Dominik Prokop Date: Thu, 12 Oct 2023 10:30:29 +0200 Subject: [PATCH 081/102] Custom variable: Initialize options from query if not present in persisted model (#76403) * Custom variable: Initialize options from query if not oresent in persisted model * Test update --- .../features/variables/state/actions.test.ts | 23 ++++++++++++++++--- .../app/features/variables/state/actions.ts | 5 ++++ .../variables/state/processVariable.test.ts | 16 ++++++++++--- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/public/app/features/variables/state/actions.test.ts b/public/app/features/variables/state/actions.test.ts index 18b79731941..d2c8c3445cd 100644 --- a/public/app/features/variables/state/actions.test.ts +++ b/public/app/features/variables/state/actions.test.ts @@ -12,6 +12,7 @@ import { variableAdapters } from '../adapters'; import { createConstantVariableAdapter } from '../constant/adapter'; import { ALL_VARIABLE_TEXT, ALL_VARIABLE_VALUE, NEW_VARIABLE_ID } from '../constants'; import { createCustomVariableAdapter } from '../custom/adapter'; +import { createCustomOptionsFromQuery } from '../custom/reducer'; import { changeVariableName } from '../editor/actions'; import { changeVariableNameFailed, changeVariableNameSucceeded, cleanEditorState } from '../editor/reducer'; import { cleanPickerState } from '../pickers/OptionsPicker/reducer'; @@ -192,7 +193,7 @@ describe('shared actions', () => { .whenAsyncActionIsDispatched(processVariables(key), true); await tester.thenDispatchedActionsPredicateShouldEqual((dispatchedActions) => { - expect(dispatchedActions.length).toEqual(5); + expect(dispatchedActions.length).toEqual(7); expect(dispatchedActions[0]).toEqual( toKeyedAction( @@ -213,7 +214,7 @@ describe('shared actions', () => { expect(dispatchedActions[2]).toEqual( toKeyedAction( key, - variableStateCompleted(toVariablePayload({ ...custom, id: dispatchedActions[2].payload.action.payload.id })) + variableStateFetching(toVariablePayload({ ...custom, id: dispatchedActions[2].payload.action.payload.id })) ) ); @@ -229,7 +230,23 @@ describe('shared actions', () => { expect(dispatchedActions[4]).toEqual( toKeyedAction( key, - variableStateCompleted(toVariablePayload({ ...query, id: dispatchedActions[4].payload.action.payload.id })) + createCustomOptionsFromQuery( + toVariablePayload({ ...custom, id: dispatchedActions[4].payload.action.payload.id }) + ) + ) + ); + + expect(dispatchedActions[5]).toEqual( + toKeyedAction( + key, + variableStateCompleted(toVariablePayload({ ...custom, id: dispatchedActions[5].payload.action.payload.id })) + ) + ); + + expect(dispatchedActions[6]).toEqual( + toKeyedAction( + key, + variableStateCompleted(toVariablePayload({ ...query, id: dispatchedActions[6].payload.action.payload.id })) ) ); diff --git a/public/app/features/variables/state/actions.ts b/public/app/features/variables/state/actions.ts index 5feea4c65dc..9e4f3d4b812 100644 --- a/public/app/features/variables/state/actions.ts +++ b/public/app/features/variables/state/actions.ts @@ -381,6 +381,11 @@ export const processVariable = ( } } + if (variable.type === 'custom') { + await dispatch(updateOptions(toKeyedVariableIdentifier(variable))); + return; + } + // for variables that aren't updated via URL or refresh, let's simulate the same state changes dispatch(completeVariableLoading(identifier)); }; diff --git a/public/app/features/variables/state/processVariable.test.ts b/public/app/features/variables/state/processVariable.test.ts index 76fbca093ca..5cea2d1de9b 100644 --- a/public/app/features/variables/state/processVariable.test.ts +++ b/public/app/features/variables/state/processVariable.test.ts @@ -6,6 +6,7 @@ import { DatasourceSrv } from 'app/features/plugins/datasource_srv'; import { reduxTester } from '../../../../test/core/redux/reduxTester'; import { variableAdapters } from '../adapters'; import { createCustomVariableAdapter } from '../custom/adapter'; +import { createCustomOptionsFromQuery } from '../custom/reducer'; import { setVariableQueryRunner, VariableQueryRunner } from '../query/VariableQueryRunner'; import { createQueryVariableAdapter } from '../query/adapter'; import { updateVariableOptions } from '../query/reducer'; @@ -125,9 +126,18 @@ describe('processVariable', () => { .whenActionIsDispatched(initDashboardTemplating(key, dashboard)) .whenAsyncActionIsDispatched(processVariable(toKeyedVariableIdentifier(custom), queryParams), true); - await tester.thenDispatchedActionsShouldEqual( - toKeyedAction(key, variableStateCompleted(toVariablePayload(custom))) - ); + await tester.thenDispatchedActionsPredicateShouldEqual((dispatchedActions) => { + expect(dispatchedActions.length).toEqual(4); + + expect(dispatchedActions[0]).toEqual(toKeyedAction(key, variableStateFetching(toVariablePayload(custom)))); + expect(dispatchedActions[1]).toEqual( + toKeyedAction(key, createCustomOptionsFromQuery(toVariablePayload(custom))) + ); + expect(dispatchedActions[2].type).toEqual('templating/keyed/shared/setCurrentVariableValue'); + expect(dispatchedActions[3]).toEqual(toKeyedAction(key, variableStateCompleted(toVariablePayload(custom)))); + + return true; + }); }); }); From df6cc96cdb9d6776c27f9d779262219ef2256886 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 09:35:59 +0100 Subject: [PATCH 082/102] Update dependency @grafana/scenes to v1.17.0 (#75862) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 0b368148317..c0c415c2153 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3344,8 +3344,8 @@ __metadata: linkType: soft "@grafana/scenes@npm:^1.15.0": - version: 1.15.0 - resolution: "@grafana/scenes@npm:1.15.0" + version: 1.17.0 + resolution: "@grafana/scenes@npm:1.17.0" dependencies: "@grafana/e2e-selectors": 10.0.2 react-grid-layout: 1.3.4 @@ -3357,7 +3357,7 @@ __metadata: "@grafana/runtime": 10.0.3 "@grafana/schema": 10.0.3 "@grafana/ui": 10.0.3 - checksum: 8416327ad9a0e3be35d5a106788248d5886f6aa220777cf0772ef73a3f64ecac889f58a8e804beea0de339acd571d6e49a10278180cf1c50a2a51f44e03bf93e + checksum: 0bd8603ae59c199c595a45a7bc31e318a1928b18307469bcd5ee5f83c3aaee89c5c27a1341a479440d928505681ab6a2479809a5cac56e8e93ba7acf06adff93 languageName: node linkType: hard From 6e0825d6e24c600f475ac036b9c950edba9b6e95 Mon Sep 17 00:00:00 2001 From: Fabrizio <135109076+fabrizio-grafana@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:39:23 +0200 Subject: [PATCH 083/102] Plugins: Fix sorting issue with expandable rows (#75553) --- .../src/components/Table/Table.test.tsx | 65 ++++++++++++++++++- .../grafana-ui/src/components/Table/Table.tsx | 52 +++++++++++---- 2 files changed, 102 insertions(+), 15 deletions(-) diff --git a/packages/grafana-ui/src/components/Table/Table.test.tsx b/packages/grafana-ui/src/components/Table/Table.test.tsx index 8d73374bc10..4b2322923d4 100644 --- a/packages/grafana-ui/src/components/Table/Table.test.tsx +++ b/packages/grafana-ui/src/components/Table/Table.test.tsx @@ -520,7 +520,7 @@ describe('Table', () => { }); describe('when mounted with nested data', () => { - it('then correct rows should be rendered and new table is rendered when expander is clicked', async () => { + beforeEach(() => { const nestedFrame = (idx: number) => applyOverrides( toDataFrame({ @@ -539,6 +539,7 @@ describe('Table', () => { ], }) ); + const defaultFrame = getDefaultDataFrame(); getTestContext({ @@ -555,6 +556,9 @@ describe('Table', () => { ], }), }); + }); + + it('then correct rows should be rendered and new table is rendered when expander is clicked', async () => { expect(getTable()).toBeInTheDocument(); expect(screen.getAllByRole('columnheader')).toHaveLength(4); expect(getColumnHeader(/time/)).toBeInTheDocument(); @@ -593,5 +597,64 @@ describe('Table', () => { expect(within(subTableRows1[2]).getByText(/17%_1/)).toBeInTheDocument(); expect(within(subTableRows1[2]).getByText(/humid_1/)).toBeInTheDocument(); }); + + it('then properly handle row expansion and sorting', async () => { + expect(getTable()).toBeInTheDocument(); + expect(screen.getAllByRole('columnheader')).toHaveLength(4); + expect(getColumnHeader(/time/)).toBeInTheDocument(); + expect(getColumnHeader(/temperature/)).toBeInTheDocument(); + expect(getColumnHeader(/img/)).toBeInTheDocument(); + + let rows = within(getTable()).getAllByRole('row'); + expect(rows).toHaveLength(5); + expect(getRowsData(rows)).toEqual([ + { time: '2021-01-01 00:00:00', temperature: '10', link: '${__value.text} interpolation' }, + { time: '2021-01-01 03:00:00', temperature: 'NaN', link: '${__value.text} interpolation' }, + { time: '2021-01-01 01:00:00', temperature: '11', link: '${__value.text} interpolation' }, + { time: '2021-01-01 02:00:00', temperature: '12', link: '${__value.text} interpolation' }, + ]); + + // Sort rows, and check the new order + const table = getTable(); + await userEvent.click(within(table).getAllByTitle('Toggle SortBy')[0]); + rows = within(table).getAllByRole('row'); + expect(rows).toHaveLength(5); + expect(getRowsData(rows)).toEqual([ + { time: '2021-01-01 00:00:00', temperature: '10', link: '${__value.text} interpolation' }, + { time: '2021-01-01 01:00:00', temperature: '11', link: '${__value.text} interpolation' }, + { time: '2021-01-01 02:00:00', temperature: '12', link: '${__value.text} interpolation' }, + { time: '2021-01-01 03:00:00', temperature: 'NaN', link: '${__value.text} interpolation' }, + ]); + + // No sub table exists before expending a row + let tables = screen.getAllByRole('table'); + expect(tables).toHaveLength(1); + + // Expand a row, and check its height + rows = within(getTable()).getAllByRole('row'); + await userEvent.click(within(rows[1]).getByLabelText('Expand row')); + tables = screen.getAllByRole('table'); + expect(tables).toHaveLength(3); + let subTable = screen.getAllByRole('table')[2]; + expect(subTable.style.height).toBe('108px'); + + // Sort again rows + tables = screen.getAllByRole('table'); + await userEvent.click(within(tables[0]).getAllByTitle('Toggle SortBy')[0]); + rows = within(table).getAllByRole('row'); + expect(rows).toHaveLength(5); + expect(getRowsData(rows)).toEqual([ + { time: '2021-01-01 03:00:00', temperature: 'NaN', link: '${__value.text} interpolation' }, + { time: '2021-01-01 02:00:00', temperature: '12', link: '${__value.text} interpolation' }, + { time: '2021-01-01 01:00:00', temperature: '11', link: '${__value.text} interpolation' }, + { time: '2021-01-01 00:00:00', temperature: '10', link: '${__value.text} interpolation' }, + ]); + + // Expand another row + rows = within(getTable()).getAllByRole('row'); + await userEvent.click(within(rows[1]).getByLabelText('Expand row')); + subTable = screen.getAllByRole('table')[2]; + expect(subTable.style.height).toBe('108px'); + }); }); }); diff --git a/packages/grafana-ui/src/components/Table/Table.tsx b/packages/grafana-ui/src/components/Table/Table.tsx index d7568efa1b7..398732e7922 100644 --- a/packages/grafana-ui/src/components/Table/Table.tsx +++ b/packages/grafana-ui/src/components/Table/Table.tsx @@ -110,8 +110,26 @@ export const Table = memo((props: Props) => { [data, width, columnMinWidth, footerItems, hasNestedData, isCountRowsSet] ); + // we need a ref to later store the `toggleAllRowsExpanded` function, returned by `useTable`. + // We cannot simply use a variable because we need to use such function in the initialization of + // `useTableStateReducer`, which is needed to construct options for `useTable` (the hook that returns + // `toggleAllRowsExpanded`), and if we used a variable, that variable would be undefined at the time + // we initialize `useTableStateReducer`. + const toggleAllRowsExpandedRef = useRef<(value?: boolean) => void>(); + // Internal react table state reducer - const stateReducer = useTableStateReducer(props); + const stateReducer = useTableStateReducer({ + ...props, + onSortByChange: (state) => { + // Collapse all rows. This prevents a known bug that causes the size of the rows to be incorrect due to + // using `VariableSizeList` and `useExpanded` together. + toggleAllRowsExpandedRef.current!(false); + + if (props.onSortByChange) { + props.onSortByChange(state); + } + }, + }); const options: any = useMemo( () => ({ @@ -142,9 +160,17 @@ export const Table = memo((props: Props) => { gotoPage, setPageSize, pageOptions, + toggleAllRowsExpanded, } = useTable(options, useFilters, useSortBy, useAbsoluteLayout, useResizeColumns, useExpanded, usePagination); const extendedState = state as GrafanaTableState; + toggleAllRowsExpandedRef.current = toggleAllRowsExpanded; + + const expandedRowsRepr = JSON.stringify(Object.keys(state.expanded)); + useEffect(() => { + // Reset the list size cache when the expanded rows change + listRef.current?.resetAfterIndex(0); + }, [expandedRowsRepr]); /* Footer value calculation is being moved in the Table component and the footerValues prop will be deprecated. @@ -213,11 +239,9 @@ export const Table = memo((props: Props) => { ); const RenderRow = useCallback( - ({ index: rowIndex, style }: { index: number; style: CSSProperties }) => { - let row = rows[rowIndex]; - if (enablePagination) { - row = page[rowIndex]; - } + ({ index, style }: { index: number; style: CSSProperties }) => { + const indexForPagination = rowIndexForPagination(index); + const row = rows[indexForPagination]; prepareRow(row); @@ -230,7 +254,7 @@ export const Table = memo((props: Props) => { @@ -251,18 +275,17 @@ export const Table = memo((props: Props) => { ); }, [ + rowIndexForPagination, rows, - enablePagination, prepareRow, + state.expanded, tableStyles, nestedDataField, - page, + width, + cellHeight, onCellFilterAdded, timeRange, data, - width, - cellHeight, - state.expanded, ] ); @@ -301,8 +324,9 @@ export const Table = memo((props: Props) => { const getItemSize = (index: number): number => { const indexForPagination = rowIndexForPagination(index); - if (state.expanded[indexForPagination] && nestedDataField) { - return getExpandedRowHeight(nestedDataField, indexForPagination, tableStyles); + const row = rows[indexForPagination]; + if (state.expanded[row.index] && nestedDataField) { + return getExpandedRowHeight(nestedDataField, index, tableStyles); } return tableStyles.rowHeight; From 420fb56fda29ad4c7402120addca90979a562284 Mon Sep 17 00:00:00 2001 From: Gabriel MABILLE Date: Thu, 12 Oct 2023 10:46:43 +0200 Subject: [PATCH 084/102] RBAC: Fix plugins pages access-control (#76321) * RBAC: Fix plugins pages access-control * Better comment Co-authored-by: Ieva * Add a small comment on connections/datasources routes --------- Co-authored-by: Ieva --- pkg/api/api.go | 13 ++++---- pkg/middleware/auth.go | 6 ++-- pkg/services/navtree/navtreeimpl/admin.go | 3 +- public/app/features/plugins/admin/routes.tsx | 31 ++++++++++++++++++-- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 215ff274848..9ff29328060 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -124,18 +124,19 @@ func (hs *HTTPServer) registerRoutes() { r.Get("/live/pipeline", reqGrafanaAdmin, hs.Index) r.Get("/live/cloud", reqGrafanaAdmin, hs.Index) - r.Get("/plugins", middleware.CanAdminPlugins(hs.Cfg), hs.Index) - r.Get("/plugins/:id/", middleware.CanAdminPlugins(hs.Cfg), hs.Index) - r.Get("/plugins/:id/edit", middleware.CanAdminPlugins(hs.Cfg), hs.Index) // deprecated - r.Get("/plugins/:id/page/:page", middleware.CanAdminPlugins(hs.Cfg), hs.Index) + r.Get("/plugins", middleware.CanAdminPlugins(hs.Cfg, hs.AccessControl), hs.Index) + r.Get("/plugins/:id/", middleware.CanAdminPlugins(hs.Cfg, hs.AccessControl), hs.Index) + r.Get("/plugins/:id/edit", middleware.CanAdminPlugins(hs.Cfg, hs.AccessControl), hs.Index) // deprecated + r.Get("/plugins/:id/page/:page", middleware.CanAdminPlugins(hs.Cfg, hs.AccessControl), hs.Index) r.Get("/connections/datasources", authorize(datasources.ConfigurationPageAccess), hs.Index) r.Get("/connections/datasources/new", authorize(datasources.NewPageAccess), hs.Index) r.Get("/connections/datasources/edit/*", authorize(datasources.EditPageAccess), hs.Index) r.Get("/connections", authorize(datasources.ConfigurationPageAccess), hs.Index) r.Get("/connections/add-new-connection", authorize(datasources.ConfigurationPageAccess), hs.Index) - r.Get("/connections/datasources/:id", middleware.CanAdminPlugins(hs.Cfg), hs.Index) - r.Get("/connections/datasources/:id/page/:page", middleware.CanAdminPlugins(hs.Cfg), hs.Index) + // Plugin details pages + r.Get("/connections/datasources/:id", middleware.CanAdminPlugins(hs.Cfg, hs.AccessControl), hs.Index) + r.Get("/connections/datasources/:id/page/:page", middleware.CanAdminPlugins(hs.Cfg, hs.AccessControl), hs.Index) // App Root Page appPluginIDScope := pluginaccesscontrol.ScopeProvider.GetResourceScope(ac.Parameter(":id")) diff --git a/pkg/middleware/auth.go b/pkg/middleware/auth.go index fccbcb409fe..d0bbe913713 100644 --- a/pkg/middleware/auth.go +++ b/pkg/middleware/auth.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/grafana/grafana/pkg/middleware/cookies" + ac "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/auth" "github.com/grafana/grafana/pkg/services/authn" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" @@ -86,9 +87,10 @@ func removeForceLoginParams(str string) string { return forceLoginParamsRegexp.ReplaceAllString(str, "") } -func CanAdminPlugins(cfg *setting.Cfg) func(c *contextmodel.ReqContext) { +func CanAdminPlugins(cfg *setting.Cfg, accessControl ac.AccessControl) func(c *contextmodel.ReqContext) { return func(c *contextmodel.ReqContext) { - if !pluginaccesscontrol.ReqCanAdminPlugins(cfg)(c) { + hasAccess := ac.HasAccess(accessControl, c) + if !pluginaccesscontrol.ReqCanAdminPlugins(cfg)(c) && !hasAccess(pluginaccesscontrol.AdminAccessEvaluator) { accessForbidden(c) return } diff --git a/pkg/services/navtree/navtreeimpl/admin.go b/pkg/services/navtree/navtreeimpl/admin.go index ace34b04ac3..c3d5602e8e4 100644 --- a/pkg/services/navtree/navtreeimpl/admin.go +++ b/pkg/services/navtree/navtreeimpl/admin.go @@ -17,7 +17,8 @@ func (s *ServiceImpl) getAdminNode(c *contextmodel.ReqContext) (*navtree.NavLink orgsAccessEvaluator := ac.EvalPermission(ac.ActionOrgsRead) authConfigUIAvailable := s.license.FeatureEnabled("saml") || s.cfg.LDAPAuthEnabled - // FIXME: while we don't have a permissions for listing plugins the legacy check has to stay as a default + // FIXME: If plugin admin is disabled or externally managed, server admins still need to access the page, this is why + // while we don't have a permissions for listing plugins the legacy check has to stay as a default if pluginaccesscontrol.ReqCanAdminPlugins(s.cfg)(c) || hasAccess(pluginaccesscontrol.AdminAccessEvaluator) { configNodes = append(configNodes, &navtree.NavLink{ Text: "Plugins", diff --git a/public/app/features/plugins/admin/routes.tsx b/public/app/features/plugins/admin/routes.tsx index 86ffb65a3d2..ab7fb2d13ce 100644 --- a/public/app/features/plugins/admin/routes.tsx +++ b/public/app/features/plugins/admin/routes.tsx @@ -1,5 +1,7 @@ import { SafeDynamicImport } from 'app/core/components/DynamicImports/SafeDynamicImport'; +import { contextSrv } from 'app/core/core'; import { RouteDescriptor } from 'app/core/navigation/types'; +import { AccessControlAction } from 'app/types'; import { PluginAdminRoutes } from './types'; @@ -7,26 +9,49 @@ const DEFAULT_ROUTES = [ { path: '/plugins', navId: 'plugins', - roles: () => ['Admin', 'ServerAdmin'], + roles: evaluateAccess( + [AccessControlAction.PluginsInstall, AccessControlAction.PluginsWrite], + ['Admin', 'ServerAdmin'] + ), routeName: PluginAdminRoutes.Home, component: SafeDynamicImport(() => import(/* webpackChunkName: "PluginListPage" */ './pages/Browse')), }, { path: '/plugins/browse', navId: 'plugins', - roles: () => ['Admin', 'ServerAdmin'], + roles: evaluateAccess( + [AccessControlAction.PluginsInstall, AccessControlAction.PluginsWrite], + ['Admin', 'ServerAdmin'] + ), routeName: PluginAdminRoutes.Browse, component: SafeDynamicImport(() => import(/* webpackChunkName: "PluginListPage" */ './pages/Browse')), }, { path: '/plugins/:pluginId/', navId: 'plugins', - roles: () => ['Admin', 'ServerAdmin'], + roles: evaluateAccess( + [AccessControlAction.PluginsInstall, AccessControlAction.PluginsWrite], + ['Admin', 'ServerAdmin'] + ), routeName: PluginAdminRoutes.Details, component: SafeDynamicImport(() => import(/* webpackChunkName: "PluginPage" */ './pages/PluginDetails')), }, ]; +// FIXME: If plugin admin is disabled or externally managed, server admins still need to access the page, this is why +// while we don't have a permissions for listing plugins the legacy check has to stay as a default +function evaluateAccess(actions: AccessControlAction[], userRoles: string[]): () => string[] { + return () => { + if (actions.some((action) => contextSrv.hasPermission(action))) { + // If the user has any of the required actions, no need to check the role + return []; + } else { + // User had no action, check the org role + return userRoles; + } + }; +} + export function getRoutes(): RouteDescriptor[] { return DEFAULT_ROUTES; } From f012b75a3ade58ccac6d03c391fade69257fd65c Mon Sep 17 00:00:00 2001 From: Marcus Andersson Date: Thu, 12 Oct 2023 10:56:08 +0200 Subject: [PATCH 085/102] PluginExtensions: Made it possible to control modal size from extension (#76232) * Added possibility to change modal size from UI extension. * added tests for openModal. * fixed typings. * added test to verify default modal size. --- packages/grafana-data/src/types/index.ts | 1 + .../src/types/pluginExtensions.ts | 18 ++-- .../plugins/extensions/utils.test.tsx | 94 ++++++++++++++++++- .../app/features/plugins/extensions/utils.tsx | 24 +++-- 4 files changed, 123 insertions(+), 14 deletions(-) diff --git a/packages/grafana-data/src/types/index.ts b/packages/grafana-data/src/types/index.ts index 6183d52d61a..96f3c8f7d39 100644 --- a/packages/grafana-data/src/types/index.ts +++ b/packages/grafana-data/src/types/index.ts @@ -63,4 +63,5 @@ export { type PluginExtensionEventHelpers, type PluginExtensionPanelContext, type PluginExtensionDataSourceConfigContext, + type PluginExtensionOpenModalOptions, } from './pluginExtensions'; diff --git a/packages/grafana-data/src/types/pluginExtensions.ts b/packages/grafana-data/src/types/pluginExtensions.ts index 568ad10d3dc..f541c56f571 100644 --- a/packages/grafana-data/src/types/pluginExtensions.ts +++ b/packages/grafana-data/src/types/pluginExtensions.ts @@ -95,15 +95,21 @@ export type PluginExtensionComponentConfig = { export type PluginExtensionConfig = PluginExtensionLinkConfig | PluginExtensionComponentConfig; +export type PluginExtensionOpenModalOptions = { + // The title of the modal + title: string; + // A React element that will be rendered inside the modal + body: React.ElementType<{ onDismiss?: () => void }>; + // Width of the modal in pixels or percentage + width?: string | number; + // Height of the modal in pixels or percentage + height?: string | number; +}; + export type PluginExtensionEventHelpers = { context?: Readonly; // Opens a modal dialog and renders the provided React component inside it - openModal: (options: { - // The title of the modal - title: string; - // A React element that will be rendered inside the modal - body: React.ElementType<{ onDismiss?: () => void }>; - }) => void; + openModal: (options: PluginExtensionOpenModalOptions) => void; }; // Extension Points & Contexts diff --git a/public/app/features/plugins/extensions/utils.test.tsx b/public/app/features/plugins/extensions/utils.test.tsx index f814bd1df3b..967286a9710 100644 --- a/public/app/features/plugins/extensions/utils.test.tsx +++ b/public/app/features/plugins/extensions/utils.test.tsx @@ -1,6 +1,12 @@ -import { PluginExtensionLinkConfig, PluginExtensionTypes } from '@grafana/data'; +import { render, screen } from '@testing-library/react'; +import React from 'react'; +import { type Unsubscribable } from 'rxjs'; -import { deepFreeze, isPluginExtensionLinkConfig, handleErrorsInFn, getReadOnlyProxy } from './utils'; +import { type PluginExtensionLinkConfig, PluginExtensionTypes } from '@grafana/data'; +import appEvents from 'app/core/app_events'; +import { ShowModalReactEvent } from 'app/types/events'; + +import { deepFreeze, isPluginExtensionLinkConfig, handleErrorsInFn, getReadOnlyProxy, getEventHelpers } from './utils'; describe('Plugin Extensions / Utils', () => { describe('deepFreeze()', () => { @@ -307,4 +313,88 @@ describe('Plugin Extensions / Utils', () => { expect(proxy.a()).toBe('testing'); }); }); + + describe('getEventHelpers', () => { + describe('openModal', () => { + let renderModalSubscription: Unsubscribable | undefined; + + beforeAll(() => { + renderModalSubscription = appEvents.subscribe(ShowModalReactEvent, (event) => { + const { payload } = event; + const Modal = payload.component; + render(); + }); + }); + + afterAll(() => { + renderModalSubscription?.unsubscribe(); + }); + + it('should open modal with provided title and body', async () => { + const { openModal } = getEventHelpers(); + + openModal({ + title: 'Title in modal', + body: () =>
    Text in body
    , + }); + + expect(screen.getByRole('dialog')).toBeVisible(); + expect(screen.getByRole('heading')).toHaveTextContent('Title in modal'); + expect(screen.getByText('Text in body')).toBeVisible(); + }); + + it('should open modal with default width if not specified', async () => { + const { openModal } = getEventHelpers(); + + openModal({ + title: 'Title in modal', + body: () =>
    Text in body
    , + }); + + const modal = screen.getByRole('dialog'); + const style = window.getComputedStyle(modal); + + expect(style.width).toBe('750px'); + expect(style.height).toBe(''); + }); + + it('should open modal with specified width', async () => { + const { openModal } = getEventHelpers(); + + openModal({ + title: 'Title in modal', + body: () =>
    Text in body
    , + width: '70%', + }); + + const modal = screen.getByRole('dialog'); + const style = window.getComputedStyle(modal); + + expect(style.width).toBe('70%'); + }); + + it('should open modal with specified height', async () => { + const { openModal } = getEventHelpers(); + + openModal({ + title: 'Title in modal', + body: () =>
    Text in body
    , + height: 600, + }); + + const modal = screen.getByRole('dialog'); + const style = window.getComputedStyle(modal); + + expect(style.height).toBe('600px'); + }); + }); + + describe('context', () => { + it('should return same object as passed to getEventHelpers', () => { + const source = {}; + const { context } = getEventHelpers(source); + expect(context).toBe(source); + }); + }); + }); }); diff --git a/public/app/features/plugins/extensions/utils.tsx b/public/app/features/plugins/extensions/utils.tsx index 96d6dd50a46..d1449484bd0 100644 --- a/public/app/features/plugins/extensions/utils.tsx +++ b/public/app/features/plugins/extensions/utils.tsx @@ -1,3 +1,4 @@ +import { css } from '@emotion/css'; import { isArray, isObject } from 'lodash'; import React from 'react'; @@ -7,6 +8,7 @@ import { type PluginExtensionConfig, type PluginExtensionEventHelpers, PluginExtensionTypes, + type PluginExtensionOpenModalOptions, } from '@grafana/data'; import { Modal } from '@grafana/ui'; import appEvents from 'app/core/app_events'; @@ -42,28 +44,38 @@ export function handleErrorsInFn(fn: Function, errorMessagePrefix = '') { // Event helpers are designed to make it easier to trigger "core actions" from an extension event handler, e.g. opening a modal or showing a notification. export function getEventHelpers(context?: Readonly): PluginExtensionEventHelpers { - const openModal: PluginExtensionEventHelpers['openModal'] = ({ title, body }) => { - appEvents.publish(new ShowModalReactEvent({ component: getModalWrapper({ title, body }) })); + const openModal: PluginExtensionEventHelpers['openModal'] = (options) => { + const { title, body, width, height } = options; + + appEvents.publish( + new ShowModalReactEvent({ + component: getModalWrapper({ title, body, width, height }), + }) + ); }; return { openModal, context }; } -export type ModalWrapperProps = { +type ModalWrapperProps = { onDismiss: () => void; }; // Wraps a component with a modal. // This way we can make sure that the modal is closable, and we also make the usage simpler. -export const getModalWrapper = ({ +const getModalWrapper = ({ // The title of the modal (appears in the header) title, // A component that serves the body of the modal body: Body, -}: Parameters[0]) => { + width, + height, +}: PluginExtensionOpenModalOptions) => { + const className = css({ width, height }); + const ModalWrapper = ({ onDismiss }: ModalWrapperProps) => { return ( - + ); From 2771fb940342aa152377b26b9554eb15082f90ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ida=20=C5=A0tambuk?= Date: Thu, 12 Oct 2023 10:59:10 +0200 Subject: [PATCH 086/102] FeatureToggle: Add awsDatasourcesNewFormStyling feature toggle (#76110) --- .../configure-grafana/feature-toggles/index.md | 1 + packages/grafana-data/src/types/featureToggles.gen.ts | 1 + pkg/services/featuremgmt/registry.go | 7 +++++++ pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 ++++ 5 files changed, 14 insertions(+) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 6b3c5206fea..40481d03655 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -149,6 +149,7 @@ Experimental features might be changed or removed without prior notice. | `kubernetesPlaylists` | Use the kubernetes API in the frontend for playlists | | `navAdminSubsections` | Splits the administration section of the nav tree into subsections | | `recoveryThreshold` | Enables feature recovery threshold (aka hysteresis) for threshold server-side expression | +| `awsDatasourcesNewFormStyling` | Applies new form styling for configuration and query editors in AWS plugins | ## Development feature toggles diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 4ef472d4b50..ef71b57f296 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -142,4 +142,5 @@ export interface FeatureToggles { kubernetesPlaylists?: boolean; navAdminSubsections?: boolean; recoveryThreshold?: boolean; + awsDatasourcesNewFormStyling?: boolean; } diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index fd84f4c3a0a..6b0b64c29dc 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -867,5 +867,12 @@ var ( Owner: grafanaAlertingSquad, RequiresRestart: true, }, + { + Name: "awsDatasourcesNewFormStyling", + Description: "Applies new form styling for configuration and query editors in AWS plugins", + Stage: FeatureStageExperimental, + FrontendOnly: true, + Owner: awsDatasourcesSquad, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 8aa3465a746..376a632aa10 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -123,3 +123,4 @@ transformationsVariableSupport,experimental,@grafana/grafana-bi-squad,false,fals kubernetesPlaylists,experimental,@grafana/grafana-app-platform-squad,false,false,false,true navAdminSubsections,experimental,@grafana/grafana-frontend-platform,false,false,false,false recoveryThreshold,experimental,@grafana/alerting-squad,false,false,true,false +awsDatasourcesNewFormStyling,experimental,@grafana/aws-datasources,false,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index c50cdb739b8..8efbf55fb58 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -502,4 +502,8 @@ const ( // FlagRecoveryThreshold // Enables feature recovery threshold (aka hysteresis) for threshold server-side expression FlagRecoveryThreshold = "recoveryThreshold" + + // FlagAwsDatasourcesNewFormStyling + // Applies new form styling for configuration and query editors in AWS plugins + FlagAwsDatasourcesNewFormStyling = "awsDatasourcesNewFormStyling" ) From 46b00666441fbe2ed6027dd02280839b6052ee1b Mon Sep 17 00:00:00 2001 From: Josh Hunt Date: Thu, 12 Oct 2023 09:00:32 +0000 Subject: [PATCH 087/102] Chore: Improve EchoSrv logging (#76385) * Chore: Allow frontend logging to be logged via localStorage * improve echo srv logging --- packages/grafana-ui/src/utils/logger.ts | 17 +++++++++++------ public/app/core/services/echo/Echo.ts | 6 +++++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/packages/grafana-ui/src/utils/logger.ts b/packages/grafana-ui/src/utils/logger.ts index 157a517de25..e52bb4ba8b9 100644 --- a/packages/grafana-ui/src/utils/logger.ts +++ b/packages/grafana-ui/src/utils/logger.ts @@ -19,17 +19,22 @@ export interface Logger { /** @internal */ export const createLogger = (name: string): Logger => { - let LOGGIN_ENABLED = false; + let loggingEnabled = false; + + if (typeof window !== 'undefined') { + loggingEnabled = window.localStorage.getItem('grafana.debug') === 'true'; + } + return { logger: (id: string, throttle = false, ...t: any[]) => { - if (process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test' || !LOGGIN_ENABLED) { + if (process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test' || !loggingEnabled) { return; } const fn = throttle ? throttledLog : console.log; - fn(`[${name}: ${id}]: `, ...t); + fn(`[${name}: ${id}]:`, ...t); }, - enable: () => (LOGGIN_ENABLED = true), - disable: () => (LOGGIN_ENABLED = false), - isEnabled: () => LOGGIN_ENABLED, + enable: () => (loggingEnabled = true), + disable: () => (loggingEnabled = false), + isEnabled: () => loggingEnabled, }; }; diff --git a/public/app/core/services/echo/Echo.ts b/public/app/core/services/echo/Echo.ts index 35e60f432ee..5640e870cc9 100644 --- a/public/app/core/services/echo/Echo.ts +++ b/public/app/core/services/echo/Echo.ts @@ -59,7 +59,11 @@ export class Echo implements EchoSrv { backend.addEvent(_event); } } - echoLog('Reporting event', false, _event); + + echoLog(`${event.type} event`, false, { + ...event.payload, + meta: _event.meta, + }); }; getMeta = (): EchoMeta => { From 5e918f031c2acbe865b5935833e196a619e12dcc Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Thu, 12 Oct 2023 11:14:30 +0200 Subject: [PATCH 088/102] Chore: Update @testing-library/dom to v9.3.3 (#76364) * Update @testing-library/dom to v9.3.3 * Update @testing-library/dom to v9.3.3 in packages * Update yarn.lock * Revert "Update yarn.lock" This reverts commit 0cc20842732f2190ded0772889723e7b1d930ea3. * point all @testing-library/dom packages to v9.3.3 --- package.json | 2 +- packages/grafana-data/package.json | 2 +- packages/grafana-runtime/package.json | 2 +- packages/grafana-ui/package.json | 2 +- yarn.lock | 16 ++++++++-------- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index 05481e3e04b..55027d7d34a 100644 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "@rtsao/plugin-proposal-class-properties": "7.0.1-patch.1", "@swc/core": "1.3.38", "@swc/helpers": "0.4.14", - "@testing-library/dom": "9.3.1", + "@testing-library/dom": "9.3.3", "@testing-library/jest-dom": "5.16.5", "@testing-library/react": "14.0.0", "@testing-library/user-event": "14.5.1", diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index 0ef93b17550..3315e50bc6e 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -66,7 +66,7 @@ "@rollup/plugin-commonjs": "25.0.2", "@rollup/plugin-json": "6.0.0", "@rollup/plugin-node-resolve": "15.2.3", - "@testing-library/dom": "9.3.1", + "@testing-library/dom": "9.3.3", "@testing-library/jest-dom": "5.16.5", "@testing-library/react": "14.0.0", "@testing-library/user-event": "14.5.1", diff --git a/packages/grafana-runtime/package.json b/packages/grafana-runtime/package.json index e1735a53209..c5284ff0dd6 100644 --- a/packages/grafana-runtime/package.json +++ b/packages/grafana-runtime/package.json @@ -52,7 +52,7 @@ "@grafana/tsconfig": "^1.2.0-rc1", "@rollup/plugin-commonjs": "25.0.2", "@rollup/plugin-node-resolve": "15.2.3", - "@testing-library/dom": "9.3.1", + "@testing-library/dom": "9.3.3", "@testing-library/react": "14.0.0", "@testing-library/user-event": "14.5.1", "@types/angular": "1.8.5", diff --git a/packages/grafana-ui/package.json b/packages/grafana-ui/package.json index 0c2c120a453..49f17edc377 100644 --- a/packages/grafana-ui/package.json +++ b/packages/grafana-ui/package.json @@ -134,7 +134,7 @@ "@storybook/react": "7.4.5", "@storybook/react-webpack5": "7.4.5", "@storybook/theming": "7.4.5", - "@testing-library/dom": "9.3.1", + "@testing-library/dom": "9.3.3", "@testing-library/jest-dom": "5.16.5", "@testing-library/react": "14.0.0", "@testing-library/user-event": "14.5.1", diff --git a/yarn.lock b/yarn.lock index c0c415c2153..deb268817e7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2992,7 +2992,7 @@ __metadata: "@rollup/plugin-commonjs": 25.0.2 "@rollup/plugin-json": 6.0.0 "@rollup/plugin-node-resolve": 15.2.3 - "@testing-library/dom": 9.3.1 + "@testing-library/dom": 9.3.3 "@testing-library/jest-dom": 5.16.5 "@testing-library/react": 14.0.0 "@testing-library/user-event": 14.5.1 @@ -3310,7 +3310,7 @@ __metadata: "@grafana/ui": 10.2.0-pre "@rollup/plugin-commonjs": 25.0.2 "@rollup/plugin-node-resolve": 15.2.3 - "@testing-library/dom": 9.3.1 + "@testing-library/dom": 9.3.3 "@testing-library/react": 14.0.0 "@testing-library/user-event": 14.5.1 "@types/angular": 1.8.5 @@ -3434,7 +3434,7 @@ __metadata: "@storybook/react": 7.4.5 "@storybook/react-webpack5": 7.4.5 "@storybook/theming": 7.4.5 - "@testing-library/dom": 9.3.1 + "@testing-library/dom": 9.3.3 "@testing-library/jest-dom": 5.16.5 "@testing-library/react": 14.0.0 "@testing-library/user-event": 14.5.1 @@ -8385,9 +8385,9 @@ __metadata: languageName: node linkType: hard -"@testing-library/dom@npm:9.3.1, @testing-library/dom@npm:>=7, @testing-library/dom@npm:^9.0.0": - version: 9.3.1 - resolution: "@testing-library/dom@npm:9.3.1" +"@testing-library/dom@npm:9.3.3, @testing-library/dom@npm:>=7, @testing-library/dom@npm:^9.0.0": + version: 9.3.3 + resolution: "@testing-library/dom@npm:9.3.3" dependencies: "@babel/code-frame": ^7.10.4 "@babel/runtime": ^7.12.5 @@ -8397,7 +8397,7 @@ __metadata: dom-accessibility-api: ^0.5.9 lz-string: ^1.5.0 pretty-format: ^27.0.2 - checksum: 8ee3136451644e39990edea93709c38cf1e8ce5306f3c66273ca00935963faa51ca74e8d92b02eb442ccb842cfa28ca62833e393e075eb269cf9bef6f5600663 + checksum: 34e0a564da7beb92aa9cc44a9080221e2412b1a132eb37be3d513fe6c58027674868deb9f86195756d98d15ba969a30fe00632a4e26e25df2a5a4f6ac0686e37 languageName: node linkType: hard @@ -17677,7 +17677,7 @@ __metadata: "@rtsao/plugin-proposal-class-properties": 7.0.1-patch.1 "@swc/core": 1.3.38 "@swc/helpers": 0.4.14 - "@testing-library/dom": 9.3.1 + "@testing-library/dom": 9.3.3 "@testing-library/jest-dom": 5.16.5 "@testing-library/react": 14.0.0 "@testing-library/react-hooks": ^8.0.1 From 334fc6a68aed2c7a6181498d71619f00c88fb3bc Mon Sep 17 00:00:00 2001 From: Krishna Dhakal <7krishna7dhakal7@gmail.com> Date: Thu, 12 Oct 2023 15:10:48 +0545 Subject: [PATCH 089/102] i18n: Markup Explorer Graph for translation (#76099) --- .../app/features/explore/Graph/GraphContainer.tsx | 14 ++++++++++---- public/locales/de-DE/grafana.json | 8 ++++++++ public/locales/en-US/grafana.json | 8 ++++++++ public/locales/es-ES/grafana.json | 8 ++++++++ public/locales/fr-FR/grafana.json | 8 ++++++++ public/locales/pseudo-LOCALE/grafana.json | 8 ++++++++ public/locales/zh-Hans/grafana.json | 8 ++++++++ 7 files changed, 58 insertions(+), 4 deletions(-) diff --git a/public/app/features/explore/Graph/GraphContainer.tsx b/public/app/features/explore/Graph/GraphContainer.tsx index bc8cc45ddbe..19edc6aff48 100644 --- a/public/app/features/explore/Graph/GraphContainer.tsx +++ b/public/app/features/explore/Graph/GraphContainer.tsx @@ -21,6 +21,7 @@ import { useStyles2, Tooltip, } from '@grafana/ui'; +import { t, Trans } from 'app/core/internationalization'; import { ExploreGraphStyle } from 'app/types'; import { storeGraphStyle } from '../state/utils'; @@ -76,19 +77,24 @@ export const GraphContainer = ({ return ( diff --git a/public/locales/de-DE/grafana.json b/public/locales/de-DE/grafana.json index 9f6aa35d392..ec68dffd9ad 100644 --- a/public/locales/de-DE/grafana.json +++ b/public/locales/de-DE/grafana.json @@ -374,6 +374,14 @@ "placeholder": "" } }, + "graph": { + "container": { + "content": "", + "show-all-series": "", + "show-only-series": "", + "title": "" + } + }, "help-modal": { "shortcuts-category": { "dashboard": "", diff --git a/public/locales/en-US/grafana.json b/public/locales/en-US/grafana.json index 40f90682684..6422a2a423f 100644 --- a/public/locales/en-US/grafana.json +++ b/public/locales/en-US/grafana.json @@ -374,6 +374,14 @@ "placeholder": "Choose" } }, + "graph": { + "container": { + "content": "Rendering too many series in a single panel may impact performance and make data harder to read. Consider refining your queries.", + "show-all-series": "Show all {{length}}", + "show-only-series": "Showing only {{MAX_NUMBER_OF_TIME_SERIES}} series", + "title": "Graph" + } + }, "help-modal": { "shortcuts-category": { "dashboard": "Dashboard", diff --git a/public/locales/es-ES/grafana.json b/public/locales/es-ES/grafana.json index 0fa848259d1..b8a17d435b2 100644 --- a/public/locales/es-ES/grafana.json +++ b/public/locales/es-ES/grafana.json @@ -379,6 +379,14 @@ "placeholder": "" } }, + "graph": { + "container": { + "content": "", + "show-all-series": "", + "show-only-series": "", + "title": "" + } + }, "help-modal": { "shortcuts-category": { "dashboard": "", diff --git a/public/locales/fr-FR/grafana.json b/public/locales/fr-FR/grafana.json index 13a678be1b2..d61babbf93d 100644 --- a/public/locales/fr-FR/grafana.json +++ b/public/locales/fr-FR/grafana.json @@ -379,6 +379,14 @@ "placeholder": "" } }, + "graph": { + "container": { + "content": "", + "show-all-series": "", + "show-only-series": "", + "title": "" + } + }, "help-modal": { "shortcuts-category": { "dashboard": "", diff --git a/public/locales/pseudo-LOCALE/grafana.json b/public/locales/pseudo-LOCALE/grafana.json index 12d900228c1..0ee6280028e 100644 --- a/public/locales/pseudo-LOCALE/grafana.json +++ b/public/locales/pseudo-LOCALE/grafana.json @@ -374,6 +374,14 @@ "placeholder": "Cĥőőşę" } }, + "graph": { + "container": { + "content": "Ŗęʼnđęřįʼnģ ŧőő mäʼny şęřįęş įʼn ä şįʼnģľę päʼnęľ mäy įmpäčŧ pęřƒőřmäʼnčę äʼnđ mäĸę đäŧä ĥäřđęř ŧő řęäđ. Cőʼnşįđęř řęƒįʼnįʼnģ yőūř qūęřįęş.", + "show-all-series": "Ŝĥőŵ äľľ {{length}}", + "show-only-series": "Ŝĥőŵįʼnģ őʼnľy {{MAX_NUMBER_OF_TIME_SERIES}} şęřįęş", + "title": "Ğřäpĥ" + } + }, "help-modal": { "shortcuts-category": { "dashboard": "Đäşĥþőäřđ", diff --git a/public/locales/zh-Hans/grafana.json b/public/locales/zh-Hans/grafana.json index 27adff77cae..d8495004414 100644 --- a/public/locales/zh-Hans/grafana.json +++ b/public/locales/zh-Hans/grafana.json @@ -369,6 +369,14 @@ "placeholder": "" } }, + "graph": { + "container": { + "content": "", + "show-all-series": "", + "show-only-series": "", + "title": "" + } + }, "help-modal": { "shortcuts-category": { "dashboard": "", From a9f7cb00e687601f7fdd68fb23212b33b695cbc4 Mon Sep 17 00:00:00 2001 From: "grafana-delivery-bot[bot]" <132647405+grafana-delivery-bot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 09:39:15 +0000 Subject: [PATCH 090/102] Changelog: Updated changelog for 9.4.17 (#76409) Co-authored-by: grafanabot --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e081823125..185e6222bf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1357,6 +1357,21 @@ Users who use LDAP role sync to only sync Viewer, Editor and Admin roles, but gr - **ContextMenu:** Fix padding and show border based on items. [#63948](https://github.com/grafana/grafana/pull/63948), [@aocenas](https://github.com/aocenas) + + +# 9.4.17 (2023-10-11) + +### Features and enhancements + +- **SSE:** DSNode to update result with names to make each value identifiable by labels (only Graphite and TestData). [#74615](https://github.com/grafana/grafana/issues/74615), [@yuri-tceretian](https://github.com/yuri-tceretian) + +### Bug fixes + +- **BrowseDashboards:** Only remember the most recent expanded folder. [#74812](https://github.com/grafana/grafana/issues/74812), [@joshhunt](https://github.com/joshhunt) +- **SQL Datasources:** Fix variable throwing error if query returns no data. [#74609](https://github.com/grafana/grafana/issues/74609), [@mdvictor](https://github.com/mdvictor) +- **RBAC:** Chore fix hasPermissionInOrg. (Enterprise) + + # 9.4.15 (2023-09-18) From 790f878eca9a0f3c756063d40dd1ea03b326746f Mon Sep 17 00:00:00 2001 From: "grafana-delivery-bot[bot]" <132647405+grafana-delivery-bot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:21:27 +0000 Subject: [PATCH 091/102] Changelog: Updated changelog for 9.5.13 (#76418) Co-authored-by: grafanabot --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 185e6222bf7..20340a4d5a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -947,6 +947,21 @@ The `database` field has been deprecated in the Elasticsearch datasource provisi - **InteractiveTable:** Updated design and minor tweak to Correlactions page. [#66443](https://github.com/grafana/grafana/issues/66443), [@torkelo](https://github.com/torkelo) + + +# 9.5.13 (2023-10-11) + +### Features and enhancements + +- **Licensing:** Updated grpc plugin factory newPlugin signature. (Enterprise) + +### Bug fixes + +- **BrowseDashboards:** Only remember the most recent expanded folder. [#74817](https://github.com/grafana/grafana/issues/74817), [@joshhunt](https://github.com/joshhunt) +- **Licensing:** Pass func to update env variables when starting plugin. [#74681](https://github.com/grafana/grafana/issues/74681), [@leandro-deveikis](https://github.com/leandro-deveikis) +- **RBAC:** Chore fix hasPermissionInOrg. (Enterprise) + + # 9.5.12 (2023-09-29) From 43928d38af2d4129a25676e4e09551dd58935d66 Mon Sep 17 00:00:00 2001 From: Mattia Buccarella Date: Thu, 12 Oct 2023 12:46:57 +0200 Subject: [PATCH 092/102] RBAC: Allow the basic role None as option of the org role selector (#76335) --- public/app/features/admin/OrgRolePicker.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/public/app/features/admin/OrgRolePicker.tsx b/public/app/features/admin/OrgRolePicker.tsx index 65f55db0f1f..bfc0752bdd1 100644 --- a/public/app/features/admin/OrgRolePicker.tsx +++ b/public/app/features/admin/OrgRolePicker.tsx @@ -13,8 +13,7 @@ interface Props { width?: number | 'auto'; } -const basicRoles = Object.values(OrgRole).filter((r) => r !== OrgRole.None); -const options = basicRoles.map((r) => ({ label: r, value: r })); +const options = Object.keys(OrgRole).map((key) => ({ label: key, value: key })); export function OrgRolePicker({ value, onChange, 'aria-label': ariaLabel, inputId, autoFocus, ...restProps }: Props) { return ( From 87ff5b66eabcf4b5a6a280c5515ea4d9ac6a46f7 Mon Sep 17 00:00:00 2001 From: Domas Date: Thu, 12 Oct 2023 13:48:22 +0300 Subject: [PATCH 093/102] TraceView/TracesPanel: Allow providing custom span link creator function (#76332) allow overriding span link creator for TraceView and TracesPanel --- .../features/explore/TraceView/TraceView.tsx | 30 +++++++++++++++++-- .../app/plugins/panel/traces/TracesPanel.tsx | 8 ++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/public/app/features/explore/TraceView/TraceView.tsx b/public/app/features/explore/TraceView/TraceView.tsx index 8f973234ef1..bbe96d1b37c 100644 --- a/public/app/features/explore/TraceView/TraceView.tsx +++ b/public/app/features/explore/TraceView/TraceView.tsx @@ -27,7 +27,14 @@ import { useDispatch, useSelector } from 'app/types'; import { changePanelState } from '../state/explorePane'; -import { SpanBarOptionsData, Trace, TracePageHeader, TraceTimelineViewer, TTraceTimeline } from './components'; +import { + SpanBarOptionsData, + SpanLinkFunc, + Trace, + TracePageHeader, + TraceTimelineViewer, + TTraceTimeline, +} from './components'; import SpanGraph from './components/TracePageHeader/SpanGraph'; import { TopOfViewRefType } from './components/TraceTimelineViewer/VirtualizedTraceView'; import { createSpanLinkFactory } from './createSpanLink'; @@ -59,10 +66,18 @@ type Props = { datasource: DataSourceApi | undefined; topOfViewRef: RefObject; topOfViewRefType: TopOfViewRefType; + createSpanLink?: SpanLinkFunc; }; export function TraceView(props: Props) { - const { traceProp, datasource, topOfViewRef, topOfViewRefType, exploreId } = props; + const { + traceProp, + datasource, + topOfViewRef, + topOfViewRefType, + exploreId, + createSpanLink: createSpanLinkFromProps, + } = props; const { detailStates, @@ -119,6 +134,7 @@ export function TraceView(props: Props) { const createSpanLink = useMemo( () => + createSpanLinkFromProps ?? createSpanLinkFactory({ splitOpenFn: props.splitOpenFn!, traceToLogsOptions, @@ -127,7 +143,15 @@ export function TraceView(props: Props) { createFocusSpanLink, trace: traceProp, }), - [props.splitOpenFn, traceToLogsOptions, traceToMetricsOptions, props.dataFrames, createFocusSpanLink, traceProp] + [ + props.splitOpenFn, + traceToLogsOptions, + traceToMetricsOptions, + props.dataFrames, + createFocusSpanLink, + traceProp, + createSpanLinkFromProps, + ] ); const timeZone = useSelector((state) => getTimeZone(state.user)); const datasourceType = datasource ? datasource?.type : 'unknown'; diff --git a/public/app/plugins/panel/traces/TracesPanel.tsx b/public/app/plugins/panel/traces/TracesPanel.tsx index 3a91bdb3a32..c09c812515e 100644 --- a/public/app/plugins/panel/traces/TracesPanel.tsx +++ b/public/app/plugins/panel/traces/TracesPanel.tsx @@ -5,6 +5,7 @@ import { useAsync } from 'react-use'; import { PanelProps } from '@grafana/data'; import { getDataSourceSrv } from '@grafana/runtime'; import { TraceView } from 'app/features/explore/TraceView/TraceView'; +import { SpanLinkFunc } from 'app/features/explore/TraceView/components'; import { TopOfViewRefType } from 'app/features/explore/TraceView/components/TraceTimelineViewer/VirtualizedTraceView'; import { transformDataFrames } from 'app/features/explore/TraceView/utils/transform'; @@ -15,7 +16,11 @@ const styles = { `, }; -export const TracesPanel = ({ data }: PanelProps) => { +export interface TracesPanelOptions { + createSpanLink?: SpanLinkFunc; +} + +export const TracesPanel = ({ data, options }: PanelProps) => { const topOfViewRef = createRef(); const traceProp = useMemo(() => transformDataFrames(data.series[0]), [data.series]); const dataSource = useAsync(async () => { @@ -41,6 +46,7 @@ export const TracesPanel = ({ data }: PanelProps) => { datasource={dataSource.value} topOfViewRef={topOfViewRef} topOfViewRefType={TopOfViewRefType.Panel} + createSpanLink={options.createSpanLink} /> ); From 38cdce526a6eb3270479589e33f146bef235be38 Mon Sep 17 00:00:00 2001 From: Eric Leijonmarck Date: Thu, 12 Oct 2023 13:00:26 +0200 Subject: [PATCH 094/102] Auth: Enable `None` role for 10.2 (#76343) * remove the feature toggle for noBasicRole * linting * remove unused --- .../configure-grafana/feature-toggles/index.md | 1 - .../src/types/featureToggles.gen.ts | 1 - pkg/services/featuremgmt/registry.go | 8 -------- pkg/services/featuremgmt/toggles_gen.csv | 1 - pkg/services/featuremgmt/toggles_gen.go | 4 ---- .../RolePicker/BuiltinRoleSelector.tsx | 16 ++++------------ .../components/RolePicker/RolePickerMenu.tsx | 7 +------ public/app/features/org/UserInviteForm.tsx | 18 ++++++------------ 8 files changed, 11 insertions(+), 45 deletions(-) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 40481d03655..b6cb3ee1031 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -131,7 +131,6 @@ Experimental features might be changed or removed without prior notice. | `featureToggleAdminPage` | Enable admin page for managing feature toggles from the Grafana front-end | | `permissionsFilterRemoveSubquery` | Alternative permission filter implementation that does not use subqueries for fetching the dashboard folder | | `influxdbSqlSupport` | Enable InfluxDB SQL query language support with new querying UI | -| `noBasicRole` | Enables a new role that has no permissions by default | | `angularDeprecationUI` | Display new Angular deprecation-related UI features | | `dashgpt` | Enable AI powered features in dashboards | | `sseGroupByDatasource` | Send query to the same datasource in a single request when using server side expressions | diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index ef71b57f296..345e9fbd3fd 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -117,7 +117,6 @@ export interface FeatureToggles { prometheusConfigOverhaulAuth?: boolean; configurableSchedulerTick?: boolean; influxdbSqlSupport?: boolean; - noBasicRole?: boolean; alertingNoDataErrorExecution?: boolean; angularDeprecationUI?: boolean; dashgpt?: boolean; diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 6b0b64c29dc..1d0fdb4db77 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -687,14 +687,6 @@ var ( Owner: grafanaObservabilityMetricsSquad, RequiresRestart: false, }, - { - Name: "noBasicRole", - Description: "Enables a new role that has no permissions by default", - Stage: FeatureStageExperimental, - FrontendOnly: true, - Owner: grafanaAuthnzSquad, - RequiresRestart: true, - }, { Name: "alertingNoDataErrorExecution", Description: "Changes how Alerting state manager handles execution of NoData/Error", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 376a632aa10..1c075935770 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -98,7 +98,6 @@ permissionsFilterRemoveSubquery,experimental,@grafana/backend-platform,false,fal prometheusConfigOverhaulAuth,GA,@grafana/observability-metrics,false,false,false,false configurableSchedulerTick,experimental,@grafana/alerting-squad,false,false,true,false influxdbSqlSupport,experimental,@grafana/observability-metrics,false,false,false,false -noBasicRole,experimental,@grafana/grafana-authnz-team,false,false,true,true alertingNoDataErrorExecution,privatePreview,@grafana/alerting-squad,false,false,true,false angularDeprecationUI,experimental,@grafana/plugins-platform-backend,false,false,false,true dashgpt,experimental,@grafana/dashboards-squad,false,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 8efbf55fb58..3240b125833 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -403,10 +403,6 @@ const ( // Enable InfluxDB SQL query language support with new querying UI FlagInfluxdbSqlSupport = "influxdbSqlSupport" - // FlagNoBasicRole - // Enables a new role that has no permissions by default - FlagNoBasicRole = "noBasicRole" - // FlagAlertingNoDataErrorExecution // Changes how Alerting state manager handles execution of NoData/Error FlagAlertingNoDataErrorExecution = "alertingNoDataErrorExecution" diff --git a/public/app/core/components/RolePicker/BuiltinRoleSelector.tsx b/public/app/core/components/RolePicker/BuiltinRoleSelector.tsx index 4b2739b6b17..075c7491fd7 100644 --- a/public/app/core/components/RolePicker/BuiltinRoleSelector.tsx +++ b/public/app/core/components/RolePicker/BuiltinRoleSelector.tsx @@ -1,23 +1,15 @@ import React from 'react'; import { SelectableValue } from '@grafana/data'; -import { config } from '@grafana/runtime'; import { Icon, RadioButtonList, Tooltip, useStyles2, useTheme2, PopoverContent } from '@grafana/ui'; -import { contextSrv } from 'app/core/core'; import { OrgRole } from 'app/types'; import { getStyles } from './styles'; -const noBasicRoleFlag = contextSrv.licensedAccessControlEnabled(); - -const noBasicRole = config.featureToggles.noBasicRole && noBasicRoleFlag; - -const BasicRoleOption: Array> = Object.values(OrgRole) - .filter((r) => noBasicRole || r !== OrgRole.None) - .map((r) => ({ - label: r === OrgRole.None ? 'No basic role' : r, - value: r, - })); +const BasicRoleOption: Array> = Object.values(OrgRole).map((r) => ({ + label: r === OrgRole.None ? 'No basic role' : r, + value: r, +})); interface Props { value?: OrgRole; diff --git a/public/app/core/components/RolePicker/RolePickerMenu.tsx b/public/app/core/components/RolePicker/RolePickerMenu.tsx index 54733ec4a29..1da2a057b52 100644 --- a/public/app/core/components/RolePicker/RolePickerMenu.tsx +++ b/public/app/core/components/RolePicker/RolePickerMenu.tsx @@ -1,10 +1,8 @@ import { css, cx } from '@emotion/css'; import React, { useEffect, useRef, useState } from 'react'; -import { config } from '@grafana/runtime'; import { Button, CustomScrollbar, HorizontalGroup, TextLink, useStyles2, useTheme2 } from '@grafana/ui'; import { getSelectStyles } from '@grafana/ui/src/components/Select/getSelectStyles'; -import { contextSrv } from 'app/core/core'; import { OrgRole, Role } from 'app/types'; import { BuiltinRoleSelector } from './BuiltinRoleSelector'; @@ -36,8 +34,7 @@ const fixedRoleGroupNames: Record = { current: 'Current org', }; -const noBasicRoleFlag = contextSrv.licensedAccessControlEnabled() && config.featureToggles.noBasicRole; -const tooltipMessage = noBasicRoleFlag ? ( +const tooltipMessage = ( <> You can now select the "No basic role" option and add permissions to your custom needs. You can find more information in  @@ -50,8 +47,6 @@ const tooltipMessage = noBasicRoleFlag ? ( . -) : ( - '' ); interface RolePickerMenuProps { diff --git a/public/app/features/org/UserInviteForm.tsx b/public/app/features/org/UserInviteForm.tsx index 86535bf40ba..6332c41e180 100644 --- a/public/app/features/org/UserInviteForm.tsx +++ b/public/app/features/org/UserInviteForm.tsx @@ -2,7 +2,7 @@ import React from 'react'; import { locationUtil, SelectableValue } from '@grafana/data'; import { Stack } from '@grafana/experimental'; -import { config, locationService } from '@grafana/runtime'; +import { locationService } from '@grafana/runtime'; import { Button, LinkButton, @@ -19,13 +19,11 @@ import { Label, } from '@grafana/ui'; import { getConfig } from 'app/core/config'; -import { contextSrv } from 'app/core/core'; import { OrgRole, useDispatch } from 'app/types'; import { addInvitee } from '../invites/state/actions'; -const noBasicRoleFlag = contextSrv.licensedAccessControlEnabled() && config.featureToggles.noBasicRole; -const tooltipMessage = noBasicRoleFlag ? ( +const tooltipMessage = ( <> You can now select the "No basic role" option and add permissions to your custom needs. You can find more information in  @@ -38,16 +36,12 @@ const tooltipMessage = noBasicRoleFlag ? ( . -) : ( - '' ); -const roles: Array> = Object.values(OrgRole) - .filter((r) => noBasicRoleFlag || r !== OrgRole.None) - .map((r) => ({ - label: r === OrgRole.None ? 'No basic role' : r, - value: r, - })); +const roles: Array> = Object.values(OrgRole).map((r) => ({ + label: r === OrgRole.None ? 'No basic role' : r, + value: r, +})); export interface FormModel { role: OrgRole; From 99e2f404387c99c699b70f761c10948babbaec62 Mon Sep 17 00:00:00 2001 From: "grafana-delivery-bot[bot]" <132647405+grafana-delivery-bot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:18:00 +0300 Subject: [PATCH 095/102] Changelog: Updated changelog for 10.0.9 (#76425) Co-authored-by: grafanabot --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20340a4d5a2..ae12cf5ebf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -446,6 +446,19 @@ Starting with 10.0, changing the folder UID is deprecated. It will be removed in - **Grafana/ui:** Fix margin in RadioButtonGroup option when only icon is present. [#68899](https://github.com/grafana/grafana/issues/68899), [@aocenas](https://github.com/aocenas) + + +# 10.0.9 (2023-10-11) + +### Bug fixes + +- **BrowseDashboards:** Only remember the most recent expanded folder. [#74819](https://github.com/grafana/grafana/issues/74819), [@joshhunt](https://github.com/joshhunt) +- **Licensing:** Pass func to update env variables when starting plugin. [#74679](https://github.com/grafana/grafana/issues/74679), [@leandro-deveikis](https://github.com/leandro-deveikis) +- **Nested folders:** Fix folder hierarchy in folder responses. [#74581](https://github.com/grafana/grafana/issues/74581), [@papagian](https://github.com/papagian) +- **RBAC:** Chore fix hasPermissionInOrg. (Enterprise) +- **Licensing:** Updated grpc plugin factory newPlugin signature. (Enterprise) + + # 10.0.8 (2023-09-29) From 625b400fafbdcd484aaf8622a829e493d3bd94ba Mon Sep 17 00:00:00 2001 From: Horst Gutmann Date: Thu, 12 Oct 2023 13:22:43 +0200 Subject: [PATCH 096/102] CI: github-release workflow should forward "latest" input (#75139) --- .github/workflows/github-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml index b7a73865264..91eb76b62cc 100644 --- a/.github/workflows/github-release.yml +++ b/.github/workflows/github-release.yml @@ -24,3 +24,4 @@ jobs: token: ${{ steps.generate_token.outputs.token }} version: ${{ inputs.version }} metrics_api_key: ${{ secrets.GRAFANA_MISC_STATS_API_KEY }} + latest: ${{ inputs.latest }} From f10b4bc1da116cac664e17a433f60806372dfefd Mon Sep 17 00:00:00 2001 From: Eric Leijonmarck Date: Thu, 12 Oct 2023 13:52:07 +0200 Subject: [PATCH 097/102] Auth: None basic role update docs (#75955) * docs: update basic role * update docs * Update docs/sources/administration/user-management/manage-org-users/index.md Co-authored-by: Ieva * Update docs/sources/administration/roles-and-permissions/access-control/plan-rbac-rollout-strategy/index.md Co-authored-by: Ieva --------- Co-authored-by: Ieva --- .../roles-and-permissions/access-control/_index.md | 3 ++- .../access-control/manage-rbac-roles/index.md | 2 +- .../access-control/plan-rbac-rollout-strategy/index.md | 5 +++++ .../administration/user-management/manage-org-users/index.md | 2 +- .../configure-grafana/enterprise-configuration/index.md | 2 +- .../configure-authentication/generic-oauth/index.md | 2 +- 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/sources/administration/roles-and-permissions/access-control/_index.md b/docs/sources/administration/roles-and-permissions/access-control/_index.md index b8e14fe227a..0bf04902b57 100644 --- a/docs/sources/administration/roles-and-permissions/access-control/_index.md +++ b/docs/sources/administration/roles-and-permissions/access-control/_index.md @@ -53,6 +53,7 @@ Grafana includes the following basic roles: - Organization administrator - Editor - Viewer +- None Each basic role is comprised of a number of _permissions_. For example, the viewer basic role contains the following permissions among others: @@ -64,7 +65,7 @@ Each basic role is comprised of a number of _permissions_. For example, the view - `Action: annotations:delete, Scope: annotations:type:dashboard`: Enables the viewer to remove annotations from a dashboard. {{% admonition type="note" %}} -You can't have a Grafana user without a basic role assigned. +You can't have a Grafana user without a basic role assigned. The `None` role contains no permissions. {{% /admonition %}} #### Basic role modification diff --git a/docs/sources/administration/roles-and-permissions/access-control/manage-rbac-roles/index.md b/docs/sources/administration/roles-and-permissions/access-control/manage-rbac-roles/index.md index f748250664a..98ec10b6308 100644 --- a/docs/sources/administration/roles-and-permissions/access-control/manage-rbac-roles/index.md +++ b/docs/sources/administration/roles-and-permissions/access-control/manage-rbac-roles/index.md @@ -32,7 +32,7 @@ To see the permissions associated with basic roles, refer to the following basic | Basic role | UID | | --------------- | --------------------- | -| `No Basic Role` | `basic_none` | +| `None` | `basic_none` | | `Viewer` | `basic_viewer` | | `Editor` | `basic_editor` | | `Admin` | `basic_admin` | diff --git a/docs/sources/administration/roles-and-permissions/access-control/plan-rbac-rollout-strategy/index.md b/docs/sources/administration/roles-and-permissions/access-control/plan-rbac-rollout-strategy/index.md index 76b01df199a..d6a4f3bc8a4 100644 --- a/docs/sources/administration/roles-and-permissions/access-control/plan-rbac-rollout-strategy/index.md +++ b/docs/sources/administration/roles-and-permissions/access-control/plan-rbac-rollout-strategy/index.md @@ -117,6 +117,11 @@ If you have a use case that you'd like to share, feel free to contribute to this 1. Add dashboards to each folder. 1. Use folder permissions to add US-based users as Editors to the `US` folder and assign EU-based users as Editors to the `EU` folder. +### Assign a user specific set of roles + +1. Create a user with the `No Basic Role` selected under organization roles. +1. Assign the user a set of fixed roles that meet your requirements. + ### Create a custom role to access alerts in a specific folder To see an alert rule in Grafana, the user must have read access to the folder that stores the alert rule, permission to read alerts in the folder, and permission to query all data sources that the rule uses. diff --git a/docs/sources/administration/user-management/manage-org-users/index.md b/docs/sources/administration/user-management/manage-org-users/index.md index 0b411b2b1a1..c01c8546679 100644 --- a/docs/sources/administration/user-management/manage-org-users/index.md +++ b/docs/sources/administration/user-management/manage-org-users/index.md @@ -74,7 +74,7 @@ If you have [server administrator]({{< relref "../../roles-and-permissions/#graf ## Invite a user to join an organization -When you invite users to join an organization, you assign the **Admin**, **Editor**, or **Viewer** role which controls user access to the dashboards and data sources owned by the organization. Users receive an email that prompts them to accept the invitation. +When you invite users to join an organization, you assign the **Admin**, **Editor**, or **Viewer** role, or select **No basic role**. Organization roles control user access to resources, such as dashboards and data sources, owned by the organization. Users receive an email that prompts them to accept the invitation. - If you know that the user already has access Grafana and you know their user name, then you issue an invitation by entering their user name. - If the user is new to Grafana, then use their email address to issue an invitation. The system automatically creates the user account on first sign in. diff --git a/docs/sources/setup-grafana/configure-grafana/enterprise-configuration/index.md b/docs/sources/setup-grafana/configure-grafana/enterprise-configuration/index.md index f3f7daebbe4..986fb532aeb 100644 --- a/docs/sources/setup-grafana/configure-grafana/enterprise-configuration/index.md +++ b/docs/sources/setup-grafana/configure-grafana/enterprise-configuration/index.md @@ -314,7 +314,7 @@ List of comma- or space-separated organizations. Each user must be a member of a ### org_mapping -List of comma- or space-separated Organization:OrgId:Role mappings. Organization can be `*` meaning "All users". Role is optional and can have the following values: `Viewer`, `Editor` or `Admin`. +List of comma- or space-separated Organization:OrgId:Role mappings. Organization can be `*` meaning "All users". Role is optional and can have the following values: `Admin`, `Editor` ,`Viewer` or `None`. ### role_values_editor diff --git a/docs/sources/setup-grafana/configure-security/configure-authentication/generic-oauth/index.md b/docs/sources/setup-grafana/configure-security/configure-authentication/generic-oauth/index.md index 34749b841e5..fb37aadfb1a 100644 --- a/docs/sources/setup-grafana/configure-security/configure-authentication/generic-oauth/index.md +++ b/docs/sources/setup-grafana/configure-security/configure-authentication/generic-oauth/index.md @@ -205,7 +205,7 @@ This section includes examples of JMESPath expressions used for role mapping. #### Map user organization role -In this example, the user has been granted the role of an `Editor`. The role assigned is based on the value of the property `role`, which must be a valid Grafana role such as `Viewer`, `Editor` or `Admin`. +In this example, the user has been granted the role of an `Editor`. The role assigned is based on the value of the property `role`, which must be a valid Grafana role such as `Admin`, `Editor`, `Viewer` or `None`. Payload: From 97d54814a3c73ac19951249f14796a83437848c9 Mon Sep 17 00:00:00 2001 From: "grafana-delivery-bot[bot]" <132647405+grafana-delivery-bot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 12:18:16 +0000 Subject: [PATCH 098/102] Changelog: Updated changelog for 10.1.5 (#76436) Co-authored-by: grafanabot --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae12cf5ebf5..acd7e844d13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,31 @@ + + +# 10.1.5 (2023-10-11) + +### Features and enhancements + +- **Cloudwatch:** Backport 73524 Bring Back Legacy Log Group Picker. [#75031](https://github.com/grafana/grafana/issues/75031), [@sarahzinger](https://github.com/sarahzinger) + +### Bug fixes + +- **Cloudwatch:** Prevent log group requests with ARNs if feature flag is off. [#75691](https://github.com/grafana/grafana/issues/75691), [@sarahzinger](https://github.com/sarahzinger) +- **Alerting:** Add support for `keep_firing_for` field from external rulers. [#75257](https://github.com/grafana/grafana/issues/75257), [@rwwiv](https://github.com/rwwiv) +- **Canvas:** Avoid conflicting stylesheets when loading SVG icons. [#75032](https://github.com/grafana/grafana/issues/75032), [@adela-almasan](https://github.com/adela-almasan) +- **Alerting:** Prevent showing "Permissions denied" alert when not accurate. [#74925](https://github.com/grafana/grafana/issues/74925), [@VikaCep](https://github.com/VikaCep) +- **BrowseDashboards:** Only remember the most recent expanded folder. [#74809](https://github.com/grafana/grafana/issues/74809), [@joshhunt](https://github.com/joshhunt) +- **Tempo Service Map:** Fix context menu links in service map when namespace is present. [#74796](https://github.com/grafana/grafana/issues/74796), [@javiruiz01](https://github.com/javiruiz01) +- **Logs Panel:** Performance issue while scrolling within panel in safari. [#74747](https://github.com/grafana/grafana/issues/74747), [@gtk-grafana](https://github.com/gtk-grafana) +- **Bug:** Allow to uninstall a deprecated plugin. [#74704](https://github.com/grafana/grafana/issues/74704), [@andresmgot](https://github.com/andresmgot) +- **Licensing:** Pass func to update env variables when starting plugin. [#74678](https://github.com/grafana/grafana/issues/74678), [@leandro-deveikis](https://github.com/leandro-deveikis) +- **Nested folders:** Fix folder hierarchy in folder responses. [#74580](https://github.com/grafana/grafana/issues/74580), [@papagian](https://github.com/papagian) +- **Share link:** Use panel relative time for direct link rendered image. [#74518](https://github.com/grafana/grafana/issues/74518), [@Clarity-89](https://github.com/Clarity-89) +- **Alerting:** Do not exit if Redis ping fails when using redis-based Alertmanager clustering. [#74399](https://github.com/grafana/grafana/issues/74399), [@alexweav](https://github.com/alexweav) +- **Alerting:** Refactor AlertRuleForm and fix annotations step description for cloud rules. [#74193](https://github.com/grafana/grafana/issues/74193), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **RBAC:** Chore fix hasPermissionInOrg. (Enterprise) +- **Licensing:** Updated grpc plugin factory newPlugin signature. (Enterprise) +- **Reporting:** Add support for old dashboard schema. (Enterprise) + + # 10.1.4 (2023-09-29) From 358b395c2bc2cde0fc0aeecfdf1e69c19c011f0d Mon Sep 17 00:00:00 2001 From: Virginia Cepeda Date: Thu, 12 Oct 2023 13:26:05 +0100 Subject: [PATCH 099/102] Alerting: text updates / replace panel (#76407) * Change text to remove "week" reference * Add alertsByState for grafana alertmanager --- .../alerting/unified/home/Insights.tsx | 5 +-- .../insights/grafana/MostFiredRulesTable.tsx | 2 +- .../grafana/alertmanager/AlertsByState.tsx | 34 +++++++++++++++++++ .../insights/mimir/rules/MostFiredRules.tsx | 2 +- 4 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 public/app/features/alerting/unified/insights/grafana/alertmanager/AlertsByState.tsx diff --git a/public/app/features/alerting/unified/home/Insights.tsx b/public/app/features/alerting/unified/home/Insights.tsx index 33ec4e4f9b7..7bd9ab986ef 100644 --- a/public/app/features/alerting/unified/home/Insights.tsx +++ b/public/app/features/alerting/unified/home/Insights.tsx @@ -28,7 +28,7 @@ import { getMostFiredInstancesScene } from '../insights/grafana/MostFiredInstanc import { getPausedGrafanaAlertsScene } from '../insights/grafana/Paused'; import { getGrafanaRulesByEvaluationScene } from '../insights/grafana/RulesByEvaluation'; import { getGrafanaRulesByEvaluationPercentageScene } from '../insights/grafana/RulesByEvaluationPercentage'; -import { getGrafanaAlertmanagerNotificationsScene } from '../insights/grafana/alertmanager/NotificationsScene'; +import { getAlertsByStateScene as getGrafanaAlertsByStateScene } from '../insights/grafana/alertmanager/AlertsByState'; import { getGrafanaAlertmanagerSilencesScene } from '../insights/grafana/alertmanager/SilencesByStateScene'; import { getAlertsByStateScene } from '../insights/mimir/AlertsByState'; import { getInvalidConfigScene } from '../insights/mimir/InvalidConfig'; @@ -267,7 +267,8 @@ function getGrafanaAlertmanagerScenes() { children: [ new SceneFlexLayout({ children: [ - getGrafanaAlertmanagerNotificationsScene(cloudUsageDs, 'Notification delivery'), + getGrafanaAlertsByStateScene(cloudUsageDs, 'Firing alerts by state'), + // getGrafanaAlertmanagerNotificationsScene(cloudUsageDs, 'Notification delivery'), getGrafanaAlertmanagerSilencesScene(cloudUsageDs, 'Silences'), ], }), diff --git a/public/app/features/alerting/unified/insights/grafana/MostFiredRulesTable.tsx b/public/app/features/alerting/unified/insights/grafana/MostFiredRulesTable.tsx index e6ef7610373..a9ff7c76d2a 100644 --- a/public/app/features/alerting/unified/insights/grafana/MostFiredRulesTable.tsx +++ b/public/app/features/alerting/unified/insights/grafana/MostFiredRulesTable.tsx @@ -44,7 +44,7 @@ export function getMostFiredRulesScene(datasource: DataSourceRef, panelTitle: st renameByName: { group: 'Group', labels_grafana_folder: 'Folder', - 'Value #A': 'Fires this week', + 'Value #A': 'Number of fires', }, }, }, diff --git a/public/app/features/alerting/unified/insights/grafana/alertmanager/AlertsByState.tsx b/public/app/features/alerting/unified/insights/grafana/alertmanager/AlertsByState.tsx new file mode 100644 index 00000000000..7f9162fa503 --- /dev/null +++ b/public/app/features/alerting/unified/insights/grafana/alertmanager/AlertsByState.tsx @@ -0,0 +1,34 @@ +import React from 'react'; + +import { PanelBuilders, SceneFlexItem, SceneQueryRunner } from '@grafana/scenes'; +import { DataSourceRef, GraphDrawStyle, TooltipDisplayMode } from '@grafana/schema'; + +import { overrideToFixedColor, PANEL_STYLES } from '../../../home/Insights'; +import { InsightsRatingModal } from '../../RatingModal'; + +export function getAlertsByStateScene(datasource: DataSourceRef, panelTitle: string) { + const query = new SceneQueryRunner({ + datasource, + queries: [ + { + refId: 'A', + expr: 'sum by (state) (grafanacloud_grafana_instance_alerting_alertmanager_alerts)', + range: true, + legendFormat: '{{state}}', + }, + ], + }); + + return new SceneFlexItem({ + ...PANEL_STYLES, + body: PanelBuilders.timeseries() + .setTitle(panelTitle) + .setDescription('A breakdown of all of your firing alert rule instances based on state') + .setData(query) + .setCustomFieldConfig('drawStyle', GraphDrawStyle.Line) + .setOption('tooltip', { mode: TooltipDisplayMode.Multi }) + .setOverrides((b) => b.matchFieldsWithName('active').overrideColor(overrideToFixedColor('active'))) + .setHeaderActions() + .build(), + }); +} diff --git a/public/app/features/alerting/unified/insights/mimir/rules/MostFiredRules.tsx b/public/app/features/alerting/unified/insights/mimir/rules/MostFiredRules.tsx index df2f00f7c38..402042c01d3 100644 --- a/public/app/features/alerting/unified/insights/mimir/rules/MostFiredRules.tsx +++ b/public/app/features/alerting/unified/insights/mimir/rules/MostFiredRules.tsx @@ -31,7 +31,7 @@ export function getMostFiredRulesScene(datasource: DataSourceRef, panelTitle: st }, indexByName: {}, renameByName: { - Value: 'Fires this week', + Value: 'Number of fires', alertname: 'Alert Rule Name', }, }, From 7d9617e07c7395e487b66d19d06965d799788e26 Mon Sep 17 00:00:00 2001 From: "grafana-delivery-bot[bot]" <132647405+grafana-delivery-bot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 12:27:27 +0000 Subject: [PATCH 100/102] Changelog: Updated changelog for 10.1.5 (#76441) Co-authored-by: grafanabot --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index acd7e844d13..94647092f67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Features and enhancements +- **Chore:** Upgrade Go to 1.20.10. [#76355](https://github.com/grafana/grafana/issues/76355), [@zerok](https://github.com/zerok) - **Cloudwatch:** Backport 73524 Bring Back Legacy Log Group Picker. [#75031](https://github.com/grafana/grafana/issues/75031), [@sarahzinger](https://github.com/sarahzinger) ### Bug fixes From afa64fc2181d797f4f1b4d80d8c034fa91cc1c87 Mon Sep 17 00:00:00 2001 From: Virginia Cepeda Date: Thu, 12 Oct 2023 13:31:18 +0100 Subject: [PATCH 101/102] Alerting: Enable Insights landing page (#76381) * Enable alerting insights * Change alertingInsights feature toggle stage * Commit autogenerated files after enabling feature flag --- .../setup-grafana/configure-grafana/feature-toggles/index.md | 2 +- pkg/services/featuremgmt/registry.go | 3 ++- pkg/services/featuremgmt/toggles_gen.csv | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index b6cb3ee1031..a2631cd9f04 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -49,6 +49,7 @@ Some features are enabled by default. You can disable these feature by setting t | `azureMonitorDataplane` | Adds dataplane compliant frame metadata in the Azure Monitor datasource | Yes | | `prometheusConfigOverhaulAuth` | Update the Prometheus configuration page with the new auth component | Yes | | `newBrowseDashboards` | New browse/manage dashboards UI | Yes | +| `alertingInsights` | Show the new alerting insights landing page | Yes | | `cloudWatchWildCardDimensionValues` | Fetches dimension values from CloudWatch to correctly label wildcard dimensions | Yes | ## Preview feature toggles @@ -137,7 +138,6 @@ Experimental features might be changed or removed without prior notice. | `requestInstrumentationStatusSource` | Include a status source label for request metrics and logs | | `libraryPanelRBAC` | Enables RBAC support for library panels | | `wargamesTesting` | Placeholder feature flag for internal testing | -| `alertingInsights` | Show the new alerting insights landing page | | `externalCorePlugins` | Allow core plugins to be loaded as external | | `pluginsAPIMetrics` | Sends metrics of public grafana packages usage by plugins | | `httpSLOLevels` | Adds SLO level to http request metrics | diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 1d0fdb4db77..5810fdb840a 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -764,8 +764,9 @@ var ( Name: "alertingInsights", Description: "Show the new alerting insights landing page", FrontendOnly: true, - Stage: FeatureStageExperimental, + Stage: FeatureStageGeneralAvailability, Owner: grafanaAlertingSquad, + Expression: "true", // enabled by default }, { Name: "externalCorePlugins", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 1c075935770..3c928a69c99 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -108,7 +108,7 @@ requestInstrumentationStatusSource,experimental,@grafana/plugins-platform-backen libraryPanelRBAC,experimental,@grafana/dashboards-squad,false,false,true,false lokiRunQueriesInParallel,privatePreview,@grafana/observability-logs,false,false,false,false wargamesTesting,experimental,@grafana/hosted-grafana-team,false,false,false,false -alertingInsights,experimental,@grafana/alerting-squad,false,false,false,true +alertingInsights,GA,@grafana/alerting-squad,false,false,false,true externalCorePlugins,experimental,@grafana/plugins-platform-backend,false,false,false,false pluginsAPIMetrics,experimental,@grafana/plugins-platform-backend,false,false,false,true httpSLOLevels,experimental,@grafana/hosted-grafana-team,false,false,true,false From f6d2c11be9ce59442e0e1a7bfe3665d10648afea Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Thu, 12 Oct 2023 13:37:26 +0100 Subject: [PATCH 102/102] Fix typo in link label (#76443) --- .../manage-notifications/template-notifications/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/alerting/manage-notifications/template-notifications/_index.md b/docs/sources/alerting/manage-notifications/template-notifications/_index.md index 34819484876..40d347fc73c 100644 --- a/docs/sources/alerting/manage-notifications/template-notifications/_index.md +++ b/docs/sources/alerting/manage-notifications/template-notifications/_index.md @@ -41,7 +41,7 @@ Learn how to write the content of your notification templates in Go’s templati Create reusable notification templates for your contact points. -[Use notification templates][use-notication-templates] +[Use notification templates][use-notification-templates] Use notification templates to send notifications to your contact points.