Alerting: Support secrets in contact points nested fields (#92035)

Back-end:
* update alerting module
* update GetSecretKeysForContactPointType to extract secret fields from nested options
* Update RemoveSecretsForContactPoint to support complex settings
* update PostableGrafanaReceiverToEmbeddedContactPoint to support nested secrets
* update Integration to support nested settings in models.Integration
* make sigv4 fields optional

Front-end:
* add UI support for encrypted subform fields
* allow emptying nested secure fields
* Omit non touched secure fields in POST payload when saving a contact point
* Use SecretInput from grafana-ui instead of the new EncryptedInput
* use produce from immer
* rename mapClone
* rename sliceClone
* Don't use produce from immer as we need to delete the fileds afterwards

---------

Co-authored-by: Gilles De Mey <gilles.de.mey@gmail.com>
Co-authored-by: Sonia Aguilar <soniaaguilarpeiron@gmail.com>
Co-authored-by: Matt Jacobson <matthew.jacobson@grafana.com>
This commit is contained in:
Yuri Tseretyan
2024-09-10 22:26:23 -04:00
committed by GitHub
co-authored by Gilles De Mey Sonia Aguilar Matt Jacobson
parent 78ce9b8f39
commit cb372d3fa8
20 changed files with 839 additions and 246 deletions
@@ -0,0 +1,55 @@
package models
import (
"encoding/binary"
"fmt"
"hash"
"hash/fnv"
"math"
"unsafe"
)
// fingerprint is a wrapper for hash.Hash64 that adds utility methods to simplify hash calculation of structs
type fingerprint struct {
h hash.Hash64
}
// creates a fingerprint that is backed by 64bit FNV-1a hash
func newFingerprint() fingerprint {
return fingerprint{h: fnv.New64a()}
}
func (f fingerprint) String() string {
return fmt.Sprintf("%016x", f.h.Sum64())
}
func (f fingerprint) writeBytes(b []byte) {
_, _ = f.h.Write(b)
// add a byte sequence that cannot happen in UTF-8 strings.
_, _ = f.h.Write([]byte{255})
}
func (f fingerprint) writeString(s string) {
if len(s) == 0 {
f.writeBytes(nil)
return
}
// #nosec G103
// avoid allocation when converting string to byte slice
f.writeBytes(unsafe.Slice(unsafe.StringData(s), len(s)))
}
func (f fingerprint) writeFloat64(num float64) {
bits := math.Float64bits(num)
bytes := make([]byte, 8)
binary.LittleEndian.PutUint64(bytes, bits)
f.writeBytes(bytes)
}
func (f fingerprint) writeBool(b bool) {
if b {
f.writeBytes([]byte{1})
} else {
f.writeBytes([]byte{0})
}
}
+274 -112
View File
@@ -2,15 +2,14 @@ package models
import (
"context"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"hash/fnv"
"maps"
"math"
"slices"
"sort"
"unsafe"
"strings"
alertingNotify "github.com/grafana/alerting/notify"
@@ -148,9 +147,39 @@ type IntegrationConfig struct {
// IntegrationField represents a field in an integration configuration.
type IntegrationField struct {
Name string
Fields map[string]IntegrationField
Secure bool
}
type IntegrationFieldPath []string
func NewIntegrationFieldPath(path string) IntegrationFieldPath {
return strings.Split(path, ".")
}
func (f IntegrationFieldPath) Head() string {
if len(f) > 0 {
return f[0]
}
return ""
}
func (f IntegrationFieldPath) Tail() IntegrationFieldPath {
return f[1:]
}
func (f IntegrationFieldPath) IsLeaf() bool {
return len(f) == 1
}
func (f IntegrationFieldPath) String() string {
return strings.Join(f, ".")
}
func (f IntegrationFieldPath) Append(segment string) IntegrationFieldPath {
return append(f, segment)
}
// IntegrationConfigFromType returns an integration configuration for a given integration type. If the integration type is
// not found an error is returned.
func IntegrationConfigFromType(integrationType string) (IntegrationConfig, error) {
@@ -160,23 +189,60 @@ func IntegrationConfigFromType(integrationType string) (IntegrationConfig, error
}
integrationConfig := IntegrationConfig{Type: config.Type, Fields: make(map[string]IntegrationField, len(config.Options))}
for _, option := range config.Options {
integrationConfig.Fields[option.PropertyName] = IntegrationField{
Name: option.PropertyName,
Secure: option.Secure,
}
integrationConfig.Fields[option.PropertyName] = notifierOptionToIntegrationField(option)
}
return integrationConfig, nil
}
func notifierOptionToIntegrationField(option channels_config.NotifierOption) IntegrationField {
f := IntegrationField{
Name: option.PropertyName,
Secure: option.Secure,
Fields: make(map[string]IntegrationField, len(option.SubformOptions)),
}
for _, subformOption := range option.SubformOptions {
f.Fields[subformOption.PropertyName] = notifierOptionToIntegrationField(subformOption)
}
return f
}
// IsSecureField returns true if the field is both known and marked as secure in the integration configuration.
func (config *IntegrationConfig) IsSecureField(field string) bool {
if config.Fields != nil {
if f, ok := config.Fields[field]; ok {
return f.Secure
func (config *IntegrationConfig) IsSecureField(path IntegrationFieldPath) bool {
f, ok := config.GetField(path)
return ok && f.Secure
}
func (config *IntegrationConfig) GetField(path IntegrationFieldPath) (IntegrationField, bool) {
for _, integrationField := range config.Fields {
if strings.EqualFold(integrationField.Name, path.Head()) {
if path.IsLeaf() {
return integrationField, true
}
return integrationField.GetField(path.Tail())
}
}
return false
return IntegrationField{}, false
}
func (config *IntegrationConfig) GetSecretFields() []IntegrationFieldPath {
return traverseFields(config.Fields, nil, func(i IntegrationField) bool {
return i.Secure
})
}
func traverseFields(flds map[string]IntegrationField, parentPath IntegrationFieldPath, predicate func(i IntegrationField) bool) []IntegrationFieldPath {
var result []IntegrationFieldPath
for key, field := range flds {
if predicate(field) {
result = append(result, parentPath.Append(key))
}
if len(field.Fields) > 0 {
result = append(result, traverseFields(field.Fields, parentPath.Append(key), predicate)...)
}
}
return result
}
func (config *IntegrationConfig) Clone() IntegrationConfig {
@@ -193,11 +259,28 @@ func (config *IntegrationConfig) Clone() IntegrationConfig {
return clone
}
func (field *IntegrationField) GetField(path IntegrationFieldPath) (IntegrationField, bool) {
for _, integrationField := range field.Fields {
if strings.EqualFold(integrationField.Name, path.Head()) {
if path.IsLeaf() {
return integrationField, true
}
return integrationField.GetField(path.Tail())
}
}
return IntegrationField{}, false
}
func (field *IntegrationField) Clone() IntegrationField {
return IntegrationField{
f := IntegrationField{
Name: field.Name,
Secure: field.Secure,
Fields: make(map[string]IntegrationField, len(field.Fields)),
}
for subName, sub := range field.Fields {
f.Fields[subName] = sub.Clone()
}
return f
}
func (integration *Integration) Clone() Integration {
@@ -206,42 +289,133 @@ func (integration *Integration) Clone() Integration {
Name: integration.Name,
Config: integration.Config.Clone(),
DisableResolveMessage: integration.DisableResolveMessage,
Settings: maps.Clone(integration.Settings),
Settings: cloneIntegrationSettings(integration.Settings),
SecureSettings: maps.Clone(integration.SecureSettings),
}
}
// cloneIntegrationSettings implements a deep copy of settings map.
// It's not a generic purpose function because settings are limited to basic types, maps and slices.
func cloneIntegrationSettings(m map[string]any) map[string]any {
result := maps.Clone(m) // do a shallow copy of the map first
for k, v := range result {
if mp, ok := v.(map[string]any); ok {
result[k] = cloneIntegrationSettings(mp)
continue
}
if mp, ok := v.([]any); ok {
result[k] = cloneIntegrationSettingsSlice(mp)
continue
}
}
return result
}
// cloneIntegrationSettingsSlice implements a deep copy of a []any in integration settings.
// It's not a generic purpose function because settings are limited to basic types, maps and slices.
func cloneIntegrationSettingsSlice(src []any) []any {
dst := slices.Clone(src)
for i, v := range dst {
if mp, ok := v.(map[string]any); ok {
dst[i] = cloneIntegrationSettings(mp)
continue
}
if mp, ok := v.([]any); ok {
dst[i] = cloneIntegrationSettingsSlice(mp)
continue
}
}
return dst
}
// Encrypt encrypts all fields in Settings that are marked as secure in the integration configuration. The encrypted values
// are stored in SecureSettings and the original values are removed from Settings.
// If a field is already in SecureSettings it is not encrypted again.
func (integration *Integration) Encrypt(encryptFn EncryptFn) error {
secretFieldPaths := integration.Config.GetSecretFields()
if len(secretFieldPaths) == 0 {
return nil
}
var errs []error
for key, val := range integration.Settings {
if isSecureField := integration.Config.IsSecureField(key); !isSecureField {
for _, path := range secretFieldPaths {
unencryptedSecureValue, ok, err := extractField(integration.Settings, path)
if err != nil {
errs = append(errs, fmt.Errorf("failed to extract secret field by path '%s': %w", path, err))
}
if !ok {
continue
}
delete(integration.Settings, key)
unencryptedSecureValue, isString := val.(string)
if !isString {
if _, exists := integration.SecureSettings[path.String()]; exists {
continue
}
if _, exists := integration.SecureSettings[key]; exists {
continue
}
encrypted, err := encryptFn(unencryptedSecureValue)
if err != nil {
errs = append(errs, fmt.Errorf("failed to encrypt secure setting '%s': %w", key, err))
errs = append(errs, fmt.Errorf("failed to encrypt secure setting '%s': %w", path.String(), err))
}
integration.SecureSettings[key] = encrypted
integration.SecureSettings[path.String()] = encrypted
}
return errors.Join(errs...)
}
func extractField(settings map[string]any, path IntegrationFieldPath) (string, bool, error) {
val, ok := settings[path.Head()]
if !ok {
return "", false, nil
}
if path.IsLeaf() {
secret, ok := val.(string)
if !ok {
return "", false, fmt.Errorf("expected string but got %T", val)
}
delete(settings, path.Head())
return secret, true, nil
}
sub, ok := val.(map[string]any)
if !ok {
return "", false, fmt.Errorf("expected nested object but got %T", val)
}
return extractField(sub, path.Tail())
}
func getFieldValue(settings map[string]any, path IntegrationFieldPath) (any, bool) {
val, ok := settings[path.Head()]
if !ok {
return nil, false
}
if path.IsLeaf() {
return val, true
}
sub, ok := val.(map[string]any)
if !ok {
return nil, false
}
return getFieldValue(sub, path.Tail())
}
func setField(settings map[string]any, path IntegrationFieldPath, valueFn func(current any) any, skipIfNotExist bool) error {
if path.IsLeaf() {
current, ok := settings[path.Head()]
if skipIfNotExist && !ok {
return nil
}
settings[path.Head()] = valueFn(current)
return nil
}
val, ok := settings[path.Head()]
if !ok {
if skipIfNotExist {
return nil
}
val = map[string]any{}
settings[path.Head()] = val
}
sub, ok := val.(map[string]any)
if !ok {
return fmt.Errorf("expected nested object but got %T", val)
}
return setField(sub, path.Tail(), valueFn, skipIfNotExist)
}
// Decrypt decrypts all fields in SecureSettings and moves them to Settings.
// The original values are removed from SecureSettings.
func (integration *Integration) Decrypt(decryptFn DecryptFn) error {
@@ -252,30 +426,35 @@ func (integration *Integration) Decrypt(decryptFn DecryptFn) error {
errs = append(errs, fmt.Errorf("failed to decrypt secure setting '%s': %w", key, err))
}
delete(integration.SecureSettings, key)
integration.Settings[key] = decrypted
}
path := NewIntegrationFieldPath(key)
err = setField(integration.Settings, path, func(current any) any {
return decrypted
}, false)
if err != nil {
errs = append(errs, fmt.Errorf("failed to set field '%s': %w", key, err))
}
}
return errors.Join(errs...)
}
// Redact redacts all fields in SecureSettings and moves them to Settings.
// The original values are removed from SecureSettings.
func (integration *Integration) Redact(redactFn RedactFn) {
for key, secureVal := range integration.SecureSettings { // TODO: Should we trust that the receiver is stored correctly or use known secure settings?
integration.Settings[key] = redactFn(secureVal)
delete(integration.SecureSettings, key)
for _, path := range integration.Config.GetSecretFields() {
_ = setField(integration.Settings, path, func(current any) any {
if s, ok := current.(string); ok && s != "" {
return redactFn(s)
}
return current
}, true)
}
// We don't trust that the receiver is stored correctly, so we redact secure fields in the settings as well.
for key, val := range integration.Settings {
if val != "" && integration.Config.IsSecureField(key) {
s, isString := val.(string)
if !isString {
continue
}
integration.Settings[key] = redactFn(s)
delete(integration.SecureSettings, key)
}
for key, secureVal := range integration.SecureSettings { // TODO: Should we trust that the receiver is stored correctly or use known secure settings?
_ = setField(integration.Settings, NewIntegrationFieldPath(key), func(any) any {
return redactFn(secureVal)
}, false)
delete(integration.SecureSettings, key)
}
}
@@ -304,11 +483,17 @@ func (integration *Integration) SecureFields() map[string]bool {
secureFields[key] = true
}
}
// We mark secure fields in the settings as well. This is to ensure legacy behaviour for redacted secure settings.
for key, val := range integration.Settings {
if val != "" && integration.Config.IsSecureField(key) {
secureFields[key] = true
for _, path := range integration.Config.GetSecretFields() {
if secureFields[path.String()] {
continue
}
value, ok := getFieldValue(integration.Settings, path)
if !ok || value == nil {
continue
}
if v, _ := value.(string); v != "" {
secureFields[path.String()] = true
}
}
@@ -369,41 +554,16 @@ func (r *Receiver) GetUID() string {
}
func (r *Receiver) Fingerprint() string {
sum := fnv.New64()
writeBytes := func(b []byte) {
_, _ = sum.Write(b)
// add a byte sequence that cannot happen in UTF-8 strings.
_, _ = sum.Write([]byte{255})
}
writeString := func(s string) {
if len(s) == 0 {
writeBytes(nil)
return
}
// #nosec G103
// avoid allocation when converting string to byte slice
writeBytes(unsafe.Slice(unsafe.StringData(s), len(s)))
}
// this temp slice is used to convert ints to bytes.
tmp := make([]byte, 8)
writeInt := func(u int) {
binary.LittleEndian.PutUint64(tmp, uint64(u))
writeBytes(tmp)
}
sum := newFingerprint()
writeIntegration := func(in *Integration) {
writeString(in.UID)
writeString(in.Name)
sum.writeString(in.UID)
sum.writeString(in.Name)
// Do not include fields in fingerprint as these are not part of the receiver definition.
writeString(in.Config.Type)
sum.writeString(in.Config.Type)
if in.DisableResolveMessage {
writeInt(1)
} else {
writeInt(0)
}
sum.writeBool(in.DisableResolveMessage)
// allocate a slice that will be used for sorting keys, so we allocate it only once
var keys []string
@@ -426,49 +586,51 @@ func (r *Receiver) Fingerprint() string {
sub := keys[:idx]
sort.Strings(sub)
for _, name := range sub {
writeString(name)
writeString(secureSettings[name])
sum.writeString(name)
sum.writeString(secureSettings[name])
}
}
writeSettings(sum, in.Settings)
writeSecureSettings(in.SecureSettings)
writeSettings := func(settings map[string]any) {
// maps do not guarantee predictable sequence of keys.
// Therefore, to make hash stable, we need to sort keys
if len(settings) == 0 {
return
}
idx := 0
for k := range settings {
keys[idx] = k
idx++
}
sub := keys[:idx]
sort.Strings(sub)
for _, name := range sub {
writeString(name)
// TODO: Improve this.
v := settings[name]
bytes, err := json.Marshal(v)
if err != nil {
writeString(fmt.Sprintf("%+v", v))
} else {
writeBytes(bytes)
}
}
}
writeSettings(in.Settings)
}
// fields that determine the rule state
writeString(r.UID)
writeString(r.Name)
writeString(string(r.Provenance))
sum.writeString(r.UID)
sum.writeString(r.Name)
sum.writeString(string(r.Provenance))
for _, integration := range r.Integrations {
writeIntegration(integration)
}
return fmt.Sprintf("%016x", sum.Sum64())
return sum.String()
}
func writeSettings(f fingerprint, m map[string]any) {
if len(m) == 0 {
f.writeBytes(nil)
return
}
keysIter := maps.Keys(m)
keys := slices.Collect(keysIter)
sort.Strings(keys)
for _, key := range keys {
f.writeString(key)
switch v := m[key].(type) {
case string:
f.writeString(v)
case bool:
f.writeBool(v)
case float64: // unmarshalling to map[string]any represents all numbers as float64
f.writeFloat64(v)
case map[string]any:
writeSettings(f, v)
default:
b, err := json.Marshal(v)
if err != nil {
f.writeString(fmt.Sprintf("%+v", v))
}
f.writeBytes(b)
}
}
}
+38 -27
View File
@@ -6,6 +6,7 @@ import (
alertingNotify "github.com/grafana/alerting/notify"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/services/ngalert/notifier/channels_config"
)
@@ -45,20 +46,19 @@ func TestReceiver_EncryptDecrypt(t *testing.T) {
secrets, err := channels_config.GetSecretKeysForContactPointType(integrationType)
assert.NoError(t, err)
for _, key := range secrets {
if val, ok := encrypted.Settings[key]; ok {
if s, isString := val.(string); isString {
encryptedVal, err := encryptFn(s)
assert.NoError(t, err)
encrypted.SecureSettings[key] = encryptedVal
delete(encrypted.Settings, key)
}
val, ok, err := extractField(encrypted.Settings, NewIntegrationFieldPath(key))
assert.NoError(t, err)
if ok {
encryptedVal, err := encryptFn(val)
assert.NoError(t, err)
encrypted.SecureSettings[key] = encryptedVal
}
}
testIntegration := decrypedIntegration.Clone()
err = testIntegration.Encrypt(encryptFn)
assert.NoError(t, err)
assert.Equal(t, encrypted, testIntegration)
require.Equal(t, encrypted, testIntegration)
err = testIntegration.Decrypt(decryptnFn)
assert.NoError(t, err)
@@ -80,12 +80,14 @@ func TestIntegration_Redact(t *testing.T) {
secrets, err := channels_config.GetSecretKeysForContactPointType(integrationType)
assert.NoError(t, err)
for _, key := range secrets {
if val, ok := expected.Settings[key]; ok {
if s, isString := val.(string); isString && s != "" {
expected.Settings[key] = redactFn(s)
err := setField(expected.Settings, NewIntegrationFieldPath(key), func(current any) any {
if s, isString := current.(string); isString && s != "" {
delete(expected.SecureSettings, key)
return redactFn(s)
}
}
return current
}, true)
require.NoError(t, err)
}
validIntegration.Redact(redactFn)
@@ -244,9 +246,9 @@ func TestIntegrationConfig(t *testing.T) {
for field := range config.Fields {
_, isSecret := allSecrets[field]
assert.Equal(t, isSecret, config.IsSecureField(field))
assert.Equalf(t, isSecret, config.IsSecureField(NewIntegrationFieldPath(field)), "field '%s' is expected to be secret", field)
}
assert.False(t, config.IsSecureField("__--**unknown_field**--__"))
assert.False(t, config.IsSecureField(IntegrationFieldPath{"__--**unknown_field**--__"}))
})
}
@@ -263,11 +265,13 @@ func TestIntegration_SecureFields(t *testing.T) {
t.Run("contains SecureSettings", func(t *testing.T) {
validIntegration := IntegrationGen(IntegrationMuts.WithValidConfig(integrationType))()
expected := make(map[string]bool, len(validIntegration.SecureSettings))
for field := range validIntegration.Config.Fields {
if validIntegration.Config.IsSecureField(field) {
expected[field] = true
validIntegration.SecureSettings[field] = "test"
delete(validIntegration.Settings, field)
for _, path := range validIntegration.Config.GetSecretFields() {
if validIntegration.Config.IsSecureField(path) {
expected[path.String()] = true
validIntegration.SecureSettings[path.String()] = "test"
_, _, err := extractField(validIntegration.Settings, path)
require.NoError(t, err)
continue
}
}
assert.Equal(t, expected, validIntegration.SecureFields())
@@ -276,11 +280,13 @@ func TestIntegration_SecureFields(t *testing.T) {
t.Run("contains secret Settings not in SecureSettings", func(t *testing.T) {
validIntegration := IntegrationGen(IntegrationMuts.WithValidConfig(integrationType))()
expected := make(map[string]bool, len(validIntegration.SecureSettings))
for field := range validIntegration.Config.Fields {
if validIntegration.Config.IsSecureField(field) {
expected[field] = true
validIntegration.Settings[field] = "test"
delete(validIntegration.SecureSettings, field)
for _, path := range validIntegration.Config.GetSecretFields() {
if validIntegration.Config.IsSecureField(path) {
expected[path.String()] = true
assert.NoError(t, setField(validIntegration.Settings, path, func(current any) any {
return "test"
}, false))
delete(validIntegration.SecureSettings, path.String())
}
}
assert.Equal(t, expected, validIntegration.SecureFields())
@@ -306,8 +312,13 @@ func TestReceiver_Fingerprint(t *testing.T) {
))()
baseReceiver.Integrations[0].UID = "stable UID"
baseReceiver.Integrations[0].DisableResolveMessage = true
baseReceiver.Integrations[0].SecureSettings = map[string]string{"test2": "test2"}
baseReceiver.Integrations[0].Settings["broken"] = broken{f1: "this"} // Add a broken type to ensure it is stable in the fingerprint.
baseReceiver.Integrations[0].SecureSettings = map[string]string{"test2": "test2", "test3": "test223", "test1": "rest22"}
baseReceiver.Integrations[0].Settings["broken"] = broken{f1: "this"} // Add a broken type to ensure it is stable in the fingerprint.
baseReceiver.Integrations[0].Settings["sub-map"] = map[string]any{
"setting": "value",
"something": 123,
"data": []string{"test"},
} // Add a broken type to ensure it is stable in the fingerprint.
baseReceiver.Integrations[0].Config = IntegrationConfig{Type: baseReceiver.Integrations[0].Config.Type} // Remove all fields except Type.
completelyDifferentReceiver := ReceiverGen(ReceiverMuts.WithName("test receiver2"), ReceiverMuts.WithIntegrations(
@@ -320,7 +331,7 @@ func TestReceiver_Fingerprint(t *testing.T) {
completelyDifferentReceiver.Integrations[0].Config = IntegrationConfig{Type: completelyDifferentReceiver.Integrations[0].Config.Type} // Remove all fields except Type.
t.Run("stable across code changes", func(t *testing.T) {
expectedFingerprint := "ae141b582965f4f5" // If this is a valid fingerprint generation change, update the expected value.
expectedFingerprint := "a3402fdaba03030c" // If this is a valid fingerprint generation change, update the expected value.
assert.Equal(t, expectedFingerprint, baseReceiver.Fingerprint())
})
t.Run("stable across clones", func(t *testing.T) {