public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Alerting: Add compressed protobuf-based alert state storage (#99193)

Browse Source
This commit is contained in:
Alexander Akhmetov
2025-01-27 18:47:33 +01:00
committed by GitHub
parent 6edd4f5a7c
commit cb43f4b696
24 changed files with 1437 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/ngalert/store/instance_database_test.go
+271 -82
View File
@@ -1,57 +1,188 @@
package store_test
import (
"bytes"
"context"
"fmt"
"testing"
"time"
"github.com/golang/snappy"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/models"
pb "github.com/grafana/grafana/pkg/services/ngalert/store/proto/v1"
"github.com/grafana/grafana/pkg/services/ngalert/tests"
"github.com/grafana/grafana/pkg/util"
)
const baseIntervalSeconds = 10
func BenchmarkAlertInstanceOperations(b *testing.B) {
b.StopTimer()
func TestIntegration_CompressedAlertRuleStateOperations(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
ctx := context.Background()
_, dbstore := tests.SetupTestEnv(b, baseIntervalSeconds)
ng, dbstore := tests.SetupTestEnv(
t,
baseIntervalSeconds,
tests.WithFeatureToggles(
featuremgmt.WithFeatures(featuremgmt.FlagAlertingSaveStateCompressed),
),
)
const mainOrgID int64 = 1
alertRule := tests.CreateTestAlertRule(b, ctx, dbstore, 60, mainOrgID)
alertRule1 := tests.CreateTestAlertRule(t, ctx, dbstore, 60, mainOrgID)
orgID := alertRule1.OrgID
alertRule2 := tests.CreateTestAlertRule(t, ctx, dbstore, 60, mainOrgID)
require.Equal(t, orgID, alertRule2.OrgID)
// Create some instances to write down and then delete.
count := 10_003
instances := make([]models.AlertInstance, 0, count)
keys := make([]models.AlertInstanceKey, 0, count)
for i := 0; i < count; i++ {
labels := models.InstanceLabels{"test": fmt.Sprint(i)}
_, labelsHash, _ := labels.StringAndHash()
instance := models.AlertInstance{
AlertInstanceKey: models.AlertInstanceKey{
RuleOrgID: alertRule.OrgID,
RuleUID: alertRule.UID,
LabelsHash: labelsHash,
tests := []struct {
name string
setupInstances func() []models.AlertInstance
listQuery *models.ListAlertInstancesQuery
validate func(t *testing.T, alerts []*models.AlertInstance)
}{
{
name: "can save and read alert rule state",
setupInstances: func() []models.AlertInstance {
return []models.AlertInstance{
createAlertInstance(alertRule1.OrgID, alertRule1.UID, "labelsHash1", string(models.InstanceStateError), models.InstanceStateFiring),
}
},
CurrentState: models.InstanceStateFiring,
CurrentReason: string(models.InstanceStateError),
Labels: labels,
}
instances = append(instances, instance)
keys = append(keys, instance.AlertInstanceKey)
listQuery: &models.ListAlertInstancesQuery{
RuleOrgID: alertRule1.OrgID,
RuleUID: alertRule1.UID,
},
validate: func(t *testing.T, alerts []*models.AlertInstance) {
require.Len(t, alerts, 1)
require.Equal(t, "labelsHash1", alerts[0].LabelsHash)
},
},
{
name: "can save and read alert rule state with multiple instances",
setupInstances: func() []models.AlertInstance {
return []models.AlertInstance{
createAlertInstance(alertRule1.OrgID, alertRule1.UID, "hash1", "", models.InstanceStateFiring),
createAlertInstance(alertRule1.OrgID, alertRule1.UID, "hash2", "", models.InstanceStateFiring),
}
},
listQuery: &models.ListAlertInstancesQuery{
RuleOrgID: alertRule1.OrgID,
RuleUID: alertRule1.UID,
},
validate: func(t *testing.T, alerts []*models.AlertInstance) {
require.Len(t, alerts, 2)
containsHash(t, alerts, "hash1")
containsHash(t, alerts, "hash2")
},
},
}
b.StartTimer()
for i := 0; i < b.N; i++ {
for _, instance := range instances {
_ = dbstore.SaveAlertInstance(ctx, instance)
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
instances := tc.setupInstances()
err := ng.InstanceStore.SaveAlertInstancesForRule(ctx, alertRule1.GetKeyWithGroup(), instances)
require.NoError(t, err)
alerts, err := ng.InstanceStore.ListAlertInstances(ctx, tc.listQuery)
require.NoError(t, err)
tc.validate(t, alerts)
})
}
}
func TestIntegration_CompressedAlertRuleStateOperations_NoNormalState(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
ctx := context.Background()
ng, dbstore := tests.SetupTestEnv(
t,
baseIntervalSeconds,
tests.WithFeatureToggles(
featuremgmt.WithFeatures(
featuremgmt.FlagAlertingSaveStateCompressed,
featuremgmt.FlagAlertingNoNormalState,
),
),
)
const mainOrgID int64 = 1
alertRule1 := tests.CreateTestAlertRule(t, ctx, dbstore, 60, mainOrgID)
orgID := alertRule1.OrgID
tests := []struct {
name string
setupInstances func() []models.AlertInstance
listQuery *models.ListAlertInstancesQuery
validate func(t *testing.T, alerts []*models.AlertInstance)
}{
{
name: "should ignore Normal state with no reason if feature flag is enabled",
setupInstances: func() []models.AlertInstance {
return []models.AlertInstance{
createAlertInstance(orgID, util.GenerateShortUID(), util.GenerateShortUID(), "", models.InstanceStateNormal),
createAlertInstance(orgID, util.GenerateShortUID(), "errorHash", "error", models.InstanceStateNormal),
}
},
listQuery: &models.ListAlertInstancesQuery{
RuleOrgID: orgID,
},
validate: func(t *testing.T, alerts []*models.AlertInstance) {
require.Len(t, alerts, 1)
containsHash(t, alerts, "errorHash")
for _, instance := range alerts {
if instance.CurrentState == models.InstanceStateNormal && instance.CurrentReason == "" {
require.Fail(t, "List operation expected to return all states except Normal but the result contains Normal states")
}
}
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
instances := tc.setupInstances()
err := ng.InstanceStore.SaveAlertInstancesForRule(ctx, alertRule1.GetKeyWithGroup(), instances)
require.NoError(t, err)
alerts, err := ng.InstanceStore.ListAlertInstances(ctx, tc.listQuery)
require.NoError(t, err)
tc.validate(t, alerts)
})
}
}
// containsHash is a helper function to check if an instance with
// a given labels hash exists in the list of alert instances.
func containsHash(t *testing.T, instances []*models.AlertInstance, hash string) {
t.Helper()
for _, i := range instances {
if i.LabelsHash == hash {
return
}
_ = dbstore.DeleteAlertInstances(ctx, keys...)
}
require.Fail(t, fmt.Sprintf("%v does not contain an instance with hash %s", instances, hash))
}
func createAlertInstance(orgID int64, ruleUID, labelsHash, reason string, state models.InstanceStateType) models.AlertInstance {
return models.AlertInstance{
AlertInstanceKey: models.AlertInstanceKey{
RuleOrgID: orgID,
RuleUID: ruleUID,
LabelsHash: labelsHash,
},
CurrentState: state,
CurrentReason: reason,
Labels: models.InstanceLabels{"label1": "value1"},
}
}
@@ -60,20 +191,10 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
t.Skip("skipping integration test")
}
ctx := context.Background()
_, dbstore := tests.SetupTestEnv(t, baseIntervalSeconds)
ng, dbstore := tests.SetupTestEnv(t, baseIntervalSeconds)
const mainOrgID int64 = 1
containsHash := func(t *testing.T, instances []*models.AlertInstance, hash string) {
t.Helper()
for _, i := range instances {
if i.LabelsHash == hash {
return
}
}
require.Fail(t, "%v does not contain an instance with hash %s", instances, hash)
}
alertRule1 := tests.CreateTestAlertRule(t, ctx, dbstore, 60, mainOrgID)
orgID := alertRule1.OrgID
@@ -99,14 +220,14 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
CurrentReason: string(models.InstanceStateError),
Labels: labels,
}
err := dbstore.SaveAlertInstance(ctx, instance)
err := ng.InstanceStore.SaveAlertInstance(ctx, instance)
require.NoError(t, err)
listCmd := &models.ListAlertInstancesQuery{
RuleOrgID: instance.RuleOrgID,
RuleUID: instance.RuleUID,
}
alerts, err := dbstore.ListAlertInstances(ctx, listCmd)
alerts, err := ng.InstanceStore.ListAlertInstances(ctx, listCmd)
require.NoError(t, err)
require.Len(t, alerts, 1)
@@ -128,7 +249,7 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
CurrentState: models.InstanceStateNormal,
Labels: labels,
}
err := dbstore.SaveAlertInstance(ctx, instance)
err := ng.InstanceStore.SaveAlertInstance(ctx, instance)
require.NoError(t, err)
listCmd := &models.ListAlertInstancesQuery{
@@ -136,7 +257,7 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
RuleUID: instance.RuleUID,
}
alerts, err := dbstore.ListAlertInstances(ctx, listCmd)
alerts, err := ng.InstanceStore.ListAlertInstances(ctx, listCmd)
require.NoError(t, err)
require.Len(t, alerts, 1)
@@ -158,7 +279,7 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
Labels: labels,
}
err := dbstore.SaveAlertInstance(ctx, instance1)
err := ng.InstanceStore.SaveAlertInstance(ctx, instance1)
require.NoError(t, err)
labels = models.InstanceLabels{"test": "testValue2"}
@@ -172,7 +293,7 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
CurrentState: models.InstanceStateFiring,
Labels: labels,
}
err = dbstore.SaveAlertInstance(ctx, instance2)
err = ng.InstanceStore.SaveAlertInstance(ctx, instance2)
require.NoError(t, err)
listQuery := &models.ListAlertInstancesQuery{
@@ -180,7 +301,7 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
RuleUID: instance1.RuleUID,
}
alerts, err := dbstore.ListAlertInstances(ctx, listQuery)
alerts, err := ng.InstanceStore.ListAlertInstances(ctx, listQuery)
require.NoError(t, err)
require.Len(t, alerts, 2)
@@ -191,13 +312,21 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
RuleOrgID: orgID,
}
alerts, err := dbstore.ListAlertInstances(ctx, listQuery)
alerts, err := ng.InstanceStore.ListAlertInstances(ctx, listQuery)
require.NoError(t, err)
require.Len(t, alerts, 4)
})
t.Run("should ignore Normal state with no reason if feature flag is enabled", func(t *testing.T) {
ng, _ := tests.SetupTestEnv(
t,
baseIntervalSeconds,
tests.WithFeatureToggles(
featuremgmt.WithFeatures(featuremgmt.FlagAlertingNoNormalState),
),
)
labels := models.InstanceLabels{"test": util.GenerateShortUID()}
instance1 := models.AlertInstance{
AlertInstanceKey: models.AlertInstanceKey{
@@ -219,27 +348,16 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
CurrentReason: models.StateReasonError,
Labels: labels,
}
err := dbstore.SaveAlertInstance(ctx, instance1)
err := ng.InstanceStore.SaveAlertInstance(ctx, instance1)
require.NoError(t, err)
err = dbstore.SaveAlertInstance(ctx, instance2)
err = ng.InstanceStore.SaveAlertInstance(ctx, instance2)
require.NoError(t, err)
listQuery := &models.ListAlertInstancesQuery{
RuleOrgID: orgID,
}
alerts, err := dbstore.ListAlertInstances(ctx, listQuery)
require.NoError(t, err)
containsHash(t, alerts, instance1.LabelsHash)
f := dbstore.FeatureToggles
dbstore.FeatureToggles = featuremgmt.WithFeatures(featuremgmt.FlagAlertingNoNormalState)
t.Cleanup(func() {
dbstore.FeatureToggles = f
})
alerts, err = dbstore.ListAlertInstances(ctx, listQuery)
alerts, err := ng.InstanceStore.ListAlertInstances(ctx, listQuery)
require.NoError(t, err)
containsHash(t, alerts, instance2.LabelsHash)
@@ -264,7 +382,7 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
Labels: labels,
}
err := dbstore.SaveAlertInstance(ctx, instance1)
err := ng.InstanceStore.SaveAlertInstance(ctx, instance1)
require.NoError(t, err)
instance2 := models.AlertInstance{
@@ -276,7 +394,7 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
CurrentState: models.InstanceStateNormal,
Labels: instance1.Labels,
}
err = dbstore.SaveAlertInstance(ctx, instance2)
err = ng.InstanceStore.SaveAlertInstance(ctx, instance2)
require.NoError(t, err)
listQuery := &models.ListAlertInstancesQuery{
@@ -284,7 +402,7 @@ func TestIntegrationAlertInstanceOperations(t *testing.T) {
RuleUID: alertRule4.UID,
}
alerts, err := dbstore.ListAlertInstances(ctx, listQuery)
alerts, err := ng.InstanceStore.ListAlertInstances(ctx, listQuery)
require.NoError(t, err)
require.Len(t, alerts, 1)
@@ -300,7 +418,7 @@ func TestIntegrationFullSync(t *testing.T) {
batchSize := 1
ctx := context.Background()
_, dbstore := tests.SetupTestEnv(t, baseIntervalSeconds)
ng, _ := tests.SetupTestEnv(t, baseIntervalSeconds)
orgID := int64(1)
@@ -312,10 +430,10 @@ func TestIntegrationFullSync(t *testing.T) {
}
t.Run("Should do a proper full sync", func(t *testing.T) {
err := dbstore.FullSync(ctx, instances, batchSize)
err := ng.InstanceStore.FullSync(ctx, instances, batchSize)
require.NoError(t, err)
res, err := dbstore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
res, err := ng.InstanceStore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
RuleOrgID: orgID,
})
require.NoError(t, err)
@@ -335,10 +453,10 @@ func TestIntegrationFullSync(t *testing.T) {
})
t.Run("Should remove non existing entries on sync", func(t *testing.T) {
err := dbstore.FullSync(ctx, instances[1:], batchSize)
err := ng.InstanceStore.FullSync(ctx, instances[1:], batchSize)
require.NoError(t, err)
res, err := dbstore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
res, err := ng.InstanceStore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
RuleOrgID: orgID,
})
require.NoError(t, err)
@@ -352,10 +470,10 @@ func TestIntegrationFullSync(t *testing.T) {
t.Run("Should add new entries on sync", func(t *testing.T) {
newRuleUID := "y"
err := dbstore.FullSync(ctx, append(instances, generateTestAlertInstance(orgID, newRuleUID)), batchSize)
err := ng.InstanceStore.FullSync(ctx, append(instances, generateTestAlertInstance(orgID, newRuleUID)), batchSize)
require.NoError(t, err)
res, err := dbstore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
res, err := ng.InstanceStore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
RuleOrgID: orgID,
})
require.NoError(t, err)
@@ -377,10 +495,10 @@ func TestIntegrationFullSync(t *testing.T) {
t.Run("Should save all instances when batch size is bigger than 1", func(t *testing.T) {
batchSize = 2
newRuleUID := "y"
err := dbstore.FullSync(ctx, append(instances, generateTestAlertInstance(orgID, newRuleUID)), batchSize)
err := ng.InstanceStore.FullSync(ctx, append(instances, generateTestAlertInstance(orgID, newRuleUID)), batchSize)
require.NoError(t, err)
res, err := dbstore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
res, err := ng.InstanceStore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
RuleOrgID: orgID,
})
require.NoError(t, err)
@@ -405,16 +523,16 @@ func TestIntegrationFullSync(t *testing.T) {
generateTestAlertInstance(orgID, "preexisting-1"),
generateTestAlertInstance(orgID, "preexisting-2"),
}
err := dbstore.FullSync(ctx, initialInstances, 5)
err := ng.InstanceStore.FullSync(ctx, initialInstances, 5)
require.NoError(t, err)
// Now call FullSync with no instances. According to the code, this should return nil
// and should not delete anything in the table.
err = dbstore.FullSync(ctx, []models.AlertInstance{}, 5)
err = ng.InstanceStore.FullSync(ctx, []models.AlertInstance{}, 5)
require.NoError(t, err)
// Check that the previously inserted instances are still present.
res, err := dbstore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
res, err := ng.InstanceStore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
RuleOrgID: orgID,
})
require.NoError(t, err)
@@ -441,11 +559,11 @@ func TestIntegrationFullSync(t *testing.T) {
// Make the invalid instance actually invalid
invalidInstance.AlertInstanceKey.RuleUID = ""
err := dbstore.FullSync(ctx, []models.AlertInstance{validInstance, invalidInstance}, 2)
err := ng.InstanceStore.FullSync(ctx, []models.AlertInstance{validInstance, invalidInstance}, 2)
require.NoError(t, err)
// Only the valid instance should be saved.
res, err := dbstore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
res, err := ng.InstanceStore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
RuleOrgID: orgID,
})
require.NoError(t, err)
@@ -460,10 +578,10 @@ func TestIntegrationFullSync(t *testing.T) {
generateTestAlertInstance(orgID, "batch-test2"),
}
err := dbstore.FullSync(ctx, smallSet, 100)
err := ng.InstanceStore.FullSync(ctx, smallSet, 100)
require.NoError(t, err)
res, err := dbstore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
res, err := ng.InstanceStore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
RuleOrgID: orgID,
})
require.NoError(t, err)
@@ -483,7 +601,7 @@ func TestIntegrationFullSync(t *testing.T) {
t.Run("Should handle a large set of instances with a moderate batchSize", func(t *testing.T) {
// Clear everything first.
err := dbstore.FullSync(ctx, []models.AlertInstance{}, 1)
err := ng.InstanceStore.FullSync(ctx, []models.AlertInstance{}, 1)
require.NoError(t, err)
largeCount := 300
@@ -492,10 +610,10 @@ func TestIntegrationFullSync(t *testing.T) {
largeSet[i] = generateTestAlertInstance(orgID, fmt.Sprintf("large-%d", i))
}
err = dbstore.FullSync(ctx, largeSet, 50)
err = ng.InstanceStore.FullSync(ctx, largeSet, 50)
require.NoError(t, err)
res, err := dbstore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
res, err := ng.InstanceStore.ListAlertInstances(ctx, &models.ListAlertInstancesQuery{
RuleOrgID: orgID,
})
require.NoError(t, err)
@@ -503,6 +621,77 @@ func TestIntegrationFullSync(t *testing.T) {
})
}
func TestIntegration_ProtoInstanceDBStore_VerifyCompressedData(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
ctx := context.Background()
ng, dbstore := tests.SetupTestEnv(
t,
baseIntervalSeconds,
tests.WithFeatureToggles(
featuremgmt.WithFeatures(
featuremgmt.FlagAlertingSaveStateCompressed,
),
),
)
alertRule := tests.CreateTestAlertRule(t, ctx, dbstore, 60, 1)
labelsHash := "hash1"
reason := "reason"
state := models.InstanceStateFiring
instances := []models.AlertInstance{
createAlertInstance(alertRule.OrgID, alertRule.UID, labelsHash, reason, state),
}
err := ng.InstanceStore.SaveAlertInstancesForRule(ctx, alertRule.GetKeyWithGroup(), instances)
require.NoError(t, err)
// Query raw data from the database
type compressedRow struct {
OrgID int64 `xorm:"org_id"`
RuleUID string `xorm:"rule_uid"`
Data []byte `xorm:"data"`
}
var rawData compressedRow
err = dbstore.SQLStore.WithDbSession(ctx, func(sess *db.Session) error {
_, err := sess.SQL("SELECT * FROM alert_rule_state").Get(&rawData)
return err
})
require.NoError(t, err)
// Decompress and compare
require.NotNil(t, rawData)
decompressedInstances, err := decompressAlertInstances(rawData.Data)
require.NoError(t, err)
require.Len(t, decompressedInstances, 1)
require.Equal(t, instances[0].LabelsHash, decompressedInstances[0].LabelsHash)
require.Equal(t, string(instances[0].CurrentState), decompressedInstances[0].CurrentState)
require.Equal(t, instances[0].CurrentReason, decompressedInstances[0].CurrentReason)
}
func decompressAlertInstances(compressed []byte) ([]*pb.AlertInstance, error) {
if len(compressed) == 0 {
return nil, nil
}
reader := snappy.NewReader(bytes.NewReader(compressed))
var b bytes.Buffer
if _, err := b.ReadFrom(reader); err != nil {
return nil, fmt.Errorf("failed to read compressed data: %w", err)
}
var instances pb.AlertInstances
if err := proto.Unmarshal(b.Bytes(), &instances); err != nil {
return nil, fmt.Errorf("failed to unmarshal protobuf: %w", err)
}
return instances.Instances, nil
}
func generateTestAlertInstance(orgID int64, ruleID string) models.AlertInstance {
return models.AlertInstance{
AlertInstanceKey: models.AlertInstanceKey{