public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cloudwatch Logs: Set Alerting timeout to datasource config's logsTimeout (#72611)

Browse Source
This commit is contained in:
Ida Štambuk
2023-08-03 19:35:30 +02:00
committed by GitHub
parent b1ef145442
commit abff6e20e9
6 changed files with 211 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/cloudwatch/log_sync_query.go
+19 -13
View File
@@ -14,14 +14,16 @@ import (
"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
)
const (
alertMaxAttempts = 8
alertPollPeriod = time.Second
)
const initialAlertPollPeriod = time.Second
var executeSyncLogQuery = func(ctx context.Context, e *cloudWatchExecutor, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
resp := backend.NewQueryDataResponse()
instance, err := e.getInstance(ctx, req.PluginContext)
if err != nil {
return nil, err
}
for _, q := range req.Queries {
var logsQuery models.LogsQuery
err := json.Unmarshal(q.JSON, &logsQuery)
@@ -36,10 +38,6 @@ var executeSyncLogQuery = func(ctx context.Context, e *cloudWatchExecutor, req *
region := logsQuery.Region
if logsQuery.Region == "" || region == defaultRegion {
instance, err := e.getInstance(ctx, req.PluginContext)
if err != nil {
return nil, err
}
logsQuery.Region = instance.Settings.Region
}
@@ -48,7 +46,7 @@ var executeSyncLogQuery = func(ctx context.Context, e *cloudWatchExecutor, req *
return nil, err
}
getQueryResultsOutput, err := e.syncQuery(ctx, logsClient, q, logsQuery)
getQueryResultsOutput, err := e.syncQuery(ctx, logsClient, q, logsQuery, instance.Settings.LogsTimeout.Duration)
if err != nil {
return nil, err
}
@@ -82,7 +80,7 @@ var executeSyncLogQuery = func(ctx context.Context, e *cloudWatchExecutor, req *
}
func (e *cloudWatchExecutor) syncQuery(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI,
queryContext backend.DataQuery, logsQuery models.LogsQuery) (*cloudwatchlogs.GetQueryResultsOutput, error) {
queryContext backend.DataQuery, logsQuery models.LogsQuery, logsTimeout time.Duration) (*cloudwatchlogs.GetQueryResultsOutput, error) {
startQueryOutput, err := e.executeStartQuery(ctx, logsClient, logsQuery, queryContext.TimeRange)
if err != nil {
return nil, err
@@ -95,7 +93,15 @@ func (e *cloudWatchExecutor) syncQuery(ctx context.Context, logsClient cloudwatc
QueryId: *startQueryOutput.QueryId,
}
ticker := time.NewTicker(alertPollPeriod)
/*
Unlike many other data sources, with Cloudwatch Logs query requests don't receive the results as the response
to the query, but rather an ID is first returned. Following this, a client is expected to send requests along
with the ID until the status of the query is complete, receiving (possibly partial) results each time. For
queries made via dashboards and Explore, the logic of making these repeated queries is handled on the
frontend, but because alerts and expressions are executed on the backend the logic needs to be reimplemented here.
*/
ticker := time.NewTicker(initialAlertPollPeriod)
defer ticker.Stop()
attemptCount := 1
@@ -107,8 +113,8 @@ func (e *cloudWatchExecutor) syncQuery(ctx context.Context, logsClient cloudwatc
if isTerminated(*res.Status) {
return res, err
}
if attemptCount >= alertMaxAttempts {
return res, fmt.Errorf("fetching of query results exceeded max number of attempts")
if time.Duration(attemptCount)*time.Second >= logsTimeout {
return res, fmt.Errorf("time to fetch query results exceeded logs timeout")
}
attemptCount++