cloudwatch: recover/handle panics when executing queries

(cherry picked from commit 879aed7d06)

pkg/tsdb/cloudwatch/cloudwatch.go

@@ -126,6 +126,18 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryCo
 		}
 
 		eg.Go(func() error {
+			defer func() {
+				if err := recover(); err != nil {
+					plog.Error("Execute Query Panic", "error", err, "stack", log.Stack(1))
+					if theErr, ok := err.(error); ok {
+						resultChan <- &tsdb.QueryResult{
+							RefId: query.RefId,
+							Error: theErr,
+						}
+					}
+				}
+			}()
+
 			queryRes, err := e.executeQuery(ectx, query, queryContext)
 			if ae, ok := err.(awserr.Error); ok && ae.Code() == "500" {
 				return err
@@ -146,6 +158,17 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryCo
 		for region, getMetricDataQuery := range getMetricDataQueries {
 			q := getMetricDataQuery
 			eg.Go(func() error {
+				defer func() {
+					if err := recover(); err != nil {
+						plog.Error("Execute Get Metric Data Query Panic", "error", err, "stack", log.Stack(1))
+						if theErr, ok := err.(error); ok {
+							resultChan <- &tsdb.QueryResult{
+								Error: theErr,
+							}
+						}
+					}
+				}()
+
 				queryResponses, err := e.executeGetMetricDataQuery(ectx, region, q, queryContext)
 				if ae, ok := err.(awserr.Error); ok && ae.Code() == "500" {
 					return err