diff --git a/docs/sources/datasources/aws-cloudwatch/query-editor/index.md b/docs/sources/datasources/aws-cloudwatch/query-editor/index.md index 9288a750742..1ca4e78b341 100644 --- a/docs/sources/datasources/aws-cloudwatch/query-editor/index.md +++ b/docs/sources/datasources/aws-cloudwatch/query-editor/index.md @@ -270,7 +270,17 @@ Click **View in CloudWatch console** to interactively view, search, and analyze ### Query Log groups with OpenSearch SQL -When querying log groups with OpenSearch SQL, you **must** explicitly state the log group identifier or ARN in the `FROM` clause: +When querying log groups with OpenSearch SQL, you can use the `$__logGroups` macro to automatically reference log groups selected in the query editor's log group selector. This is the recommended approach as it allows you to manage log groups through the UI. + +```sql +SELECT window.start, COUNT(*) AS exceptionCount +FROM $__logGroups +WHERE `@message` LIKE '%Exception%' +``` + +The `$__logGroups` macro expands to the proper `logGroups(logGroupIdentifier: [...])` syntax with the log groups you've selected in the UI. + +Alternatively, you can manually specify a single log group directly in the `FROM` clause: ```sql SELECT window.start, COUNT(*) AS exceptionCount @@ -278,7 +288,7 @@ FROM `log_group` WHERE `@message` LIKE '%Exception%' ``` -or, when querying multiple log groups: +or, when querying multiple log groups you **must** use the `logGroups(logGroupIdentifier: [...])` syntax: ```sql SELECT window.start, COUNT(*) AS exceptionCount @@ -286,6 +296,8 @@ FROM `logGroups( logGroupIdentifier: ['LogGroup1', 'LogGroup2'])` WHERE `@message` LIKE '%Exception%' ``` +To reference log groups in a monitoring account, use ARNs instead of LogGroup names. + You can also write queries returning time series data by using the [`stats` command](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CWL_Insights-Visualizing-Log-Data.html). When making `stats` queries in [Explore](ref:explore), ensure you are in Metrics Explore mode. diff --git a/pkg/tsdb/cloudwatch/log_actions.go b/pkg/tsdb/cloudwatch/log_actions.go index 60ebad06442..9e7ca734cea 100644 --- a/pkg/tsdb/cloudwatch/log_actions.go +++ b/pkg/tsdb/cloudwatch/log_actions.go @@ -30,6 +30,7 @@ const ( defaultLogGroupLimit = int32(50) logIdentifierInternal = "__log__grafana_internal__" logStreamIdentifierInternal = "__logstream__grafana_internal__" + logGroupsMacro = "$__logGroups" ) type AWSError struct { @@ -189,6 +190,47 @@ func (ds *DataSource) executeStartQuery(ctx context.Context, logsClient models.C logsQuery.QueryLanguage = &cwli } + region := logsQuery.Region + if region == "" || region == defaultRegion { + region = ds.Settings.Region + } + + useARN := false + if len(logsQuery.LogGroups) > 0 && features.IsEnabled(ctx, features.FlagCloudWatchCrossAccountQuerying) && region != "" { + isMonitoringAccount, err := ds.isMonitoringAccount(ctx, region) + if err != nil { + ds.logger.FromContext(ctx).Debug("failed to determine monitoring account status", "err", err) + } else { + useARN = isMonitoringAccount + } + } + + var logGroupIdentifiers []string + if len(logsQuery.LogGroups) > 0 { + // Log queries should use ARNs when querying a monitoring account because log group names are not unique across accounts. + if useARN { + for _, lg := range logsQuery.LogGroups { + if lg.Arn != "" { + // The startQuery api does not support arns with a trailing * so we need to remove it + logGroupIdentifiers = append(logGroupIdentifiers, strings.TrimSuffix(lg.Arn, "*")) + } + } + } else { + // deduplicate log group names because we only deduplicate log groups by their ARNs instead of their names when the query is created + seen := make(map[string]struct{}, len(logsQuery.LogGroups)) + for _, lg := range logsQuery.LogGroups { + if lg.Name == "" { + continue + } + if _, exists := seen[lg.Name]; exists { + continue + } + seen[lg.Name] = struct{}{} + logGroupIdentifiers = append(logGroupIdentifiers, lg.Name) + } + } + } + finalQueryString := logsQuery.QueryString // Only for CWLI queries // The fields @log and @logStream are always included in the results of a user's query @@ -200,6 +242,21 @@ func (ds *DataSource) executeStartQuery(ctx context.Context, logsClient models.C logStreamIdentifierInternal + "|" + logsQuery.QueryString } + // Expand $__logGroups macro for SQL queries + if *logsQuery.QueryLanguage == dataquery.LogsQueryLanguageSQL { + if strings.Contains(finalQueryString, logGroupsMacro) { + if len(logGroupIdentifiers) == 0 { + return nil, backend.DownstreamError(fmt.Errorf("query contains %s but no log groups are selected", logGroupsMacro)) + } + quoted := make([]string, len(logGroupIdentifiers)) + for i, id := range logGroupIdentifiers { + quoted[i] = fmt.Sprintf("'%s'", id) + } + replacement := fmt.Sprintf("`logGroups(logGroupIdentifier: [%s])`", strings.Join(quoted, ", ")) + finalQueryString = strings.Replace(finalQueryString, logGroupsMacro, replacement, 1) + } + } + startQueryInput := &cloudwatchlogs.StartQueryInput{ StartTime: aws.Int64(startTime.Unix()), // Usually grafana time range allows only second precision, but you can create ranges with milliseconds @@ -213,47 +270,13 @@ func (ds *DataSource) executeStartQuery(ctx context.Context, logsClient models.C // log group identifiers can be left out if the query is an SQL query if *logsQuery.QueryLanguage != dataquery.LogsQueryLanguageSQL { - useLogGroupIdentifiers := false - logGroupsFromQuery := len(logsQuery.LogGroups) > 0 - if logGroupsFromQuery && features.IsEnabled(ctx, features.FlagCloudWatchCrossAccountQuerying) { - region := logsQuery.Region - if region == "" || region == defaultRegion { - region = ds.Settings.Region - } - if region != "" { - isMonitoringAccount, err := ds.isMonitoringAccount(ctx, region) - if err != nil { - ds.logger.FromContext(ctx).Debug("failed to determine monitoring account status", "err", err) - } else if isMonitoringAccount { - // monitoring accounts require querying by log group identifiers because log group names are not unique across accounts. - var logGroupIdentifiers []string - for _, lg := range logsQuery.LogGroups { - // due to a bug in the startQuery api, we remove * from the arn, otherwise it throws an error - arn := strings.TrimSuffix(lg.Arn, "*") - logGroupIdentifiers = append(logGroupIdentifiers, arn) - } - startQueryInput.LogGroupIdentifiers = logGroupIdentifiers - useLogGroupIdentifiers = true - } - } - } - - if !useLogGroupIdentifiers { + if useARN { + startQueryInput.LogGroupIdentifiers = logGroupIdentifiers + } else { // even though logsQuery.LogGroupNames is deprecated, we still need to support it for backwards compatibility and alert queries startQueryInput.LogGroupNames = append([]string(nil), logsQuery.LogGroupNames...) - if len(startQueryInput.LogGroupNames) == 0 && logGroupsFromQuery { - // deduplicate log group names because we only deduplicate log groups by their ARNs instead of their names when the query is created - seenLogGroupNames := make(map[string]struct{}, len(logsQuery.LogGroups)) - for _, lg := range logsQuery.LogGroups { - if lg.Name == "" { - continue - } - if _, exists := seenLogGroupNames[lg.Name]; exists { - continue - } - seenLogGroupNames[lg.Name] = struct{}{} - startQueryInput.LogGroupNames = append(startQueryInput.LogGroupNames, lg.Name) - } + if len(startQueryInput.LogGroupNames) == 0 && len(logGroupIdentifiers) > 0 { + startQueryInput.LogGroupNames = logGroupIdentifiers } } } diff --git a/pkg/tsdb/cloudwatch/log_actions_test.go b/pkg/tsdb/cloudwatch/log_actions_test.go index 142b1d0e24d..377f42d74d5 100644 --- a/pkg/tsdb/cloudwatch/log_actions_test.go +++ b/pkg/tsdb/cloudwatch/log_actions_test.go @@ -873,6 +873,204 @@ func TestQuery_GetQueryResults(t *testing.T) { }, resp) } +func Test_expandLogGroupsMacro(t *testing.T) { + origNewCWLogsClient := NewCWLogsClient + t.Cleanup(func() { + NewCWLogsClient = origNewCWLogsClient + }) + + var cli fakeCWLogsClient + + NewCWLogsClient = func(cfg aws.Config) models.CWLogsClient { + return &cli + } + + t.Run("expands $__logGroups macro with log group names when not a monitoring account", func(t *testing.T) { + cli = fakeCWLogsClient{} + ds := newTestDatasource() + + _, err := ds.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}}, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Unix(0, 0), To: time.Unix(1, 0)}, + JSON: json.RawMessage(`{ + "type": "logAction", + "subtype": "StartQuery", + "queryLanguage": "SQL", + "queryString":"SELECT * FROM $__logGroups", + "logGroups":[{"arn": "arn:aws:logs:us-east-1:123456789012:log-group:group1", "name": "group1"}, {"arn": "arn:aws:logs:us-east-1:123456789012:log-group:group2", "name": "group2"}] + }`), + }, + }, + }) + + assert.NoError(t, err) + require.Len(t, cli.calls.startQuery, 1) + assert.Equal(t, "SELECT * FROM `logGroups(logGroupIdentifier: ['group1', 'group2'])`", *cli.calls.startQuery[0].QueryString) + }) + + t.Run("expands $__logGroups macro with ARNs when monitoring account", func(t *testing.T) { + cli = fakeCWLogsClient{} + ds := newTestDatasource(func(ds *DataSource) { + ds.monitoringAccountCache.Store("us-east-1", true) + }) + + _, err := ds.QueryData(contextWithFeaturesEnabled(features.FlagCloudWatchCrossAccountQuerying), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}}, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Unix(0, 0), To: time.Unix(1, 0)}, + JSON: json.RawMessage(`{ + "type": "logAction", + "subtype": "StartQuery", + "queryLanguage": "SQL", + "queryString":"SELECT * FROM $__logGroups", + "logGroups":[{"arn": "arn:aws:logs:us-east-1:123456789012:log-group:group1", "name": "group1"}, {"arn": "arn:aws:logs:us-east-1:123456789012:log-group:group2", "name": "group2"}], + "region": "us-east-1" + }`), + }, + }, + }) + + assert.NoError(t, err) + require.Len(t, cli.calls.startQuery, 1) + assert.Equal(t, "SELECT * FROM `logGroups(logGroupIdentifier: ['arn:aws:logs:us-east-1:123456789012:log-group:group1', 'arn:aws:logs:us-east-1:123456789012:log-group:group2'])`", *cli.calls.startQuery[0].QueryString) + }) + + t.Run("strips trailing * from ARNs when expanding macro", func(t *testing.T) { + cli = fakeCWLogsClient{} + ds := newTestDatasource(func(ds *DataSource) { + ds.monitoringAccountCache.Store("us-east-1", true) + }) + + _, err := ds.QueryData(contextWithFeaturesEnabled(features.FlagCloudWatchCrossAccountQuerying), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}}, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Unix(0, 0), To: time.Unix(1, 0)}, + JSON: json.RawMessage(`{ + "type": "logAction", + "subtype": "StartQuery", + "queryLanguage": "SQL", + "queryString":"SELECT * FROM $__logGroups", + "logGroups":[{"arn": "arn:aws:logs:us-east-1:123456789012:log-group:group1*", "name": "group1"}], + "region": "us-east-1" + }`), + }, + }, + }) + + assert.NoError(t, err) + require.Len(t, cli.calls.startQuery, 1) + assert.Equal(t, "SELECT * FROM `logGroups(logGroupIdentifier: ['arn:aws:logs:us-east-1:123456789012:log-group:group1'])`", *cli.calls.startQuery[0].QueryString) + }) + + t.Run("returns error when $__logGroups macro is used but no log groups are selected", func(t *testing.T) { + cli = fakeCWLogsClient{} + ds := newTestDatasource() + + resp, err := ds.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}}, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Unix(0, 0), To: time.Unix(1, 0)}, + JSON: json.RawMessage(`{ + "type": "logAction", + "subtype": "StartQuery", + "queryLanguage": "SQL", + "queryString":"SELECT * FROM $__logGroups" + }`), + }, + }, + }) + + assert.NoError(t, err) + assert.Contains(t, resp.Responses["A"].Error.Error(), "query contains $__logGroups but no log groups are selected") + }) + + t.Run("does not expand macro when query does not contain $__logGroups", func(t *testing.T) { + cli = fakeCWLogsClient{} + ds := newTestDatasource() + + _, err := ds.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}}, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Unix(0, 0), To: time.Unix(1, 0)}, + JSON: json.RawMessage(`{ + "type": "logAction", + "subtype": "StartQuery", + "queryLanguage": "SQL", + "queryString":"SELECT * FROM ` + "`logGroups(logGroupIdentifier: ['my-log-group'])`" + `" + }`), + }, + }, + }) + + assert.NoError(t, err) + require.Len(t, cli.calls.startQuery, 1) + assert.Equal(t, "SELECT * FROM `logGroups(logGroupIdentifier: ['my-log-group'])`", *cli.calls.startQuery[0].QueryString) + }) + + t.Run("does not expand macro for non-SQL query languages", func(t *testing.T) { + cli = fakeCWLogsClient{} + ds := newTestDatasource() + + _, err := ds.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}}, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Unix(0, 0), To: time.Unix(1, 0)}, + JSON: json.RawMessage(`{ + "type": "logAction", + "subtype": "StartQuery", + "queryLanguage": "CWLI", + "queryString":"fields @message | $__logGroups", + "logGroups":[{"arn": "arn:aws:logs:us-east-1:123456789012:log-group:group1", "name": "group1"}] + }`), + }, + }, + }) + + assert.NoError(t, err) + require.Len(t, cli.calls.startQuery, 1) + assert.Contains(t, *cli.calls.startQuery[0].QueryString, "$__logGroups") + }) + + t.Run("expands macro with single log group", func(t *testing.T) { + cli = fakeCWLogsClient{} + ds := newTestDatasource() + + _, err := ds.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}}, + Queries: []backend.DataQuery{ + { + RefID: "A", + TimeRange: backend.TimeRange{From: time.Unix(0, 0), To: time.Unix(1, 0)}, + JSON: json.RawMessage(`{ + "type": "logAction", + "subtype": "StartQuery", + "queryLanguage": "SQL", + "queryString":"SELECT * FROM $__logGroups", + "logGroups":[{"arn": "arn:aws:logs:us-east-1:123456789012:log-group:single-group", "name": "single-group"}] + }`), + }, + }, + }) + + assert.NoError(t, err) + require.Len(t, cli.calls.startQuery, 1) + assert.Equal(t, "SELECT * FROM `logGroups(logGroupIdentifier: ['single-group'])`", *cli.calls.startQuery[0].QueryString) + }) +} + func TestGroupResponseFrame(t *testing.T) { t.Run("Doesn't group results without time field", func(t *testing.T) { frame := data.NewFrameOfFieldTypes("test", 0, data.FieldTypeString, data.FieldTypeInt32) diff --git a/public/app/plugins/datasource/cloudwatch/defaultQueries.ts b/public/app/plugins/datasource/cloudwatch/defaultQueries.ts index c8935405f6d..f4a73491aba 100644 --- a/public/app/plugins/datasource/cloudwatch/defaultQueries.ts +++ b/public/app/plugins/datasource/cloudwatch/defaultQueries.ts @@ -36,7 +36,7 @@ export const DEFAULT_ANNOTATIONS_QUERY: Omit export const DEFAULT_CWLI_QUERY_STRING = 'fields @timestamp, @message |\nsort @timestamp desc |\nlimit 20'; export const DEFAULT_PPL_QUERY_STRING = 'fields `@timestamp`, `@message`\n| sort - `@timestamp`\n| head 25s'; export const DEFAULT_SQL_QUERY_STRING = - 'SELECT `@timestamp`, `@message`\nFROM `log_group`\nORDER BY `@timestamp` DESC\nLIMIT 25;'; + 'SELECT `@timestamp`, `@message`\nFROM $__logGroups\nORDER BY `@timestamp` DESC\nLIMIT 25;'; export const getDefaultLogsQuery = ( defaultLogGroups?: LogGroup[], diff --git a/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/completion/CompletionItemProvider.test.ts b/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/completion/CompletionItemProvider.test.ts index 606527f4711..d4d83599c28 100644 --- a/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/completion/CompletionItemProvider.test.ts +++ b/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/completion/CompletionItemProvider.test.ts @@ -97,14 +97,22 @@ describe('LogsSQLCompletionItemProvider', () => { const suggestions = await getSuggestions(singleLineFullQuery.query, { lineNumber: 1, column: 103 }); const suggestionLabels = suggestions.map((s) => s.label); expect(suggestionLabels).toEqual( - expect.arrayContaining([FROM, `${FROM} \`logGroups(logGroupIdentifier: [...])\``, CASE, ...ALL_FUNCTIONS]) + expect.arrayContaining([ + FROM, + `${FROM} $__logGroups`, + `${FROM} \`logGroups(logGroupIdentifier: [...])\``, + CASE, + ...ALL_FUNCTIONS, + ]) ); }); - it('returns logGroups suggestion after from keyword', async () => { + it('returns logGroups and $__logGroups suggestion after from keyword', async () => { const suggestions = await getSuggestions(singleLineFullQuery.query, { lineNumber: 1, column: 108 }); const suggestionLabels = suggestions.map((s) => s.label); - expect(suggestionLabels).toEqual(expect.arrayContaining(['`logGroups(logGroupIdentifier: [...])`'])); + expect(suggestionLabels).toEqual( + expect.arrayContaining(['$__logGroups', '`logGroups(logGroupIdentifier: [...])`']) + ); }); it('returns where, having, limit, group by, order by, and join suggestions after from arguments', async () => { diff --git a/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/completion/CompletionItemProvider.ts b/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/completion/CompletionItemProvider.ts index cdba699b7a0..0d1311cf36d 100644 --- a/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/completion/CompletionItemProvider.ts +++ b/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/completion/CompletionItemProvider.ts @@ -142,6 +142,12 @@ export class LogsSQLCompletionItemProvider extends CompletionItemProvider { command: TRIGGER_SUGGEST, sortText: CompletionItemPriority.MediumHigh, }); + addSuggestion(`${FROM} $__logGroups`, { + insertText: `${FROM} $__logGroups`, + kind: monaco.languages.CompletionItemKind.Snippet, + sortText: CompletionItemPriority.High, + detail: 'Use selected log groups from the selector', + }); addSuggestion(`${FROM} \`logGroups(logGroupIdentifier: [...])\``, { insertText: `${FROM} \`logGroups(logGroupIdentifier: [$0])\``, insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, @@ -152,6 +158,12 @@ export class LogsSQLCompletionItemProvider extends CompletionItemProvider { break; case SuggestionKind.AfterFromKeyword: + addSuggestion('$__logGroups', { + insertText: '$__logGroups', + kind: monaco.languages.CompletionItemKind.Variable, + sortText: CompletionItemPriority.High, + detail: 'Expands to selected log groups', + }); addSuggestion('`logGroups(logGroupIdentifier: [...])`', { insertText: '`logGroups(logGroupIdentifier: [$0])`', insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet, diff --git a/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/language.ts b/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/language.ts index 419684e9748..6a0fea55678 100644 --- a/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/language.ts +++ b/public/app/plugins/datasource/cloudwatch/language/cloudwatch-logs-sql/language.ts @@ -488,6 +488,7 @@ export const language: CloudWatchLanguage = { root: [ { include: '@comments' }, { include: '@whitespace' }, + { include: '@macros' }, { include: '@customParams' }, { include: '@numbers' }, { include: '@binaries' }, @@ -519,6 +520,7 @@ export const language: CloudWatchLanguage = { [/\*\//, { token: 'comment.quote', next: '@pop' }], [/./, 'comment'], ], + macros: [[/\$__[a-zA-Z0-9_]+/, 'type']], customParams: [ [/\${[A-Za-z0-9._-]*}/, 'variable'], [/\@\@{[A-Za-z0-9._-]*}/, 'variable'],