diff --git a/docs/sources/datasources/azuremonitor.md b/docs/sources/datasources/azuremonitor.md index 5a41790a5d8..b9518b87733 100644 --- a/docs/sources/datasources/azuremonitor.md +++ b/docs/sources/datasources/azuremonitor.md @@ -69,6 +69,7 @@ In the query editor for a panel, after choosing your Azure Monitor data source, - `Application Insights` - `Logs` - `Insights Analytics` +- `Azure Resource Graph` The query editor changes depending on which one you pick. Metrics is the default. @@ -413,3 +414,65 @@ Application Insights and Insights Analytics are two ways to query the same Azure > **Note** In Grafana 8.0, Application Insights and Insights Analytics will be deprecated and made read-only in favor of querying this data through Metrics and Logs. Existing queries will continue to work, but you cannot edit them. To prepare for this upcoming change, Application Insights queries can now be made in Metrics, under the "microsoft.insights/components" Namespace. Insights Analytics queries cannot be made within Logs with KQL at this time. + +## Query Azure Resource Graph service + +Azure Resource Graph (ARG) is a service in Azure that is designed to extend Azure Resource Management by providing efficient and performant resource exploration with the ability to query at scale across a given set of subscriptions so that you can effectively govern your environment. By querying ARG, you can query resources with complex filtering, iteratively explore resources based on governance requirements, and assess the impact of applying policies in a vast cloud environment. + +{{< docs-imagebox img="/img/docs/azure-monitor/azure-resource-graph-8-0.png" class="docs-image--no-shadow" caption="Azure Resource Graph Editor" >}} + +### Table queries + +Queries are written in the [Kusto Query Language](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/). An Azure Resource Graph query is formatted as table data. + +If your credentials give you access to multiple subscriptions, then you can choose multiple subscriptions before entering queries. + +### Sort results by resource properties + +Here is an example query that returns any type of resource, but only the name, type, and location properties: + +```kusto +Resources +| project name, type, location +| order by name asc +``` + +The query uses `order by` to sort the properties by the `name` property in ascending (asc) order. You can change what property to sort by and the order (`asc` or `desc`). The query uses `project` to show the listed properties in the results. You can add or remove properties. + +### Query resources with complex filtering + +Filtering for Azure resources with a tag name of `Environment` that have a value of `Internal`. You can change these to any desired tag key and value. The `=~` in the `type` match tells Resource Graph to be case insensitive. You can project by other properties or add/ remove more. + +The tag key is case sensitive. `Environment` and `environment` will give different results. For example, a query that returns a list of resources with a specified tag value: + +```kusto +Resources +| where tags.environment=~'internal' +| project name +``` + +### Group and aggregate the values by property + +You can also use `summarize` and `count` to define how to group and aggregate the values by property. For example, returning count of healthy, unhealthy, and not applicable resources per recommendation.: + +```kusto +securityresources +| where type == 'microsoft.security/assessments' +| extend resourceId=id, +    recommendationId=name, +    resourceType=type, +    recommendationName=properties.displayName, +    source=properties.resourceDetails.Source, +    recommendationState=properties.status.code, +    description=properties.metadata.description, +    assessmentType=properties.metadata.assessmentType, +    remediationDescription=properties.metadata.remediationDescription, +    policyDefinitionId=properties.metadata.policyDefinitionId, +    implementationEffort=properties.metadata.implementationEffort, +    recommendationSeverity=properties.metadata.severity, +    category=properties.metadata.categories, +    userImpact=properties.metadata.userImpact, +    threats=properties.metadata.threats, +    portalLink=properties.links.azurePortal +| summarize numberOfResources=count(resourceId) by tostring(recommendationName), tostring(recommendationState) +``` diff --git a/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go b/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go index 59b47983b6d..9357b0b114f 100644 --- a/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go +++ b/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go @@ -86,7 +86,7 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(queries []plugins.DataSubQuer resultFormat := azureLogAnalyticsTarget.ResultFormat if resultFormat == "" { - resultFormat = "time_series" + resultFormat = timeSeries } urlComponents := map[string]string{} @@ -173,7 +173,7 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A return queryResultErrorWithExecuted(err) } - frame, err := LogTableToFrame(t) + frame, err := ResponseTableToFrame(t) if err != nil { return queryResultErrorWithExecuted(err) } @@ -187,7 +187,7 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A azlog.Warn("failed to add custom metadata to azure log analytics response", err) } - if query.ResultFormat == "time_series" { + if query.ResultFormat == timeSeries { tsSchema := frame.TimeSeriesSchema() if tsSchema.Type == data.TimeSeriesTypeLong { wideFrame, err := data.LongToWide(frame, nil) @@ -259,7 +259,7 @@ func (e *AzureLogAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourceP // GetPrimaryResultTable returns the first table in the response named "PrimaryResult", or an // error if there is no table by that name. -func (ar *AzureLogAnalyticsResponse) GetPrimaryResultTable() (*AzureLogAnalyticsTable, error) { +func (ar *AzureLogAnalyticsResponse) GetPrimaryResultTable() (*AzureResponseTable, error) { for _, t := range ar.Tables { if t.Name == "PrimaryResult" { return &t, nil diff --git a/pkg/tsdb/azuremonitor/azure-log-analytics-datasource_test.go b/pkg/tsdb/azuremonitor/azure-log-analytics-datasource_test.go index d5401d6a011..d89ff4f9d30 100644 --- a/pkg/tsdb/azuremonitor/azure-log-analytics-datasource_test.go +++ b/pkg/tsdb/azuremonitor/azure-log-analytics-datasource_test.go @@ -41,7 +41,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) { "azureLogAnalytics": map[string]interface{}{ "workspace": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", "query": "query=Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer", - "resultFormat": "time_series", + "resultFormat": timeSeries, }, }), RefID: "A", @@ -50,12 +50,12 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) { azureLogAnalyticsQueries: []*AzureLogAnalyticsQuery{ { RefID: "A", - ResultFormat: "time_series", + ResultFormat: timeSeries, URL: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/query", Model: simplejson.NewFromAny(map[string]interface{}{ "azureLogAnalytics": map[string]interface{}{ "query": "query=Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer", - "resultFormat": "time_series", + "resultFormat": timeSeries, "workspace": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", }, }), diff --git a/pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go b/pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go new file mode 100644 index 00000000000..2f86a791f3e --- /dev/null +++ b/pkg/tsdb/azuremonitor/azure-resource-graph-datasource.go @@ -0,0 +1,271 @@ +package azuremonitor + +import ( + "bytes" + + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "path" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/api/pluginproxy" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util/errutil" + "github.com/opentracing/opentracing-go" + "golang.org/x/net/context/ctxhttp" +) + +// AzureResourceGraphDatasource calls the Azure Resource Graph API's +type AzureResourceGraphDatasource struct { + httpClient *http.Client + dsInfo *models.DataSource + pluginManager plugins.Manager + cfg *setting.Cfg +} + +// AzureResourceGraphQuery is the query request that is built from the saved values for +// from the UI +type AzureResourceGraphQuery struct { + RefID string + ResultFormat string + URL string + Model *simplejson.Json + InterpolatedQuery string +} + +const argAPIVersion = "2018-09-01-preview" +const argQueryProviderName = "/providers/Microsoft.ResourceGraph/resources" + +// executeTimeSeriesQuery does the following: +// 1. builds the AzureMonitor url and querystring for each query +// 2. executes each query by calling the Azure Monitor API +// 3. parses the responses for each query into the timeseries format +func (e *AzureResourceGraphDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []plugins.DataSubQuery, + timeRange plugins.DataTimeRange) (backend.QueryDataResponse, error) { + result := backend.QueryDataResponse{ + Responses: map[string]backend.DataResponse{}, + } + + queries, err := e.buildQueries(originalQueries, timeRange) + if err != nil { + return backend.QueryDataResponse{}, err + } + + for _, query := range queries { + result.Responses[query.RefID] = e.executeQuery(ctx, query, timeRange) + } + + return result, nil +} + +func (e *AzureResourceGraphDatasource) buildQueries(queries []plugins.DataSubQuery, + timeRange plugins.DataTimeRange) ([]*AzureResourceGraphQuery, error) { + var azureResourceGraphQueries []*AzureResourceGraphQuery + + for _, query := range queries { + queryBytes, err := query.Model.Encode() + if err != nil { + return nil, fmt.Errorf("failed to re-encode the Azure Resource Graph query into JSON: %w", err) + } + + queryJSONModel := argJSONQuery{} + err = json.Unmarshal(queryBytes, &queryJSONModel) + if err != nil { + return nil, fmt.Errorf("failed to decode the Azure Resource Graph query object from JSON: %w", err) + } + + azureResourceGraphTarget := queryJSONModel.AzureResourceGraph + azlog.Debug("AzureResourceGraph", "target", azureResourceGraphTarget) + + resultFormat := azureResourceGraphTarget.ResultFormat + if resultFormat == "" { + resultFormat = "table" + } + + interpolatedQuery, err := KqlInterpolate(query, timeRange, azureResourceGraphTarget.Query) + + if err != nil { + return nil, err + } + + azureResourceGraphQueries = append(azureResourceGraphQueries, &AzureResourceGraphQuery{ + RefID: query.RefID, + ResultFormat: resultFormat, + Model: query.Model, + InterpolatedQuery: interpolatedQuery, + }) + } + + return azureResourceGraphQueries, nil +} + +func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *AzureResourceGraphQuery, + timeRange plugins.DataTimeRange) backend.DataResponse { + queryResult := backend.DataResponse{} + + params := url.Values{} + params.Add("api-version", argAPIVersion) + + queryResultErrorWithExecuted := func(err error) backend.DataResponse { + queryResult = backend.DataResponse{Error: err} + frames := data.Frames{ + &data.Frame{ + RefID: query.RefID, + Meta: &data.FrameMeta{ + ExecutedQueryString: query.InterpolatedQuery, + }, + }, + } + queryResult.Frames = frames + return queryResult + } + + reqBody, err := json.Marshal(map[string]interface{}{ + "subscriptions": query.Model.Get("subscriptions").MustStringArray(), + "query": query.InterpolatedQuery, + }) + + if err != nil { + queryResult.Error = err + return queryResult + } + + req, err := e.createRequest(ctx, e.dsInfo, reqBody) + + if err != nil { + queryResult.Error = err + return queryResult + } + + req.URL.Path = path.Join(req.URL.Path, argQueryProviderName) + req.URL.RawQuery = params.Encode() + + span, ctx := opentracing.StartSpanFromContext(ctx, "azure resource graph query") + span.SetTag("interpolated_query", query.InterpolatedQuery) + span.SetTag("from", timeRange.From) + span.SetTag("until", timeRange.To) + span.SetTag("datasource_id", e.dsInfo.Id) + span.SetTag("org_id", e.dsInfo.OrgId) + + defer span.Finish() + + if err := opentracing.GlobalTracer().Inject( + span.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header)); err != nil { + return queryResultErrorWithExecuted(err) + } + + azlog.Debug("AzureResourceGraph", "Request ApiURL", req.URL.String()) + res, err := ctxhttp.Do(ctx, e.httpClient, req) + if err != nil { + return queryResultErrorWithExecuted(err) + } + + argResponse, err := e.unmarshalResponse(res) + if err != nil { + return queryResultErrorWithExecuted(err) + } + + frame, err := ResponseTableToFrame(&argResponse.Data) + if err != nil { + return queryResultErrorWithExecuted(err) + } + if frame.Meta == nil { + frame.Meta = &data.FrameMeta{} + } + frame.Meta.ExecutedQueryString = req.URL.RawQuery + + queryResult.Frames = data.Frames{frame} + return queryResult +} + +func (e *AzureResourceGraphDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource, reqBody []byte) (*http.Request, error) { + u, err := url.Parse(dsInfo.Url) + if err != nil { + return nil, err + } + u.Path = path.Join(u.Path, "render") + req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewBuffer(reqBody)) + if err != nil { + azlog.Debug("Failed to create request", "error", err) + return nil, errutil.Wrap("failed to create request", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) + + // find plugin + plugin := e.pluginManager.GetDataSource(dsInfo.Type) + if plugin == nil { + return nil, errors.New("unable to find datasource plugin Azure Monitor") + } + cloudName := dsInfo.JsonData.Get("cloudName").MustString("azuremonitor") + + argRoute, proxypass, err := e.getPluginRoute(plugin, cloudName) + if err != nil { + return nil, err + } + pluginproxy.ApplyRoute(ctx, req, proxypass, argRoute, dsInfo, e.cfg) + + return req, nil +} + +func (e *AzureResourceGraphDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, cloudName string) ( + *plugins.AppPluginRoute, string, error) { + pluginRouteName := "azureresourcegraph" + + switch cloudName { + case "chinaazuremonitor": + pluginRouteName = "chinaazureresourcegraph" + case "govazuremonitor": + pluginRouteName = "govazureresourcegraph" + } + + var argRoute *plugins.AppPluginRoute + for _, route := range plugin.Routes { + if route.Path == pluginRouteName { + argRoute = route + break + } + } + + return argRoute, pluginRouteName, nil +} + +func (e *AzureResourceGraphDatasource) unmarshalResponse(res *http.Response) (AzureResourceGraphResponse, error) { + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return AzureResourceGraphResponse{}, err + } + defer func() { + if err := res.Body.Close(); err != nil { + azlog.Warn("Failed to close response body", "err", err) + } + }() + + if res.StatusCode/100 != 2 { + azlog.Debug("Request failed", "status", res.Status, "body", string(body)) + return AzureResourceGraphResponse{}, fmt.Errorf("request failed, status: %s, body: %s", res.Status, string(body)) + } + + var data AzureResourceGraphResponse + d := json.NewDecoder(bytes.NewReader(body)) + d.UseNumber() + err = d.Decode(&data) + if err != nil { + azlog.Debug("Failed to unmarshal azure resource graph response", "error", err, "status", res.Status, "body", string(body)) + return AzureResourceGraphResponse{}, err + } + + return data, nil +} diff --git a/pkg/tsdb/azuremonitor/azure-resource-graph-datasource_test.go b/pkg/tsdb/azuremonitor/azure-resource-graph-datasource_test.go new file mode 100644 index 00000000000..27b7c597d19 --- /dev/null +++ b/pkg/tsdb/azuremonitor/azure-resource-graph-datasource_test.go @@ -0,0 +1,75 @@ +package azuremonitor + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/stretchr/testify/require" +) + +func TestBuildingAzureResourceGraphQueries(t *testing.T) { + datasource := &AzureResourceGraphDatasource{} + fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) + + tests := []struct { + name string + queryModel []plugins.DataSubQuery + timeRange plugins.DataTimeRange + azureResourceGraphQueries []*AzureResourceGraphQuery + Err require.ErrorAssertionFunc + }{ + { + name: "Query with macros should be interpolated", + timeRange: plugins.DataTimeRange{ + From: fmt.Sprintf("%v", fromStart.Unix()*1000), + To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000), + }, + queryModel: []plugins.DataSubQuery{ + { + DataSource: &models.DataSource{ + JsonData: simplejson.NewFromAny(map[string]interface{}{}), + }, + Model: simplejson.NewFromAny(map[string]interface{}{ + "queryType": "Azure Resource Graph", + "azureResourceGraph": map[string]interface{}{ + "query": "resources | where $__contains(name,'res1','res2')", + "resultFormat": "table", + }, + }), + RefID: "A", + }, + }, + azureResourceGraphQueries: []*AzureResourceGraphQuery{ + { + RefID: "A", + ResultFormat: "table", + URL: "", + Model: simplejson.NewFromAny(map[string]interface{}{ + "azureResourceGraph": map[string]interface{}{ + "query": "resources | where $__contains(name,'res1','res2')", + "resultFormat": "table", + }, + }), + InterpolatedQuery: "resources | where ['name'] in ('res1','res2')", + }, + }, + Err: require.NoError, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + queries, err := datasource.buildQueries(tt.queryModel, tt.timeRange) + tt.Err(t, err) + if diff := cmp.Diff(tt.azureResourceGraphQueries, queries, cmpopts.IgnoreUnexported(simplejson.Json{})); diff != "" { + t.Errorf("Result mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/tsdb/azuremonitor/azure-log-analytics-table-frame.go b/pkg/tsdb/azuremonitor/azure-response-table-frame.go similarity index 88% rename from pkg/tsdb/azuremonitor/azure-log-analytics-table-frame.go rename to pkg/tsdb/azuremonitor/azure-response-table-frame.go index 65add176731..cf9ff5b5c48 100644 --- a/pkg/tsdb/azuremonitor/azure-log-analytics-table-frame.go +++ b/pkg/tsdb/azuremonitor/azure-response-table-frame.go @@ -10,8 +10,8 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" ) -// LogTableToFrame converts an AzureLogAnalyticsTable to a data.Frame. -func LogTableToFrame(table *AzureLogAnalyticsTable) (*data.Frame, error) { +// ResponseTableToFrame converts an AzureResponseTable to a data.Frame. +func ResponseTableToFrame(table *AzureResponseTable) (*data.Frame, error) { converterFrame, err := converterFrameForTable(table) if err != nil { return nil, err @@ -27,7 +27,7 @@ func LogTableToFrame(table *AzureLogAnalyticsTable) (*data.Frame, error) { return converterFrame.Frame, nil } -func converterFrameForTable(t *AzureLogAnalyticsTable) (*data.FrameInputConverter, error) { +func converterFrameForTable(t *AzureResponseTable) (*data.FrameInputConverter, error) { converters := []data.FieldConverter{} colNames := make([]string, len(t.Columns)) colTypes := make([]string, len(t.Columns)) // for metadata @@ -64,12 +64,14 @@ var converterMap = map[string]data.FieldConverter{ "guid": stringConverter, "timespan": stringConverter, "dynamic": stringConverter, + "object": objectToStringConverter, "datetime": timeConverter, "int": intConverter, "long": longConverter, "real": realConverter, "bool": boolConverter, "decimal": decimalConverter, + "integer": intConverter, } var stringConverter = data.FieldConverter{ @@ -88,6 +90,26 @@ var stringConverter = data.FieldConverter{ }, } +var objectToStringConverter = data.FieldConverter{ + OutputFieldType: data.FieldTypeNullableString, + Converter: func(kustoValue interface{}) (interface{}, error) { + var output *string + if kustoValue == nil { + return output, nil + } + + data, err := json.Marshal(kustoValue) + if err != nil { + fmt.Printf("failed to marshal column value: %s", err) + } + + asString := string(data) + output = &asString + + return output, nil + }, +} + var timeConverter = data.FieldConverter{ OutputFieldType: data.FieldTypeNullableTime, Converter: func(v interface{}) (interface{}, error) { diff --git a/pkg/tsdb/azuremonitor/azure-log-analytics-table-frame_test.go b/pkg/tsdb/azuremonitor/azure-response-table-frame_test.go similarity index 96% rename from pkg/tsdb/azuremonitor/azure-log-analytics-table-frame_test.go rename to pkg/tsdb/azuremonitor/azure-response-table-frame_test.go index eb5c3995972..3892d167550 100644 --- a/pkg/tsdb/azuremonitor/azure-log-analytics-table-frame_test.go +++ b/pkg/tsdb/azuremonitor/azure-response-table-frame_test.go @@ -114,10 +114,11 @@ func TestLogTableToFrame(t *testing.T) { data.NewField("XReal", nil, []*float64{pointer.Float64(1.797693134862315708145274237317043567981e+308)}), data.NewField("XTimeSpan", nil, []*string{pointer.String("00:00:00.0000001")}), data.NewField("XDecimal", nil, []*float64{pointer.Float64(79228162514264337593543950335)}), + data.NewField("XObject", nil, []*string{pointer.String(`"{\"person\": \"Daniel\", \"cats\": 23, \"diagnosis\": \"cat problem\"}"`)}), ) frame.Meta = &data.FrameMeta{ Custom: &LogAnalyticsMeta{ColumnTypes: []string{"bool", "string", "datetime", - "dynamic", "guid", "int", "long", "real", "timespan", "decimal"}}, + "dynamic", "guid", "int", "long", "real", "timespan", "decimal", "object"}}, } return frame }, @@ -142,7 +143,7 @@ func TestLogTableToFrame(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { res := loadLogAnalyticsTestFileWithNumber(t, tt.testFile) - frame, err := LogTableToFrame(&res.Tables[0]) + frame, err := ResponseTableToFrame(&res.Tables[0]) require.NoError(t, err) if diff := cmp.Diff(tt.expectedFrame(), frame, data.FrameTestCompareOptions()...); diff != "" { diff --git a/pkg/tsdb/azuremonitor/azuremonitor.go b/pkg/tsdb/azuremonitor/azuremonitor.go index 87a6c48c2e0..4d6a1b0331a 100644 --- a/pkg/tsdb/azuremonitor/azuremonitor.go +++ b/pkg/tsdb/azuremonitor/azuremonitor.go @@ -13,6 +13,8 @@ import ( "github.com/grafana/grafana/pkg/setting" ) +const timeSeries = "time_series" + var ( azlog = log.New("tsdb.azuremonitor") legendKeyFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`) @@ -72,6 +74,7 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat var applicationInsightsQueries []plugins.DataSubQuery var azureLogAnalyticsQueries []plugins.DataSubQuery var insightsAnalyticsQueries []plugins.DataSubQuery + var azureResourceGraphQueries []plugins.DataSubQuery for _, query := range tsdbQuery.Queries { queryType := query.Model.Get("queryType").MustString("") @@ -85,6 +88,8 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat azureLogAnalyticsQueries = append(azureLogAnalyticsQueries, query) case "Insights Analytics": insightsAnalyticsQueries = append(insightsAnalyticsQueries, query) + case "Azure Resource Graph": + azureResourceGraphQueries = append(azureResourceGraphQueries, query) default: return plugins.DataResponse{}, fmt.Errorf("alerting not supported for %q", queryType) } @@ -118,6 +123,12 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat cfg: e.cfg, } + argDatasource := &AzureResourceGraphDatasource{ + httpClient: e.httpClient, + dsInfo: e.dsInfo, + pluginManager: e.pluginManager, + } + azResult, err := azDatasource.executeTimeSeriesQuery(ctx, azureMonitorQueries, *tsdbQuery.TimeRange) if err != nil { return plugins.DataResponse{}, err @@ -138,6 +149,11 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat return plugins.DataResponse{}, err } + argResult, err := argDatasource.executeTimeSeriesQuery(ctx, azureResourceGraphQueries, *tsdbQuery.TimeRange) + if err != nil { + return plugins.DataResponse{}, err + } + for k, v := range aiResult.Results { azResult.Results[k] = v } @@ -150,5 +166,9 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat azResult.Results[k] = v } + for k, v := range argResult.Responses { + azResult.Results[k] = plugins.DataQueryResult{Error: v.Error, Dataframes: plugins.NewDecodedDataFrames(v.Frames)} + } + return azResult, nil } diff --git a/pkg/tsdb/azuremonitor/insights-analytics-datasource.go b/pkg/tsdb/azuremonitor/insights-analytics-datasource.go index 85b92031a86..b9a770b06fd 100644 --- a/pkg/tsdb/azuremonitor/insights-analytics-datasource.go +++ b/pkg/tsdb/azuremonitor/insights-analytics-datasource.go @@ -163,12 +163,12 @@ func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *I return queryResultError(err) } - frame, err := LogTableToFrame(t) + frame, err := ResponseTableToFrame(t) if err != nil { return queryResultError(err) } - if query.ResultFormat == "time_series" { + if query.ResultFormat == timeSeries { tsSchema := frame.TimeSeriesSchema() if tsSchema.Type == data.TimeSeriesTypeLong { wideFrame, err := data.LongToWide(frame, nil) diff --git a/pkg/tsdb/azuremonitor/testdata/loganalytics/7-log-analytics-all-types-table.json b/pkg/tsdb/azuremonitor/testdata/loganalytics/7-log-analytics-all-types-table.json index 676c1a15102..d25a87b6859 100644 --- a/pkg/tsdb/azuremonitor/testdata/loganalytics/7-log-analytics-all-types-table.json +++ b/pkg/tsdb/azuremonitor/testdata/loganalytics/7-log-analytics-all-types-table.json @@ -42,6 +42,10 @@ { "name":"XDecimal", "type":"decimal" + }, + { + "name":"XObject", + "type":"object" } ], "rows": [ @@ -55,10 +59,11 @@ 9223372036854775807, 1.7976931348623157e+308, "00:00:00.0000001", - "79228162514264337593543950335" + "79228162514264337593543950335", + "{\"person\": \"Daniel\", \"cats\": 23, \"diagnosis\": \"cat problem\"}" ] ] } ] } - + diff --git a/pkg/tsdb/azuremonitor/types.go b/pkg/tsdb/azuremonitor/types.go index 63294e52f69..77d35d3678f 100644 --- a/pkg/tsdb/azuremonitor/types.go +++ b/pkg/tsdb/azuremonitor/types.go @@ -56,11 +56,16 @@ type AzureMonitorResponse struct { // AzureLogAnalyticsResponse is the json response object from the Azure Log Analytics API. type AzureLogAnalyticsResponse struct { - Tables []AzureLogAnalyticsTable `json:"tables"` + Tables []AzureResponseTable `json:"tables"` } -// AzureLogAnalyticsTable is the table format for Log Analytics responses -type AzureLogAnalyticsTable struct { +// AzureResourceGraphResponse is the json response object from the Azure Resource Graph Analytics API. +type AzureResourceGraphResponse struct { + Data AzureResponseTable `json:"data"` +} + +// AzureResponseTable is the table format for Azure responses +type AzureResponseTable struct { Name string `json:"name"` Columns []struct { Name string `json:"name"` @@ -137,6 +142,13 @@ type logJSONQuery struct { } `json:"azureLogAnalytics"` } +type argJSONQuery struct { + AzureResourceGraph struct { + Query string `json:"query"` + ResultFormat string `json:"resultFormat"` + } `json:"azureResourceGraph"` +} + // InsightsDimensions will unmarshal from a JSON string, or an array of strings, // into a string array. This exists to support an older query format which is updated // when a user saves the query or it is sent from the front end, but may not be when diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/__mocks__/query.ts b/public/app/plugins/datasource/grafana-azure-monitor-datasource/__mocks__/query.ts index ff72138ff38..56c2cf54de6 100644 --- a/public/app/plugins/datasource/grafana-azure-monitor-datasource/__mocks__/query.ts +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/__mocks__/query.ts @@ -11,6 +11,11 @@ export default function createMockQuery(): AzureMonitorQuery { workspace: 'e3fe4fde-ad5e-4d60-9974-e2f3562ffdf2', }, + azureResourceGraph: { + query: 'Resources | summarize count()', + resultFormat: 'time_series', + }, + azureMonitor: { // aggOptions: [], aggregation: 'Average', @@ -35,8 +40,8 @@ export default function createMockQuery(): AzureMonitorQuery { queryType: AzureQueryType.AzureMonitor, refId: 'A', - subscription: 'abc-123', - + subscription: '99999999-cccc-bbbb-aaaa-9106972f9572', + subscriptions: ['99999999-cccc-bbbb-aaaa-9106972f9572'], format: 'dunno lol', // unsure what this value should be. It's not there at runtime, but it's in the ts interface }; } diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/azure_resource_graph/azure_resource_graph_datasource.test.ts b/public/app/plugins/datasource/grafana-azure-monitor-datasource/azure_resource_graph/azure_resource_graph_datasource.test.ts new file mode 100644 index 00000000000..d53df0f0222 --- /dev/null +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/azure_resource_graph/azure_resource_graph_datasource.test.ts @@ -0,0 +1,149 @@ +import { TemplateSrv } from 'app/features/templating/template_srv'; +import { backendSrv } from 'app/core/services/backend_srv'; +import AzureResourceGraphDatasource from './azure_resource_graph_datasource'; +import { CustomVariableModel, initialVariableModelState, VariableHide } from 'app/features/variables/types'; +import { initialCustomVariableModelState } from 'app/features/variables/custom/reducer'; + +const templateSrv = new TemplateSrv(); + +const single: CustomVariableModel = { + ...initialVariableModelState, + id: 'var1', + name: 'var1', + index: 0, + current: { value: 'var1-foo', text: 'var1-foo', selected: true }, + options: [{ value: 'var1-foo', text: 'var1-foo', selected: true }], + multi: false, + includeAll: false, + query: '', + hide: VariableHide.dontHide, + type: 'custom', +}; + +const multi: CustomVariableModel = { + ...initialVariableModelState, + id: 'var3', + name: 'var3', + index: 2, + current: { value: ['var3-foo', 'var3-baz'], text: 'var3-foo + var3-baz', selected: true }, + options: [ + { selected: true, value: 'var3-foo', text: 'var3-foo' }, + { selected: false, value: 'var3-bar', text: 'var3-bar' }, + { selected: true, value: 'var3-baz', text: 'var3-baz' }, + ], + multi: true, + includeAll: false, + query: '', + hide: VariableHide.dontHide, + type: 'custom', +}; + +templateSrv.init([single, multi]); + +jest.mock('app/core/services/backend_srv'); +jest.mock('@grafana/runtime', () => ({ + ...((jest.requireActual('@grafana/runtime') as unknown) as object), + getBackendSrv: () => backendSrv, + getTemplateSrv: () => templateSrv, +})); + +describe('AzureResourceGraphDatasource', () => { + const datasourceRequestMock = jest.spyOn(backendSrv, 'datasourceRequest'); + + beforeEach(() => { + jest.clearAllMocks(); + datasourceRequestMock.mockImplementation(jest.fn()); + }); + + const ctx: any = {}; + + beforeEach(() => { + ctx.instanceSettings = { + url: 'http://azureresourcegraphapi', + }; + + ctx.ds = new AzureResourceGraphDatasource(ctx.instanceSettings); + }); + + describe('When applying template variables', () => { + it('should expand single value template variable', () => { + const target = { + azureResourceGraph: { + query: 'Resources | $var1', + resultFormat: '', + }, + }; + expect(ctx.ds.applyTemplateVariables(target)).toStrictEqual({ + azureResourceGraph: { query: 'Resources | var1-foo', resultFormat: 'table' }, + format: undefined, + queryType: 'Azure Resource Graph', + refId: undefined, + subscriptions: undefined, + }); + }); + + it('should expand multi value template variable', () => { + const target = { + azureResourceGraph: { + query: 'resources | where $__contains(name, $var3)', + resultFormat: '', + }, + }; + expect(ctx.ds.applyTemplateVariables(target)).toStrictEqual({ + azureResourceGraph: { + query: `resources | where $__contains(name, 'var3-foo','var3-baz')`, + resultFormat: 'table', + }, + format: undefined, + queryType: 'Azure Resource Graph', + refId: undefined, + subscriptions: undefined, + }); + }); + }); + + describe('When interpolating variables', () => { + beforeEach(() => { + ctx.variable = { ...initialCustomVariableModelState }; + }); + + describe('and value is a string', () => { + it('should return an unquoted value', () => { + expect(ctx.ds.interpolateVariable('abc', ctx.variable)).toEqual('abc'); + }); + }); + + describe('and value is a number', () => { + it('should return an unquoted value', () => { + expect(ctx.ds.interpolateVariable(1000, ctx.variable)).toEqual(1000); + }); + }); + + describe('and value is an array of strings', () => { + it('should return comma separated quoted values', () => { + expect(ctx.ds.interpolateVariable(['a', 'b', 'c'], ctx.variable)).toEqual("'a','b','c'"); + }); + }); + + describe('and variable allows multi-value and value is a string', () => { + it('should return a quoted value', () => { + ctx.variable.multi = true; + expect(ctx.ds.interpolateVariable('abc', ctx.variable)).toEqual("'abc'"); + }); + }); + + describe('and variable contains single quote', () => { + it('should return a quoted value', () => { + ctx.variable.multi = true; + expect(ctx.ds.interpolateVariable("a'bc", ctx.variable)).toEqual("'a'bc'"); + }); + }); + + describe('and variable allows all and value is a string', () => { + it('should return a quoted value', () => { + ctx.variable.includeAll = true; + expect(ctx.ds.interpolateVariable('abc', ctx.variable)).toEqual("'abc'"); + }); + }); + }); +}); diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/azure_resource_graph/azure_resource_graph_datasource.ts b/public/app/plugins/datasource/grafana-azure-monitor-datasource/azure_resource_graph/azure_resource_graph_datasource.ts new file mode 100644 index 00000000000..da71c28c160 --- /dev/null +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/azure_resource_graph/azure_resource_graph_datasource.ts @@ -0,0 +1,56 @@ +// eslint-disable-next-line lodash/import-scope +import _ from 'lodash'; +import { AzureMonitorQuery, AzureDataSourceJsonData, AzureQueryType } from '../types'; +import { ScopedVars } from '@grafana/data'; +import { getTemplateSrv, DataSourceWithBackend } from '@grafana/runtime'; + +export default class AzureResourceGraphDatasource extends DataSourceWithBackend< + AzureMonitorQuery, + AzureDataSourceJsonData +> { + filterQuery(item: AzureMonitorQuery): boolean { + return !!item.azureResourceGraph?.query; + } + + applyTemplateVariables(target: AzureMonitorQuery, scopedVars: ScopedVars): Record { + const item = target.azureResourceGraph; + + const templateSrv = getTemplateSrv(); + + const query = templateSrv.replace(item.query, scopedVars, this.interpolateVariable); + + return { + refId: target.refId, + format: target.format, + queryType: AzureQueryType.AzureResourceGraph, + subscriptions: target.subscriptions, + azureResourceGraph: { + resultFormat: 'table', + query, + }, + }; + } + + interpolateVariable(value: string, variable: { multi: any; includeAll: any }) { + if (typeof value === 'string') { + if (variable.multi || variable.includeAll) { + return "'" + value + "'"; + } else { + return value; + } + } + + if (typeof value === 'number') { + return value; + } + + const quotedValues = _.map(value, (val) => { + if (typeof value === 'number') { + return value; + } + + return "'" + val + "'"; + }); + return quotedValues.join(','); + } +} diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/ArgQueryEditor/ArgQueryEditor.tsx b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/ArgQueryEditor/ArgQueryEditor.tsx new file mode 100644 index 00000000000..27b05b063bc --- /dev/null +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/ArgQueryEditor/ArgQueryEditor.tsx @@ -0,0 +1,51 @@ +import React from 'react'; +import { AzureMonitorErrorish, AzureMonitorOption, AzureMonitorQuery } from '../../types'; +import Datasource from '../../datasource'; +import { InlineFieldRow } from '@grafana/ui'; +import SubscriptionField from '../SubscriptionField'; +import QueryField from './QueryField'; + +interface LogsQueryEditorProps { + query: AzureMonitorQuery; + datasource: Datasource; + subscriptionId: string; + onChange: (newQuery: AzureMonitorQuery) => void; + variableOptionGroup: { label: string; options: AzureMonitorOption[] }; + setError: (source: string, error: AzureMonitorErrorish | undefined) => void; +} + +const ArgQueryEditor: React.FC = ({ + query, + datasource, + subscriptionId, + variableOptionGroup, + onChange, + setError, +}) => { + return ( +
+ + + + + +
+ ); +}; + +export default ArgQueryEditor; diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/ArgQueryEditor/QueryField.tsx b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/ArgQueryEditor/QueryField.tsx new file mode 100644 index 00000000000..c4330fa7db5 --- /dev/null +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/ArgQueryEditor/QueryField.tsx @@ -0,0 +1,32 @@ +import { CodeEditor } from '@grafana/ui'; +import React, { useCallback } from 'react'; +import { AzureQueryEditorFieldProps } from '../../types'; + +const QueryField: React.FC = ({ query, onQueryChange }) => { + const onChange = useCallback( + (newQuery: string) => { + onQueryChange({ + ...query, + azureResourceGraph: { + ...query.azureResourceGraph, + query: newQuery, + }, + }); + }, + [onQueryChange, query] + ); + + return ( + + ); +}; + +export default QueryField; diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/ArgQueryEditor/index.tsx b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/ArgQueryEditor/index.tsx new file mode 100644 index 00000000000..9f6ff9eebc8 --- /dev/null +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/ArgQueryEditor/index.tsx @@ -0,0 +1 @@ +export { default } from './ArgQueryEditor'; diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/LogsQueryEditor/FormatAsField.tsx b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/LogsQueryEditor/FormatAsField.tsx index 1530f8b7ce6..2de756fe29a 100644 --- a/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/LogsQueryEditor/FormatAsField.tsx +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/LogsQueryEditor/FormatAsField.tsx @@ -1,6 +1,6 @@ +import React, { useCallback, useMemo } from 'react'; import { SelectableValue } from '@grafana/data'; import { Select } from '@grafana/ui'; -import React, { useCallback, useMemo } from 'react'; import { AzureMonitorOption, AzureQueryEditorFieldProps, AzureResultFormat } from '../../types'; import { findOption } from '../../utils/common'; import { Field } from '../Field'; diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/QueryEditor.tsx b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/QueryEditor.tsx index 3ac5224b849..42d6af3b9fb 100644 --- a/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/QueryEditor.tsx +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/QueryEditor.tsx @@ -6,6 +6,7 @@ import MetricsQueryEditor from '../MetricsQueryEditor'; import QueryTypeField from './QueryTypeField'; import useLastError from '../../utils/useLastError'; import LogsQueryEditor from '../LogsQueryEditor'; +import ArgQueryEditor from '../ArgQueryEditor'; import ApplicationInsightsEditor from '../ApplicationInsightsEditor'; import InsightsAnalyticsEditor from '../InsightsAnalyticsEditor'; import { Space } from '../Space'; @@ -93,6 +94,18 @@ const EditorForQueryType: React.FC = ({ case AzureQueryType.InsightsAnalytics: return ; + + case AzureQueryType.AzureResourceGraph: + return ( + + ); } return null; diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/QueryTypeField.tsx b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/QueryTypeField.tsx index e32435d25c2..043030ea2d7 100644 --- a/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/QueryTypeField.tsx +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/QueryTypeField.tsx @@ -10,6 +10,7 @@ const QUERY_TYPES = [ { value: AzureQueryType.LogAnalytics, label: 'Logs' }, { value: AzureQueryType.ApplicationInsights, label: 'Application Insights' }, { value: AzureQueryType.InsightsAnalytics, label: 'Insights Analytics' }, + { value: AzureQueryType.AzureResourceGraph, label: 'Azure Resource Graph' }, ]; interface QueryTypeFieldProps { diff --git a/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/SubscriptionField.tsx b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/SubscriptionField.tsx index 0c0e889136f..46db33ad2ad 100644 --- a/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/SubscriptionField.tsx +++ b/public/app/plugins/datasource/grafana-azure-monitor-datasource/components/SubscriptionField.tsx @@ -1,13 +1,14 @@ import React, { useCallback, useEffect, useState, useMemo } from 'react'; import { SelectableValue } from '@grafana/data'; -import { Select } from '@grafana/ui'; +import { Select, MultiSelect } from '@grafana/ui'; import { AzureMonitorQuery, AzureQueryType, AzureQueryEditorFieldProps, AzureMonitorOption } from '../types'; -import { findOption } from '../utils/common'; +import { findOption, findOptions } from '../utils/common'; import { Field } from './Field'; interface SubscriptionFieldProps extends AzureQueryEditorFieldProps { onQueryChange: (newQuery: AzureMonitorQuery) => void; + multiSelect?: boolean; } const ERROR_SOURCE = 'metrics-subscription'; @@ -17,6 +18,7 @@ const SubscriptionField: React.FC = ({ variableOptionGroup, onQueryChange, setError, + multiSelect = false, }) => { const [subscriptions, setSubscriptions] = useState([]); @@ -92,9 +94,33 @@ const SubscriptionField: React.FC = ({ [query, onQueryChange] ); + const onSubscriptionsChange = useCallback( + (change: Array>) => { + if (!change) { + return; + } + + query.subscriptions = change.map((c) => c.value ?? ''); + + onQueryChange(query); + }, + [query, onQueryChange] + ); + const options = useMemo(() => [...subscriptions, variableOptionGroup], [subscriptions, variableOptionGroup]); - return ( + return multiSelect ? ( + + + + ) : (