AzureMonitor: Add Azure Resource Graph (#33293)

* Add Azure Resource Graph in Azure Plugin

* fix lodash import

* Fix mock queries

* use "backend" sdk

* Address comments

* add converter for object type

* Query error should cause 400 & apply template var

* fix backend test & add documentation

* update image

* Address comments

* marshal body from map

* use interpolated query instead of raw query

* fix test

* filter out empty queries

* fix go linting problem

* use new query field language name

* improve variable tests

* add better tests for interpolate variable

Co-authored-by: joshhunt <josh@trtr.co>
Co-authored-by: Erik Sundell <erik.sundell87@gmail.com>
This commit is contained in:
shuotli
2021-05-19 10:31:27 +02:00
committed by GitHub
co-authored by joshhunt Erik Sundell
parent 4e31169a43
commit 71fd0981ca
26 changed files with 934 additions and 25 deletions
@@ -86,7 +86,7 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(queries []plugins.DataSubQuer
resultFormat := azureLogAnalyticsTarget.ResultFormat
if resultFormat == "" {
resultFormat = "time_series"
resultFormat = timeSeries
}
urlComponents := map[string]string{}
@@ -173,7 +173,7 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
return queryResultErrorWithExecuted(err)
}
frame, err := LogTableToFrame(t)
frame, err := ResponseTableToFrame(t)
if err != nil {
return queryResultErrorWithExecuted(err)
}
@@ -187,7 +187,7 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
azlog.Warn("failed to add custom metadata to azure log analytics response", err)
}
if query.ResultFormat == "time_series" {
if query.ResultFormat == timeSeries {
tsSchema := frame.TimeSeriesSchema()
if tsSchema.Type == data.TimeSeriesTypeLong {
wideFrame, err := data.LongToWide(frame, nil)
@@ -259,7 +259,7 @@ func (e *AzureLogAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourceP
// GetPrimaryResultTable returns the first table in the response named "PrimaryResult", or an
// error if there is no table by that name.
func (ar *AzureLogAnalyticsResponse) GetPrimaryResultTable() (*AzureLogAnalyticsTable, error) {
func (ar *AzureLogAnalyticsResponse) GetPrimaryResultTable() (*AzureResponseTable, error) {
for _, t := range ar.Tables {
if t.Name == "PrimaryResult" {
return &t, nil
@@ -41,7 +41,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"azureLogAnalytics": map[string]interface{}{
"workspace": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
"query": "query=Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer",
"resultFormat": "time_series",
"resultFormat": timeSeries,
},
}),
RefID: "A",
@@ -50,12 +50,12 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
azureLogAnalyticsQueries: []*AzureLogAnalyticsQuery{
{
RefID: "A",
ResultFormat: "time_series",
ResultFormat: timeSeries,
URL: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/query",
Model: simplejson.NewFromAny(map[string]interface{}{
"azureLogAnalytics": map[string]interface{}{
"query": "query=Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer",
"resultFormat": "time_series",
"resultFormat": timeSeries,
"workspace": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
},
}),
@@ -0,0 +1,271 @@
package azuremonitor
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"path"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util/errutil"
"github.com/opentracing/opentracing-go"
"golang.org/x/net/context/ctxhttp"
)
// AzureResourceGraphDatasource calls the Azure Resource Graph API's
type AzureResourceGraphDatasource struct {
httpClient *http.Client
dsInfo *models.DataSource
pluginManager plugins.Manager
cfg *setting.Cfg
}
// AzureResourceGraphQuery is the query request that is built from the saved values for
// from the UI
type AzureResourceGraphQuery struct {
RefID string
ResultFormat string
URL string
Model *simplejson.Json
InterpolatedQuery string
}
const argAPIVersion = "2018-09-01-preview"
const argQueryProviderName = "/providers/Microsoft.ResourceGraph/resources"
// executeTimeSeriesQuery does the following:
// 1. builds the AzureMonitor url and querystring for each query
// 2. executes each query by calling the Azure Monitor API
// 3. parses the responses for each query into the timeseries format
func (e *AzureResourceGraphDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []plugins.DataSubQuery,
timeRange plugins.DataTimeRange) (backend.QueryDataResponse, error) {
result := backend.QueryDataResponse{
Responses: map[string]backend.DataResponse{},
}
queries, err := e.buildQueries(originalQueries, timeRange)
if err != nil {
return backend.QueryDataResponse{}, err
}
for _, query := range queries {
result.Responses[query.RefID] = e.executeQuery(ctx, query, timeRange)
}
return result, nil
}
func (e *AzureResourceGraphDatasource) buildQueries(queries []plugins.DataSubQuery,
timeRange plugins.DataTimeRange) ([]*AzureResourceGraphQuery, error) {
var azureResourceGraphQueries []*AzureResourceGraphQuery
for _, query := range queries {
queryBytes, err := query.Model.Encode()
if err != nil {
return nil, fmt.Errorf("failed to re-encode the Azure Resource Graph query into JSON: %w", err)
}
queryJSONModel := argJSONQuery{}
err = json.Unmarshal(queryBytes, &queryJSONModel)
if err != nil {
return nil, fmt.Errorf("failed to decode the Azure Resource Graph query object from JSON: %w", err)
}
azureResourceGraphTarget := queryJSONModel.AzureResourceGraph
azlog.Debug("AzureResourceGraph", "target", azureResourceGraphTarget)
resultFormat := azureResourceGraphTarget.ResultFormat
if resultFormat == "" {
resultFormat = "table"
}
interpolatedQuery, err := KqlInterpolate(query, timeRange, azureResourceGraphTarget.Query)
if err != nil {
return nil, err
}
azureResourceGraphQueries = append(azureResourceGraphQueries, &AzureResourceGraphQuery{
RefID: query.RefID,
ResultFormat: resultFormat,
Model: query.Model,
InterpolatedQuery: interpolatedQuery,
})
}
return azureResourceGraphQueries, nil
}
func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *AzureResourceGraphQuery,
timeRange plugins.DataTimeRange) backend.DataResponse {
queryResult := backend.DataResponse{}
params := url.Values{}
params.Add("api-version", argAPIVersion)
queryResultErrorWithExecuted := func(err error) backend.DataResponse {
queryResult = backend.DataResponse{Error: err}
frames := data.Frames{
&data.Frame{
RefID: query.RefID,
Meta: &data.FrameMeta{
ExecutedQueryString: query.InterpolatedQuery,
},
},
}
queryResult.Frames = frames
return queryResult
}
reqBody, err := json.Marshal(map[string]interface{}{
"subscriptions": query.Model.Get("subscriptions").MustStringArray(),
"query": query.InterpolatedQuery,
})
if err != nil {
queryResult.Error = err
return queryResult
}
req, err := e.createRequest(ctx, e.dsInfo, reqBody)
if err != nil {
queryResult.Error = err
return queryResult
}
req.URL.Path = path.Join(req.URL.Path, argQueryProviderName)
req.URL.RawQuery = params.Encode()
span, ctx := opentracing.StartSpanFromContext(ctx, "azure resource graph query")
span.SetTag("interpolated_query", query.InterpolatedQuery)
span.SetTag("from", timeRange.From)
span.SetTag("until", timeRange.To)
span.SetTag("datasource_id", e.dsInfo.Id)
span.SetTag("org_id", e.dsInfo.OrgId)
defer span.Finish()
if err := opentracing.GlobalTracer().Inject(
span.Context(),
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header)); err != nil {
return queryResultErrorWithExecuted(err)
}
azlog.Debug("AzureResourceGraph", "Request ApiURL", req.URL.String())
res, err := ctxhttp.Do(ctx, e.httpClient, req)
if err != nil {
return queryResultErrorWithExecuted(err)
}
argResponse, err := e.unmarshalResponse(res)
if err != nil {
return queryResultErrorWithExecuted(err)
}
frame, err := ResponseTableToFrame(&argResponse.Data)
if err != nil {
return queryResultErrorWithExecuted(err)
}
if frame.Meta == nil {
frame.Meta = &data.FrameMeta{}
}
frame.Meta.ExecutedQueryString = req.URL.RawQuery
queryResult.Frames = data.Frames{frame}
return queryResult
}
func (e *AzureResourceGraphDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource, reqBody []byte) (*http.Request, error) {
u, err := url.Parse(dsInfo.Url)
if err != nil {
return nil, err
}
u.Path = path.Join(u.Path, "render")
req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewBuffer(reqBody))
if err != nil {
azlog.Debug("Failed to create request", "error", err)
return nil, errutil.Wrap("failed to create request", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
// find plugin
plugin := e.pluginManager.GetDataSource(dsInfo.Type)
if plugin == nil {
return nil, errors.New("unable to find datasource plugin Azure Monitor")
}
cloudName := dsInfo.JsonData.Get("cloudName").MustString("azuremonitor")
argRoute, proxypass, err := e.getPluginRoute(plugin, cloudName)
if err != nil {
return nil, err
}
pluginproxy.ApplyRoute(ctx, req, proxypass, argRoute, dsInfo, e.cfg)
return req, nil
}
func (e *AzureResourceGraphDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, cloudName string) (
*plugins.AppPluginRoute, string, error) {
pluginRouteName := "azureresourcegraph"
switch cloudName {
case "chinaazuremonitor":
pluginRouteName = "chinaazureresourcegraph"
case "govazuremonitor":
pluginRouteName = "govazureresourcegraph"
}
var argRoute *plugins.AppPluginRoute
for _, route := range plugin.Routes {
if route.Path == pluginRouteName {
argRoute = route
break
}
}
return argRoute, pluginRouteName, nil
}
func (e *AzureResourceGraphDatasource) unmarshalResponse(res *http.Response) (AzureResourceGraphResponse, error) {
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return AzureResourceGraphResponse{}, err
}
defer func() {
if err := res.Body.Close(); err != nil {
azlog.Warn("Failed to close response body", "err", err)
}
}()
if res.StatusCode/100 != 2 {
azlog.Debug("Request failed", "status", res.Status, "body", string(body))
return AzureResourceGraphResponse{}, fmt.Errorf("request failed, status: %s, body: %s", res.Status, string(body))
}
var data AzureResourceGraphResponse
d := json.NewDecoder(bytes.NewReader(body))
d.UseNumber()
err = d.Decode(&data)
if err != nil {
azlog.Debug("Failed to unmarshal azure resource graph response", "error", err, "status", res.Status, "body", string(body))
return AzureResourceGraphResponse{}, err
}
return data, nil
}
@@ -0,0 +1,75 @@
package azuremonitor
import (
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/stretchr/testify/require"
)
func TestBuildingAzureResourceGraphQueries(t *testing.T) {
datasource := &AzureResourceGraphDatasource{}
fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local)
tests := []struct {
name string
queryModel []plugins.DataSubQuery
timeRange plugins.DataTimeRange
azureResourceGraphQueries []*AzureResourceGraphQuery
Err require.ErrorAssertionFunc
}{
{
name: "Query with macros should be interpolated",
timeRange: plugins.DataTimeRange{
From: fmt.Sprintf("%v", fromStart.Unix()*1000),
To: fmt.Sprintf("%v", fromStart.Add(34*time.Minute).Unix()*1000),
},
queryModel: []plugins.DataSubQuery{
{
DataSource: &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{}),
},
Model: simplejson.NewFromAny(map[string]interface{}{
"queryType": "Azure Resource Graph",
"azureResourceGraph": map[string]interface{}{
"query": "resources | where $__contains(name,'res1','res2')",
"resultFormat": "table",
},
}),
RefID: "A",
},
},
azureResourceGraphQueries: []*AzureResourceGraphQuery{
{
RefID: "A",
ResultFormat: "table",
URL: "",
Model: simplejson.NewFromAny(map[string]interface{}{
"azureResourceGraph": map[string]interface{}{
"query": "resources | where $__contains(name,'res1','res2')",
"resultFormat": "table",
},
}),
InterpolatedQuery: "resources | where ['name'] in ('res1','res2')",
},
},
Err: require.NoError,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
queries, err := datasource.buildQueries(tt.queryModel, tt.timeRange)
tt.Err(t, err)
if diff := cmp.Diff(tt.azureResourceGraphQueries, queries, cmpopts.IgnoreUnexported(simplejson.Json{})); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff)
}
})
}
}
@@ -10,8 +10,8 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
)
// LogTableToFrame converts an AzureLogAnalyticsTable to a data.Frame.
func LogTableToFrame(table *AzureLogAnalyticsTable) (*data.Frame, error) {
// ResponseTableToFrame converts an AzureResponseTable to a data.Frame.
func ResponseTableToFrame(table *AzureResponseTable) (*data.Frame, error) {
converterFrame, err := converterFrameForTable(table)
if err != nil {
return nil, err
@@ -27,7 +27,7 @@ func LogTableToFrame(table *AzureLogAnalyticsTable) (*data.Frame, error) {
return converterFrame.Frame, nil
}
func converterFrameForTable(t *AzureLogAnalyticsTable) (*data.FrameInputConverter, error) {
func converterFrameForTable(t *AzureResponseTable) (*data.FrameInputConverter, error) {
converters := []data.FieldConverter{}
colNames := make([]string, len(t.Columns))
colTypes := make([]string, len(t.Columns)) // for metadata
@@ -64,12 +64,14 @@ var converterMap = map[string]data.FieldConverter{
"guid": stringConverter,
"timespan": stringConverter,
"dynamic": stringConverter,
"object": objectToStringConverter,
"datetime": timeConverter,
"int": intConverter,
"long": longConverter,
"real": realConverter,
"bool": boolConverter,
"decimal": decimalConverter,
"integer": intConverter,
}
var stringConverter = data.FieldConverter{
@@ -88,6 +90,26 @@ var stringConverter = data.FieldConverter{
},
}
var objectToStringConverter = data.FieldConverter{
OutputFieldType: data.FieldTypeNullableString,
Converter: func(kustoValue interface{}) (interface{}, error) {
var output *string
if kustoValue == nil {
return output, nil
}
data, err := json.Marshal(kustoValue)
if err != nil {
fmt.Printf("failed to marshal column value: %s", err)
}
asString := string(data)
output = &asString
return output, nil
},
}
var timeConverter = data.FieldConverter{
OutputFieldType: data.FieldTypeNullableTime,
Converter: func(v interface{}) (interface{}, error) {
@@ -114,10 +114,11 @@ func TestLogTableToFrame(t *testing.T) {
data.NewField("XReal", nil, []*float64{pointer.Float64(1.797693134862315708145274237317043567981e+308)}),
data.NewField("XTimeSpan", nil, []*string{pointer.String("00:00:00.0000001")}),
data.NewField("XDecimal", nil, []*float64{pointer.Float64(79228162514264337593543950335)}),
data.NewField("XObject", nil, []*string{pointer.String(`"{\"person\": \"Daniel\", \"cats\": 23, \"diagnosis\": \"cat problem\"}"`)}),
)
frame.Meta = &data.FrameMeta{
Custom: &LogAnalyticsMeta{ColumnTypes: []string{"bool", "string", "datetime",
"dynamic", "guid", "int", "long", "real", "timespan", "decimal"}},
"dynamic", "guid", "int", "long", "real", "timespan", "decimal", "object"}},
}
return frame
},
@@ -142,7 +143,7 @@ func TestLogTableToFrame(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
res := loadLogAnalyticsTestFileWithNumber(t, tt.testFile)
frame, err := LogTableToFrame(&res.Tables[0])
frame, err := ResponseTableToFrame(&res.Tables[0])
require.NoError(t, err)
if diff := cmp.Diff(tt.expectedFrame(), frame, data.FrameTestCompareOptions()...); diff != "" {
+20
View File
@@ -13,6 +13,8 @@ import (
"github.com/grafana/grafana/pkg/setting"
)
const timeSeries = "time_series"
var (
azlog = log.New("tsdb.azuremonitor")
legendKeyFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
@@ -72,6 +74,7 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat
var applicationInsightsQueries []plugins.DataSubQuery
var azureLogAnalyticsQueries []plugins.DataSubQuery
var insightsAnalyticsQueries []plugins.DataSubQuery
var azureResourceGraphQueries []plugins.DataSubQuery
for _, query := range tsdbQuery.Queries {
queryType := query.Model.Get("queryType").MustString("")
@@ -85,6 +88,8 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat
azureLogAnalyticsQueries = append(azureLogAnalyticsQueries, query)
case "Insights Analytics":
insightsAnalyticsQueries = append(insightsAnalyticsQueries, query)
case "Azure Resource Graph":
azureResourceGraphQueries = append(azureResourceGraphQueries, query)
default:
return plugins.DataResponse{}, fmt.Errorf("alerting not supported for %q", queryType)
}
@@ -118,6 +123,12 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat
cfg: e.cfg,
}
argDatasource := &AzureResourceGraphDatasource{
httpClient: e.httpClient,
dsInfo: e.dsInfo,
pluginManager: e.pluginManager,
}
azResult, err := azDatasource.executeTimeSeriesQuery(ctx, azureMonitorQueries, *tsdbQuery.TimeRange)
if err != nil {
return plugins.DataResponse{}, err
@@ -138,6 +149,11 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat
return plugins.DataResponse{}, err
}
argResult, err := argDatasource.executeTimeSeriesQuery(ctx, azureResourceGraphQueries, *tsdbQuery.TimeRange)
if err != nil {
return plugins.DataResponse{}, err
}
for k, v := range aiResult.Results {
azResult.Results[k] = v
}
@@ -150,5 +166,9 @@ func (e *AzureMonitorExecutor) DataQuery(ctx context.Context, dsInfo *models.Dat
azResult.Results[k] = v
}
for k, v := range argResult.Responses {
azResult.Results[k] = plugins.DataQueryResult{Error: v.Error, Dataframes: plugins.NewDecodedDataFrames(v.Frames)}
}
return azResult, nil
}
@@ -163,12 +163,12 @@ func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *I
return queryResultError(err)
}
frame, err := LogTableToFrame(t)
frame, err := ResponseTableToFrame(t)
if err != nil {
return queryResultError(err)
}
if query.ResultFormat == "time_series" {
if query.ResultFormat == timeSeries {
tsSchema := frame.TimeSeriesSchema()
if tsSchema.Type == data.TimeSeriesTypeLong {
wideFrame, err := data.LongToWide(frame, nil)
@@ -42,6 +42,10 @@
{
"name":"XDecimal",
"type":"decimal"
},
{
"name":"XObject",
"type":"object"
}
],
"rows": [
@@ -55,10 +59,11 @@
9223372036854775807,
1.7976931348623157e+308,
"00:00:00.0000001",
"79228162514264337593543950335"
"79228162514264337593543950335",
"{\"person\": \"Daniel\", \"cats\": 23, \"diagnosis\": \"cat problem\"}"
]
]
}
]
}
+15 -3
View File
@@ -56,11 +56,16 @@ type AzureMonitorResponse struct {
// AzureLogAnalyticsResponse is the json response object from the Azure Log Analytics API.
type AzureLogAnalyticsResponse struct {
Tables []AzureLogAnalyticsTable `json:"tables"`
Tables []AzureResponseTable `json:"tables"`
}
// AzureLogAnalyticsTable is the table format for Log Analytics responses
type AzureLogAnalyticsTable struct {
// AzureResourceGraphResponse is the json response object from the Azure Resource Graph Analytics API.
type AzureResourceGraphResponse struct {
Data AzureResponseTable `json:"data"`
}
// AzureResponseTable is the table format for Azure responses
type AzureResponseTable struct {
Name string `json:"name"`
Columns []struct {
Name string `json:"name"`
@@ -137,6 +142,13 @@ type logJSONQuery struct {
} `json:"azureLogAnalytics"`
}
type argJSONQuery struct {
AzureResourceGraph struct {
Query string `json:"query"`
ResultFormat string `json:"resultFormat"`
} `json:"azureResourceGraph"`
}
// InsightsDimensions will unmarshal from a JSON string, or an array of strings,
// into a string array. This exists to support an older query format which is updated
// when a user saves the query or it is sent from the front end, but may not be when