Azure: Split insights into two services (#25410)

Azure Application Insights Analytics is no longer accessed by the edit button from within the Application Insights service. Instead, there is now an Insights Analytics option in the Service drop down.

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
This commit is contained in:
Kyle Brandt
2020-06-25 12:48:18 -04:00
committed by GitHub
co-authored by Ryan McKinley
parent af0c73720e
commit bc9c53389c
18 changed files with 601 additions and 574 deletions
@@ -92,72 +92,43 @@ func (e *ApplicationInsightsDatasource) buildQueries(queries []*tsdb.Query, time
insightsJSONModel := queryJSONModel.AppInsights
azlog.Debug("Application Insights", "target", insightsJSONModel)
if insightsJSONModel.RawQuery == nil {
return nil, fmt.Errorf("missing the 'rawQuery' property")
}
if *insightsJSONModel.RawQuery {
var rawQueryString string
if insightsJSONModel.RawQueryString == "" {
return nil, errors.New("rawQuery requires rawQueryString")
}
rawQueryString, err := KqlInterpolate(query, timeRange, insightsJSONModel.RawQueryString)
azureURL := fmt.Sprintf("metrics/%s", insightsJSONModel.MetricName)
timeGrain := insightsJSONModel.TimeGrain
timeGrains := insightsJSONModel.AllowedTimeGrainsMs
if timeGrain == "auto" {
timeGrain, err = setAutoTimeGrain(query.IntervalMs, timeGrains)
if err != nil {
return nil, err
}
params := url.Values{}
params.Add("query", rawQueryString)
applicationInsightsQueries = append(applicationInsightsQueries, &ApplicationInsightsQuery{
RefID: query.RefId,
IsRaw: true,
ApiURL: "query",
Params: params,
TimeColumnName: insightsJSONModel.TimeColumn,
ValueColumnName: insightsJSONModel.ValueColumn,
SegmentColumnName: insightsJSONModel.SegmentColumn,
Target: params.Encode(),
})
} else {
azureURL := fmt.Sprintf("metrics/%s", insightsJSONModel.MetricName)
timeGrain := insightsJSONModel.TimeGrain
timeGrains := insightsJSONModel.AllowedTimeGrainsMs
if timeGrain == "auto" {
timeGrain, err = setAutoTimeGrain(query.IntervalMs, timeGrains)
if err != nil {
return nil, err
}
}
params := url.Values{}
params.Add("timespan", fmt.Sprintf("%v/%v", startTime.UTC().Format(time.RFC3339), endTime.UTC().Format(time.RFC3339)))
if timeGrain != "none" {
params.Add("interval", timeGrain)
}
params.Add("aggregation", insightsJSONModel.Aggregation)
dimension := strings.TrimSpace(insightsJSONModel.Dimension)
// Azure Monitor combines this and the following logic such that if dimensionFilter, must also Dimension, should that be done here as well?
if dimension != "" && !strings.EqualFold(dimension, "none") {
params.Add("segment", dimension)
}
dimensionFilter := strings.TrimSpace(insightsJSONModel.DimensionFilter)
if dimensionFilter != "" {
params.Add("filter", dimensionFilter)
}
applicationInsightsQueries = append(applicationInsightsQueries, &ApplicationInsightsQuery{
RefID: query.RefId,
IsRaw: false,
ApiURL: azureURL,
Params: params,
Alias: insightsJSONModel.Alias,
Target: params.Encode(),
})
}
params := url.Values{}
params.Add("timespan", fmt.Sprintf("%v/%v", startTime.UTC().Format(time.RFC3339), endTime.UTC().Format(time.RFC3339)))
if timeGrain != "none" {
params.Add("interval", timeGrain)
}
params.Add("aggregation", insightsJSONModel.Aggregation)
dimension := strings.TrimSpace(insightsJSONModel.Dimension)
// Azure Monitor combines this and the following logic such that if dimensionFilter, must also Dimension, should that be done here as well?
if dimension != "" && !strings.EqualFold(dimension, "none") {
params.Add("segment", dimension)
}
dimensionFilter := strings.TrimSpace(insightsJSONModel.DimensionFilter)
if dimensionFilter != "" {
params.Add("filter", dimensionFilter)
}
applicationInsightsQueries = append(applicationInsightsQueries, &ApplicationInsightsQuery{
RefID: query.RefId,
IsRaw: false,
ApiURL: azureURL,
Params: params,
Alias: insightsJSONModel.Alias,
Target: params.Encode(),
})
}
return applicationInsightsQueries, nil
@@ -209,18 +180,10 @@ func (e *ApplicationInsightsDatasource) executeQuery(ctx context.Context, query
return nil, fmt.Errorf("Request failed status: %v", res.Status)
}
if query.IsRaw {
queryResult.Series, queryResult.Meta, err = e.parseTimeSeriesFromQuery(body, query)
if err != nil {
queryResult.Error = err
return queryResult, nil
}
} else {
queryResult.Series, err = e.parseTimeSeriesFromMetrics(body, query)
if err != nil {
queryResult.Error = err
return queryResult, nil
}
queryResult.Series, err = e.parseTimeSeriesFromMetrics(body, query)
if err != nil {
queryResult.Error = err
return queryResult, nil
}
return queryResult, nil
@@ -280,96 +243,6 @@ func (e *ApplicationInsightsDatasource) getPluginRoute(plugin *plugins.DataSourc
return pluginRoute, pluginRouteName, nil
}
func (e *ApplicationInsightsDatasource) parseTimeSeriesFromQuery(body []byte, query *ApplicationInsightsQuery) (tsdb.TimeSeriesSlice, *simplejson.Json, error) {
var data ApplicationInsightsQueryResponse
err := json.Unmarshal(body, &data)
if err != nil {
azlog.Debug("Failed to unmarshal Application Insights response", "error", err, "body", string(body))
return nil, nil, err
}
type Metadata struct {
Columns []string `json:"columns"`
}
meta := Metadata{}
for _, t := range data.Tables {
if t.Name == "PrimaryResult" {
timeIndex, valueIndex, segmentIndex := -1, -1, -1
meta.Columns = make([]string, 0)
for i, v := range t.Columns {
meta.Columns = append(meta.Columns, v.Name)
switch v.Name {
case query.TimeColumnName:
timeIndex = i
case query.ValueColumnName:
valueIndex = i
case query.SegmentColumnName:
segmentIndex = i
}
}
if timeIndex == -1 {
azlog.Info("no time column specified, returning existing columns, no data")
return nil, simplejson.NewFromAny(meta), nil
}
if valueIndex == -1 {
azlog.Info("no value column specified, returning existing columns, no data")
return nil, simplejson.NewFromAny(meta), nil
}
var getPoints func([]interface{}) *tsdb.TimeSeriesPoints
slice := tsdb.TimeSeriesSlice{}
if segmentIndex == -1 {
legend := formatApplicationInsightsLegendKey(query.Alias, query.ValueColumnName, "", "")
series := tsdb.NewTimeSeries(legend, []tsdb.TimePoint{})
slice = append(slice, series)
getPoints = func(row []interface{}) *tsdb.TimeSeriesPoints {
return &series.Points
}
} else {
mapping := map[string]*tsdb.TimeSeriesPoints{}
getPoints = func(row []interface{}) *tsdb.TimeSeriesPoints {
segment := fmt.Sprintf("%v", row[segmentIndex])
if points, ok := mapping[segment]; ok {
return points
}
legend := formatApplicationInsightsLegendKey(query.Alias, query.ValueColumnName, query.SegmentColumnName, segment)
series := tsdb.NewTimeSeries(legend, []tsdb.TimePoint{})
slice = append(slice, series)
mapping[segment] = &series.Points
return &series.Points
}
}
for _, r := range t.Rows {
timeStr, ok := r[timeIndex].(string)
if !ok {
return nil, simplejson.NewFromAny(meta), errors.New("invalid time value")
}
timeValue, err := time.Parse(time.RFC3339Nano, timeStr)
if err != nil {
return nil, simplejson.NewFromAny(meta), err
}
var value float64
if value, err = getFloat(r[valueIndex]); err != nil {
return nil, simplejson.NewFromAny(meta), err
}
points := getPoints(r)
*points = append(*points, tsdb.NewTimePoint(null.FloatFrom(value), float64(timeValue.Unix()*1000)))
}
return slice, simplejson.NewFromAny(meta), nil
}
}
return nil, nil, errors.New("could not find table")
}
func (e *ApplicationInsightsDatasource) parseTimeSeriesFromMetrics(body []byte, query *ApplicationInsightsQuery) (tsdb.TimeSeriesSlice, error) {
doc, err := simplejson.NewJson(body)
if err != nil {
@@ -142,98 +142,6 @@ func TestApplicationInsightsDatasource(t *testing.T) {
So(queries[0].Target, ShouldEqual, "aggregation=Average&interval=PT1M&timespan=2018-03-15T13%3A00%3A00Z%2F2018-03-15T13%3A34%3A00Z")
})
Convey("id a raw query", func() {
tsdbQuery.Queries[0].Model = simplejson.NewFromAny(map[string]interface{}{
"appInsights": map[string]interface{}{
"rawQuery": true,
"rawQueryString": "exceptions | where $__timeFilter(timestamp) | summarize count=count() by bin(timestamp, $__interval)",
"timeColumn": "timestamp",
"valueColumn": "count",
},
})
queries, err := datasource.buildQueries(tsdbQuery.Queries, tsdbQuery.TimeRange)
So(err, ShouldBeNil)
So(queries[0].Params["query"][0], ShouldEqual, "exceptions | where ['timestamp'] >= datetime('2018-03-15T13:00:00Z') and ['timestamp'] <= datetime('2018-03-15T13:34:00Z') | summarize count=count() by bin(timestamp, 1234ms)")
So(queries[0].Target, ShouldEqual, "query=exceptions+%7C+where+%5B%27timestamp%27%5D+%3E%3D+datetime%28%272018-03-15T13%3A00%3A00Z%27%29+and+%5B%27timestamp%27%5D+%3C%3D+datetime%28%272018-03-15T13%3A34%3A00Z%27%29+%7C+summarize+count%3Dcount%28%29+by+bin%28timestamp%2C+1234ms%29")
})
})
Convey("Parse Application Insights query API response in the time series format", func() {
Convey("no segments", func() {
data, err := ioutil.ReadFile("testdata/applicationinsights/1-application-insights-response-raw-query.json")
So(err, ShouldBeNil)
query := &ApplicationInsightsQuery{
IsRaw: true,
TimeColumnName: "timestamp",
ValueColumnName: "value",
}
series, _, err := datasource.parseTimeSeriesFromQuery(data, query)
So(err, ShouldBeNil)
So(len(series), ShouldEqual, 1)
So(series[0].Name, ShouldEqual, "value")
So(len(series[0].Points), ShouldEqual, 2)
So(series[0].Points[0][0].Float64, ShouldEqual, 1)
So(series[0].Points[0][1].Float64, ShouldEqual, int64(1568336523000))
So(series[0].Points[1][0].Float64, ShouldEqual, 2)
So(series[0].Points[1][1].Float64, ShouldEqual, int64(1568340123000))
})
Convey("with segments", func() {
data, err := ioutil.ReadFile("testdata/applicationinsights/2-application-insights-response-raw-query-segmented.json")
So(err, ShouldBeNil)
query := &ApplicationInsightsQuery{
IsRaw: true,
TimeColumnName: "timestamp",
ValueColumnName: "value",
SegmentColumnName: "segment",
}
series, _, err := datasource.parseTimeSeriesFromQuery(data, query)
So(err, ShouldBeNil)
So(len(series), ShouldEqual, 2)
So(series[0].Name, ShouldEqual, "{segment=a}.value")
So(len(series[0].Points), ShouldEqual, 2)
So(series[0].Points[0][0].Float64, ShouldEqual, 1)
So(series[0].Points[0][1].Float64, ShouldEqual, int64(1568336523000))
So(series[0].Points[1][0].Float64, ShouldEqual, 3)
So(series[0].Points[1][1].Float64, ShouldEqual, int64(1568426523000))
So(series[1].Name, ShouldEqual, "{segment=b}.value")
So(series[1].Points[0][0].Float64, ShouldEqual, 2)
So(series[1].Points[0][1].Float64, ShouldEqual, int64(1568336523000))
So(series[1].Points[1][0].Float64, ShouldEqual, 4)
So(series[1].Points[1][1].Float64, ShouldEqual, int64(1568426523000))
Convey("with alias", func() {
data, err := ioutil.ReadFile("testdata/applicationinsights/2-application-insights-response-raw-query-segmented.json")
So(err, ShouldBeNil)
query := &ApplicationInsightsQuery{
IsRaw: true,
TimeColumnName: "timestamp",
ValueColumnName: "value",
SegmentColumnName: "segment",
Alias: "{{metric}} {{dimensionname}} {{dimensionvalue}}",
}
series, _, err := datasource.parseTimeSeriesFromQuery(data, query)
So(err, ShouldBeNil)
So(len(series), ShouldEqual, 2)
So(series[0].Name, ShouldEqual, "value segment a")
So(series[1].Name, ShouldEqual, "value segment b")
})
})
})
Convey("Parse Application Insights metrics API", func() {
+17
View File
@@ -51,6 +51,7 @@ func (e *AzureMonitorExecutor) Query(ctx context.Context, dsInfo *models.DataSou
var azureMonitorQueries []*tsdb.Query
var applicationInsightsQueries []*tsdb.Query
var azureLogAnalyticsQueries []*tsdb.Query
var insightsAnalyticsQueries []*tsdb.Query
for _, query := range tsdbQuery.Queries {
queryType := query.Model.Get("queryType").MustString("")
@@ -62,6 +63,8 @@ func (e *AzureMonitorExecutor) Query(ctx context.Context, dsInfo *models.DataSou
applicationInsightsQueries = append(applicationInsightsQueries, query)
case "Azure Log Analytics":
azureLogAnalyticsQueries = append(azureLogAnalyticsQueries, query)
case "Insights Analytics":
insightsAnalyticsQueries = append(insightsAnalyticsQueries, query)
default:
return nil, fmt.Errorf("Alerting not supported for %s", queryType)
}
@@ -82,6 +85,11 @@ func (e *AzureMonitorExecutor) Query(ctx context.Context, dsInfo *models.DataSou
dsInfo: e.dsInfo,
}
iaDatasource := &InsightsAnalyticsDatasource{
httpClient: e.httpClient,
dsInfo: e.dsInfo,
}
azResult, err := azDatasource.executeTimeSeriesQuery(ctx, azureMonitorQueries, tsdbQuery.TimeRange)
if err != nil {
return nil, err
@@ -97,6 +105,11 @@ func (e *AzureMonitorExecutor) Query(ctx context.Context, dsInfo *models.DataSou
return nil, err
}
iaResult, err := iaDatasource.executeTimeSeriesQuery(ctx, insightsAnalyticsQueries, tsdbQuery.TimeRange)
if err != nil {
return nil, err
}
for k, v := range aiResult.Results {
azResult.Results[k] = v
}
@@ -105,5 +118,9 @@ func (e *AzureMonitorExecutor) Query(ctx context.Context, dsInfo *models.DataSou
azResult.Results[k] = v
}
for k, v := range iaResult.Results {
azResult.Results[k] = v
}
return azResult, nil
}
@@ -0,0 +1,232 @@
package azuremonitor
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"path"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/util/errutil"
"github.com/opentracing/opentracing-go"
"golang.org/x/net/context/ctxhttp"
)
type InsightsAnalyticsDatasource struct {
httpClient *http.Client
dsInfo *models.DataSource
}
type InsightsAnalyticsQuery struct {
RefID string
RawQuery string
InterpolatedQuery string
ResultFormat string
Params url.Values
Target string
}
func (e *InsightsAnalyticsDatasource) executeTimeSeriesQuery(ctx context.Context, originalQueries []*tsdb.Query, timeRange *tsdb.TimeRange) (*tsdb.Response, error) {
result := &tsdb.Response{
Results: map[string]*tsdb.QueryResult{},
}
queries, err := e.buildQueries(originalQueries, timeRange)
if err != nil {
return nil, err
}
for _, query := range queries {
result.Results[query.RefID] = e.executeQuery(ctx, query)
}
return result, nil
}
func (e *InsightsAnalyticsDatasource) buildQueries(queries []*tsdb.Query, timeRange *tsdb.TimeRange) ([]*InsightsAnalyticsQuery, error) {
iaQueries := []*InsightsAnalyticsQuery{}
for _, query := range queries {
queryBytes, err := query.Model.Encode()
if err != nil {
return nil, fmt.Errorf("failed to re-encode the Azure Application Insights Analytics query into JSON: %w", err)
}
qm := InsightsAnalyticsQuery{}
queryJSONModel := insightsAnalyticsJSONQuery{}
err = json.Unmarshal(queryBytes, &queryJSONModel)
if err != nil {
return nil, fmt.Errorf("failed to decode the Azure Application Insights Analytics query object from JSON: %w", err)
}
qm.RawQuery = queryJSONModel.InsightsAnalytics.Query
qm.ResultFormat = queryJSONModel.InsightsAnalytics.ResultFormat
qm.RefID = query.RefId
if qm.RawQuery == "" {
return nil, fmt.Errorf("query is missing query string property")
}
qm.InterpolatedQuery, err = KqlInterpolate(query, timeRange, qm.RawQuery)
if err != nil {
return nil, err
}
qm.Params = url.Values{}
qm.Params.Add("query", qm.InterpolatedQuery)
qm.Target = qm.Params.Encode()
iaQueries = append(iaQueries, &qm)
}
return iaQueries, nil
}
func (e *InsightsAnalyticsDatasource) executeQuery(ctx context.Context, query *InsightsAnalyticsQuery) *tsdb.QueryResult {
queryResult := &tsdb.QueryResult{RefId: query.RefID}
queryResultError := func(err error) *tsdb.QueryResult {
queryResult.Error = err
return queryResult
}
req, err := e.createRequest(ctx, e.dsInfo)
if err != nil {
queryResultError(err)
}
req.URL.Path = path.Join(req.URL.Path, "query")
req.URL.RawQuery = query.Params.Encode()
span, ctx := opentracing.StartSpanFromContext(ctx, "application insights analytics query")
span.SetTag("target", query.Target)
span.SetTag("datasource_id", e.dsInfo.Id)
span.SetTag("org_id", e.dsInfo.OrgId)
defer span.Finish()
err = opentracing.GlobalTracer().Inject(
span.Context(),
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header))
if err != nil {
azlog.Warn("failed to inject global tracer")
}
azlog.Debug("ApplicationInsights", "Request URL", req.URL.String())
res, err := ctxhttp.Do(ctx, e.httpClient, req)
if err != nil {
queryResultError(err)
}
body, err := ioutil.ReadAll(res.Body)
defer res.Body.Close()
if err != nil {
queryResultError(err)
}
if res.StatusCode/100 != 2 {
azlog.Debug("Request failed", "status", res.Status, "body", string(body))
queryResultError(fmt.Errorf("Request failed status: %v", res.Status))
}
var logResponse AzureLogAnalyticsResponse
d := json.NewDecoder(bytes.NewReader(body))
d.UseNumber()
err = d.Decode(&logResponse)
if err != nil {
queryResultError(err)
}
t, err := logResponse.GetPrimaryResultTable()
if err != nil {
queryResultError(err)
}
frame, err := LogTableToFrame(t)
if err != nil {
return queryResultError(err)
}
if query.ResultFormat == "time_series" {
tsSchema := frame.TimeSeriesSchema()
if tsSchema.Type == data.TimeSeriesTypeLong {
wideFrame, err := data.LongToWide(frame, &data.FillMissing{})
if err == nil {
frame = wideFrame
} else {
frame.AppendNotices(data.Notice{Severity: data.NoticeSeverityWarning, Text: "could not convert frame to time series, returning raw table: " + err.Error()})
}
}
}
frames := data.Frames{frame}
queryResult.Dataframes = tsdb.NewDecodedDataFrames(frames)
return queryResult
}
func (e *InsightsAnalyticsDatasource) createRequest(ctx context.Context, dsInfo *models.DataSource) (*http.Request, error) {
// find plugin
plugin, ok := plugins.DataSources[dsInfo.Type]
if !ok {
return nil, errors.New("Unable to find datasource plugin Azure Application Insights")
}
cloudName := dsInfo.JsonData.Get("cloudName").MustString("azuremonitor")
appInsightsRoute, pluginRouteName, err := e.getPluginRoute(plugin, cloudName)
if err != nil {
return nil, err
}
appInsightsAppID := dsInfo.JsonData.Get("appInsightsAppId").MustString()
proxyPass := fmt.Sprintf("%s/v1/apps/%s", pluginRouteName, appInsightsAppID)
u, err := url.Parse(dsInfo.Url)
if err != nil {
return nil, fmt.Errorf("unable to parse url for Application Insights Analytics datasource: %w", err)
}
u.Path = path.Join(u.Path, fmt.Sprintf("/v1/apps/%s", appInsightsAppID))
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
azlog.Debug("Failed to create request", "error", err)
return nil, errutil.Wrap("Failed to create request", err)
}
req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
pluginproxy.ApplyRoute(ctx, req, proxyPass, appInsightsRoute, dsInfo)
return req, nil
}
func (e *InsightsAnalyticsDatasource) getPluginRoute(plugin *plugins.DataSourcePlugin, cloudName string) (*plugins.AppPluginRoute, string, error) {
pluginRouteName := "appinsights"
if cloudName == "chinaazuremonitor" {
pluginRouteName = "chinaappinsights"
}
var pluginRoute *plugins.AppPluginRoute
for _, route := range plugin.Routes {
if route.Path == pluginRouteName {
pluginRoute = route
break
}
}
return pluginRoute, pluginRouteName, nil
}
+7 -5
View File
@@ -107,16 +107,18 @@ type insightsJSONQuery struct {
Dimension string `json:"dimension"`
DimensionFilter string `json:"dimensionFilter"`
MetricName string `json:"metricName"`
RawQuery *bool `json:"rawQuery"`
RawQueryString string `json:"rawQueryString"`
TimeGrain string `json:"timeGrain"`
TimeColumn string `json:"timeColumn"`
ValueColumn string `json:"valueColumn"`
SegmentColumn string `json:"segmentColumn"`
} `json:"appInsights"`
Raw *bool `json:"raw"`
}
type insightsAnalyticsJSONQuery struct {
InsightsAnalytics struct {
Query string `json:"query"`
ResultFormat string `json:"resultFormat"`
} `json:"insightsAnalytics"`
}
// logJSONQuery is the frontend JSON query model for an Azure Log Analytics query.
type logJSONQuery struct {
AzureLogAnalytics struct {