public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AzureMonitor: Improve Log Analytics query efficiency (#74675)

Browse Source 
* Promisify loading schema

- Move schema loading to LogsQueryEditor
- Improve typing
- Switch callbacks to promises

* Update types

* Refactor backend for new props

- Rename intersectTime
- Support setting timeColumn
- Add additional properties to logs request body

* Update applyTemplateVariables

* Update set functions

* Add new TimeManagement component

* Update LogsQueryEditor

* Hardcode timestamp column for traces queries

* Ensure timeColumn is always set for log queries

* Update tests

* Update frontend tests

* Readd type to make migration easier

* Add migration

* Add fake schema

* Use predefined type

* Update checks and defaults

* Add tests

* README updates

* README update

* Type update

* Lint

* More linting and type fixing

* Error silently

* More linting and typing

* Update betterer

* Update test

* Simplify default column setting

* Fix default column setting

* Add tracking

* Review

- Fix typo on comment
- Destructure and remove type assertion
- Break out await into two variables
- Remove lets and rename variable for clarity
This commit is contained in:
Andreas Christou
2023-09-18 18:38:39 +01:00
committed by GitHub
parent 025979df75
commit b779ce5687
25 changed files with 693 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/azuremonitor/loganalytics/azure-log-analytics-datasource_test.go
+93 -56
View File
@@ -107,7 +107,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
RefID: "A",
@@ -126,7 +126,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
Query: "Perf | where ['TimeGenerated'] >= datetime('2018-03-15T13:00:00Z') and ['TimeGenerated'] <= datetime('2018-03-15T13:34:00Z') | where ['Computer'] in ('comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, 34000ms), Computer",
@@ -134,7 +134,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TimeRange: timeRange,
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
},
},
Err: require.NoError,
@@ -172,7 +172,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
Resources: []string{},
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
},
},
Err: require.NoError,
@@ -210,7 +210,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
Resources: []string{},
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
},
},
Err: require.NoError,
@@ -225,7 +225,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "Perf",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
RefID: "A",
@@ -243,14 +243,14 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "Perf",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
Query: "Perf",
Resources: []string{"/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace"},
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
},
},
Err: require.NoError,
@@ -265,7 +265,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resources": ["/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace", "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace2"],
"query": "Perf",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
RefID: "A",
@@ -284,7 +284,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resources": ["/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace", "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace2"],
"query": "Perf",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
Query: "Perf",
@@ -292,7 +292,52 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TimeRange: timeRange,
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
},
},
Err: require.NoError,
},
{
name: "Query that uses dashboard time",
queryModel: []backend.DataQuery{
{
JSON: []byte(fmt.Sprintf(`{
"queryType": "Azure Log Analytics",
"azureLogAnalytics": {
"resources": ["/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace"],
"query": "Perf",
"resultFormat": "%s",
"dashboardTime": true,
"timeColumn": "TimeGenerated"
}
}`, types.TimeSeries)),
RefID: "A",
TimeRange: timeRange,
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
},
},
azureLogAnalyticsQueries: []*AzureLogAnalyticsQuery{
{
RefID: "A",
ResultFormat: types.TimeSeries,
URL: "v1/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace/query",
JSON: []byte(fmt.Sprintf(`{
"queryType": "Azure Log Analytics",
"azureLogAnalytics": {
"resources": ["/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace"],
"query": "Perf",
"resultFormat": "%s",
"dashboardTime": true,
"timeColumn": "TimeGenerated"
}
}`, types.TimeSeries)),
Query: "Perf",
Resources: []string{"/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace"},
TimeRange: timeRange,
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
DashboardTime: true,
TimeColumn: "TimeGenerated",
},
},
Err: require.NoError,
@@ -373,7 +418,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -451,7 +497,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -526,7 +573,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"${__data.fields.traceID}\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -604,7 +652,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -687,7 +736,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -770,7 +820,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -853,7 +904,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -928,7 +980,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"${__data.fields.traceID}\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1006,7 +1059,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1052,7 +1106,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1134,7 +1189,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"app('/subscriptions/test-sub/resourcegroups/test-rg/providers/microsoft.insights/components/r2').traces\n" +
"| where operation_Id == \"op-id-multi\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1213,7 +1269,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"app('/subscriptions/test-sub/resourcegroups/test-rg/providers/microsoft.insights/components/r2').traces\n" +
"| where operation_Id == \"${__data.fields.traceID}\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1295,7 +1352,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"app('/subscriptions/test-sub/resourcegroups/test-rg/providers/microsoft.insights/components/r2').traces\n" +
"| where operation_Id == \"op-id-multi\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1384,7 +1442,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"app('/subscriptions/test-sub/resourcegroups/test-rg/providers/microsoft.insights/components/r3').traces\n" +
"| where operation_Id == \"op-id-non-overlapping\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1411,7 +1470,7 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
req, err := ds.createRequest(ctx, url, &AzureLogAnalyticsQuery{
Resources: []string{"r"},
Query: "Perf",
IntersectTime: false,
DashboardTime: false,
AppInsightsQuery: false,
})
require.NoError(t, err)
@@ -1430,30 +1489,6 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
}
})
t.Run("creates a request with timespan", func(t *testing.T) {
ds := AzureLogAnalyticsDatasource{}
req, err := ds.createRequest(ctx, url, &AzureLogAnalyticsQuery{
Resources: []string{"r"},
Query: "Perf",
IntersectTime: true,
AppInsightsQuery: false,
})
require.NoError(t, err)
if req.URL.String() != url {
t.Errorf("Expecting %s, got %s", url, req.URL.String())
}
expectedHeaders := http.Header{"Content-Type": []string{"application/json"}}
if !cmp.Equal(req.Header, expectedHeaders) {
t.Errorf("Unexpected HTTP headers: %v", cmp.Diff(req.Header, expectedHeaders))
}
expectedBody := `{"query":"Perf","timespan":"0001-01-01T00:00:00Z/0001-01-01T00:00:00Z"}`
body, err := io.ReadAll(req.Body)
require.NoError(t, err)
if !cmp.Equal(string(body), expectedBody) {
t.Errorf("Unexpected Body: %v", cmp.Diff(string(body), expectedBody))
}
})
t.Run("creates a request with multiple resources", func(t *testing.T) {
ds := AzureLogAnalyticsDatasource{}
req, err := ds.createRequest(ctx, url, &AzureLogAnalyticsQuery{
@@ -1461,7 +1496,7 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
Query: "Perf",
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
})
require.NoError(t, err)
expectedBody := `{"query":"Perf","workspaces":["/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r1","/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r2"]}`
@@ -1472,7 +1507,7 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
}
})
t.Run("creates a request with timerange from query", func(t *testing.T) {
t.Run("creates a request with timerange from dashboard", func(t *testing.T) {
ds := AzureLogAnalyticsDatasource{}
from := time.Now()
to := from.Add(3 * time.Hour)
@@ -1485,10 +1520,11 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
To: to,
},
AppInsightsQuery: false,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "TimeGenerated",
})
require.NoError(t, err)
expectedBody := fmt.Sprintf(`{"query":"Perf","timespan":"%s/%s","workspaces":["/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r1","/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r2"]}`, from.Format(time.RFC3339), to.Format(time.RFC3339))
expectedBody := fmt.Sprintf(`{"query":"Perf","query_datetimescope_column":"TimeGenerated","query_datetimescope_from":"%s","query_datetimescope_to":"%s","timespan":"%s/%s","workspaces":["/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r1","/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r2"]}`, from.Format(time.RFC3339), to.Format(time.RFC3339), from.Format(time.RFC3339), to.Format(time.RFC3339))
body, err := io.ReadAll(req.Body)
require.NoError(t, err)
if !cmp.Equal(string(body), expectedBody) {
@@ -1508,10 +1544,11 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
To: to,
},
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
})
require.NoError(t, err)
expectedBody := fmt.Sprintf(`{"applications":["/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.Insights/components/r1","/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.Insights/components/r2"],"query":"","timespan":"%s/%s"}`, from.Format(time.RFC3339), to.Format(time.RFC3339))
expectedBody := fmt.Sprintf(`{"applications":["/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.Insights/components/r1","/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.Insights/components/r2"],"query":"","query_datetimescope_column":"timestamp","query_datetimescope_from":"%s","query_datetimescope_to":"%s","timespan":"%s/%s"}`, from.Format(time.RFC3339), to.Format(time.RFC3339), from.Format(time.RFC3339), to.Format(time.RFC3339))
body, err := io.ReadAll(req.Body)
require.NoError(t, err)
if !cmp.Equal(string(body), expectedBody) {