public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AzureMonitor: Improve Log Analytics query efficiency (#74675)

Browse Source 
* Promisify loading schema

- Move schema loading to LogsQueryEditor
- Improve typing
- Switch callbacks to promises

* Update types

* Refactor backend for new props

- Rename intersectTime
- Support setting timeColumn
- Add additional properties to logs request body

* Update applyTemplateVariables

* Update set functions

* Add new TimeManagement component

* Update LogsQueryEditor

* Hardcode timestamp column for traces queries

* Ensure timeColumn is always set for log queries

* Update tests

* Update frontend tests

* Readd type to make migration easier

* Add migration

* Add fake schema

* Use predefined type

* Update checks and defaults

* Add tests

* README updates

* README update

* Type update

* Lint

* More linting and type fixing

* Error silently

* More linting and typing

* Update betterer

* Update test

* Simplify default column setting

* Fix default column setting

* Add tracking

* Review

- Fix typo on comment
- Destructure and remove type assertion
- Break out await into two variables
- Remove lets and rename variable for clarity
This commit is contained in:
Andreas Christou
2023-09-18 18:38:39 +01:00
committed by GitHub
parent 025979df75
commit b779ce5687
25 changed files with 693 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/azuremonitor/loganalytics/azure-log-analytics-datasource.go
+22 -7
View File
@@ -47,7 +47,8 @@ type AzureLogAnalyticsQuery struct {
Resources []string
QueryType string
AppInsightsQuery bool
IntersectTime bool
DashboardTime bool
TimeColumn string
}
func (e *AzureLogAnalyticsDatasource) ResourceRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client) (http.ResponseWriter, error) {
@@ -107,7 +108,8 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(ctx context.Context, queries
traceExploreQuery := ""
traceParentExploreQuery := ""
traceLogsExploreQuery := ""
intersectTime := false
dashboardTime := false
timeColumn := ""
if query.QueryType == string(dataquery.AzureQueryTypeAzureLogAnalytics) {
queryJSONModel := types.LogJSONQuery{}
err := json.Unmarshal(query.JSON, &queryJSONModel)
@@ -143,8 +145,16 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(ctx context.Context, queries
queryString = *azureLogAnalyticsTarget.Query
}
if azureLogAnalyticsTarget.IntersectTime != nil {
intersectTime = *azureLogAnalyticsTarget.IntersectTime
if azureLogAnalyticsTarget.DashboardTime != nil {
dashboardTime = *azureLogAnalyticsTarget.DashboardTime
if dashboardTime {
if azureLogAnalyticsTarget.TimeColumn != nil {
timeColumn = *azureLogAnalyticsTarget.TimeColumn
} else {
// Final fallback to TimeGenerated if no column is provided
timeColumn = "TimeGenerated"
}
}
}
}
@@ -218,7 +228,8 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(ctx context.Context, queries
return nil, fmt.Errorf("failed to create traces logs explore query: %s", err)
}
intersectTime = true
dashboardTime = true
timeColumn = "timestamp"
}
apiURL := getApiURL(resourceOrWorkspace, appInsightsQuery)
@@ -241,7 +252,8 @@ func (e *AzureLogAnalyticsDatasource) buildQueries(ctx context.Context, queries
TraceParentExploreQuery: traceParentExploreQuery,
TraceLogsExploreQuery: traceLogsExploreQuery,
AppInsightsQuery: appInsightsQuery,
IntersectTime: intersectTime,
DashboardTime: dashboardTime,
TimeColumn: timeColumn,
})
}
@@ -432,11 +444,14 @@ func (e *AzureLogAnalyticsDatasource) createRequest(ctx context.Context, queryUR
"query": query.Query,
}
if query.IntersectTime {
if query.DashboardTime {
from := query.TimeRange.From.Format(time.RFC3339)
to := query.TimeRange.To.Format(time.RFC3339)
timespan := fmt.Sprintf("%s/%s", from, to)
body["timespan"] = timespan
body["query_datetimescope_from"] = from
body["query_datetimescope_to"] = to
body["query_datetimescope_column"] = query.TimeColumn
}
if len(query.Resources) > 1 && query.QueryType == string(dataquery.AzureQueryTypeAzureLogAnalytics) && !query.AppInsightsQuery {
pkg/tsdb/azuremonitor/loganalytics/azure-log-analytics-datasource_test.go
+93 -56
View File
@@ -107,7 +107,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
RefID: "A",
@@ -126,7 +126,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "Perf | where $__timeFilter() | where $__contains(Computer, 'comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, $__interval), Computer",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
Query: "Perf | where ['TimeGenerated'] >= datetime('2018-03-15T13:00:00Z') and ['TimeGenerated'] <= datetime('2018-03-15T13:34:00Z') | where ['Computer'] in ('comp1','comp2') | summarize avg(CounterValue) by bin(TimeGenerated, 34000ms), Computer",
@@ -134,7 +134,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TimeRange: timeRange,
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
},
},
Err: require.NoError,
@@ -172,7 +172,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
Resources: []string{},
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
},
},
Err: require.NoError,
@@ -210,7 +210,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
Resources: []string{},
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
},
},
Err: require.NoError,
@@ -225,7 +225,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "Perf",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
RefID: "A",
@@ -243,14 +243,14 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resource": "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace",
"query": "Perf",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
Query: "Perf",
Resources: []string{"/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace"},
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
},
},
Err: require.NoError,
@@ -265,7 +265,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resources": ["/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace", "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace2"],
"query": "Perf",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
RefID: "A",
@@ -284,7 +284,7 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"resources": ["/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace", "/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace2"],
"query": "Perf",
"resultFormat": "%s",
"intersectTime": false
"dashboardTime": false
}
}`, types.TimeSeries)),
Query: "Perf",
@@ -292,7 +292,52 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TimeRange: timeRange,
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
},
},
Err: require.NoError,
},
{
name: "Query that uses dashboard time",
queryModel: []backend.DataQuery{
{
JSON: []byte(fmt.Sprintf(`{
"queryType": "Azure Log Analytics",
"azureLogAnalytics": {
"resources": ["/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace"],
"query": "Perf",
"resultFormat": "%s",
"dashboardTime": true,
"timeColumn": "TimeGenerated"
}
}`, types.TimeSeries)),
RefID: "A",
TimeRange: timeRange,
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
},
},
azureLogAnalyticsQueries: []*AzureLogAnalyticsQuery{
{
RefID: "A",
ResultFormat: types.TimeSeries,
URL: "v1/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace/query",
JSON: []byte(fmt.Sprintf(`{
"queryType": "Azure Log Analytics",
"azureLogAnalytics": {
"resources": ["/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace"],
"query": "Perf",
"resultFormat": "%s",
"dashboardTime": true,
"timeColumn": "TimeGenerated"
}
}`, types.TimeSeries)),
Query: "Perf",
Resources: []string{"/subscriptions/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/resourceGroups/cloud-datasources/providers/Microsoft.OperationalInsights/workspaces/AppInsightsTestDataWorkspace"},
TimeRange: timeRange,
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
DashboardTime: true,
TimeColumn: "TimeGenerated",
},
},
Err: require.NoError,
@@ -373,7 +418,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -451,7 +497,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -526,7 +573,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"${__data.fields.traceID}\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -604,7 +652,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -687,7 +736,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -770,7 +820,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -853,7 +904,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -928,7 +980,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"${__data.fields.traceID}\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1006,7 +1059,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1052,7 +1106,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
TraceLogsExploreQuery: "union availabilityResults,\n" + "customEvents,\n" + "dependencies,\n" + "exceptions,\n" + "pageViews,\n" + "requests,\n" + "traces\n" +
"| where operation_Id == \"test-op-id\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1134,7 +1189,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"app('/subscriptions/test-sub/resourcegroups/test-rg/providers/microsoft.insights/components/r2').traces\n" +
"| where operation_Id == \"op-id-multi\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1213,7 +1269,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"app('/subscriptions/test-sub/resourcegroups/test-rg/providers/microsoft.insights/components/r2').traces\n" +
"| where operation_Id == \"${__data.fields.traceID}\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1295,7 +1352,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"app('/subscriptions/test-sub/resourcegroups/test-rg/providers/microsoft.insights/components/r2').traces\n" +
"| where operation_Id == \"op-id-multi\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1384,7 +1442,8 @@ func TestBuildingAzureLogAnalyticsQueries(t *testing.T) {
"app('/subscriptions/test-sub/resourcegroups/test-rg/providers/microsoft.insights/components/r3').traces\n" +
"| where operation_Id == \"op-id-non-overlapping\"",
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
},
},
Err: require.NoError,
@@ -1411,7 +1470,7 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
req, err := ds.createRequest(ctx, url, &AzureLogAnalyticsQuery{
Resources: []string{"r"},
Query: "Perf",
IntersectTime: false,
DashboardTime: false,
AppInsightsQuery: false,
})
require.NoError(t, err)
@@ -1430,30 +1489,6 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
}
})
t.Run("creates a request with timespan", func(t *testing.T) {
ds := AzureLogAnalyticsDatasource{}
req, err := ds.createRequest(ctx, url, &AzureLogAnalyticsQuery{
Resources: []string{"r"},
Query: "Perf",
IntersectTime: true,
AppInsightsQuery: false,
})
require.NoError(t, err)
if req.URL.String() != url {
t.Errorf("Expecting %s, got %s", url, req.URL.String())
}
expectedHeaders := http.Header{"Content-Type": []string{"application/json"}}
if !cmp.Equal(req.Header, expectedHeaders) {
t.Errorf("Unexpected HTTP headers: %v", cmp.Diff(req.Header, expectedHeaders))
}
expectedBody := `{"query":"Perf","timespan":"0001-01-01T00:00:00Z/0001-01-01T00:00:00Z"}`
body, err := io.ReadAll(req.Body)
require.NoError(t, err)
if !cmp.Equal(string(body), expectedBody) {
t.Errorf("Unexpected Body: %v", cmp.Diff(string(body), expectedBody))
}
})
t.Run("creates a request with multiple resources", func(t *testing.T) {
ds := AzureLogAnalyticsDatasource{}
req, err := ds.createRequest(ctx, url, &AzureLogAnalyticsQuery{
@@ -1461,7 +1496,7 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
Query: "Perf",
QueryType: string(dataquery.AzureQueryTypeAzureLogAnalytics),
AppInsightsQuery: false,
IntersectTime: false,
DashboardTime: false,
})
require.NoError(t, err)
expectedBody := `{"query":"Perf","workspaces":["/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r1","/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r2"]}`
@@ -1472,7 +1507,7 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
}
})
t.Run("creates a request with timerange from query", func(t *testing.T) {
t.Run("creates a request with timerange from dashboard", func(t *testing.T) {
ds := AzureLogAnalyticsDatasource{}
from := time.Now()
to := from.Add(3 * time.Hour)
@@ -1485,10 +1520,11 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
To: to,
},
AppInsightsQuery: false,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "TimeGenerated",
})
require.NoError(t, err)
expectedBody := fmt.Sprintf(`{"query":"Perf","timespan":"%s/%s","workspaces":["/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r1","/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r2"]}`, from.Format(time.RFC3339), to.Format(time.RFC3339))
expectedBody := fmt.Sprintf(`{"query":"Perf","query_datetimescope_column":"TimeGenerated","query_datetimescope_from":"%s","query_datetimescope_to":"%s","timespan":"%s/%s","workspaces":["/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r1","/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.OperationalInsights/workspaces/r2"]}`, from.Format(time.RFC3339), to.Format(time.RFC3339), from.Format(time.RFC3339), to.Format(time.RFC3339))
body, err := io.ReadAll(req.Body)
require.NoError(t, err)
if !cmp.Equal(string(body), expectedBody) {
@@ -1508,10 +1544,11 @@ func TestLogAnalyticsCreateRequest(t *testing.T) {
To: to,
},
AppInsightsQuery: true,
IntersectTime: true,
DashboardTime: true,
TimeColumn: "timestamp",
})
require.NoError(t, err)
expectedBody := fmt.Sprintf(`{"applications":["/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.Insights/components/r1","/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.Insights/components/r2"],"query":"","timespan":"%s/%s"}`, from.Format(time.RFC3339), to.Format(time.RFC3339))
expectedBody := fmt.Sprintf(`{"applications":["/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.Insights/components/r1","/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.Insights/components/r2"],"query":"","query_datetimescope_column":"timestamp","query_datetimescope_from":"%s","query_datetimescope_to":"%s","timespan":"%s/%s"}`, from.Format(time.RFC3339), to.Format(time.RFC3339), from.Format(time.RFC3339), to.Format(time.RFC3339))
body, err := io.ReadAll(req.Body)
require.NoError(t, err)
if !cmp.Equal(string(body), expectedBody) {