diff --git a/docs/sources/administration/provisioning/index.md b/docs/sources/administration/provisioning/index.md index 289e4b4ab1d..f1c59989121 100644 --- a/docs/sources/administration/provisioning/index.md +++ b/docs/sources/administration/provisioning/index.md @@ -169,56 +169,57 @@ Common settings in the [built-in core data sources]({{< relref "../../datasource > **Note:** Data sources tagged with _HTTP\*_ communicate using the HTTP protocol, which includes all core data source plugins except MySQL, PostgreSQL, and MSSQL. -| Name | Type | Data source | Description | -| -------------------------- | ------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | -| tlsAuth | boolean | _HTTP\*_, MySQL | Enable TLS authentication using client cert configured in secure json data | -| tlsAuthWithCACert | boolean | _HTTP\*_, MySQL, PostgreSQL | Enable TLS authentication using CA cert | -| tlsSkipVerify | boolean | _HTTP\*_, MySQL, PostgreSQL, MSSQL | Controls whether a client verifies the server's certificate chain and host name. | -| serverName | string | _HTTP\*_, MSSQL | Optional. Controls the server name used for certificate common name/subject alternative name verification. Defaults to using the data source URL. | -| timeout | string | _HTTP\*_ | Request timeout in seconds. Overrides dataproxy.timeout option | -| graphiteVersion | string | Graphite | Graphite version | -| timeInterval | string | Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL and MSSQL | Lowest interval/step value that should be used for this data source. | -| httpMode | string | Influxdb | HTTP Method. 'GET', 'POST', defaults to GET | -| maxSeries | number | Influxdb | Max number of series/tables that Grafana processes | -| httpMethod | string | Prometheus | HTTP Method. 'GET', 'POST', defaults to POST | -| customQueryParameters | string | Prometheus | Query parameters to add, as a URL-encoded string. | -| manageAlerts | boolean | Prometheus and Loki | Manage alerts via Alerting UI | -| alertmanagerUid | string | Prometheus and Loki | UID of Alert Manager that manages Alert for this data source. | -| timeField | string | Elasticsearch | Which field that should be used as timestamp | -| interval | string | Elasticsearch | Index date time format. nil(No Pattern), 'Hourly', 'Daily', 'Weekly', 'Monthly' or 'Yearly' | -| logMessageField | string | Elasticsearch | Which field should be used as the log message | -| logLevelField | string | Elasticsearch | Which field should be used to indicate the priority of the log message | -| maxConcurrentShardRequests | number | Elasticsearch | Maximum number of concurrent shard requests that each sub-search request executes per node | -| sigV4Auth | boolean | Elasticsearch and Prometheus | Enable usage of SigV4 | -| sigV4AuthType | string | Elasticsearch and Prometheus | SigV4 auth provider. default/credentials/keys | -| sigV4ExternalId | string | Elasticsearch and Prometheus | Optional SigV4 External ID | -| sigV4AssumeRoleArn | string | Elasticsearch and Prometheus | Optional SigV4 ARN role to assume | -| sigV4Region | string | Elasticsearch and Prometheus | SigV4 AWS region | -| sigV4Profile | string | Elasticsearch and Prometheus | Optional SigV4 credentials profile | -| authType | string | Cloudwatch | Auth provider. default/credentials/keys | -| externalId | string | Cloudwatch | Optional External ID | -| assumeRoleArn | string | Cloudwatch | Optional ARN role to assume | -| defaultRegion | string | Cloudwatch | Optional default AWS region | -| customMetricsNamespaces | string | Cloudwatch | Namespaces of Custom Metrics | -| profile | string | Cloudwatch | Optional credentials profile | -| tsdbVersion | string | OpenTSDB | Version | -| tsdbResolution | string | OpenTSDB | Resolution | -| sslmode | string | PostgreSQL | SSLmode. 'disable', 'require', 'verify-ca' or 'verify-full' | -| tlsConfigurationMethod | string | PostgreSQL | SSL Certificate configuration, either by 'file-path' or 'file-content' | -| sslRootCertFile | string | PostgreSQL, MSSQL | SSL server root certificate file, must be readable by the Grafana user | -| sslCertFile | string | PostgreSQL | SSL client certificate file, must be readable by the Grafana user | -| sslKeyFile | string | PostgreSQL | SSL client key file, must be readable by _only_ the Grafana user | -| encrypt | string | MSSQL | Connection SSL encryption handling. 'disable', 'false' or 'true' | -| postgresVersion | number | PostgreSQL | Postgres version as a number (903/904/905/906/1000) meaning v9.3, v9.4, ..., v10 | -| timescaledb | boolean | PostgreSQL | Enable usage of TimescaleDB extension | -| maxOpenConns | number | MySQL, PostgreSQL and MSSQL | Maximum number of open connections to the database (Grafana v5.4+) | -| maxIdleConns | number | MySQL, PostgreSQL and MSSQL | Maximum number of connections in the idle connection pool (Grafana v5.4+) | -| connMaxLifetime | number | MySQL, PostgreSQL and MSSQL | Maximum amount of time in seconds a connection may be reused (Grafana v5.4+) | -| keepCookies | array | _HTTP\*_ | Cookies that needs to be passed along while communicating with data sources | -| prometheusVersion | string | Prometheus | The version of the Prometheus data source, such as `2.37.0`, `2.24.0` | -| prometheusType | string | Prometheus | The type of the Prometheus data sources. such as `Prometheus`, `Cortex`, `Thanos`, `Mimir` | -| implementation | string | AlertManager | The implementation of the AlertManager data source, such as `prometheus`, `cortex` or `mimir` | -| handleGrafanaManagedAlerts | boolean | AlertManager | When enabled, Grafana-managed alerts are sent to this Alertmanager | +| Name | Type | Data source | Description | +| -------------------------- | ------- | ---------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| tlsAuth | boolean | _HTTP\*_, MySQL | Enable TLS authentication using client cert configured in secure json data | +| tlsAuthWithCACert | boolean | _HTTP\*_, MySQL, PostgreSQL | Enable TLS authentication using CA cert | +| tlsSkipVerify | boolean | _HTTP\*_, MySQL, PostgreSQL, MSSQL | Controls whether a client verifies the server's certificate chain and host name. | +| serverName | string | _HTTP\*_, MSSQL | Optional. Controls the server name used for certificate common name/subject alternative name verification. Defaults to using the data source URL. | +| timeout | string | _HTTP\*_ | Request timeout in seconds. Overrides dataproxy.timeout option | +| graphiteVersion | string | Graphite | Graphite version | +| timeInterval | string | Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL and MSSQL | Lowest interval/step value that should be used for this data source. | +| httpMode | string | Influxdb | HTTP Method. 'GET', 'POST', defaults to GET | +| maxSeries | number | Influxdb | Max number of series/tables that Grafana processes | +| httpMethod | string | Prometheus | HTTP Method. 'GET', 'POST', defaults to POST | +| customQueryParameters | string | Prometheus | Query parameters to add, as a URL-encoded string. | +| manageAlerts | boolean | Prometheus and Loki | Manage alerts via Alerting UI | +| alertmanagerUid | string | Prometheus and Loki | UID of Alert Manager that manages Alert for this data source. | +| timeField | string | Elasticsearch | Which field that should be used as timestamp | +| interval | string | Elasticsearch | Index date time format. nil(No Pattern), 'Hourly', 'Daily', 'Weekly', 'Monthly' or 'Yearly' | +| logMessageField | string | Elasticsearch | Which field should be used as the log message | +| logLevelField | string | Elasticsearch | Which field should be used to indicate the priority of the log message | +| maxConcurrentShardRequests | number | Elasticsearch | Maximum number of concurrent shard requests that each sub-search request executes per node | +| sigV4Auth | boolean | Elasticsearch and Prometheus | Enable usage of SigV4 | +| sigV4AuthType | string | Elasticsearch and Prometheus | SigV4 auth provider. default/credentials/keys | +| sigV4ExternalId | string | Elasticsearch and Prometheus | Optional SigV4 External ID | +| sigV4AssumeRoleArn | string | Elasticsearch and Prometheus | Optional SigV4 ARN role to assume | +| sigV4Region | string | Elasticsearch and Prometheus | SigV4 AWS region | +| sigV4Profile | string | Elasticsearch and Prometheus | Optional SigV4 credentials profile | +| authType | string | Cloudwatch | Auth provider. default/credentials/keys | +| externalId | string | Cloudwatch | Optional External ID | +| assumeRoleArn | string | Cloudwatch | Optional ARN role to assume | +| defaultRegion | string | Cloudwatch | Optional default AWS region | +| customMetricsNamespaces | string | Cloudwatch | Namespaces of Custom Metrics | +| profile | string | Cloudwatch | Optional credentials profile | +| tsdbVersion | string | OpenTSDB | Version | +| tsdbResolution | string | OpenTSDB | Resolution | +| sslmode | string | PostgreSQL | SSLmode. 'disable', 'require', 'verify-ca' or 'verify-full' | +| tlsConfigurationMethod | string | PostgreSQL | SSL Certificate configuration, either by 'file-path' or 'file-content' | +| sslRootCertFile | string | PostgreSQL, MSSQL | SSL server root certificate file, must be readable by the Grafana user | +| sslCertFile | string | PostgreSQL | SSL client certificate file, must be readable by the Grafana user | +| sslKeyFile | string | PostgreSQL | SSL client key file, must be readable by _only_ the Grafana user | +| encrypt | string | MSSQL | Connection SSL encryption handling. 'disable', 'false' or 'true' | +| postgresVersion | number | PostgreSQL | Postgres version as a number (903/904/905/906/1000) meaning v9.3, v9.4, ..., v10 | +| timescaledb | boolean | PostgreSQL | Enable usage of TimescaleDB extension | +| maxOpenConns | number | MySQL, PostgreSQL and MSSQL | Maximum number of open connections to the database (Grafana v5.4+) | +| maxIdleConns | number | MySQL, PostgreSQL and MSSQL | Maximum number of connections in the idle connection pool (Grafana v5.4+) | +| connMaxLifetime | number | MySQL, PostgreSQL and MSSQL | Maximum amount of time in seconds a connection may be reused (Grafana v5.4+) | +| keepCookies | array | _HTTP\*_ | Cookies that needs to be passed along while communicating with data sources | +| prometheusVersion | string | Prometheus | The version of the Prometheus data source, such as `2.37.0`, `2.24.0` | +| prometheusType | string | Prometheus | The type of the Prometheus data sources. such as `Prometheus`, `Cortex`, `Thanos`, `Mimir` | +| cacheLevel | string | Prometheus | This determines the duration of the browser cache. Valid values include: `Low`, `Medium`, `High`, and `None`. This field is configurable when you enable the `prometheusResourceBrowserCache` feature flag. | +| implementation | string | AlertManager | The implementation of the AlertManager data source, such as `prometheus`, `cortex` or `mimir` | +| handleGrafanaManagedAlerts | boolean | AlertManager | When enabled, Grafana-managed alerts are sent to this Alertmanager | For examples of specific data sources' JSON data, refer to that [data source's documentation]({{< relref "../../datasources" >}}). diff --git a/docs/sources/datasources/prometheus/_index.md b/docs/sources/datasources/prometheus/_index.md index fbba28629be..376489eaf79 100644 --- a/docs/sources/datasources/prometheus/_index.md +++ b/docs/sources/datasources/prometheus/_index.md @@ -96,6 +96,7 @@ datasources: manageAlerts: true prometheusType: Prometheus prometheusVersion: 2.37.0 + cacheLevel: 'High' exemplarTraceIdDestinations: # Field with internal link pointing to data source in Grafana. # datasourceUid value can be anything, but it should be unique across all defined data source uids. diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 6916434c112..fb6508ceb67 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -96,6 +96,7 @@ Alpha features might be changed or removed without prior notice. | `traceqlSearch` | Enables the 'TraceQL Search' tab for the Tempo datasource which provides a UI to generate TraceQL queries | | `prometheusMetricEncyclopedia` | Replaces the Prometheus query builder metric select option with a paginated and filterable component | | `timeSeriesTable` | Enable time series table transformer & sparkline cell type | +| `prometheusResourceBrowserCache` | Displays browser caching options in Prometheus data source configuration | | `influxdbBackendMigration` | Query InfluxDB InfluxQL without the proxy | | `clientTokenRotation` | Replaces the current in-request token rotation so that the client initiates the rotation | | `prometheusDataplane` | Changes responses to from Prometheus to be compliant with the dataplane specification. In particular it sets the numeric Field.Name from 'Value' to the value of the `__name__` label when present. | diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index d157af930b1..200907cb6e9 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -83,6 +83,7 @@ export interface FeatureToggles { traceqlSearch?: boolean; prometheusMetricEncyclopedia?: boolean; timeSeriesTable?: boolean; + prometheusResourceBrowserCache?: boolean; influxdbBackendMigration?: boolean; clientTokenRotation?: boolean; disableElasticsearchBackendExploreQuery?: boolean; diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 35e63af4483..c222cdd39da 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -438,6 +438,13 @@ var ( FrontendOnly: true, Owner: appO11ySquad, }, + { + Name: "prometheusResourceBrowserCache", + Description: "Displays browser caching options in Prometheus data source configuration", + State: FeatureStateAlpha, + FrontendOnly: true, + Owner: grafanaObservabilityMetricsSquad, + }, { Name: "influxdbBackendMigration", Description: "Query InfluxDB InfluxQL without the proxy", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 89f4f467682..1c261bb6fbe 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -64,6 +64,7 @@ drawerDataSourcePicker,alpha,@grafana/grafana-bi-squad,false,false,false,true traceqlSearch,alpha,@grafana/observability-traces-and-profiling,false,false,false,true prometheusMetricEncyclopedia,alpha,@grafana/observability-metrics,false,false,false,true timeSeriesTable,alpha,@grafana/app-o11y,false,false,false,true +prometheusResourceBrowserCache,alpha,@grafana/observability-metrics,false,false,false,true influxdbBackendMigration,alpha,@grafana/observability-metrics,false,false,false,true clientTokenRotation,alpha,@grafana/grafana-authnz-team,false,false,false,false disableElasticsearchBackendExploreQuery,beta,@grafana/observability-logs,false,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index bc47fc1ab16..75b1d0599db 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -267,6 +267,10 @@ const ( // Enable time series table transformer & sparkline cell type FlagTimeSeriesTable = "timeSeriesTable" + // FlagPrometheusResourceBrowserCache + // Displays browser caching options in Prometheus data source configuration + FlagPrometheusResourceBrowserCache = "prometheusResourceBrowserCache" + // FlagInfluxdbBackendMigration // Query InfluxDB InfluxQL without the proxy FlagInfluxdbBackendMigration = "influxdbBackendMigration" diff --git a/pkg/tsdb/prometheus/resource/resource.go b/pkg/tsdb/prometheus/resource/resource.go index 7d7f7d7ce05..3328be895b7 100644 --- a/pkg/tsdb/prometheus/resource/resource.go +++ b/pkg/tsdb/prometheus/resource/resource.go @@ -7,7 +7,6 @@ import ( "net/http" "github.com/grafana/grafana-plugin-sdk-go/backend" - "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/tsdb/prometheus/client" "github.com/grafana/grafana/pkg/tsdb/prometheus/utils" @@ -47,6 +46,12 @@ func (r *Resource) Execute(ctx context.Context, req *backend.CallResourceRequest return nil, fmt.Errorf("error querying resource: %v", err) } + // frontend sets the X-Grafana-Cache with the desired response cache control value + if len(req.GetHTTPHeaders().Get("X-Grafana-Cache")) > 0 { + resp.Header.Set("X-Grafana-Cache", "y") + resp.Header.Set("Cache-Control", req.GetHTTPHeaders().Get("X-Grafana-Cache")) + } + defer func() { tmpErr := resp.Body.Close() if tmpErr != nil && err == nil { diff --git a/public/app/plugins/datasource/prometheus/components/PromLink.test.tsx b/public/app/plugins/datasource/prometheus/components/PromLink.test.tsx index 5edb830120e..bb115283ada 100644 --- a/public/app/plugins/datasource/prometheus/components/PromLink.test.tsx +++ b/public/app/plugins/datasource/prometheus/components/PromLink.test.tsx @@ -1,7 +1,7 @@ import { render, screen } from '@testing-library/react'; import React from 'react'; -import { PanelData } from '@grafana/data'; +import { dateTime, PanelData, TimeRange } from '@grafana/data'; import { PrometheusDatasource } from '../datasource'; import { PromQuery } from '../types'; @@ -15,6 +15,10 @@ jest.mock('@grafana/data', () => ({ }, })); +const now = dateTime().valueOf(); +const intervalInSeconds = 60 * 5; +const endInput = encodeURIComponent(dateTime(now).add(5, 'hours').format('Y-MM-DD HH:mm')); + const getPanelData = (panelDataOverrides?: Partial) => { const panelData = { request: { @@ -24,12 +28,10 @@ const getPanelData = (panelDataOverrides?: Partial) => { { refId: 'B', datasource: 'prom2' }, ], range: { - to: { - utc: () => ({ - format: jest.fn(), - }), - }, - }, + raw: {}, + to: dateTime(now), // "now" + from: dateTime(now - 1000 * intervalInSeconds), // 5 minutes ago from "now" + } as TimeRange, }, }; @@ -38,7 +40,6 @@ const getPanelData = (panelDataOverrides?: Partial) => { const getDataSource = (datasourceOverrides?: Partial) => { const datasource = { - getPrometheusTime: () => 123, createQuery: () => ({ expr: 'up', step: 15 }), directUrl: 'prom1', getRateIntervalScopedVariable: jest.fn(() => ({ __rate_interval: { text: '60s', value: '60s' } })), @@ -49,7 +50,7 @@ const getDataSource = (datasourceOverrides?: Partial) => { const getDataSourceWithCustomQueryParameters = (datasourceOverrides?: Partial) => { const datasource = { - getPrometheusTime: () => 124, + getPrometheusTime: () => 1677870470, createQuery: () => ({ expr: 'up', step: 20 }), directUrl: 'prom3', getRateIntervalScopedVariable: jest.fn(() => ({ __rate_interval: { text: '60s', value: '60s' } })), @@ -68,7 +69,7 @@ describe('PromLink', () => { ); expect(screen.getByText('Prometheus')).toHaveAttribute( 'href', - 'prom1/graph?g0.expr=up&g0.range_input=0s&g0.end_input=undefined&g0.step_input=15&g0.tab=0' + `prom1/graph?g0.expr=up&g0.range_input=${intervalInSeconds}s&g0.end_input=${endInput}&g0.step_input=15&g0.tab=0` ); }); it('should show different link when there are 2 components with the same panel data', () => { @@ -85,11 +86,11 @@ describe('PromLink', () => { const promLinkButtons = screen.getAllByText('Prometheus'); expect(promLinkButtons[0]).toHaveAttribute( 'href', - 'prom1/graph?g0.expr=up&g0.range_input=0s&g0.end_input=undefined&g0.step_input=15&g0.tab=0' + `prom1/graph?g0.expr=up&g0.range_input=${intervalInSeconds}s&g0.end_input=${endInput}&g0.step_input=15&g0.tab=0` ); expect(promLinkButtons[1]).toHaveAttribute( 'href', - 'prom2/graph?g0.expr=up&g0.range_input=0s&g0.end_input=undefined&g0.step_input=15&g0.tab=0' + `prom2/graph?g0.expr=up&g0.range_input=${intervalInSeconds}s&g0.end_input=${endInput}&g0.step_input=15&g0.tab=0` ); }); it('should create sanitized link', async () => { @@ -116,7 +117,7 @@ describe('PromLink', () => { ); expect(screen.getByText('Prometheus')).toHaveAttribute( 'href', - 'prom3/graph?g0.foo=1&g0.expr=up&g0.range_input=0s&g0.end_input=undefined&g0.step_input=20&g0.tab=0' + `prom3/graph?g0.foo=1&g0.expr=up&g0.range_input=${intervalInSeconds}s&g0.end_input=${endInput}&g0.step_input=20&g0.tab=0` ); }); }); diff --git a/public/app/plugins/datasource/prometheus/components/PromLink.tsx b/public/app/plugins/datasource/prometheus/components/PromLink.tsx index 52f5405da4d..5b01ed63837 100644 --- a/public/app/plugins/datasource/prometheus/components/PromLink.tsx +++ b/public/app/plugins/datasource/prometheus/components/PromLink.tsx @@ -4,6 +4,7 @@ import React, { useEffect, useState, memo } from 'react'; import { DataQueryRequest, PanelData, ScopedVars, textUtil, rangeUtil } from '@grafana/data'; import { PrometheusDatasource } from '../datasource'; +import { getPrometheusTime } from '../language_utils'; import { PromQuery } from '../types'; interface Props { @@ -26,8 +27,8 @@ const PromLink = ({ panelData, query, datasource }: Props) => { request: { range, interval, scopedVars }, } = panelData; - const start = datasource.getPrometheusTime(range.from, false); - const end = datasource.getPrometheusTime(range.to, true); + const start = getPrometheusTime(range.from, false); + const end = getPrometheusTime(range.to, true); const rangeDiff = Math.ceil(end - start); const endTime = range.to.utc().format('YYYY-MM-DD HH:mm'); diff --git a/public/app/plugins/datasource/prometheus/components/PromQueryEditorByApp.test.tsx b/public/app/plugins/datasource/prometheus/components/PromQueryEditorByApp.test.tsx index 93cc8173792..e8da197c087 100644 --- a/public/app/plugins/datasource/prometheus/components/PromQueryEditorByApp.test.tsx +++ b/public/app/plugins/datasource/prometheus/components/PromQueryEditorByApp.test.tsx @@ -28,6 +28,7 @@ function setup(app: CoreApp): { onRunQuery: jest.Mock } { getInitHints: () => [], getPrometheusTime: jest.fn((date, roundup) => 123), getQueryHints: jest.fn(() => []), + getDebounceTimeInMilliseconds: jest.fn(() => 300), languageProvider: { start: () => Promise.resolve([]), syntax: () => {}, diff --git a/public/app/plugins/datasource/prometheus/components/monaco-query-field/MonacoQueryField.tsx b/public/app/plugins/datasource/prometheus/components/monaco-query-field/MonacoQueryField.tsx index 7952a7c0e82..6527f583de2 100644 --- a/public/app/plugins/datasource/prometheus/components/monaco-query-field/MonacoQueryField.tsx +++ b/public/app/plugins/datasource/prometheus/components/monaco-query-field/MonacoQueryField.tsx @@ -165,6 +165,7 @@ const MonacoQueryField = (props: Props) => { const getSeriesValues = lpRef.current.getSeriesValues; const getSeriesLabels = lpRef.current.getSeriesLabels; + const dataProvider = { getHistory, getAllMetricNames, @@ -230,7 +231,7 @@ const MonacoQueryField = (props: Props) => { const updateCurrentEditorValue = debounce(() => { const editorValue = editor.getValue(); onChangeRef.current(editorValue); - }, 300); + }, lpRef.current.datasource.getDebounceTimeInMilliseconds()); editor.getModel()?.onDidChangeContent(() => { updateCurrentEditorValue(); diff --git a/public/app/plugins/datasource/prometheus/configuration/PromSettings.tsx b/public/app/plugins/datasource/prometheus/configuration/PromSettings.tsx index 7302e732991..50e1083230b 100644 --- a/public/app/plugins/datasource/prometheus/configuration/PromSettings.tsx +++ b/public/app/plugins/datasource/prometheus/configuration/PromSettings.tsx @@ -19,10 +19,11 @@ import { Select, } from '@grafana/ui'; +import config from '../../../../core/config'; import { useUpdateDatasource } from '../../../../features/datasources/state'; import { PromApplication, PromBuildInfoResponse } from '../../../../types/unified-alerting-dto'; import { QueryEditorMode } from '../querybuilder/shared/types'; -import { PromOptions } from '../types'; +import { PrometheusCacheLevel, PromOptions } from '../types'; import { ExemplarsSettings } from './ExemplarsSettings'; import { PromFlavorVersions } from './PromFlavorVersions'; @@ -39,6 +40,13 @@ const editorOptions = [ { value: QueryEditorMode.Code, label: 'Code' }, ]; +const cacheValueOptions = [ + { value: PrometheusCacheLevel.Low, label: 'Low' }, + { value: PrometheusCacheLevel.Medium, label: 'Medium' }, + { value: PrometheusCacheLevel.High, label: 'High' }, + { value: PrometheusCacheLevel.None, label: 'None' }, +]; + type PrometheusSelectItemsType = Array<{ value: PromApplication; label: PromApplication }>; const prometheusFlavorSelectItems: PrometheusSelectItemsType = [ @@ -301,7 +309,7 @@ export const PromSettings = (props: Props) => {
{ />
+ {config.featureToggles.prometheusResourceBrowserCache && ( +
+
+ o.value === options.jsonData.cacheLevel)} + /> + } + /> +
+
+ )} { jest.clearAllMocks(); @@ -73,7 +86,8 @@ describe('PrometheusDatasource', () => { password: 'mupp', jsonData: { customQueryParameters: '', - }, + cacheLevel: PrometheusCacheLevel.Low, + } as Partial, } as unknown as DataSourceInstanceSettings; beforeEach(() => { @@ -421,6 +435,106 @@ describe('PrometheusDatasource', () => { }); }); + // Remove when prometheusResourceBrowserCache is removed + describe('When prometheusResourceBrowserCache feature flag is off, there should be no change to the query intervals ', () => { + beforeEach(() => { + config.featureToggles.prometheusResourceBrowserCache = false; + }); + + it('test default 1 minute quantization', () => { + const dataSource = new PrometheusDatasource( + { + ...instanceSettings, + jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.Low }, + }, + templateSrvStub as unknown as TemplateSrv, + timeSrvStub as unknown as TimeSrv + ); + const quantizedRange = dataSource.getAdjustedInterval(); + const oldRange = dataSource.getTimeRangeParams(); + // For "1 minute" the window is unchanged + expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(60); + expect(parseInt(oldRange.end, 10) - parseInt(oldRange.start, 10)).toBe(60); + }); + + it('test 10 minute quantization', () => { + const dataSource = new PrometheusDatasource( + { + ...instanceSettings, + jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.Medium }, + }, + templateSrvStub as unknown as TemplateSrv, + timeSrvStub as unknown as TimeSrv + ); + const quantizedRange = dataSource.getAdjustedInterval(); + const oldRange = dataSource.getTimeRangeParams(); + + expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(60); + expect(parseInt(oldRange.end, 10) - parseInt(oldRange.start, 10)).toBe(60); + }); + }); + + describe('Test query range snapping', () => { + beforeEach(() => { + config.featureToggles.prometheusResourceBrowserCache = true; + }); + + it('test default 1 minute quantization', () => { + const dataSource = new PrometheusDatasource( + { + ...instanceSettings, + jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.Low }, + }, + templateSrvStub as unknown as TemplateSrv, + timeSrvStub as unknown as TimeSrv + ); + const quantizedRange = dataSource.getAdjustedInterval(); + // For "1 minute" the window contains all the minutes, so a query from 1:11:09 - 1:12:09 becomes 1:11 - 1:13 + expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(120); + }); + + it('test 10 minute quantization', () => { + const dataSource = new PrometheusDatasource( + { + ...instanceSettings, + jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.Medium }, + }, + templateSrvStub as unknown as TemplateSrv, + timeSrvStub as unknown as TimeSrv + ); + const quantizedRange = dataSource.getAdjustedInterval(); + expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(600); + }); + + it('test 60 minute quantization', () => { + const dataSource = new PrometheusDatasource( + { + ...instanceSettings, + jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.High }, + }, + templateSrvStub as unknown as TemplateSrv, + timeSrvStub as unknown as TimeSrv + ); + const quantizedRange = dataSource.getAdjustedInterval(); + expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(3600); + }); + + it('test quantization turned off', () => { + const dataSource = new PrometheusDatasource( + { + ...instanceSettings, + jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.None }, + }, + templateSrvStub as unknown as TemplateSrv, + timeSrvStub as unknown as TimeSrv + ); + const quantizedRange = dataSource.getAdjustedInterval(); + expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe( + (toSeconds - fromSeconds) / 1000 + ); + }); + }); + describe('alignRange', () => { it('does not modify already aligned intervals with perfect step', () => { const range = alignRange(0, 3, 3, 0); @@ -716,8 +830,13 @@ describe('PrometheusDatasource', () => { describe('metricFindQuery', () => { beforeEach(() => { + const prometheusDatasource = new PrometheusDatasource( + { ...instanceSettings, jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.None } }, + templateSrvStub, + timeSrvStubOld + ); const query = 'query_result(topk(5,rate(http_request_duration_microseconds_count[$__interval])))'; - ds.metricFindQuery(query); + prometheusDatasource.metricFindQuery(query); }); it('should call templateSrv.replace with scopedVars', () => { @@ -756,7 +875,7 @@ describe('PrometheusDatasource2', () => { directUrl: 'direct', user: 'test', password: 'mupp', - jsonData: { httpMethod: 'GET' }, + jsonData: { httpMethod: 'GET', cacheLevel: PrometheusCacheLevel.None }, } as unknown as DataSourceInstanceSettings; let ds: PrometheusDatasource; @@ -1398,7 +1517,13 @@ describe('PrometheusDatasource2', () => { const step = 55; const adjusted = alignRange(start, end, step, timeSrvStub.timeRange().to.utcOffset() * 60); const urlExpected = - 'proxied/api/v1/query_range?query=test' + '&start=' + adjusted.start + '&end=' + adjusted.end + '&step=' + step; + 'proxied/api/v1/query_range?query=test' + + '&start=' + + adjusted.start + + '&end=' + + (adjusted.end + step) + + '&step=' + + step; fetchMock.mockImplementation(() => of(response)); ds.query(query); const res = fetchMock.mock.calls[0][0]; @@ -1651,7 +1776,7 @@ describe('PrometheusDatasource2', () => { '&start=' + adjusted.start + '&end=' + - adjusted.end + + (adjusted.end + step) + '&step=' + step; fetchMock.mockImplementation(() => of(response)); diff --git a/public/app/plugins/datasource/prometheus/datasource.tsx b/public/app/plugins/datasource/prometheus/datasource.tsx index a1e8fba13ea..bc82353972e 100644 --- a/public/app/plugins/datasource/prometheus/datasource.tsx +++ b/public/app/plugins/datasource/prometheus/datasource.tsx @@ -17,8 +17,6 @@ import { DataSourceInstanceSettings, DataSourceWithQueryExportSupport, DataSourceWithQueryImportSupport, - dateMath, - DateTime, dateTime, LoadingState, QueryFixAction, @@ -43,10 +41,17 @@ import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv'; import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv'; import { PromApiFeatures, PromApplication } from 'app/types/unified-alerting-dto'; +import config from '../../../core/config'; + import { addLabelToQuery } from './add_label_to_query'; import { AnnotationQueryEditor } from './components/AnnotationQueryEditor'; import PrometheusLanguageProvider from './language_provider'; -import { expandRecordingRules } from './language_utils'; +import { + expandRecordingRules, + getClientCacheDurationInMinutes, + getPrometheusTime, + getRangeSnapInterval, +} from './language_utils'; import { renderLegendFormat } from './legend'; import PrometheusMetricFindQuery from './metric_find_query'; import { getInitHints, getQueryHints } from './query_hints'; @@ -57,6 +62,7 @@ import { ExemplarTraceIdDestination, PromDataErrorResponse, PromDataSuccessResponse, + PrometheusCacheLevel, PromExemplarData, PromMatrixData, PromOptions, @@ -98,6 +104,7 @@ export class PrometheusDatasource exemplarsAvailable: boolean; subType: PromApplication; rulerEnabled: boolean; + cacheLevel: PrometheusCacheLevel; constructor( instanceSettings: DataSourceInstanceSettings, @@ -131,6 +138,7 @@ export class PrometheusDatasource this.defaultEditor = instanceSettings.jsonData.defaultEditor; this.variables = new PrometheusVariableSupport(this, this.templateSrv, this.timeSrv); this.exemplarsAvailable = true; + this.cacheLevel = instanceSettings.jsonData.cacheLevel ?? PrometheusCacheLevel.Low; // This needs to be here and cannot be static because of how annotations typing affects casting of data source // objects to DataSourceApi types. @@ -451,8 +459,8 @@ export class PrometheusDatasource ); // Run queries trough browser/proxy } else { - const start = this.getPrometheusTime(request.range.from, false); - const end = this.getPrometheusTime(request.range.to, true); + const start = getPrometheusTime(request.range.from, false); + const end = getPrometheusTime(request.range.to, true); const { queries, activeTargets } = this.prepareTargets(request, start, end); // No valid targets, return the empty result to save a round trip. @@ -794,8 +802,8 @@ export class PrometheusDatasource method: 'POST', headers: this.getRequestHeaders(), data: { - from: (this.getPrometheusTime(options.range.from, false) * 1000).toString(), - to: (this.getPrometheusTime(options.range.to, true) * 1000).toString(), + from: (getPrometheusTime(options.range.from, false) * 1000).toString(), + to: (getPrometheusTime(options.range.to, true) * 1000).toString(), queries: [this.applyTemplateVariables(queryModel, {})], }, requestId: `prom-query-${annotation.name}`, @@ -1164,19 +1172,32 @@ export class PrometheusDatasource return { ...query, expr: expression }; } - getPrometheusTime(date: string | DateTime, roundUp: boolean) { - if (typeof date === 'string') { - date = dateMath.parse(date, roundUp)!; + /** + * Returns the adjusted "snapped" interval parameters + */ + getAdjustedInterval(): { start: string; end: string } { + if (!config.featureToggles.prometheusResourceBrowserCache) { + return this.getTimeRangeParams(); } - - return Math.ceil(date.valueOf() / 1000); + const range = this.timeSrv.timeRange(); + return getRangeSnapInterval(this.cacheLevel, range); } + /** + * This will return a time range that always includes the users current time range, + * and then a little extra padding to round up/down to the nearest nth minute, + * defined by the result of the getCacheDurationInMinutes. + * + * For longer cache durations, and shorter query durations, the window we're calculating might be much bigger then the user's current window, + * resulting in us returning labels/values that might not be applicable for the given window, this is a necessary trade off if we want to cache larger durations + * + */ + getTimeRangeParams(): { start: string; end: string } { const range = this.timeSrv.timeRange(); return { - start: this.getPrometheusTime(range.from, false).toString(), - end: this.getPrometheusTime(range.to, true).toString(), + start: getPrometheusTime(range.from, false).toString(), + end: getPrometheusTime(range.to, true).toString(), }; } @@ -1232,6 +1253,32 @@ export class PrometheusDatasource interpolateString(string: string) { return this.templateSrv.replace(string, undefined, this.interpolateQueryExpr); } + + getDebounceTimeInMilliseconds(): number { + switch (this.cacheLevel) { + case PrometheusCacheLevel.Medium: + return 600; + case PrometheusCacheLevel.High: + return 1200; + default: + return 350; + } + } + + getDaysToCacheMetadata(): number { + switch (this.cacheLevel) { + case PrometheusCacheLevel.Medium: + return 7; + case PrometheusCacheLevel.High: + return 30; + default: + return 1; + } + } + + getCacheDurationInMinutes(): number { + return getClientCacheDurationInMinutes(this.cacheLevel); + } } /** diff --git a/public/app/plugins/datasource/prometheus/language_provider.test.ts b/public/app/plugins/datasource/prometheus/language_provider.test.ts index 311779fe6d7..dbe70fb8a0e 100644 --- a/public/app/plugins/datasource/prometheus/language_provider.test.ts +++ b/public/app/plugins/datasource/prometheus/language_provider.test.ts @@ -1,24 +1,54 @@ import { Editor as SlateEditor } from 'slate'; import Plain from 'slate-plain-serializer'; -import { AbstractLabelOperator, HistoryItem } from '@grafana/data'; +import { AbstractLabelOperator, dateTime, HistoryItem, TimeRange } from '@grafana/data'; +import { config } from '@grafana/runtime'; import { SearchFunctionType } from '@grafana/ui'; import { Label } from './components/monaco-query-field/monaco-completion-provider/situation'; import { PrometheusDatasource } from './datasource'; import LanguageProvider from './language_provider'; -import { PromQuery } from './types'; +import { getClientCacheDurationInMinutes, getPrometheusTime, getRangeSnapInterval } from './language_utils'; +import { PrometheusCacheLevel, PromQuery } from './types'; + +const now = new Date().getTime(); +const timeRangeDurationSeconds = 1; +const toPrometheusTime = getPrometheusTime(dateTime(now), false); +const fromPrometheusTime = getPrometheusTime(dateTime(now - timeRangeDurationSeconds * 1000), false); +const toPrometheusTimeString = toPrometheusTime.toString(10); +const fromPrometheusTimeString = fromPrometheusTime.toString(10); + +const getTimeRangeParams = (override?: Partial<{ start: string; end: string }>): { start: string; end: string } => ({ + start: fromPrometheusTimeString, + end: toPrometheusTimeString, + ...override, +}); + +const getMockQuantizedTimeRangeParams = (override?: Partial): TimeRange => ({ + from: dateTime(fromPrometheusTime * 1000), + to: dateTime(toPrometheusTime * 1000), + raw: { + from: `now-${timeRangeDurationSeconds}s`, + to: 'now', + }, + ...override, +}); describe('Language completion provider', () => { - const datasource: PrometheusDatasource = { + const defaultDatasource: PrometheusDatasource = { metadataRequest: () => ({ data: { data: [] } }), - getTimeRangeParams: () => ({ start: '0', end: '1' }), + getTimeRangeParams: getTimeRangeParams, interpolateString: (string: string) => string, hasLabelsMatchAPISupport: () => false, + getQuantizedTimeRangeParams: () => + getRangeSnapInterval(PrometheusCacheLevel.None, getMockQuantizedTimeRangeParams()), + getDaysToCacheMetadata: () => 1, + getAdjustedInterval: () => getRangeSnapInterval(PrometheusCacheLevel.None, getMockQuantizedTimeRangeParams()), + cacheLevel: PrometheusCacheLevel.None, } as unknown as PrometheusDatasource; describe('cleanText', () => { - const cleanText = new LanguageProvider(datasource).cleanText; + const cleanText = new LanguageProvider(defaultDatasource).cleanText; it('does not remove metric or label keys', () => { expect(cleanText('foo')).toBe('foo'); expect(cleanText('foo_bar')).toBe('foo_bar'); @@ -68,26 +98,14 @@ describe('Language completion provider', () => { }); }); - describe('getSeriesLabels', () => { - it('should call series endpoint', () => { - const languageProvider = new LanguageProvider({ ...datasource } as PrometheusDatasource); - const getSeriesLabels = languageProvider.getSeriesLabels; - const requestSpy = jest.spyOn(languageProvider, 'request'); - - const labelName = 'job'; - const labelValue = 'grafana'; - getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]); - expect(requestSpy).toHaveBeenCalled(); - expect(requestSpy).toHaveBeenCalledWith('/api/v1/series', [], { - end: '1', - 'match[]': '{job="grafana"}', - start: '0', - }); + // @todo clean up prometheusResourceBrowserCache feature flag + describe('getSeriesLabelsDeprecatedLRU', () => { + beforeEach(() => { + config.featureToggles.prometheusResourceBrowserCache = false; }); - it('should call labels endpoint', () => { const languageProvider = new LanguageProvider({ - ...datasource, + ...defaultDatasource, hasLabelsMatchAPISupport: () => true, } as PrometheusDatasource); const getSeriesLabels = languageProvider.getSeriesLabels; @@ -98,30 +116,145 @@ describe('Language completion provider', () => { getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]); expect(requestSpy).toHaveBeenCalled(); expect(requestSpy).toHaveBeenCalledWith(`/api/v1/labels`, [], { - end: '1', + end: toPrometheusTimeString, 'match[]': '{job="grafana"}', - start: '0', + start: fromPrometheusTimeString, }); }); + + it('should call series endpoint', () => { + const languageProvider = new LanguageProvider({ + ...defaultDatasource, + getAdjustedInterval: () => getRangeSnapInterval(PrometheusCacheLevel.None, getMockQuantizedTimeRangeParams()), + } as PrometheusDatasource); + const getSeriesLabels = languageProvider.getSeriesLabels; + const requestSpy = jest.spyOn(languageProvider, 'request'); + + const labelName = 'job'; + const labelValue = 'grafana'; + getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]); + expect(requestSpy).toHaveBeenCalled(); + expect(requestSpy).toHaveBeenCalledWith('/api/v1/series', [], { + end: toPrometheusTimeString, + 'match[]': '{job="grafana"}', + start: fromPrometheusTimeString, + }); + }); + }); + describe('getSeriesLabels', () => { + beforeEach(() => { + config.featureToggles.prometheusResourceBrowserCache = true; + }); + it('should call labels endpoint', () => { + const languageProvider = new LanguageProvider({ + ...defaultDatasource, + hasLabelsMatchAPISupport: () => true, + } as PrometheusDatasource); + const getSeriesLabels = languageProvider.getSeriesLabels; + const requestSpy = jest.spyOn(languageProvider, 'request'); + + const labelName = 'job'; + const labelValue = 'grafana'; + getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]); + expect(requestSpy).toHaveBeenCalled(); + expect(requestSpy).toHaveBeenCalledWith( + `/api/v1/labels`, + [], + { + end: toPrometheusTimeString, + 'match[]': '{job="grafana"}', + start: fromPrometheusTimeString, + }, + undefined + ); + }); + + it('should call series endpoint', () => { + const languageProvider = new LanguageProvider({ + ...defaultDatasource, + getAdjustedInterval: () => getRangeSnapInterval(PrometheusCacheLevel.None, getMockQuantizedTimeRangeParams()), + } as PrometheusDatasource); + const getSeriesLabels = languageProvider.getSeriesLabels; + const requestSpy = jest.spyOn(languageProvider, 'request'); + + const labelName = 'job'; + const labelValue = 'grafana'; + getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]); + expect(requestSpy).toHaveBeenCalled(); + expect(requestSpy).toHaveBeenCalledWith( + '/api/v1/series', + [], + { + end: toPrometheusTimeString, + 'match[]': '{job="grafana"}', + start: fromPrometheusTimeString, + }, + undefined + ); + }); + + it('should call labels endpoint with quantized start', () => { + const timeSnapMinutes = getClientCacheDurationInMinutes(PrometheusCacheLevel.Low); + const languageProvider = new LanguageProvider({ + ...defaultDatasource, + hasLabelsMatchAPISupport: () => true, + cacheLevel: PrometheusCacheLevel.Low, + getAdjustedInterval: () => getRangeSnapInterval(PrometheusCacheLevel.Low, getMockQuantizedTimeRangeParams()), + getCacheDurationInMinutes: () => timeSnapMinutes, + } as PrometheusDatasource); + const getSeriesLabels = languageProvider.getSeriesLabels; + const requestSpy = jest.spyOn(languageProvider, 'request'); + + const labelName = 'job'; + const labelValue = 'grafana'; + getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]); + expect(requestSpy).toHaveBeenCalled(); + expect(requestSpy).toHaveBeenCalledWith( + `/api/v1/labels`, + [], + { + end: ( + dateTime(fromPrometheusTime * 1000) + .add(timeSnapMinutes, 'minute') + .startOf('minute') + .valueOf() / 1000 + ).toString(), + 'match[]': '{job="grafana"}', + start: ( + dateTime(toPrometheusTime * 1000) + .startOf('minute') + .valueOf() / 1000 + ).toString(), + }, + { headers: { 'X-Grafana-Cache': `private, max-age=${timeSnapMinutes * 60}` } } + ); + }); }); describe('getSeriesValues', () => { it('should call old series endpoint and should use match[] parameter', () => { - const languageProvider = new LanguageProvider(datasource); + const languageProvider = new LanguageProvider({ + ...defaultDatasource, + } as PrometheusDatasource); const getSeriesValues = languageProvider.getSeriesValues; const requestSpy = jest.spyOn(languageProvider, 'request'); getSeriesValues('job', '{job="grafana"}'); expect(requestSpy).toHaveBeenCalled(); - expect(requestSpy).toHaveBeenCalledWith('/api/v1/series', [], { - end: '1', - 'match[]': '{job="grafana"}', - start: '0', - }); + expect(requestSpy).toHaveBeenCalledWith( + '/api/v1/series', + [], + { + end: toPrometheusTimeString, + 'match[]': '{job="grafana"}', + start: fromPrometheusTimeString, + }, + undefined + ); }); it('should call new series endpoint and should use match[] parameter', () => { const languageProvider = new LanguageProvider({ - ...datasource, + ...defaultDatasource, hasLabelsMatchAPISupport: () => true, } as PrometheusDatasource); const getSeriesValues = languageProvider.getSeriesValues; @@ -130,17 +263,22 @@ describe('Language completion provider', () => { const labelValue = 'grafana'; getSeriesValues(labelName, `{${labelName}="${labelValue}"}`); expect(requestSpy).toHaveBeenCalled(); - expect(requestSpy).toHaveBeenCalledWith(`/api/v1/label/${labelName}/values`, [], { - end: '1', - 'match[]': `{${labelName}="${labelValue}"}`, - start: '0', - }); + expect(requestSpy).toHaveBeenCalledWith( + `/api/v1/label/${labelName}/values`, + [], + { + end: toPrometheusTimeString, + 'match[]': `{${labelName}="${labelValue}"}`, + start: fromPrometheusTimeString, + }, + undefined + ); }); }); describe('fetchSeries', () => { it('should use match[] parameter', () => { - const languageProvider = new LanguageProvider(datasource); + const languageProvider = new LanguageProvider(defaultDatasource); const fetchSeries = languageProvider.fetchSeries; const requestSpy = jest.spyOn(languageProvider, 'request'); fetchSeries('{job="grafana"}'); @@ -148,7 +286,8 @@ describe('Language completion provider', () => { expect(requestSpy).toHaveBeenCalledWith( '/api/v1/series', {}, - { end: '1', 'match[]': '{job="grafana"}', start: '0' } + { end: toPrometheusTimeString, 'match[]': '{job="grafana"}', start: fromPrometheusTimeString }, + undefined ); }); }); @@ -156,41 +295,51 @@ describe('Language completion provider', () => { describe('fetchSeriesLabels', () => { it('should interpolate variable in series', () => { const languageProvider = new LanguageProvider({ - ...datasource, + ...defaultDatasource, interpolateString: (string: string) => string.replace(/\$/, 'interpolated-'), } as PrometheusDatasource); const fetchSeriesLabels = languageProvider.fetchSeriesLabels; const requestSpy = jest.spyOn(languageProvider, 'request'); fetchSeriesLabels('$metric'); expect(requestSpy).toHaveBeenCalled(); - expect(requestSpy).toHaveBeenCalledWith('/api/v1/series', [], { - end: '1', - 'match[]': 'interpolated-metric', - start: '0', - }); + expect(requestSpy).toHaveBeenCalledWith( + '/api/v1/series', + [], + { + end: toPrometheusTimeString, + 'match[]': 'interpolated-metric', + start: fromPrometheusTimeString, + }, + undefined + ); }); }); describe('fetchLabelValues', () => { it('should interpolate variable in series', () => { const languageProvider = new LanguageProvider({ - ...datasource, + ...defaultDatasource, interpolateString: (string: string) => string.replace(/\$/, 'interpolated-'), } as PrometheusDatasource); const fetchLabelValues = languageProvider.fetchLabelValues; const requestSpy = jest.spyOn(languageProvider, 'request'); fetchLabelValues('$job'); expect(requestSpy).toHaveBeenCalled(); - expect(requestSpy).toHaveBeenCalledWith('/api/v1/label/interpolated-job/values', [], { - end: '1', - start: '0', - }); + expect(requestSpy).toHaveBeenCalledWith( + '/api/v1/label/interpolated-job/values', + [], + { + end: toPrometheusTimeString, + start: fromPrometheusTimeString, + }, + undefined + ); }); }); describe('empty query suggestions', () => { it('returns no suggestions on empty context', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); const value = Plain.deserialize(''); const result = await instance.provideCompletionItems({ text: '', prefix: '', value, wrapperClasses: [] }); expect(result.context).toBeUndefined(); @@ -198,7 +347,7 @@ describe('Language completion provider', () => { }); it('returns no suggestions with metrics on empty context even when metrics were provided', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); instance.metrics = ['foo', 'bar']; const value = Plain.deserialize(''); const result = await instance.provideCompletionItems({ text: '', prefix: '', value, wrapperClasses: [] }); @@ -207,7 +356,7 @@ describe('Language completion provider', () => { }); it('returns history on empty context when history was provided', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); const value = Plain.deserialize(''); const history: Array> = [ { @@ -236,7 +385,7 @@ describe('Language completion provider', () => { describe('range suggestions', () => { it('returns range suggestions in range context', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); const value = Plain.deserialize('1'); const result = await instance.provideCompletionItems({ text: '1', @@ -266,7 +415,7 @@ describe('Language completion provider', () => { describe('metric suggestions', () => { it('returns history, metrics and function suggestions in an uknown context ', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); instance.metrics = ['foo', 'bar']; const history: Array> = [ { @@ -301,7 +450,7 @@ describe('Language completion provider', () => { }); it('returns no suggestions directly after a binary operator', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); instance.metrics = ['foo', 'bar']; const value = Plain.deserialize('*'); const result = await instance.provideCompletionItems({ text: '*', prefix: '', value, wrapperClasses: [] }); @@ -310,7 +459,7 @@ describe('Language completion provider', () => { }); it('returns metric suggestions with prefix after a binary operator', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); instance.metrics = ['foo', 'bar']; const value = Plain.deserialize('foo + b'); const ed = new SlateEditor({ value }); @@ -333,7 +482,7 @@ describe('Language completion provider', () => { }); it('returns no suggestions at the beginning of a non-empty function', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); const value = Plain.deserialize('sum(up)'); const ed = new SlateEditor({ value }); @@ -351,7 +500,7 @@ describe('Language completion provider', () => { describe('label suggestions', () => { it('returns default label suggestions on label context and no metric', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); const value = Plain.deserialize('{}'); const ed = new SlateEditor({ value }); const valueWithSelection = ed.moveForward(1).value; @@ -373,9 +522,8 @@ describe('Language completion provider', () => { it('returns label suggestions on label context and metric', async () => { const datasources: PrometheusDatasource = { + ...defaultDatasource, metadataRequest: () => ({ data: { data: [{ __name__: 'metric', bar: 'bazinga' }] } }), - getTimeRangeParams: () => ({ start: '0', end: '1' }), - interpolateString: (string: string) => string, } as unknown as PrometheusDatasource; const instance = new LanguageProvider(datasources); const value = Plain.deserialize('metric{}'); @@ -394,7 +542,8 @@ describe('Language completion provider', () => { }); it('returns label suggestions on label context but leaves out labels that already exist', async () => { - const datasource: PrometheusDatasource = { + const testDatasource: PrometheusDatasource = { + ...defaultDatasource, metadataRequest: () => ({ data: { data: [ @@ -408,10 +557,8 @@ describe('Language completion provider', () => { ], }, }), - getTimeRangeParams: () => ({ start: '0', end: '1' }), - interpolateString: (string: string) => string, } as unknown as PrometheusDatasource; - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(testDatasource); const value = Plain.deserialize('{job1="foo",job2!="foo",job3=~"foo",__name__="metric",}'); const ed = new SlateEditor({ value }); const valueWithSelection = ed.moveForward(54).value; @@ -429,7 +576,7 @@ describe('Language completion provider', () => { it('returns label value suggestions inside a label value context after a negated matching operator', async () => { const instance = new LanguageProvider({ - ...datasource, + ...defaultDatasource, metadataRequest: () => { return { data: { data: ['value1', 'value2'] } }; }, @@ -456,7 +603,7 @@ describe('Language completion provider', () => { it('returns a refresher on label context and unavailable metric', async () => { jest.spyOn(console, 'warn').mockImplementation(() => {}); - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); const value = Plain.deserialize('metric{}'); const ed = new SlateEditor({ value }); const valueWithSelection = ed.moveForward(7).value; @@ -473,7 +620,7 @@ describe('Language completion provider', () => { it('returns label values on label context when given a metric and a label key', async () => { const instance = new LanguageProvider({ - ...datasource, + ...defaultDatasource, metadataRequest: () => simpleMetricLabelsResponse, } as unknown as PrometheusDatasource); const value = Plain.deserialize('metric{bar=ba}'); @@ -494,7 +641,7 @@ describe('Language completion provider', () => { it('returns label suggestions on aggregation context and metric w/ selector', async () => { const instance = new LanguageProvider({ - ...datasource, + ...defaultDatasource, metadataRequest: () => simpleMetricLabelsResponse, } as unknown as PrometheusDatasource); const value = Plain.deserialize('sum(metric{foo="xx"}) by ()'); @@ -514,7 +661,7 @@ describe('Language completion provider', () => { it('returns label suggestions on aggregation context and metric w/o selector', async () => { const instance = new LanguageProvider({ - ...datasource, + ...defaultDatasource, metadataRequest: () => simpleMetricLabelsResponse, } as unknown as PrometheusDatasource); const value = Plain.deserialize('sum(metric) by ()'); @@ -534,7 +681,7 @@ describe('Language completion provider', () => { it('returns label suggestions inside a multi-line aggregation context', async () => { const instance = new LanguageProvider({ - ...datasource, + ...defaultDatasource, metadataRequest: () => simpleMetricLabelsResponse, } as unknown as PrometheusDatasource); const value = Plain.deserialize('sum(\nmetric\n)\nby ()'); @@ -560,7 +707,7 @@ describe('Language completion provider', () => { it('returns label suggestions inside an aggregation context with a range vector', async () => { const instance = new LanguageProvider({ - ...datasource, + ...defaultDatasource, metadataRequest: () => simpleMetricLabelsResponse, } as unknown as PrometheusDatasource); const value = Plain.deserialize('sum(rate(metric[1h])) by ()'); @@ -584,7 +731,7 @@ describe('Language completion provider', () => { it('returns label suggestions inside an aggregation context with a range vector and label', async () => { const instance = new LanguageProvider({ - ...datasource, + ...defaultDatasource, metadataRequest: () => simpleMetricLabelsResponse, } as unknown as PrometheusDatasource); const value = Plain.deserialize('sum(rate(metric{label1="value"}[1h])) by ()'); @@ -607,7 +754,7 @@ describe('Language completion provider', () => { }); it('returns no suggestions inside an unclear aggregation context using alternate syntax', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); const value = Plain.deserialize('sum by ()'); const ed = new SlateEditor({ value }); const valueWithSelection = ed.moveForward(8).value; @@ -623,7 +770,7 @@ describe('Language completion provider', () => { it('returns label suggestions inside an aggregation context using alternate syntax', async () => { const instance = new LanguageProvider({ - ...datasource, + ...defaultDatasource, metadataRequest: () => simpleMetricLabelsResponse, } as unknown as PrometheusDatasource); const value = Plain.deserialize('sum by () (metric)'); @@ -646,15 +793,16 @@ describe('Language completion provider', () => { }); it('does not re-fetch default labels', async () => { - const datasource: PrometheusDatasource = { + const testDatasource: PrometheusDatasource = { + ...defaultDatasource, metadataRequest: jest.fn(() => ({ data: { data: [] } })), - getTimeRangeParams: jest.fn(() => ({ start: '0', end: '1' })), interpolateString: (string: string) => string, + getQuantizedTimeRangeParams: getMockQuantizedTimeRangeParams, } as unknown as PrometheusDatasource; - const mockedMetadataRequest = jest.mocked(datasource.metadataRequest); + const mockedMetadataRequest = jest.mocked(testDatasource.metadataRequest); - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(testDatasource); const value = Plain.deserialize('{}'); const ed = new SlateEditor({ value }); const valueWithSelection = ed.moveForward(1).value; @@ -678,7 +826,6 @@ describe('Language completion provider', () => { jest.spyOn(console, 'warn').mockImplementation(() => {}); const datasource: PrometheusDatasource = { metadataRequest: jest.fn(() => ({ data: { data: ['foo', 'bar'] as string[] } })), - getTimeRangeParams: jest.fn(() => ({ start: '0', end: '1' })), lookupsDisabled: true, } as unknown as PrometheusDatasource; const mockedMetadataRequest = jest.mocked(datasource.metadataRequest); @@ -702,10 +849,9 @@ describe('Language completion provider', () => { }); it('issues metadata requests when lookup is not disabled', async () => { const datasource: PrometheusDatasource = { + ...defaultDatasource, metadataRequest: jest.fn(() => ({ data: { data: ['foo', 'bar'] as string[] } })), - getTimeRangeParams: jest.fn(() => ({ start: '0', end: '1' })), lookupsDisabled: false, - interpolateString: (string: string) => string, } as unknown as PrometheusDatasource; const mockedMetadataRequest = jest.mocked(datasource.metadataRequest); const instance = new LanguageProvider(datasource); @@ -714,18 +860,35 @@ describe('Language completion provider', () => { await instance.start(); expect(mockedMetadataRequest.mock.calls.length).toBeGreaterThan(0); }); + + it('doesnt blow up if metadata or fetchLabels rejects', async () => { + jest.spyOn(console, 'error').mockImplementation(); + const datasource: PrometheusDatasource = { + ...defaultDatasource, + metadataRequest: jest.fn(() => Promise.reject('rejected')), + lookupsDisabled: false, + } as unknown as PrometheusDatasource; + const mockedMetadataRequest = jest.mocked(datasource.metadataRequest); + const instance = new LanguageProvider(datasource); + + expect(mockedMetadataRequest.mock.calls.length).toBe(0); + const result = await instance.start(); + expect(result[0]).toBeUndefined(); + expect(result[1]).toEqual([]); + expect(mockedMetadataRequest.mock.calls.length).toBe(3); + }); }); describe('Query imports', () => { it('returns empty queries', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); const result = await instance.importFromAbstractQuery({ refId: 'bar', labelMatchers: [] }); expect(result).toEqual({ refId: 'bar', expr: '', range: true }); }); describe('exporting to abstract query', () => { it('exports labels with metric name', async () => { - const instance = new LanguageProvider(datasource); + const instance = new LanguageProvider(defaultDatasource); const abstractQuery = instance.exportToAbstractQuery({ refId: 'bar', expr: 'metric_name{label1="value1", label2!="value2", label3=~"value3", label4!~"value4"}', diff --git a/public/app/plugins/datasource/prometheus/language_provider.ts b/public/app/plugins/datasource/prometheus/language_provider.ts index 4b2251ac7e1..b104c0a1df6 100644 --- a/public/app/plugins/datasource/prometheus/language_provider.ts +++ b/public/app/plugins/datasource/prometheus/language_provider.ts @@ -11,7 +11,7 @@ import { HistoryItem, LanguageProvider, } from '@grafana/data'; -import { BackendSrvRequest } from '@grafana/runtime'; +import { BackendSrvRequest, config } from '@grafana/runtime'; import { CompletionItem, CompletionItemGroup, SearchFunctionType, TypeaheadInput, TypeaheadOutput } from '@grafana/ui'; import { Label } from './components/monaco-query-field/monaco-completion-provider/situation'; @@ -27,7 +27,7 @@ import { toPromLikeQuery, } from './language_utils'; import PromqlSyntax, { FUNCTIONS, RATE_RANGES } from './promql'; -import { PromMetricsMetadata, PromQuery } from './types'; +import { PrometheusCacheLevel, PromMetricsMetadata, PromQuery } from './types'; const DEFAULT_KEYS = ['job', 'instance']; const EMPTY_SELECTOR = '{}'; @@ -43,6 +43,14 @@ const setFunctionKind = (suggestion: CompletionItem): CompletionItem => { return suggestion; }; +const buildCacheHeaders = (durationInSeconds: number) => { + return { + headers: { + 'X-Grafana-Cache': `private, max-age=${durationInSeconds}`, + }, + }; +}; + export function addHistoryMetadata(item: CompletionItem, history: any[]): CompletionItem { const cutoffTs = Date.now() - HISTORY_COUNT_CUTOFF; const historyForItem = history.filter((h) => h.ts > cutoffTs && h.query === item.label); @@ -97,6 +105,8 @@ const PREFIX_DELIMITER_REGEX = interface AutocompleteContext { history?: Array>; } + +const secondsInDay = 86400; export default class PromQlLanguageProvider extends LanguageProvider { histogramMetrics: string[]; timeRange?: { start: number; end: number }; @@ -106,7 +116,6 @@ export default class PromQlLanguageProvider extends LanguageProvider { datasource: PrometheusDatasource; labelKeys: string[] = []; declare labelFetchTs: number; - /** * Cache for labels of series. This is bit simplistic in the sense that it just counts responses each as a 1 and does * not account for different size of a response. If that is needed a `length` function can be added in the options. @@ -114,7 +123,6 @@ export default class PromQlLanguageProvider extends LanguageProvider { */ private labelsCache = new LRU>({ max: 10 }); private labelValuesCache = new LRU({ max: 10 }); - constructor(datasource: PrometheusDatasource, initialValues?: Partial) { super(); @@ -126,6 +134,16 @@ export default class PromQlLanguageProvider extends LanguageProvider { Object.assign(this, initialValues); } + getDefaultCacheHeaders() { + // @todo clean up prometheusResourceBrowserCache feature flag + if (config.featureToggles.prometheusResourceBrowserCache) { + if (this.datasource.cacheLevel !== PrometheusCacheLevel.None) { + return buildCacheHeaders(this.datasource.getCacheDurationInMinutes() * 60); + } + } + return; + } + // Strip syntax chars so that typeahead suggestions can work on clean inputs cleanText(s: string) { const parts = s.split(PREFIX_DELIMITER_REGEX); @@ -153,17 +171,26 @@ export default class PromQlLanguageProvider extends LanguageProvider { return []; } - // TODO #33976: make those requests parallel - await this.fetchLabels(); this.metrics = (await this.fetchLabelValues('__name__')) || []; - await this.loadMetricsMetadata(); this.histogramMetrics = processHistogramMetrics(this.metrics).sort(); - return []; + return Promise.all([this.loadMetricsMetadata(), this.fetchLabels()]); }; async loadMetricsMetadata() { + // @todo clean up prometheusResourceBrowserCache feature flag + const headers = config.featureToggles.prometheusResourceBrowserCache + ? buildCacheHeaders(this.datasource.getDaysToCacheMetadata() * secondsInDay) + : {}; this.metricsMetadata = fixSummariesMetadata( - await this.request('/api/v1/metadata', {}, {}, { showErrorAlert: false }) + await this.request( + '/api/v1/metadata', + {}, + {}, + { + showErrorAlert: false, + ...headers, + } + ) ); } @@ -488,13 +515,14 @@ export default class PromQlLanguageProvider extends LanguageProvider { } /** - * @todo cache * @param key */ fetchLabelValues = async (key: string): Promise => { - const params = this.datasource.getTimeRangeParams(); - const url = `/api/v1/label/${this.datasource.interpolateString(key)}/values`; - return await this.request(url, [], params); + const params = this.datasource.getAdjustedInterval(); + const interpolatedName = this.datasource.interpolateString(key); + const url = `/api/v1/label/${interpolatedName}/values`; + const value = await this.request(url, [], params, this.getDefaultCacheHeaders()); + return value ?? []; }; async getLabelValues(key: string): Promise { @@ -506,10 +534,10 @@ export default class PromQlLanguageProvider extends LanguageProvider { */ async fetchLabels(): Promise { const url = '/api/v1/labels'; - const params = this.datasource.getTimeRangeParams(); + const params = this.datasource.getAdjustedInterval(); this.labelFetchTs = Date.now().valueOf(); - const res = await this.request(url, [], params); + const res = await this.request(url, [], params, this.getDefaultCacheHeaders()); if (Array.isArray(res)) { this.labelKeys = res.slice().sort(); } @@ -539,12 +567,41 @@ export default class PromQlLanguageProvider extends LanguageProvider { */ fetchSeriesValuesWithMatch = async (name: string, match?: string): Promise => { const interpolatedName = name ? this.datasource.interpolateString(name) : null; - const range = this.datasource.getTimeRangeParams(); + const range = this.datasource.getAdjustedInterval(); const urlParams = { ...range, ...(match && { 'match[]': match }), }; + // @todo clean up prometheusResourceBrowserCache feature flag + if (!config.featureToggles.prometheusResourceBrowserCache) { + return await this.fetchSeriesValuesLRUCache(interpolatedName, range, name, urlParams); + } + + const value = await this.request( + `/api/v1/label/${interpolatedName}/values`, + [], + urlParams, + this.getDefaultCacheHeaders() + ); + return value ?? []; + }; + + /** + * @deprecated + * @todo clean up prometheusResourceBrowserCache feature flag + * @param interpolatedName + * @param range + * @param name + * @param urlParams + * @private + */ + private async fetchSeriesValuesLRUCache( + interpolatedName: string | null, + range: { start: string; end: string }, + name: string, + urlParams: { start: string; end: string } + ) { const cacheParams = new URLSearchParams({ 'match[]': interpolatedName ?? '', start: roundSecToMin(parseInt(range.start, 10)).toString(), @@ -561,7 +618,7 @@ export default class PromQlLanguageProvider extends LanguageProvider { } } return value ?? []; - }; + } /** * Gets series labels @@ -597,12 +654,38 @@ export default class PromQlLanguageProvider extends LanguageProvider { */ fetchSeriesLabels = async (name: string, withName?: boolean): Promise> => { const interpolatedName = this.datasource.interpolateString(name); - const range = this.datasource.getTimeRangeParams(); + const range = this.datasource.getAdjustedInterval(); const urlParams = { ...range, 'match[]': interpolatedName, }; const url = `/api/v1/series`; + + if (!config.featureToggles.prometheusResourceBrowserCache) { + return await this.fetchSeriesLabelsLRUCache(interpolatedName, range, withName, url, urlParams); + } + + const data = await this.request(url, [], urlParams, this.getDefaultCacheHeaders()); + const { values } = processLabels(data, withName); + return values; + }; + + /** + * @deprecated + * @param interpolatedName + * @param range + * @param withName + * @param url + * @param urlParams + * @private + */ + private async fetchSeriesLabelsLRUCache( + interpolatedName: string, + range: { start: string; end: string }, + withName: boolean | undefined, + url: string, + urlParams: { start: string; 'match[]': string; end: string } + ) { // Cache key is a bit different here. We add the `withName` param and also round up to a minute the intervals. // The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every // millisecond while still actually getting all the keys for the correct interval. This still can create problems @@ -623,7 +706,7 @@ export default class PromQlLanguageProvider extends LanguageProvider { this.labelsCache.set(cacheKey, value); } return value; - }; + } /** * Fetch labels for a series using /labels endpoint. This is cached by its args but also by the global timeRange currently selected as @@ -633,12 +716,37 @@ export default class PromQlLanguageProvider extends LanguageProvider { */ fetchSeriesLabelsMatch = async (name: string, withName?: boolean): Promise> => { const interpolatedName = this.datasource.interpolateString(name); - const range = this.datasource.getTimeRangeParams(); + const range = this.datasource.getAdjustedInterval(); const urlParams = { ...range, 'match[]': interpolatedName, }; const url = `/api/v1/labels`; + if (!config.featureToggles.prometheusResourceBrowserCache) { + return await this.fetchSeriesLabelMatchLRUCache(interpolatedName, range, withName, url, urlParams); + } + + const data: string[] = await this.request(url, [], urlParams, this.getDefaultCacheHeaders()); + // Convert string array to Record + return data.reduce((ac, a) => ({ ...ac, [a]: '' }), {}); + }; + + /** + * @deprecated + * @param interpolatedName + * @param range + * @param withName + * @param url + * @param urlParams + * @private + */ + private async fetchSeriesLabelMatchLRUCache( + interpolatedName: string, + range: { start: string; end: string }, + withName: boolean | undefined, + url: string, + urlParams: { start: string; 'match[]': string; end: string } + ) { // Cache key is a bit different here. We add the `withName` param and also round up to a minute the intervals. // The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every // millisecond while still actually getting all the keys for the correct interval. This still can create problems @@ -659,7 +767,7 @@ export default class PromQlLanguageProvider extends LanguageProvider { this.labelsCache.set(cacheKey, value); } return value; - }; + } /** * Fetch series for a selector. Use this for raw results. Use fetchSeriesLabels() to get labels. @@ -669,7 +777,7 @@ export default class PromQlLanguageProvider extends LanguageProvider { const url = '/api/v1/series'; const range = this.datasource.getTimeRangeParams(); const params = { ...range, 'match[]': match }; - return await this.request(url, {}, params); + return await this.request(url, {}, params, this.getDefaultCacheHeaders()); }; /** diff --git a/public/app/plugins/datasource/prometheus/language_utils.test.ts b/public/app/plugins/datasource/prometheus/language_utils.test.ts index 5066ef72169..cdf497ae8ed 100644 --- a/public/app/plugins/datasource/prometheus/language_utils.test.ts +++ b/public/app/plugins/datasource/prometheus/language_utils.test.ts @@ -1,13 +1,18 @@ -import { AbstractLabelOperator, AbstractQuery } from '@grafana/data'; +import { Moment } from 'moment'; + +import { AbstractLabelOperator, AbstractQuery, DateTime, dateTime, TimeRange } from '@grafana/data'; import { escapeLabelValueInExactSelector, escapeLabelValueInRegexSelector, expandRecordingRules, fixSummariesMetadata, + getPrometheusTime, + getRangeSnapInterval, parseSelector, toPromLikeQuery, } from './language_utils'; +import { PrometheusCacheLevel } from './types'; describe('parseSelector()', () => { let parsed; @@ -222,6 +227,234 @@ describe('escapeLabelValueInRegexSelector()', () => { }); }); +describe('getRangeSnapInterval', () => { + it('will not change input if set to no cache', () => { + const intervalSeconds = 10 * 60; // 10 minutes + const now = new Date().valueOf(); + + const expectedFrom = dateTime(now - intervalSeconds * 1000); + const expectedTo = dateTime(now); + + const range: TimeRange = { + from: expectedFrom, + to: expectedTo, + } as TimeRange; + + expect(getRangeSnapInterval(PrometheusCacheLevel.None, range)).toEqual({ + start: getPrometheusTime(expectedFrom, false).toString(), + end: getPrometheusTime(expectedTo, true).toString(), + }); + }); + + it('will snap range to closest minute', () => { + const queryDurationMinutes = 10; + const intervalSeconds = queryDurationMinutes * 60; // 10 minutes + const now = new Date().valueOf(); + const nowPlusOneMinute = now + 1000 * 60; + const nowPlusTwoMinute = now + 1000 * 60 * 2; + + const nowTime = dateTime(now) as Moment; + + const expectedFrom = nowTime.clone().startOf('minute').subtract(queryDurationMinutes, 'minute'); + const expectedTo = nowTime.clone().startOf('minute').add(1, 'minute'); + + const range: TimeRange = { + from: dateTime(now - intervalSeconds * 1000), + to: dateTime(now), + } as TimeRange; + + const range2: TimeRange = { + from: dateTime(nowPlusOneMinute - intervalSeconds * 1000), + to: dateTime(nowPlusOneMinute), + raw: { + from: dateTime(nowPlusOneMinute - intervalSeconds * 1000), + to: dateTime(nowPlusOneMinute), + }, + }; + const range3: TimeRange = { + from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000), + to: dateTime(nowPlusTwoMinute), + raw: { + from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000), + to: dateTime(nowPlusTwoMinute), + }, + }; + + const first = getRangeSnapInterval(PrometheusCacheLevel.Low, range); + const second = getRangeSnapInterval(PrometheusCacheLevel.Low, range2); + const third = getRangeSnapInterval(PrometheusCacheLevel.Low, range3); + + expect(first).toEqual({ + start: getPrometheusTime(expectedFrom as DateTime, false).toString(10), + end: getPrometheusTime(expectedTo as DateTime, false).toString(10), + }); + + expect(second).toEqual({ + start: getPrometheusTime(expectedFrom.clone().add(1, 'minute') as DateTime, false).toString(10), + end: getPrometheusTime(expectedTo.clone().add(1, 'minute') as DateTime, false).toString(10), + }); + + expect(third).toEqual({ + start: getPrometheusTime(expectedFrom.clone().add(2, 'minute') as DateTime, false).toString(10), + end: getPrometheusTime(expectedTo.clone().add(2, 'minute') as DateTime, false).toString(10), + }); + }); + + it('will snap range to closest 10 minute', () => { + const queryDurationMinutes = 60; + const intervalSeconds = queryDurationMinutes * 60; // 10 minutes + const now = new Date().valueOf(); + const nowPlusOneMinute = now + 1000 * 60; + const nowPlusTwoMinute = now + 1000 * 60 * 2; + + const nowTime = dateTime(now) as Moment; + const nowTimePlusOne = dateTime(nowPlusOneMinute) as Moment; + const nowTimePlusTwo = dateTime(nowPlusTwoMinute) as Moment; + + const calculateClosest10 = (date: Moment): Moment => { + const numberOfMinutes = Math.floor(date.minutes() / 10) * 10; + const numberOfHours = numberOfMinutes < 60 ? date.hours() : date.hours() + 1; + return date + .clone() + .minutes(numberOfMinutes % 60) + .hours(numberOfHours); + }; + + const expectedFromFirst = calculateClosest10( + nowTime.clone().startOf('minute').subtract(queryDurationMinutes, 'minute') + ); + const expectedToFirst = calculateClosest10(nowTime.clone().startOf('minute').add(1, 'minute')); + + const expectedFromSecond = calculateClosest10( + nowTimePlusOne.clone().startOf('minute').subtract(queryDurationMinutes, 'minute') + ); + const expectedToSecond = calculateClosest10(nowTimePlusOne.clone().startOf('minute').add(1, 'minute')); + + const expectedFromThird = calculateClosest10( + nowTimePlusTwo.clone().startOf('minute').subtract(queryDurationMinutes, 'minute') + ); + const expectedToThird = calculateClosest10(nowTimePlusTwo.clone().startOf('minute').add(1, 'minute')); + + const range: TimeRange = { + from: dateTime(now - intervalSeconds * 1000), + to: dateTime(now), + } as TimeRange; + + const range2: TimeRange = { + from: dateTime(nowPlusOneMinute - intervalSeconds * 1000), + to: dateTime(nowPlusOneMinute), + raw: { + from: dateTime(nowPlusOneMinute - intervalSeconds * 1000), + to: dateTime(nowPlusOneMinute), + }, + }; + const range3: TimeRange = { + from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000), + to: dateTime(nowPlusTwoMinute), + raw: { + from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000), + to: dateTime(nowPlusTwoMinute), + }, + }; + + const first = getRangeSnapInterval(PrometheusCacheLevel.Medium, range); + const second = getRangeSnapInterval(PrometheusCacheLevel.Medium, range2); + const third = getRangeSnapInterval(PrometheusCacheLevel.Medium, range3); + + expect(first).toEqual({ + start: getPrometheusTime(expectedFromFirst as DateTime, false).toString(10), + end: getPrometheusTime(expectedToFirst as DateTime, false).toString(10), + }); + + expect(second).toEqual({ + start: getPrometheusTime(expectedFromSecond.clone() as DateTime, false).toString(10), + end: getPrometheusTime(expectedToSecond.clone() as DateTime, false).toString(10), + }); + + expect(third).toEqual({ + start: getPrometheusTime(expectedFromThird.clone() as DateTime, false).toString(10), + end: getPrometheusTime(expectedToThird.clone() as DateTime, false).toString(10), + }); + }); + + it('will snap range to closest 60 minute', () => { + const queryDurationMinutes = 120; + const intervalSeconds = queryDurationMinutes * 60; + const now = new Date().valueOf(); + const nowPlusOneMinute = now + 1000 * 60; + const nowPlusTwoMinute = now + 1000 * 60 * 2; + + const nowTime = dateTime(now) as Moment; + const nowTimePlusOne = dateTime(nowPlusOneMinute) as Moment; + const nowTimePlusTwo = dateTime(nowPlusTwoMinute) as Moment; + + const calculateClosest60 = (date: Moment): Moment => { + const numberOfMinutes = Math.floor(date.minutes() / 60) * 60; + const numberOfHours = numberOfMinutes < 60 ? date.hours() : date.hours() + 1; + return date + .clone() + .minutes(numberOfMinutes % 60) + .hours(numberOfHours); + }; + + const expectedFromFirst = calculateClosest60( + nowTime.clone().startOf('minute').subtract(queryDurationMinutes, 'minute') + ); + const expectedToFirst = calculateClosest60(nowTime.clone().startOf('minute').add(1, 'minute')); + + const expectedFromSecond = calculateClosest60( + nowTimePlusOne.clone().startOf('minute').subtract(queryDurationMinutes, 'minute') + ); + const expectedToSecond = calculateClosest60(nowTimePlusOne.clone().startOf('minute').add(1, 'minute')); + + const expectedFromThird = calculateClosest60( + nowTimePlusTwo.clone().startOf('minute').subtract(queryDurationMinutes, 'minute') + ); + const expectedToThird = calculateClosest60(nowTimePlusTwo.clone().startOf('minute').add(1, 'minute')); + + const range: TimeRange = { + from: dateTime(now - intervalSeconds * 1000), + to: dateTime(now), + } as TimeRange; + + const range2: TimeRange = { + from: dateTime(nowPlusOneMinute - intervalSeconds * 1000), + to: dateTime(nowPlusOneMinute), + raw: { + from: dateTime(nowPlusOneMinute - intervalSeconds * 1000), + to: dateTime(nowPlusOneMinute), + }, + }; + const range3: TimeRange = { + from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000), + to: dateTime(nowPlusTwoMinute), + raw: { + from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000), + to: dateTime(nowPlusTwoMinute), + }, + }; + + const first = getRangeSnapInterval(PrometheusCacheLevel.High, range); + const second = getRangeSnapInterval(PrometheusCacheLevel.High, range2); + const third = getRangeSnapInterval(PrometheusCacheLevel.High, range3); + + expect(first).toEqual({ + start: getPrometheusTime(expectedFromFirst as DateTime, false).toString(10), + end: getPrometheusTime(expectedToFirst as DateTime, false).toString(10), + }); + + expect(second).toEqual({ + start: getPrometheusTime(expectedFromSecond.clone() as DateTime, false).toString(10), + end: getPrometheusTime(expectedToSecond.clone() as DateTime, false).toString(10), + }); + + expect(third).toEqual({ + start: getPrometheusTime(expectedFromThird.clone() as DateTime, false).toString(10), + end: getPrometheusTime(expectedToThird.clone() as DateTime, false).toString(10), + }); + }); +}); + describe('toPromLikeQuery', () => { it('export abstract query to PromQL-like query', () => { const abstractQuery: AbstractQuery = { diff --git a/public/app/plugins/datasource/prometheus/language_utils.ts b/public/app/plugins/datasource/prometheus/language_utils.ts index 978866163f7..ba3a4f54fc1 100644 --- a/public/app/plugins/datasource/prometheus/language_utils.ts +++ b/public/app/plugins/datasource/prometheus/language_utils.ts @@ -1,11 +1,20 @@ import { invert } from 'lodash'; import { Token } from 'prismjs'; -import { DataQuery, AbstractQuery, AbstractLabelOperator, AbstractLabelMatcher } from '@grafana/data'; +import { + AbstractLabelMatcher, + AbstractLabelOperator, + AbstractQuery, + DataQuery, + dateMath, + DateTime, + incrRoundDn, + TimeRange, +} from '@grafana/data'; import { addLabelToQuery } from './add_label_to_query'; import { SUGGESTIONS_LIMIT } from './language_provider'; -import { PromMetricsMetadata, PromMetricsMetadataItem } from './types'; +import { PrometheusCacheLevel, PromMetricsMetadata, PromMetricsMetadataItem } from './types'; export const processHistogramMetrics = (metrics: string[]) => { const resultSet: Set = new Set(); @@ -55,6 +64,7 @@ export function processLabels(labels: Array<{ [key: string]: string }>, withName // const cleanSelectorRegexp = /\{(\w+="[^"\n]*?")(,\w+="[^"\n]*?")*\}/; export const selectorRegexp = /\{[^}]*?(\}|$)/; export const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g; + export function parseSelector(query: string, cursorOffset = 1): { labelKeys: any[]; selector: string } { if (!query.match(selectorRegexp)) { // Special matcher for metrics @@ -231,6 +241,11 @@ export function roundSecToMin(seconds: number): number { return Math.floor(seconds / 60); } +// Returns number of minutes rounded up to the nearest nth minute +export function roundSecToNextMin(seconds: number, secondsToRound = 1): number { + return Math.ceil(seconds / 60) - (Math.ceil(seconds / 60) % secondsToRound); +} + export function limitSuggestions(items: string[]) { return items.slice(0, SUGGESTIONS_LIMIT); } @@ -248,6 +263,7 @@ export function addLimitInfo(items: any[] | undefined): string { // the list of metacharacters is: *+?()|\.[]{}^$ // we make a javascript regular expression that matches those characters: const RE2_METACHARACTERS = /[*+?()|\\.\[\]{}^$]/g; + function escapePrometheusRegexp(value: string): string { return value.replace(RE2_METACHARACTERS, '\\$&'); } @@ -344,3 +360,59 @@ export function extractLabelMatchers(tokens: Array): AbstractLab return labelMatchers; } + +/** + * Calculates new interval "snapped" to the closest Nth minute, depending on cache level datasource setting + * @param cacheLevel + * @param range + */ +export function getRangeSnapInterval( + cacheLevel: PrometheusCacheLevel, + range: TimeRange +): { start: string; end: string } { + // Don't round the range if we're not caching + if (cacheLevel === PrometheusCacheLevel.None) { + return { + start: getPrometheusTime(range.from, false).toString(), + end: getPrometheusTime(range.to, true).toString(), + }; + } + // Otherwise round down to the nearest nth minute for the start time + const startTime = getPrometheusTime(range.from, false); + // const startTimeQuantizedSeconds = roundSecToLastMin(startTime, getClientCacheDurationInMinutes(cacheLevel)) * 60; + const startTimeQuantizedSeconds = incrRoundDn(startTime, getClientCacheDurationInMinutes(cacheLevel) * 60); + + // And round up to the nearest nth minute for the end time + const endTime = getPrometheusTime(range.to, true); + const endTimeQuantizedSeconds = roundSecToNextMin(endTime, getClientCacheDurationInMinutes(cacheLevel)) * 60; + + // If the interval was too short, we could have rounded both start and end to the same time, if so let's add one step to the end + if (startTimeQuantizedSeconds === endTimeQuantizedSeconds) { + const endTimePlusOneStep = endTimeQuantizedSeconds + getClientCacheDurationInMinutes(cacheLevel) * 60; + return { start: startTimeQuantizedSeconds.toString(), end: endTimePlusOneStep.toString() }; + } + + const start = startTimeQuantizedSeconds.toString(); + const end = endTimeQuantizedSeconds.toString(); + + return { start, end }; +} + +export function getClientCacheDurationInMinutes(cacheLevel: PrometheusCacheLevel) { + switch (cacheLevel) { + case PrometheusCacheLevel.Medium: + return 10; + case PrometheusCacheLevel.High: + return 60; + default: + return 1; + } +} + +export function getPrometheusTime(date: string | DateTime, roundUp: boolean) { + if (typeof date === 'string') { + date = dateMath.parse(date, roundUp)!; + } + + return Math.ceil(date.valueOf() / 1000); +} diff --git a/public/app/plugins/datasource/prometheus/metric_find_query.ts b/public/app/plugins/datasource/prometheus/metric_find_query.ts index 2c6bea204c8..7ce5f1f50d8 100644 --- a/public/app/plugins/datasource/prometheus/metric_find_query.ts +++ b/public/app/plugins/datasource/prometheus/metric_find_query.ts @@ -6,6 +6,7 @@ import { MetricFindValue, TimeRange } from '@grafana/data'; import { getTimeSrv } from 'app/features/dashboard/services/TimeSrv'; import { PrometheusDatasource } from './datasource'; +import { getPrometheusTime } from './language_utils'; import { PromQueryRequest } from './types'; export default class PrometheusMetricFindQuery { @@ -56,8 +57,8 @@ export default class PrometheusMetricFindQuery { } labelValuesQuery(label: string, metric?: string) { - const start = this.datasource.getPrometheusTime(this.range.from, false); - const end = this.datasource.getPrometheusTime(this.range.to, true); + const start = getPrometheusTime(this.range.from, false); + const end = getPrometheusTime(this.range.to, true); const params = { ...(metric && { 'match[]': metric }), start: start.toString(), end: end.toString() }; if (!metric || this.datasource.hasLabelsMatchAPISupport()) { @@ -89,8 +90,8 @@ export default class PrometheusMetricFindQuery { } metricNameQuery(metricFilterPattern: string) { - const start = this.datasource.getPrometheusTime(this.range.from, false); - const end = this.datasource.getPrometheusTime(this.range.to, true); + const start = getPrometheusTime(this.range.from, false); + const end = getPrometheusTime(this.range.to, true); const params = { start: start.toString(), end: end.toString(), @@ -114,7 +115,7 @@ export default class PrometheusMetricFindQuery { } queryResultQuery(query: string) { - const end = this.datasource.getPrometheusTime(this.range.to, true); + const end = getPrometheusTime(this.range.to, true); const instantQuery: PromQueryRequest = { expr: query } as PromQueryRequest; return this.datasource.performInstantQuery(instantQuery, end).pipe( map((result) => { @@ -152,8 +153,8 @@ export default class PrometheusMetricFindQuery { } metricNameAndLabelsQuery(query: string): Promise { - const start = this.datasource.getPrometheusTime(this.range.from, false); - const end = this.datasource.getPrometheusTime(this.range.to, true); + const start = getPrometheusTime(this.range.from, false); + const end = getPrometheusTime(this.range.to, true); const params = { 'match[]': query, start: start.toString(), diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilterItem.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilterItem.tsx index d5e31f675e3..645339f8fea 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilterItem.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilterItem.tsx @@ -20,6 +20,7 @@ export interface Props { invalidLabel?: boolean; invalidValue?: boolean; getLabelValuesAutofillSuggestions: (query: string, labelName?: string) => Promise; + debounceDuration: number; } export function LabelFilterItem({ @@ -32,6 +33,7 @@ export function LabelFilterItem({ invalidLabel, invalidValue, getLabelValuesAutofillSuggestions, + debounceDuration, }: Props) { const [state, setState] = useState<{ labelNames?: SelectableValue[]; @@ -54,7 +56,10 @@ export function LabelFilterItem({ return []; }; - const labelValueSearch = debounce((query: string) => getLabelValuesAutofillSuggestions(query, item.label), 350); + const labelValueSearch = debounce( + (query: string) => getLabelValuesAutofillSuggestions(query, item.label), + debounceDuration + ); return (
diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilters.test.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilters.test.tsx index 6cf9ec1d16a..4248b73ed46 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilters.test.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilters.test.tsx @@ -5,7 +5,7 @@ import { selectOptionInTest } from 'test/helpers/selectOptionInTest'; import { getLabelSelects } from '../testUtils'; -import { LabelFilters, MISSING_LABEL_FILTER_ERROR_MESSAGE } from './LabelFilters'; +import { LabelFilters, MISSING_LABEL_FILTER_ERROR_MESSAGE, Props } from './LabelFilters'; describe('LabelFilters', () => { it('renders empty input without labels', async () => { @@ -81,6 +81,7 @@ describe('LabelFilters', () => { getLabelValuesAutofillSuggestions={jest.fn()} onGetLabelValues={jest.fn()} labelsFilters={[]} + debounceDuration={300} /> ); expect(screen.getAllByText('Select label')).toHaveLength(1); @@ -101,9 +102,8 @@ describe('LabelFilters', () => { }); function setup(propOverrides?: Partial>) { - const defaultProps = { + const defaultProps: Props = { onChange: jest.fn(), - getLabelValues: jest.fn(), getLabelValuesAutofillSuggestions: async (query: string, labelName?: string) => [ { label: 'bar', value: 'bar' }, { label: 'qux', value: 'qux' }, @@ -119,6 +119,7 @@ function setup(propOverrides?: Partial>) { { label: 'qux', value: 'qux' }, { label: 'quux', value: 'quux' }, ], + debounceDuration: 300, labelsFilters: [], }; diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilters.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilters.tsx index bc40754de99..195b2367fea 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilters.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilters.tsx @@ -18,6 +18,7 @@ export interface Props { /** If set to true, component will show error message until at least 1 filter is selected */ labelFilterRequired?: boolean; getLabelValuesAutofillSuggestions: (query: string, labelName?: string) => Promise; + debounceDuration: number; } export function LabelFilters({ @@ -27,6 +28,7 @@ export function LabelFilters({ onGetLabelValues, labelFilterRequired, getLabelValuesAutofillSuggestions, + debounceDuration, }: Props) { const defaultOp = '='; const [items, setItems] = useState>>([{ op: defaultOp }]); @@ -63,6 +65,7 @@ export function LabelFilters({ onChange={onLabelsChange} renderItem={(item: Partial, onChangeItem, onDelete) => ( { setMetrics(metrics); setFilteredMetricCount(metrics.length); setIsLoading(false); - }, 300), + }, datasource.getDebounceTimeInMilliseconds()), [datasource, query.labels] ); diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx index 43fe9936bb8..ced806d0c77 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx @@ -114,7 +114,10 @@ export function MetricSelect({ datasource, query, onChange, onGetMetrics, labels }); }; - const debouncedSearch = debounce((query: string) => getMetricLabels(query), 300); + const debouncedSearch = debounce( + (query: string) => getMetricLabels(query), + datasource.getDebounceTimeInMilliseconds() + ); return ( diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx index 868f7059e0d..894f9d97298 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx @@ -255,6 +255,7 @@ export const PromQueryBuilder = React.memo((props) => { /> )}