From 2a714601a71883a51314b8329f5291f72e3059b3 Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Thu, 20 Jun 2024 15:04:22 +0200 Subject: [PATCH] Chore: Remove prometheusIncrementalQueryInstrumentation feature toggle (#89463) * remove prometheusIncrementalQueryInstrumentation feature toggle * remove profile method * remove more about profiling --- .../feature-toggles/index.md | 1 - .../src/types/featureToggles.gen.ts | 1 - packages/grafana-prometheus/src/datasource.ts | 9 - .../src/querycache/QueryCache.test.ts | 15 +- .../src/querycache/QueryCache.ts | 187 ------------------ pkg/services/featuremgmt/registry.go | 7 - pkg/services/featuremgmt/toggles_gen.csv | 1 - pkg/services/featuremgmt/toggles_gen.go | 4 - pkg/services/featuremgmt/toggles_gen.json | 3 +- 9 files changed, 3 insertions(+), 225 deletions(-) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index b2413585105..5f6237ff724 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -138,7 +138,6 @@ Experimental features might be changed or removed without prior notice. | `pluginsFrontendSandbox` | Enables the plugins frontend sandbox | | `frontendSandboxMonitorOnly` | Enables monitor only in the plugin frontend sandbox (if enabled) | | `vizAndWidgetSplit` | Split panels between visualizations and widgets | -| `prometheusIncrementalQueryInstrumentation` | Adds RudderStack events to incremental queries | | `awsDatasourcesTempCredentials` | Support temporary security credentials in AWS plugins for Grafana Cloud customers | | `mlExpressions` | Enable support for Machine Learning in server-side expressions | | `metricsSummary` | Enables metrics summary queries in the Tempo data source | diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index a7b45c16968..8af56fc11c3 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -82,7 +82,6 @@ export interface FeatureToggles { sqlDatasourceDatabaseSelection?: boolean; recordedQueriesMulti?: boolean; vizAndWidgetSplit?: boolean; - prometheusIncrementalQueryInstrumentation?: boolean; logsExploreTableVisualisation?: boolean; awsDatasourcesTempCredentials?: boolean; transformationsRedesign?: boolean; diff --git a/packages/grafana-prometheus/src/datasource.ts b/packages/grafana-prometheus/src/datasource.ts index 34688f47465..3296a8da674 100644 --- a/packages/grafana-prometheus/src/datasource.ts +++ b/packages/grafana-prometheus/src/datasource.ts @@ -139,7 +139,6 @@ export class PrometheusDatasource this.cache = new QueryCache({ getTargetSignature: this.getPrometheusTargetSignature.bind(this), overlapString: instanceSettings.jsonData.incrementalQueryOverlapWindow ?? defaultPrometheusQueryOverlapWindow, - profileFunction: this.getPrometheusProfileData.bind(this), }); // This needs to be here and cannot be static because of how annotations typing affects casting of data source @@ -162,14 +161,6 @@ export class PrometheusDatasource return query.expr; } - getPrometheusProfileData(request: DataQueryRequest, targ: PromQuery) { - return { - interval: targ.interval ?? request.interval, - expr: this.interpolateString(targ.expr), - datasource: 'Prometheus', - }; - } - /** * Get target signature for query caching * @param request diff --git a/packages/grafana-prometheus/src/querycache/QueryCache.test.ts b/packages/grafana-prometheus/src/querycache/QueryCache.test.ts index 78ff585ec20..9efe6a73288 100644 --- a/packages/grafana-prometheus/src/querycache/QueryCache.test.ts +++ b/packages/grafana-prometheus/src/querycache/QueryCache.test.ts @@ -6,7 +6,7 @@ import { DataFrame, DataQueryRequest, DateTime, dateTime, TimeRange } from '@gra import { QueryEditorMode } from '../querybuilder/shared/types'; import { PromQuery } from '../types'; -import { DatasourceProfileData, QueryCache } from './QueryCache'; +import { QueryCache } from './QueryCache'; import { IncrementalStorageDataFrameScenarios } from './QueryCacheTestData'; // Will not interpolate vars! @@ -60,14 +60,6 @@ const mockPromRequest = (request?: Partial>): DataQu }; }; -const getPromProfileData = (request: DataQueryRequest, targ: PromQuery): DatasourceProfileData => { - return { - expr: targ.expr, - interval: targ.interval ?? request.interval, - datasource: 'prom', - }; -}; - describe('QueryCache: Generic', function () { it('instantiates', () => { const storage = new QueryCache({ @@ -192,7 +184,6 @@ describe('QueryCache: Prometheus', function () { const storage = new QueryCache({ getTargetSignature: getPrometheusTargetSignature, overlapString: '10m', - profileFunction: getPromProfileData, }); const firstFrames = scenario.first.dataFrames as unknown as DataFrame[]; const secondFrames = scenario.second.dataFrames as unknown as DataFrame[]; @@ -328,7 +319,6 @@ describe('QueryCache: Prometheus', function () { const storage = new QueryCache({ getTargetSignature: getPrometheusTargetSignature, overlapString: '10m', - profileFunction: getPromProfileData, }); // Initial request with all data for time range @@ -489,7 +479,6 @@ describe('QueryCache: Prometheus', function () { const storage = new QueryCache({ getTargetSignature: getPrometheusTargetSignature, overlapString: '10m', - profileFunction: getPromProfileData, }); const cacheRequest = storage.requestInfo(request); expect(cacheRequest.requests[0]).toBe(request); @@ -501,7 +490,6 @@ describe('QueryCache: Prometheus', function () { const storage = new QueryCache({ getTargetSignature: getPrometheusTargetSignature, overlapString: '10m', - profileFunction: getPromProfileData, }); const cacheRequest = storage.requestInfo(request); expect(cacheRequest.requests[0]).toBe(request); @@ -513,7 +501,6 @@ describe('QueryCache: Prometheus', function () { const storage = new QueryCache({ getTargetSignature: getPrometheusTargetSignature, overlapString: '10m', - profileFunction: getPromProfileData, }); const cacheRequest = storage.requestInfo(request); expect(cacheRequest.requests[0]).toBe(request); diff --git a/packages/grafana-prometheus/src/querycache/QueryCache.ts b/packages/grafana-prometheus/src/querycache/QueryCache.ts index 05ad2c3f1d3..f4b955b82a6 100644 --- a/packages/grafana-prometheus/src/querycache/QueryCache.ts +++ b/packages/grafana-prometheus/src/querycache/QueryCache.ts @@ -9,8 +9,6 @@ import { isValidDuration, parseDuration, } from '@grafana/data'; -import { faro } from '@grafana/faro-web-sdk'; -import { config, reportInteraction } from '@grafana/runtime'; import { amendTable, Table, trimTable } from '../gcopypaste/app/features/live/data/amendTimeSeries'; import { PromQuery } from '../types'; @@ -19,8 +17,6 @@ import { PromQuery } from '../types'; // (must be stable across query changes, time range changes / interval changes / panel resizes / template variable changes) type TargetIdent = string; -type RequestID = string; - // query + template variables + interval + raw time range // used for full target cache busting -> full range re-query type TargetSig = string; @@ -44,22 +40,6 @@ export interface CacheRequestInfo { shouldCache: boolean; } -export interface DatasourceProfileData { - interval?: string; - expr: string; - datasource: string; -} - -interface ProfileData extends DatasourceProfileData { - identity: string; - bytes: number | null; - dashboardUID: string; - panelId?: number; - from: string; - queryRangeSeconds: number; - refreshIntervalMs: number; -} - /** * Get field identity * This is the string used to uniquely identify a field within a "target" @@ -76,40 +56,12 @@ export const getFieldIdent = (field: Field) => `${field.type}|${field.name}|${JS export class QueryCache { private overlapWindowMs: number; private getTargetSignature: (request: DataQueryRequest, target: T) => string; - private getProfileData?: (request: DataQueryRequest, target: T) => DatasourceProfileData; - - private perfObeserver?: PerformanceObserver; - private shouldProfile: boolean; - - // send profile events every 10 minutes - sendEventsInterval = 60000 * 10; - - pendingRequestIdsToTargSigs = new Map(); - - pendingAccumulatedEvents = new Map< - string, - { - requestCount: number; - savedBytesTotal: number; - initialRequestSize: number; - lastRequestSize: number; - panelId: string; - dashId: string; - expr: string; - refreshIntervalMs: number; - sent: boolean; - datasource: string; - from: string; - queryRangeSeconds: number; - } - >(); cache = new Map(); constructor(options: { getTargetSignature: (request: DataQueryRequest, target: T) => string; overlapString: string; - profileFunction?: (request: DataQueryRequest, target: T) => DatasourceProfileData; }) { const unverifiedOverlap = options.overlapString; if (isValidDuration(unverifiedOverlap)) { @@ -120,132 +72,9 @@ export class QueryCache { this.overlapWindowMs = durationToMilliseconds(duration); } - if ( - (config.grafanaJavascriptAgent.enabled || config.featureToggles?.prometheusIncrementalQueryInstrumentation) && - options.profileFunction !== undefined - ) { - this.profile(); - this.shouldProfile = true; - } else { - this.shouldProfile = false; - } - this.getProfileData = options.profileFunction; this.getTargetSignature = options.getTargetSignature; } - private profile() { - // Check if PerformanceObserver is supported, and if we have Faro enabled for internal profiling - if (typeof PerformanceObserver === 'function') { - this.perfObeserver = new PerformanceObserver((list: PerformanceObserverEntryList) => { - list.getEntries().forEach((entry) => { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const entryTypeCast: PerformanceResourceTiming = entry as PerformanceResourceTiming; - - // Safari support for this is coming in 16.4: - // https://caniuse.com/mdn-api_performanceresourcetiming_transfersize - // Gating that this exists to prevent runtime errors - const isSupported = typeof entryTypeCast?.transferSize === 'number'; - - if (entryTypeCast?.initiatorType === 'fetch' && isSupported) { - let fetchUrl = entryTypeCast.name; - - if (fetchUrl.includes('/api/ds/query')) { - let match = fetchUrl.match(/requestId=([a-z\d]+)/i); - - if (match) { - let requestId = match[1]; - - const requestTransferSize = Math.round(entryTypeCast.transferSize); - const currentRequest = this.pendingRequestIdsToTargSigs.get(requestId); - - if (currentRequest) { - const entries = this.pendingRequestIdsToTargSigs.entries(); - - for (let [, value] of entries) { - if (value.identity === currentRequest.identity && value.bytes !== null) { - const previous = this.pendingAccumulatedEvents.get(value.identity); - - const savedBytes = value.bytes - requestTransferSize; - - this.pendingAccumulatedEvents.set(value.identity, { - datasource: value.datasource ?? 'N/A', - requestCount: (previous?.requestCount ?? 0) + 1, - savedBytesTotal: (previous?.savedBytesTotal ?? 0) + savedBytes, - initialRequestSize: value.bytes, - lastRequestSize: requestTransferSize, - panelId: currentRequest.panelId?.toString() ?? '', - dashId: currentRequest.dashboardUID ?? '', - expr: currentRequest.expr ?? '', - refreshIntervalMs: currentRequest.refreshIntervalMs ?? 0, - sent: false, - from: currentRequest.from ?? '', - queryRangeSeconds: currentRequest.queryRangeSeconds ?? 0, - }); - - // We don't need to save each subsequent request, only the first one - this.pendingRequestIdsToTargSigs.delete(requestId); - - return; - } - } - - // If we didn't return above, this should be the first request, let's save the observed size - this.pendingRequestIdsToTargSigs.set(requestId, { ...currentRequest, bytes: requestTransferSize }); - } - } - } - } - }); - }); - - this.perfObeserver.observe({ type: 'resource', buffered: false }); - - setInterval(this.sendPendingTrackingEvents, this.sendEventsInterval); - - // Send any pending profile information when the user navigates away - window.addEventListener('beforeunload', this.sendPendingTrackingEvents); - } - } - - sendPendingTrackingEvents = () => { - const entries = this.pendingAccumulatedEvents.entries(); - - for (let [key, value] of entries) { - if (!value.sent) { - const event = { - datasource: value.datasource.toString(), - requestCount: value.requestCount.toString(), - savedBytesTotal: value.savedBytesTotal.toString(), - initialRequestSize: value.initialRequestSize.toString(), - lastRequestSize: value.lastRequestSize.toString(), - panelId: value.panelId.toString(), - dashId: value.dashId.toString(), - expr: value.expr.toString(), - refreshIntervalMs: value.refreshIntervalMs.toString(), - from: value.from.toString(), - queryRangeSeconds: value.queryRangeSeconds.toString(), - }; - - if (config.featureToggles.prometheusIncrementalQueryInstrumentation) { - reportInteraction('grafana_incremental_queries_profile', event); - } else if (faro.api.pushEvent) { - faro.api.pushEvent('incremental query response size', event, 'no-interaction', { - skipDedupe: true, - }); - } - - this.pendingAccumulatedEvents.set(key, { - ...value, - sent: true, - requestCount: 0, - savedBytesTotal: 0, - initialRequestSize: 0, - lastRequestSize: 0, - }); - } - } - }; - // can be used to change full range request to partial, split into multiple requests requestInfo(request: DataQueryRequest): CacheRequestInfo { // TODO: align from/to to interval to increase probability of hitting backend cache @@ -260,27 +89,11 @@ export class QueryCache { let doPartialQuery = shouldCache; let prevTo: TimestampMs | undefined = undefined; - const refreshIntervalMs = request.intervalMs; - // pre-compute reqTargSigs const reqTargSigs = new Map(); request.targets.forEach((targ) => { let targIdent = `${request.dashboardUID}|${request.panelId}|${targ.refId}`; let targSig = this.getTargetSignature(request, targ); // ${request.maxDataPoints} ? - - if (this.shouldProfile && this.getProfileData) { - this.pendingRequestIdsToTargSigs.set(request.requestId, { - ...this.getProfileData(request, targ), - identity: targIdent + '|' + targSig, - bytes: null, - panelId: request.panelId, - dashboardUID: request.dashboardUID ?? '', - from: request.rangeRaw?.from.toString() ?? '', - queryRangeSeconds: request.range.to.diff(request.range.from, 'seconds') ?? '', - refreshIntervalMs: refreshIntervalMs ?? 0, - }); - } - reqTargSigs.set(targIdent, targSig); }); diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 9117d55c666..3d763445ea2 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -480,13 +480,6 @@ var ( FrontendOnly: true, Owner: grafanaDashboardsSquad, }, - { - Name: "prometheusIncrementalQueryInstrumentation", - Description: "Adds RudderStack events to incremental queries", - FrontendOnly: true, - Stage: FeatureStageExperimental, - Owner: grafanaObservabilityMetricsSquad, - }, { Name: "logsExploreTableVisualisation", Description: "A table visualisation for logs in Explore", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index da88637ca63..c0cf306fe54 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -63,7 +63,6 @@ frontendSandboxMonitorOnly,experimental,@grafana/plugins-platform-backend,false, sqlDatasourceDatabaseSelection,preview,@grafana/dataviz-squad,false,false,true recordedQueriesMulti,GA,@grafana/observability-metrics,false,false,false vizAndWidgetSplit,experimental,@grafana/dashboards-squad,false,false,true -prometheusIncrementalQueryInstrumentation,experimental,@grafana/observability-metrics,false,false,true logsExploreTableVisualisation,GA,@grafana/observability-logs,false,false,true awsDatasourcesTempCredentials,experimental,@grafana/aws-datasources,false,false,false transformationsRedesign,GA,@grafana/observability-metrics,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index b56aa2c1e46..01c91f4b044 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -263,10 +263,6 @@ const ( // Split panels between visualizations and widgets FlagVizAndWidgetSplit = "vizAndWidgetSplit" - // FlagPrometheusIncrementalQueryInstrumentation - // Adds RudderStack events to incremental queries - FlagPrometheusIncrementalQueryInstrumentation = "prometheusIncrementalQueryInstrumentation" - // FlagLogsExploreTableVisualisation // A table visualisation for logs in Explore FlagLogsExploreTableVisualisation = "logsExploreTableVisualisation" diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index e26f18d188b..4650fac0999 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -1832,7 +1832,8 @@ "metadata": { "name": "prometheusIncrementalQueryInstrumentation", "resourceVersion": "1718727528075", - "creationTimestamp": "2023-07-05T19:39:49Z" + "creationTimestamp": "2023-07-05T19:39:49Z", + "deletionTimestamp": "2024-06-20T11:30:37Z" }, "spec": { "description": "Adds RudderStack events to incremental queries",